Make nbin bundling work and improve watching

Author: code-asher

package.json

@@ -9,6 +9,7 @@
   "scripts": {
     "postinstall": "yarn build:rules",
     "build": "yarn task build",
+    "bundle": "yarn task bundle",
     "start": "npm-run-all --parallel watch build:run",
     "watch": "yarn task build true",
     "build:run": "cd ./out && node ./packages/server/src/cli # TODO: restart on change",

packages/server/src/bootstrap.ts

@@ -1,11 +1,33 @@
-import * as tsConfig from "../../../tsconfig.json";
+import * as path from "path";
+
+const rootPath = path.resolve(__dirname, "../../../..");
+
+// tslint:disable-next-line no-any
+if ((process.versions as any).nbin) {
+	require("nbin").shimNativeFs(rootPath);
+}
+
 import * as tsConfigPaths from "tsconfig-paths";
 
 // Prevent vscode from trying to load original-fs from electron.
 delete process.env.ELECTRON_RUN_AS_NODE;
 
 // This will advise Node so it can resolve paths based on our aliases.
 tsConfigPaths.register({
-	baseUrl: tsConfig.compilerOptions.baseUrl,
-	paths: tsConfig.compilerOptions.paths,
+	baseUrl: rootPath,
+	paths: {
+		"@coder/*": [
+			"./out/packages/*/src",
+		],
+		"vs/*": [
+			"./lib/vscode/out/vs/*",
+		],
+		electron: [
+			"./out/packages/server/src/modules/electron",
+		],
+	},
 });
+
+if (!process.env.IS_FORK) {
+	require("./cli");
+}

packages/server/src/cli.ts

@@ -8,7 +8,7 @@ import * as fse from "fs-extra";
 import * as os from "os";
 import * as path from "path";
 import * as WebSocket from "ws";
-import { buildDir, cacheHome, dataHome, isCli } from "./constants";
+import { libPath, tmpPath, cacheHome, dataHome } from "./constants";
 import { createApp } from "./server";
 import { forkModule } from "./vscode/bootstrapFork";
 import { SharedProcess, SharedProcessState } from "./vscode/sharedProcess";
@@ -41,11 +41,6 @@ commander.version(process.env.VERSION || "development")
 	.arguments("Specify working directory.")
 	.parse(process.argv);
 
-Error.stackTraceLimit = Infinity;
-if (isCli) {
-	require("nbin").shimNativeFs(buildDir);
-	require("nbin").shimNativeFs("/node_modules");
-}
 // Makes strings or numbers bold in stdout
 const bold = (text: string | number): string | number => {
 	return `\u001B[1m${text}\u001B[0m`;
@@ -86,11 +81,11 @@ const bold = (text: string | number): string | number => {
 
 	const dataDir = path.resolve(options.userDataDir || options.dataDir || path.join(dataHome, "code-server"));
 	const extensionsDir = options.extensionsDir ? path.resolve(options.extensionsDir) : path.resolve(dataDir, "extensions");
-	const builtInExtensionsDir = path.resolve(buildDir || path.join(__dirname, ".."), "build/extensions");
+	const builtInExtensionsDir = path.resolve(path.resolve(libPath, "extensions"));
 	const extraExtensionDirs = options.extraExtensionsDir ? options.extraExtensionsDir.map((p) => path.resolve(p)) : [];
 	const extraBuiltinExtensionDirs = options.extraBuiltinExtensionsDir ? options.extraBuiltinExtensionsDir.map((p) => path.resolve(p)) : [];
 	const workingDir = path.resolve(args[0] || process.cwd());
-	const dependenciesDir = path.join(os.tmpdir(), "code-server/dependencies");
+	const dependenciesDir = path.join(tmpPath, "dependencies");
 
 	if (!fs.existsSync(dataDir)) {
 		const oldDataDir = path.resolve(path.join(os.homedir(), ".code-server"));
@@ -110,21 +105,6 @@ const bold = (text: string | number): string | number => {
 		...extraBuiltinExtensionDirs.map((p) => fse.mkdirp(p)),
 	]);
 
-	// const unpackExecutable = (binaryName: string): void => {
-	// 	const memFile = path.join(isCli ? buildDir! : path.join(__dirname, ".."), "build/dependencies", binaryName);
-	// 	const diskFile = path.join(dependenciesDir, binaryName);
-	// 	if (!fse.existsSync(diskFile)) {
-	// 		fse.writeFileSync(diskFile, fse.readFileSync(memFile));
-	// 	}
-	// 	fse.chmodSync(diskFile, "755");
-	// };
-
-	// TODO: should this just be a dependency instead of bundled? If not, we'll
-	// still need to patch (maybe we can get it working without it though).
-	// unpackExecutable("rg");
-	// tslint:disable-next-line no-any
-	// (<any>global).RIPGREP_LOCATION = path.join(dependenciesDir, "rg");
-
 	if (!process.env.VSCODE_LOGS) {
 		process.env.VSCODE_LOGS = path.join(cacheHome, "code-server/logs", new Date().toISOString().replace(/[-:.TZ]/g, ""));
 	}

packages/server/src/constants.ts

@@ -1,8 +1,21 @@
 import * as path from "path";
 import * as os from "os";
 
-export const isCli = typeof process.env.CLI !== "undefined" && process.env.CLI !== "false";
-export const buildDir = process.env.BUILD_DIR ? path.resolve(process.env.BUILD_DIR) : "";
+/**
+ * Application root.
+ */
+export const rootPath = path.resolve(__dirname, "../../../..");
+
+/**
+ * Contains vscode and built-in extensions.
+ */
+export const libPath = path.join(rootPath, "lib");
+
+/**
+ * Place all temporary files here.
+ */
+export const tmpPath = path.join(os.tmpdir(), "code-server");
+
 const xdgResolve = (primary: string | undefined, fallback: string): string => {
 	return primary ? path.resolve(primary) : path.resolve(process.env.HOME || os.homedir(), fallback);
 };

packages/server/src/index.ts

@@ -18,7 +18,7 @@ import * as pem from "pem";
 import * as util from "util";
 import * as url from "url";
 import * as ws from "ws";
-import { buildDir } from "./constants";
+import { rootPath } from "./constants";
 import { createPortScanner } from "./portScanner";
 import safeCompare = require("safe-compare");
 
@@ -208,8 +208,7 @@ export const createApp = async (options: CreateAppOptions): Promise<{
 		return res.redirect(code, newUrlString);
 	};
 
-	const baseDir = buildDir || path.join(__dirname, "..");
-	const staticGzip = expressStaticGzip(path.join(baseDir, "build/web"));
+	const staticGzip = expressStaticGzip(path.join(rootPath, "out/packages/web/src"));
 
 	app.use((req, res, next) => {
 		logger.trace(`\u001B[1m${req.method} ${res.statusCode} \u001B[0m${req.originalUrl}`, field("host", req.hostname), field("ip", req.ip));

packages/server/src/server.ts

@@ -27,6 +27,7 @@ export const forkModule = (modulePath: string, args?: string[], options?: cp.For
 		env: {
 			...process.env, ...(options && options.env || {}),
 			AMD_ENTRYPOINT: modulePath,
+			IS_FORK: "true",
 		},
 	};
 

packages/server/src/vscode/bootstrapFork.ts

@@ -4,6 +4,7 @@ import { retry, withEnv } from "@coder/protocol";
 import { fork, ChildProcess } from "child_process";
 import * as os from "os";
 import * as path from "path";
+import { tmpPath } from "../constants";
 import { IpcHandler } from "../ipc";
 
 // tslint:disable-next-line completed-docs pretty obvious what this is
@@ -33,8 +34,8 @@ export type SharedProcessEvent = {
  */
 export class SharedProcess {
 	public readonly socketPath: string = os.platform() === "win32"
-		? path.join("\\\\?\\pipe", os.tmpdir(), `.code-server${Math.random().toString()}`)
-		: path.join(os.tmpdir(), `.code-server${Math.random().toString()}`);
+		? path.join("\\\\?\\pipe", tmpPath, `shared-process-${Math.random().toString()}`)
+		: path.join(tmpPath, `shared-process-${Math.random().toString()}`);
 	private _state: SharedProcessState = SharedProcessState.Stopped;
 	private activeProcess: ChildProcess | undefined;
 	private ipcHandler: IpcHandler | undefined;
@@ -100,9 +101,13 @@ export class SharedProcess {
 			this.activeProcess.kill();
 		}
 
-		const activeProcess = fork(path.join(__dirname, "shared-process"), [
-				"--user-data-dir", this.userDataDir,
-			], withEnv({ env: { VSCODE_ALLOW_IO: "true" } }),
+		const activeProcess = fork(
+			path.join(__dirname, "shared-process"),
+			[ "--user-data-dir", this.userDataDir ],
+			withEnv({ env: {
+				VSCODE_ALLOW_IO: "true",
+				IS_FORK: "true",
+			}}),
 		);
 		this.activeProcess = activeProcess;
 

packages/server/src/vscode/sharedProcess.ts

@@ -2,6 +2,7 @@ import { Binary } from "@coder/nbin";
 import { field } from "@coder/logger";
 import { register, run } from "@coder/runner";
 
+import * as cp from "child_process";
 import * as fs from "fs";
 import * as fse from "fs-extra";
 import * as https from "https";
@@ -11,8 +12,9 @@ import * as tar from "tar";
 
 import { platform } from "./platform";
 
-const libPath = path.join(__dirname, "../lib");
-const releasePath = path.resolve(__dirname, "../release");
+const rootPath = path.resolve(__dirname, "..");
+const libPath = path.join(rootPath, "lib");
+const releasePath = path.join(rootPath, "release");
 const target = `${platform()}-${os.arch()}`;
 const vscodeVersion = process.env.VSCODE_VERSION || "1.35.0";
 
@@ -30,25 +32,34 @@ register("build", async (runner, logger, shouldWatch: string) => {
 	const outPath = path.join(__dirname, "../out");
 	const compile = async (): Promise<void> => {
 		fse.removeSync(path.resolve(outPath));
-
-		runner.cwd = path.resolve(__dirname, "..");
-		const resp = await runner.execute(
-			"tsc",
-			["--project", "tsconfig.build.json"].concat(watch ? ["--watch"] : []),
-		);
-		if (resp.exitCode !== 0) {
-			throw new Error(`Failed to build: ${resp.stderr}`);
+		if (watch) {
+			const proc = cp.spawn("tsc", ["--project", "tsconfig.build.json", "--watch", "--preserveWatchOutput"], {
+				cwd: rootPath,
+			});
+			await new Promise((resolve, reject): void => {
+				proc.stdout.setEncoding("utf8");
+				proc.stdout.on("data", (data: string) => {
+					logger.info(data.split("\n").filter((l) => !!l).join("\n"));
+				});
+				proc.on("exit", resolve);
+				proc.on("error", reject);
+			});
+		} else {
+			runner.cwd = rootPath;
+			const resp = await runner.execute("tsc", ["--project", "tsconfig.build.json"]);
+			if (resp.exitCode !== 0) {
+				throw new Error(`Failed to build: ${resp.stderr}`);
+			}
 		}
 	};
 
 	const copy = async (): Promise<void> => {
 		// TODO: If watching, copy every time they change.
 		await Promise.all([
 			"packages/protocol/src/proto",
-			["tsconfig.runtime.json", "tsconfig.json"],
 		].map((p) => fse.copy(
-			path.resolve(__dirname, "..", Array.isArray(p) ? p[0] : p),
-			path.resolve(outPath, Array.isArray(p) ? p[1] : p),
+			path.resolve(rootPath, p),
+			path.resolve(outPath, p),
 		)));
 		fse.unlinkSync(path.resolve(outPath, "packages/protocol/src/proto/index.ts"));
 	};
@@ -64,15 +75,14 @@ register("build", async (runner, logger, shouldWatch: string) => {
  * Bundle built code into a binary with nbin.
  */
 register("bundle", async () => {
-	const root = path.join(__dirname, "..");
 	const bin = new Binary({
-		mainFile: path.join(root, "out/packages/server/src/cli.js"),
+		mainFile: path.join(rootPath, "out/packages/server/src/bootstrap.js"),
 		target: platform() === "darwin" ? "darwin" : platform() === "musl" ? "alpine" : "linux",
 	});
 
-	bin.writeFiles(path.join(root, "lib/**"));
-	bin.writeFiles(path.join(root, "out/**"));
-	bin.writeFiles(path.join(root, "**/node_modules"));
+	bin.writeFiles(path.join(rootPath, "lib/**"));
+	bin.writeFiles(path.join(rootPath, "out/**"));
+	bin.writeFiles(path.join(rootPath, "**/node_modules/**"));
 
 	fse.mkdirpSync(releasePath);
 

scripts/tasks.ts

@@ -4,7 +4,7 @@
 		"incremental": true
 	},
 	"include": [
-		"packages/server/src/index.ts",
+		"packages/server/src/cli.ts",
 		"packages/server/src/vscode/bootstrap-fork.ts",
 		"packages/server/src/vscode/shared-process.ts",
 		"packages/server/src/modules/electron.ts"