Bundle etcher-util with main app

.github/actions/publish/action.yml

@@ -118,7 +118,7 @@ runs:
       # IMPORTANT: before making changes to this step please consult @engineering in balena's chat.
       run: |
         if [[ '${{ inputs.VERBOSE }}' =~ on|On|Yes|yes|true|True ]]; then
-          export DEBUG='electron-forge:*,electron-packager,electron-rebuild'
+          export DEBUG='electron-forge:*,sidecar'
         fi
 
         APPLICATION_VERSION="$(jq -r '.version' package.json)"

.github/actions/test/action.yml

@@ -50,7 +50,7 @@ runs:
       shell: bash
       run: |
         if [[ '${{ inputs.VERBOSE }}' =~ on|On|Yes|yes|true|True ]]; then
-          export DEBUG='electron-forge:*,electron-packager,electron-rebuild'
+          export DEBUG='electron-forge:*,sidecar'
         fi
 
         runner_os="$(echo "${RUNNER_OS}" | tr '[:upper:]' '[:lower:]')"

forge.config.ts

@@ -9,6 +9,7 @@ import { AutoUnpackNativesPlugin } from '@electron-forge/plugin-auto-unpack-nati
 import { WebpackPlugin } from '@electron-forge/plugin-webpack';
 
 import { mainConfig, rendererConfig } from './webpack.config';
+import * as sidecar from './forge.sidecar';
 
 import { hostDependencies, productDescription } from './package.json';
 
@@ -129,6 +130,7 @@ const config: ForgeConfig = {
 				],
 			},
 		}),
+		new sidecar.SidecarPlugin(),
 	],
 	hooks: {
 		readPackageJson: async (_config, packageJson) => {

forge.sidecar.ts

@@ -0,0 +1,168 @@
+import { PluginBase } from '@electron-forge/plugin-base';
+import {
+	ForgeHookMap,
+	ResolvedForgeConfig,
+} from '@electron-forge/shared-types';
+import { WebpackPlugin } from '@electron-forge/plugin-webpack';
+import { DefinePlugin } from 'webpack';
+
+import { execFileSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import * as d from 'debug';
+
+const debug = d('sidecar');
+
+function isStartScrpt(): boolean {
+	return process.env.npm_lifecycle_event === 'start';
+}
+
+function addWebpackDefine(
+	config: ResolvedForgeConfig,
+	defineName: string,
+	binDir: string,
+	binName: string,
+): ResolvedForgeConfig {
+	config.plugins.forEach((plugin) => {
+		if (plugin.name !== 'webpack' || !(plugin instanceof WebpackPlugin)) {
+			return;
+		}
+
+		const { mainConfig } = plugin.config as any;
+		if (mainConfig.plugins == null) {
+			mainConfig.plugins = [];
+		}
+
+		const value = isStartScrpt()
+			? // on `npm start`, point directly to the binary
+			  path.resolve(binDir, binName)
+			: // otherwise point relative to the resources folder of the bundled app
+			  binName;
+
+		debug(`define '${defineName}'='${value}'`);
+
+		mainConfig.plugins.push(
+			new DefinePlugin({
+				// expose path to helper via this webpack define
+				[defineName]: JSON.stringify(value),
+			}),
+		);
+	});
+
+	return config;
+}
+
+function build(
+	sourcesDir: string,
+	buildForArchs: string,
+	binDir: string,
+	binName: string,
+) {
+	const commands: Array<[string, string[], object?]> = [
+		['tsc', ['--project', 'tsconfig.sidecar.json', '--outDir', sourcesDir]],
+	];
+
+	buildForArchs.split(',').forEach((arch) => {
+		const binPath = isStartScrpt()
+			? // on `npm start`, we don't know the arch we're building for at the time we're
+			  // adding the webpack define, so we just build under binDir
+			  path.resolve(binDir, binName)
+			: // otherwise build in arch-specific directory within binDir
+			  path.resolve(binDir, arch, binName);
+
+		// FIXME: rebuilding mountutils shouldn't be necessary, but it is.
+		// It's coming from etcher-sdk, a fix has been upstreamed but to use
+		// the latest etcher-sdk we need to upgrade axios at the same time.
+		commands.push(['npm', ['rebuild', 'mountutils', `--arch=${arch}`]]);
+
+		commands.push([
+			'pkg',
+			[
+				path.join(sourcesDir, 'util', 'api.js'),
+				'-c',
+				'pkg-sidecar.json',
+				// `--no-bytecode` so that we can cross-compile for arm64 on x64
+				'--no-bytecode',
+				'--public',
+				'--public-packages',
+				'"*"',
+				// always build for host platform and node version
+				// https://github.com/vercel/pkg-fetch/releases
+				'--target',
+				`node18-${arch}`,
+				'--output',
+				binPath,
+			],
+		]);
+	});
+
+	commands.forEach(([cmd, args, opt]) => {
+		debug('running command:', cmd, args.join(' '));
+		execFileSync(cmd, args, { shell: true, stdio: 'inherit', ...opt });
+	});
+}
+
+function copyArtifact(
+	buildPath: string,
+	arch: string,
+	binDir: string,
+	binName: string,
+) {
+	const binPath = isStartScrpt()
+		? // on `npm start`, we don't know the arch we're building for at the time we're
+		  // adding the webpack define, so look for the binary directly under binDir
+		  path.resolve(binDir, binName)
+		: // otherwise look into arch-specific directory within binDir
+		  path.resolve(binDir, arch, binName);
+
+	// buildPath points to appPath, which is inside resources dir which is the one we actually want
+	const resourcesPath = path.dirname(buildPath);
+	const dest = path.resolve(resourcesPath, path.basename(binPath));
+	debug(`copying '${binPath}' to '${dest}'`);
+	fs.copyFileSync(binPath, dest);
+}
+
+export class SidecarPlugin extends PluginBase<void> {
+	name = 'sidecar';
+
+	constructor() {
+		super();
+		this.getHooks = this.getHooks.bind(this);
+		debug('isStartScript:', isStartScrpt());
+	}
+
+	getHooks(): ForgeHookMap {
+		const DEFINE_NAME = 'ETCHER_UTIL_BIN_PATH';
+		const BASE_DIR = path.join('out', 'sidecar');
+		const SRC_DIR = path.join(BASE_DIR, 'src');
+		const BIN_DIR = path.join(BASE_DIR, 'bin');
+		const BIN_NAME = `etcher-util${process.platform === 'win32' ? '.exe' : ''}`;
+
+		return {
+			resolveForgeConfig: async (currentConfig) => {
+				debug('resolveForgeConfig');
+				return addWebpackDefine(currentConfig, DEFINE_NAME, BIN_DIR, BIN_NAME);
+			},
+			generateAssets: async (_config, platform, arch) => {
+				debug('generateAssets', { platform, arch });
+				build(SRC_DIR, arch, BIN_DIR, BIN_NAME);
+			},
+			packageAfterCopy: async (
+				_config,
+				buildPath,
+				electronVersion,
+				platform,
+				arch,
+			) => {
+				debug('packageAfterCopy', {
+					buildPath,
+					electronVersion,
+					platform,
+					arch,
+				});
+				copyArtifact(buildPath, arch, BIN_DIR, BIN_NAME);
+			},
+		};
+	}
+}

lib/gui/app/modules/api.ts

@@ -18,7 +18,6 @@ import * as os from 'os';
 import * as path from 'path';
 import * as packageJSON from '../../../../package.json';
 import * as permissions from '../../../shared/permissions';
-import { getAppPath } from '../../../shared/get-app-path';
 import * as errors from '../../../shared/errors';
 
 const THREADS_PER_CPU = 16;
@@ -27,8 +26,8 @@ const THREADS_PER_CPU = 16;
 // the stdout maxBuffer size to be exceeded when flashing
 ipc.config.silent = true;
 
-function writerArgv(): string[] {
-	let entryPoint = path.join(getAppPath(), 'generated', 'etcher-util');
+async function writerArgv(): Promise<string[]> {
+	let entryPoint = await window.etcher.getEtcherUtilPath();
 	// AppImages run over FUSE, so the files inside the mount point
 	// can only be accessed by the user that mounted the AppImage.
 	// This means we can't re-spawn Etcher as root from the same
@@ -75,7 +74,7 @@ async function spawnChild({
 	IPC_SERVER_ID: string;
 	IPC_SOCKET_ROOT: string;
 }) {
-	const argv = writerArgv();
+	const argv = await writerArgv();
 	const env = writerEnv(IPC_CLIENT_ID, IPC_SERVER_ID, IPC_SOCKET_ROOT);
 	if (withPrivileges) {
 		return await permissions.elevateCommand(argv, {

lib/gui/app/preload.ts

@@ -1,2 +1,12 @@
 // See the Electron documentation for details on how to use preload scripts:
 // https://www.electronjs.org/docs/latest/tutorial/process-model#preload-scripts
+
+import * as webapi from '../webapi';
+
+declare global {
+	interface Window {
+		etcher: typeof webapi;
+	}
+}
+
+window['etcher'] = webapi;

lib/gui/etcher.ts

@@ -242,6 +242,20 @@ electron.app.on('before-quit', () => {
 	process.exit(EXIT_CODES.SUCCESS);
 });
 
+// this is replaced at build-time with the path to helper binary,
+// relative to the app resources directory.
+declare const ETCHER_UTIL_BIN_PATH: string;
+
+electron.ipcMain.handle('get-util-path', () => {
+	if (process.env.NODE_ENV === 'development') {
+		// In development there is no "app bundle" and we're working directly with
+		// artifacts from the "out" directory, where this value point to.
+		return ETCHER_UTIL_BIN_PATH;
+	}
+	// In any other case, resolve the helper relative to resources path.
+	return path.resolve(process.resourcesPath, ETCHER_UTIL_BIN_PATH);
+});
+
 async function main(): Promise<void> {
 	if (!electron.app.requestSingleInstanceLock()) {
 		electron.app.quit();

lib/gui/webapi.ts

@@ -0,0 +1,15 @@
+//
+// Anything exported from this module will become available to the
+// renderer process via preload. They're accessible as `window.etcher.foo()`.
+//
+
+import { ipcRenderer } from 'electron';
+
+// FIXME: this is a workaround for the renderer to be able to find the etcher-util
+// binary. We should instead export a function that asks the main process to launch
+// the binary itself.
+export async function getEtcherUtilPath(): Promise<string> {
+	const utilPath = await ipcRenderer.invoke('get-util-path');
+	console.log(utilPath);
+	return utilPath;
+}

lib/pkg-sidekick.json

@@ -1,10 +0,0 @@
-{
-  "bin": "build/util/child-writer.js",
-  "pkg": {
-    "assets": [
-      "node_modules/usb/prebuilds/darwin-x64+arm64/node.napi.node",
-      "node_modules/lzma-native/prebuilds/darwin-arm64/node.napi.node",
-      "node_modules/drivelist/build/Release/drivelist.node"
-    ]
-  }
-}

npm-shrinkwrap.json

@@ -52,6 +52,7 @@
         "@svgr/webpack": "5.5.0",
         "@types/chai": "4.3.4",
         "@types/copy-webpack-plugin": "6.4.3",
+        "@types/debug": "^4.1.12",
         "@types/mime-types": "2.1.1",
         "@types/mini-css-extract-plugin": "1.4.3",
         "@types/mocha": "^9.1.1",
@@ -5709,6 +5710,15 @@
         "@types/webpack": "^4"
       }
     },
+    "node_modules/@types/debug": {
+      "version": "4.1.12",
+      "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
+      "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/ms": "*"
+      }
+    },
     "node_modules/@types/eslint": {
       "version": "8.44.2",
       "dev": true,
@@ -5867,6 +5877,12 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/ms": {
+      "version": "0.7.34",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz",
+      "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==",
+      "dev": true
+    },
     "node_modules/@types/node": {
       "version": "16.18.46",
       "license": "MIT"
@@ -8751,7 +8767,8 @@
     },
     "node_modules/debug": {
       "version": "4.3.4",
-      "license": "MIT",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
       "dependencies": {
         "ms": "2.1.2"
       },
@@ -24676,6 +24693,15 @@
         "@types/webpack": "^4"
       }
     },
+    "@types/debug": {
+      "version": "4.1.12",
+      "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
+      "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==",
+      "dev": true,
+      "requires": {
+        "@types/ms": "*"
+      }
+    },
     "@types/eslint": {
       "version": "8.44.2",
       "dev": true,
@@ -24813,6 +24839,12 @@
       "version": "9.1.1",
       "dev": true
     },
+    "@types/ms": {
+      "version": "0.7.34",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz",
+      "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==",
+      "dev": true
+    },
     "@types/node": {
       "version": "16.18.46"
     },
@@ -26833,6 +26865,8 @@
     },
     "debug": {
       "version": "4.3.4",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
       "requires": {
         "ms": "2.1.2"
       }

package.json

@@ -14,10 +14,8 @@
     "url": "git@github.com:balena-io/etcher.git"
   },
   "scripts": {
-    "build:rebuild-mountutils": "cd node_modules/mountutils && npm rebuild",
-    "build:sidecar": "npm run build:rebuild-mountutils && tsc --project tsconfig.sidecar.json && pkg build/util/api.js -c pkg-sidecar.json --target node18 --output generated/etcher-util",
     "lint-css": "prettier --write lib/**/*.css",
-    "lint-ts": "balena-lint --fix --typescript typings lib tests forge.config.ts webpack.config.ts",
+    "lint-ts": "balena-lint --fix --typescript typings lib tests forge.config.ts forge.sidecar.ts webpack.config.ts",
     "lint": "npm run lint-ts && npm run lint-css",
     "test-gui": "electron-mocha --recursive --reporter spec --window-config tests/gui/window-config.json --require ts-node/register/transpile-only --require-main tests/gui/allow-renderer-process-reuse.ts --full-trace --no-sandbox --renderer tests/gui/**/*.ts",
     "test-shared": "electron-mocha --recursive --reporter spec --require ts-node/register/transpile-only --require-main tests/gui/allow-renderer-process-reuse.ts --full-trace --no-sandbox tests/shared/**/*.ts",
@@ -87,6 +85,7 @@
     "@svgr/webpack": "5.5.0",
     "@types/chai": "4.3.4",
     "@types/copy-webpack-plugin": "6.4.3",
+    "@types/debug": "^4.1.12",
     "@types/mime-types": "2.1.1",
     "@types/mini-css-extract-plugin": "1.4.3",
     "@types/mocha": "^9.1.1",

tsconfig.sidecar.json

@@ -11,8 +11,7 @@
     "module": "CommonJS",
     "moduleResolution": "Node",
     "resolveJsonModule": true,
-    "isolatedModules": true,
-    "outDir": "build"
+    "isolatedModules": true
   },
   "include": ["lib/util"]
 }