From da4f3ca28e5d8a7bde9099fd9d3afff840fa7990 Mon Sep 17 00:00:00 2001 From: Akis Kesoglou Date: Wed, 20 Dec 2023 10:58:02 +0200 Subject: [PATCH] Bundle etcher-util with main app --- .github/actions/publish/action.yml | 2 +- .github/actions/test/action.yml | 2 +- forge.config.ts | 2 + forge.sidecar.ts | 168 +++++++++++++++++++++++++++++ lib/gui/app/modules/api.ts | 7 +- lib/gui/app/preload.ts | 10 ++ lib/gui/etcher.ts | 14 +++ lib/gui/webapi.ts | 15 +++ lib/pkg-sidekick.json | 10 -- npm-shrinkwrap.json | 36 ++++++- package.json | 5 +- tsconfig.sidecar.json | 3 +- 12 files changed, 252 insertions(+), 22 deletions(-) create mode 100644 forge.sidecar.ts create mode 100644 lib/gui/webapi.ts delete mode 100644 lib/pkg-sidekick.json diff --git a/.github/actions/publish/action.yml b/.github/actions/publish/action.yml index 55feef2f..80a0acd4 100644 --- a/.github/actions/publish/action.yml +++ b/.github/actions/publish/action.yml @@ -118,7 +118,7 @@ runs: # IMPORTANT: before making changes to this step please consult @engineering in balena's chat. run: | if [[ '${{ inputs.VERBOSE }}' =~ on|On|Yes|yes|true|True ]]; then - export DEBUG='electron-forge:*,electron-packager,electron-rebuild' + export DEBUG='electron-forge:*,sidecar' fi APPLICATION_VERSION="$(jq -r '.version' package.json)" diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index f40fba33..4874faa1 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -50,7 +50,7 @@ runs: shell: bash run: | if [[ '${{ inputs.VERBOSE }}' =~ on|On|Yes|yes|true|True ]]; then - export DEBUG='electron-forge:*,electron-packager,electron-rebuild' + export DEBUG='electron-forge:*,sidecar' fi runner_os="$(echo "${RUNNER_OS}" | tr '[:upper:]' '[:lower:]')" diff --git a/forge.config.ts b/forge.config.ts index d9662ec0..28817de7 100644 --- a/forge.config.ts +++ b/forge.config.ts @@ -9,6 +9,7 @@ import { AutoUnpackNativesPlugin } from '@electron-forge/plugin-auto-unpack-nati import { WebpackPlugin } from '@electron-forge/plugin-webpack'; import { mainConfig, rendererConfig } from './webpack.config'; +import * as sidecar from './forge.sidecar'; import { hostDependencies, productDescription } from './package.json'; @@ -129,6 +130,7 @@ const config: ForgeConfig = { ], }, }), + new sidecar.SidecarPlugin(), ], hooks: { readPackageJson: async (_config, packageJson) => { diff --git a/forge.sidecar.ts b/forge.sidecar.ts new file mode 100644 index 00000000..675fadb3 --- /dev/null +++ b/forge.sidecar.ts @@ -0,0 +1,168 @@ +import { PluginBase } from '@electron-forge/plugin-base'; +import { + ForgeHookMap, + ResolvedForgeConfig, +} from '@electron-forge/shared-types'; +import { WebpackPlugin } from '@electron-forge/plugin-webpack'; +import { DefinePlugin } from 'webpack'; + +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; + +import * as d from 'debug'; + +const debug = d('sidecar'); + +function isStartScrpt(): boolean { + return process.env.npm_lifecycle_event === 'start'; +} + +function addWebpackDefine( + config: ResolvedForgeConfig, + defineName: string, + binDir: string, + binName: string, +): ResolvedForgeConfig { + config.plugins.forEach((plugin) => { + if (plugin.name !== 'webpack' || !(plugin instanceof WebpackPlugin)) { + return; + } + + const { mainConfig } = plugin.config as any; + if (mainConfig.plugins == null) { + mainConfig.plugins = []; + } + + const value = isStartScrpt() + ? // on `npm start`, point directly to the binary + path.resolve(binDir, binName) + : // otherwise point relative to the resources folder of the bundled app + binName; + + debug(`define '${defineName}'='${value}'`); + + mainConfig.plugins.push( + new DefinePlugin({ + // expose path to helper via this webpack define + [defineName]: JSON.stringify(value), + }), + ); + }); + + return config; +} + +function build( + sourcesDir: string, + buildForArchs: string, + binDir: string, + binName: string, +) { + const commands: Array<[string, string[], object?]> = [ + ['tsc', ['--project', 'tsconfig.sidecar.json', '--outDir', sourcesDir]], + ]; + + buildForArchs.split(',').forEach((arch) => { + const binPath = isStartScrpt() + ? // on `npm start`, we don't know the arch we're building for at the time we're + // adding the webpack define, so we just build under binDir + path.resolve(binDir, binName) + : // otherwise build in arch-specific directory within binDir + path.resolve(binDir, arch, binName); + + // FIXME: rebuilding mountutils shouldn't be necessary, but it is. + // It's coming from etcher-sdk, a fix has been upstreamed but to use + // the latest etcher-sdk we need to upgrade axios at the same time. + commands.push(['npm', ['rebuild', 'mountutils', `--arch=${arch}`]]); + + commands.push([ + 'pkg', + [ + path.join(sourcesDir, 'util', 'api.js'), + '-c', + 'pkg-sidecar.json', + // `--no-bytecode` so that we can cross-compile for arm64 on x64 + '--no-bytecode', + '--public', + '--public-packages', + '"*"', + // always build for host platform and node version + // https://github.com/vercel/pkg-fetch/releases + '--target', + `node18-${arch}`, + '--output', + binPath, + ], + ]); + }); + + commands.forEach(([cmd, args, opt]) => { + debug('running command:', cmd, args.join(' ')); + execFileSync(cmd, args, { shell: true, stdio: 'inherit', ...opt }); + }); +} + +function copyArtifact( + buildPath: string, + arch: string, + binDir: string, + binName: string, +) { + const binPath = isStartScrpt() + ? // on `npm start`, we don't know the arch we're building for at the time we're + // adding the webpack define, so look for the binary directly under binDir + path.resolve(binDir, binName) + : // otherwise look into arch-specific directory within binDir + path.resolve(binDir, arch, binName); + + // buildPath points to appPath, which is inside resources dir which is the one we actually want + const resourcesPath = path.dirname(buildPath); + const dest = path.resolve(resourcesPath, path.basename(binPath)); + debug(`copying '${binPath}' to '${dest}'`); + fs.copyFileSync(binPath, dest); +} + +export class SidecarPlugin extends PluginBase { + name = 'sidecar'; + + constructor() { + super(); + this.getHooks = this.getHooks.bind(this); + debug('isStartScript:', isStartScrpt()); + } + + getHooks(): ForgeHookMap { + const DEFINE_NAME = 'ETCHER_UTIL_BIN_PATH'; + const BASE_DIR = path.join('out', 'sidecar'); + const SRC_DIR = path.join(BASE_DIR, 'src'); + const BIN_DIR = path.join(BASE_DIR, 'bin'); + const BIN_NAME = `etcher-util${process.platform === 'win32' ? '.exe' : ''}`; + + return { + resolveForgeConfig: async (currentConfig) => { + debug('resolveForgeConfig'); + return addWebpackDefine(currentConfig, DEFINE_NAME, BIN_DIR, BIN_NAME); + }, + generateAssets: async (_config, platform, arch) => { + debug('generateAssets', { platform, arch }); + build(SRC_DIR, arch, BIN_DIR, BIN_NAME); + }, + packageAfterCopy: async ( + _config, + buildPath, + electronVersion, + platform, + arch, + ) => { + debug('packageAfterCopy', { + buildPath, + electronVersion, + platform, + arch, + }); + copyArtifact(buildPath, arch, BIN_DIR, BIN_NAME); + }, + }; + } +} diff --git a/lib/gui/app/modules/api.ts b/lib/gui/app/modules/api.ts index a1ec7fcf..dd5fa905 100644 --- a/lib/gui/app/modules/api.ts +++ b/lib/gui/app/modules/api.ts @@ -18,7 +18,6 @@ import * as os from 'os'; import * as path from 'path'; import * as packageJSON from '../../../../package.json'; import * as permissions from '../../../shared/permissions'; -import { getAppPath } from '../../../shared/get-app-path'; import * as errors from '../../../shared/errors'; const THREADS_PER_CPU = 16; @@ -27,8 +26,8 @@ const THREADS_PER_CPU = 16; // the stdout maxBuffer size to be exceeded when flashing ipc.config.silent = true; -function writerArgv(): string[] { - let entryPoint = path.join(getAppPath(), 'generated', 'etcher-util'); +async function writerArgv(): Promise { + let entryPoint = await window.etcher.getEtcherUtilPath(); // AppImages run over FUSE, so the files inside the mount point // can only be accessed by the user that mounted the AppImage. // This means we can't re-spawn Etcher as root from the same @@ -75,7 +74,7 @@ async function spawnChild({ IPC_SERVER_ID: string; IPC_SOCKET_ROOT: string; }) { - const argv = writerArgv(); + const argv = await writerArgv(); const env = writerEnv(IPC_CLIENT_ID, IPC_SERVER_ID, IPC_SOCKET_ROOT); if (withPrivileges) { return await permissions.elevateCommand(argv, { diff --git a/lib/gui/app/preload.ts b/lib/gui/app/preload.ts index 5e9d369c..ad1b81aa 100644 --- a/lib/gui/app/preload.ts +++ b/lib/gui/app/preload.ts @@ -1,2 +1,12 @@ // See the Electron documentation for details on how to use preload scripts: // https://www.electronjs.org/docs/latest/tutorial/process-model#preload-scripts + +import * as webapi from '../webapi'; + +declare global { + interface Window { + etcher: typeof webapi; + } +} + +window['etcher'] = webapi; diff --git a/lib/gui/etcher.ts b/lib/gui/etcher.ts index d67c3e5c..dfd6453c 100644 --- a/lib/gui/etcher.ts +++ b/lib/gui/etcher.ts @@ -242,6 +242,20 @@ electron.app.on('before-quit', () => { process.exit(EXIT_CODES.SUCCESS); }); +// this is replaced at build-time with the path to helper binary, +// relative to the app resources directory. +declare const ETCHER_UTIL_BIN_PATH: string; + +electron.ipcMain.handle('get-util-path', () => { + if (process.env.NODE_ENV === 'development') { + // In development there is no "app bundle" and we're working directly with + // artifacts from the "out" directory, where this value point to. + return ETCHER_UTIL_BIN_PATH; + } + // In any other case, resolve the helper relative to resources path. + return path.resolve(process.resourcesPath, ETCHER_UTIL_BIN_PATH); +}); + async function main(): Promise { if (!electron.app.requestSingleInstanceLock()) { electron.app.quit(); diff --git a/lib/gui/webapi.ts b/lib/gui/webapi.ts new file mode 100644 index 00000000..01a9d80b --- /dev/null +++ b/lib/gui/webapi.ts @@ -0,0 +1,15 @@ +// +// Anything exported from this module will become available to the +// renderer process via preload. They're accessible as `window.etcher.foo()`. +// + +import { ipcRenderer } from 'electron'; + +// FIXME: this is a workaround for the renderer to be able to find the etcher-util +// binary. We should instead export a function that asks the main process to launch +// the binary itself. +export async function getEtcherUtilPath(): Promise { + const utilPath = await ipcRenderer.invoke('get-util-path'); + console.log(utilPath); + return utilPath; +} diff --git a/lib/pkg-sidekick.json b/lib/pkg-sidekick.json deleted file mode 100644 index 45673835..00000000 --- a/lib/pkg-sidekick.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "bin": "build/util/child-writer.js", - "pkg": { - "assets": [ - "node_modules/usb/prebuilds/darwin-x64+arm64/node.napi.node", - "node_modules/lzma-native/prebuilds/darwin-arm64/node.napi.node", - "node_modules/drivelist/build/Release/drivelist.node" - ] - } -} diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index c43807d9..42b650ad 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -52,6 +52,7 @@ "@svgr/webpack": "5.5.0", "@types/chai": "4.3.4", "@types/copy-webpack-plugin": "6.4.3", + "@types/debug": "^4.1.12", "@types/mime-types": "2.1.1", "@types/mini-css-extract-plugin": "1.4.3", "@types/mocha": "^9.1.1", @@ -5709,6 +5710,15 @@ "@types/webpack": "^4" } }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "dev": true, + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/eslint": { "version": "8.44.2", "dev": true, @@ -5867,6 +5877,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==", + "dev": true + }, "node_modules/@types/node": { "version": "16.18.46", "license": "MIT" @@ -8751,7 +8767,8 @@ }, "node_modules/debug": { "version": "4.3.4", - "license": "MIT", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dependencies": { "ms": "2.1.2" }, @@ -24676,6 +24693,15 @@ "@types/webpack": "^4" } }, + "@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "dev": true, + "requires": { + "@types/ms": "*" + } + }, "@types/eslint": { "version": "8.44.2", "dev": true, @@ -24813,6 +24839,12 @@ "version": "9.1.1", "dev": true }, + "@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==", + "dev": true + }, "@types/node": { "version": "16.18.46" }, @@ -26833,6 +26865,8 @@ }, "debug": { "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "requires": { "ms": "2.1.2" } diff --git a/package.json b/package.json index f282e5bd..23337f35 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,8 @@ "url": "git@github.com:balena-io/etcher.git" }, "scripts": { - "build:rebuild-mountutils": "cd node_modules/mountutils && npm rebuild", - "build:sidecar": "npm run build:rebuild-mountutils && tsc --project tsconfig.sidecar.json && pkg build/util/api.js -c pkg-sidecar.json --target node18 --output generated/etcher-util", "lint-css": "prettier --write lib/**/*.css", - "lint-ts": "balena-lint --fix --typescript typings lib tests forge.config.ts webpack.config.ts", + "lint-ts": "balena-lint --fix --typescript typings lib tests forge.config.ts forge.sidecar.ts webpack.config.ts", "lint": "npm run lint-ts && npm run lint-css", "test-gui": "electron-mocha --recursive --reporter spec --window-config tests/gui/window-config.json --require ts-node/register/transpile-only --require-main tests/gui/allow-renderer-process-reuse.ts --full-trace --no-sandbox --renderer tests/gui/**/*.ts", "test-shared": "electron-mocha --recursive --reporter spec --require ts-node/register/transpile-only --require-main tests/gui/allow-renderer-process-reuse.ts --full-trace --no-sandbox tests/shared/**/*.ts", @@ -87,6 +85,7 @@ "@svgr/webpack": "5.5.0", "@types/chai": "4.3.4", "@types/copy-webpack-plugin": "6.4.3", + "@types/debug": "^4.1.12", "@types/mime-types": "2.1.1", "@types/mini-css-extract-plugin": "1.4.3", "@types/mocha": "^9.1.1", diff --git a/tsconfig.sidecar.json b/tsconfig.sidecar.json index e3c943a1..1a52525e 100644 --- a/tsconfig.sidecar.json +++ b/tsconfig.sidecar.json @@ -11,8 +11,7 @@ "module": "CommonJS", "moduleResolution": "Node", "resolveJsonModule": true, - "isolatedModules": true, - "outDir": "build" + "isolatedModules": true }, "include": ["lib/util"] }