From 631dc0e362e302c1c0bc422b844f4518b820ec20 Mon Sep 17 00:00:00 2001 From: Alexis Svinartchouk Date: Mon, 7 Oct 2019 18:11:58 +0200 Subject: [PATCH] Webpack everything except native modules WIP --- Makefile | 1 - electron-builder.yml | 15 +++- etcher-pro.Dockerfile | 121 +++++++++++++++++++++++++-- lib/start.js | 9 +- npm-shrinkwrap.json | 25 ++++-- package.json | 7 +- webpack.config.js | 185 ++++++++++++++++++++++++------------------ 7 files changed, 260 insertions(+), 103 deletions(-) diff --git a/Makefile b/Makefile index 25708218..5c0102be 100644 --- a/Makefile +++ b/Makefile @@ -147,7 +147,6 @@ webpack: .PHONY: $(TARGETS) sass: - npm rebuild node-sass node-sass lib/gui/app/scss/main.scss > lib/gui/css/main.css lint-ts: diff --git a/electron-builder.yml b/electron-builder.yml index e7cbdfc6..f72767ce 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -5,12 +5,17 @@ npmRebuild: true nodeGypRebuild: true publish: null files: - - lib + - "!**/*" - lib/gui/app/index.html - generated - - build/**/*.node - assets/icon.png - - node_modules/**/* + - build/Release/elevator.node + - node_modules/xxhash/build/Release/hash.node + - node_modules/lzma-native/binding-v4.0.5-electron-v6.0-linux-x64/lzma_native.node + - node_modules/mountutils/build/Release/MountUtils.node + - node_modules/ext2fs/build/Release/bindings.node + - node_modules/usb/build/Release/usb_bindings.node + - node_modules/drivelist/build/Release/drivelist.node mac: icon: assets/icon.icns category: public.app-category.developer-tools @@ -47,6 +52,10 @@ linux: executableName: balena-etcher-electron synopsis: balenaEtcher is a powerful OS image flasher built with web technologies to ensure flashing an SDCard or USB drive is a pleasant and safe experience. It protects you from accidentally writing to your hard-drives, ensures every byte of data was written correctly and much more. icon: assets/iconset + target: + - AppImage + - rpm + - deb deb: priority: optional depends: diff --git a/etcher-pro.Dockerfile b/etcher-pro.Dockerfile index 7c57f588..b7536d9f 100644 --- a/etcher-pro.Dockerfile +++ b/etcher-pro.Dockerfile @@ -8,12 +8,119 @@ RUN \ && \ apt-get clean WORKDIR /usr/src/app -ADD . ./ -RUN npm config set unsafe-perm true -RUN npm config set arch armv7l -ENV npm_config_arch=armv7l -RUN make electron-develop +ENV npm_config_runtime=electron +ENV npm_config_disturl=https://electronjs.org/headers +ENV npm_config_target=6.0.10 +ENV npm_config_build_from_source=true +ENV npm_config_unsafe_perm=true +ADD package.json npm-shrinkwrap.json binding.gyp ./ +RUN npm install + +# TODO: makefile only needed for sass +ADD Makefile webpack.config.js tsconfig.json ./ +ADD lib ./lib +ADD typings ./typings +ADD assets ./assets +RUN npm run webpack +RUN make sass FROM alexisresinio/balena-electronjs-amd64 -WORKDIR /usr/src/app -COPY --from=builder /usr/src/app /usr/src/app +# TODO: only elevator.target on the next line? +COPY --from=builder /usr/src/app/build/Release /usr/src/app/build/Release +COPY --from=builder /usr/src/app/lib/start.js /usr/src/app/lib/start.js +COPY --from=builder /usr/src/app/lib/gui/app/index.html /usr/src/app/lib/gui/app/index.html +COPY --from=builder /usr/src/app/lib/gui/css /usr/src/app/lib/gui/css +COPY --from=builder /usr/src/app/lib/gui/assets /usr/src/app/lib/gui/assets +COPY --from=builder /usr/src/app/lib/shared /usr/src/app/lib/shared +COPY --from=builder /usr/src/app/package.json /usr/src/app/package.json +COPY --from=builder /usr/src/app/generated /usr/src/app/generated +COPY --from=builder /usr/src/app/node_modules/electron /usr/src/app/node_modules/electron +# TODO: be more restrictive on lodash +#COPY --from=builder /usr/src/app/node_modules /usr/src/app/node_modules +# copy only native modules and their dependencies +COPY --from=builder /usr/src/app/node_modules/xxhash /usr/src/app/node_modules/xxhash +COPY --from=builder /usr/src/app/node_modules/lzma-native /usr/src/app/node_modules/lzma-native +COPY --from=builder /usr/src/app/node_modules/drivelist /usr/src/app/node_modules/drivelist +COPY --from=builder /usr/src/app/node_modules/electron /usr/src/app/node_modules/electron +COPY --from=builder /usr/src/app/node_modules/mountutils /usr/src/app/node_modules/mountutils +COPY --from=builder /usr/src/app/node_modules/ext2fs /usr/src/app/node_modules/ext2fs +COPY --from=builder /usr/src/app/node_modules/usb /usr/src/app/node_modules/usb +# used by drivelist +COPY --from=builder /usr/src/app/node_modules/bindings /usr/src/app/node_modules/bindings +COPY --from=builder /usr/src/app/node_modules/mz /usr/src/app/node_modules/mz +# used by mz/child_process +COPY --from=builder /usr/src/app/node_modules/thenify-all /usr/src/app/node_modules/thenify-all +# used my thenify-all +COPY --from=builder /usr/src/app/node_modules/thenify /usr/src/app/node_modules/thenify +# used my thenify +COPY --from=builder /usr/src/app/node_modules/any-promise /usr/src/app/node_modules/any-promise +# used by bindings +COPY --from=builder /usr/src/app/node_modules/file-uri-to-path /usr/src/app/node_modules/file-uri-to-path +# used by ext2fs +COPY --from=builder /usr/src/app/node_modules/bluebird /usr/src/app/node_modules/bluebird +# used by lzma-native +COPY --from=builder /usr/src/app/node_modules/readable-stream /usr/src/app/node_modules/readable-stream +COPY --from=builder /usr/src/app/node_modules/node-pre-gyp /usr/src/app/node_modules/node-pre-gyp +# used by readable-stream +COPY --from=builder /usr/src/app/node_modules/process-nextick-args /usr/src/app/node_modules/process-nextick-args +COPY --from=builder /usr/src/app/node_modules/isarray /usr/src/app/node_modules/isarray +COPY --from=builder /usr/src/app/node_modules/safe-buffer /usr/src/app/node_modules/safe-buffer +COPY --from=builder /usr/src/app/node_modules/core-util-is /usr/src/app/node_modules/core-util-is +COPY --from=builder /usr/src/app/node_modules/inherits /usr/src/app/node_modules/inherits +COPY --from=builder /usr/src/app/node_modules/util-deprecate /usr/src/app/node_modules/util-deprecate +# used by node-pre-gyp +COPY --from=builder /usr/src/app/node_modules/nopt /usr/src/app/node_modules/nopt +COPY --from=builder /usr/src/app/node_modules/npmlog /usr/src/app/node_modules/npmlog +COPY --from=builder /usr/src/app/node_modules/rimraf /usr/src/app/node_modules/rimraf +COPY --from=builder /usr/src/app/node_modules/semver /usr/src/app/node_modules/semver +COPY --from=builder /usr/src/app/node_modules/detect-libc /usr/src/app/node_modules/detect-libc +# used by rimraf +COPY --from=builder /usr/src/app/node_modules/glob /usr/src/app/node_modules/glob +# used by glob +COPY --from=builder /usr/src/app/node_modules/fs.realpath /usr/src/app/node_modules/fs.realpath +COPY --from=builder /usr/src/app/node_modules/minimatch /usr/src/app/node_modules/minimatch +COPY --from=builder /usr/src/app/node_modules/path-is-absolute /usr/src/app/node_modules/path-is-absolute +COPY --from=builder /usr/src/app/node_modules/inflight /usr/src/app/node_modules/inflight +# used by inflight +COPY --from=builder /usr/src/app/node_modules/wrappy /usr/src/app/node_modules/wrappy +COPY --from=builder /usr/src/app/node_modules/once /usr/src/app/node_modules/once +# used by minimatch +COPY --from=builder /usr/src/app/node_modules/brace-expansion /usr/src/app/node_modules/brace-expansion +# used by brance-expansion +COPY --from=builder /usr/src/app/node_modules/concat-map /usr/src/app/node_modules/concat-map +COPY --from=builder /usr/src/app/node_modules/balanced-match /usr/src/app/node_modules/balanced-match +# used by npmlog +COPY --from=builder /usr/src/app/node_modules/are-we-there-yet /usr/src/app/node_modules/are-we-there-yet +COPY --from=builder /usr/src/app/node_modules/gauge /usr/src/app/node_modules/gauge +COPY --from=builder /usr/src/app/node_modules/set-blocking /usr/src/app/node_modules/set-blocking +# used by gauge +COPY --from=builder /usr/src/app/node_modules/console-control-strings /usr/src/app/node_modules/console-control-strings +COPY --from=builder /usr/src/app/node_modules/wide-align /usr/src/app/node_modules/wide-align +COPY --from=builder /usr/src/app/node_modules/aproba /usr/src/app/node_modules/aproba +COPY --from=builder /usr/src/app/node_modules/object-assign /usr/src/app/node_modules/object-assign +COPY --from=builder /usr/src/app/node_modules/has-unicode /usr/src/app/node_modules/has-unicode +COPY --from=builder /usr/src/app/node_modules/signal-exit /usr/src/app/node_modules/signal-exit +# used by wide-align +COPY --from=builder /usr/src/app/node_modules/string-width /usr/src/app/node_modules/string-width +# used by string-width +COPY --from=builder /usr/src/app/node_modules/strip-ansi /usr/src/app/node_modules/strip-ansi +COPY --from=builder /usr/src/app/node_modules/code-point-at /usr/src/app/node_modules/code-point-at +COPY --from=builder /usr/src/app/node_modules/is-fullwidth-code-point /usr/src/app/node_modules/is-fullwidth-code-point +# used by is-fullwidth-code-point +COPY --from=builder /usr/src/app/node_modules/number-is-nan /usr/src/app/node_modules/number-is-nan +# used by strip-ansi +COPY --from=builder /usr/src/app/node_modules/ansi-regex /usr/src/app/node_modules/ansi-regex +# used by are-we-there-yet +COPY --from=builder /usr/src/app/node_modules/delegates /usr/src/app/node_modules/delegates +# used by nopt +COPY --from=builder /usr/src/app/node_modules/abbrev /usr/src/app/node_modules/abbrev +COPY --from=builder /usr/src/app/node_modules/osenv /usr/src/app/node_modules/osenv +# used by osenv +COPY --from=builder /usr/src/app/node_modules/os-tmpdir /usr/src/app/node_modules/os-tmpdir +COPY --from=builder /usr/src/app/node_modules/os-homedir /usr/src/app/node_modules/os-homedir +# icons +COPY --from=builder /usr/src/app/node_modules/bootstrap-sass/assets/fonts/bootstrap /usr/src/app/node_modules/bootstrap-sass/assets/fonts/bootstrap +# icons +COPY --from=builder /usr/src/app/node_modules/bootstrap-sass/assets/fonts/bootstrap /usr/src/app/node_modules/bootstrap-sass/assets/fonts/bootstrap +# flexboxgrid +COPY --from=builder /usr/src/app/node_modules/flexboxgrid/dist/flexboxgrid.css /usr/src/app/node_modules/flexboxgrid/dist/flexboxgrid.css diff --git a/lib/start.js b/lib/start.js index 562e6ec1..7dfe96d9 100644 --- a/lib/start.js +++ b/lib/start.js @@ -22,12 +22,9 @@ // *won't* attempt to load the `app.asar` application by default, therefore // if passing `ELECTRON_RUN_AS_NODE`, you have to pass the path to the asar // or the entry point file (this file) manually as an argument. -// -// We also consider `ATOM_SHELL_INTERNAL_RUN_AS_NODE`, which is basically -// an older equivalent of `ELECTRON_RUN_AS_NODE` that still gets set when -// using `child_process.fork()`. -if (process.env.ELECTRON_RUN_AS_NODE || process.env.ATOM_SHELL_INTERNAL_RUN_AS_NODE) { + +if (process.env.ELECTRON_RUN_AS_NODE) { require('./gui/modules/child-writer') } else { - require('../generated/etcher') + require('./gui/etcher') } diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 8037c222..4e9058eb 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -11631,6 +11631,15 @@ "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz", "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=" }, + "simple-functional-loader": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/simple-functional-loader/-/simple-functional-loader-1.1.0.tgz", + "integrity": "sha512-ynj69HhFvOdISrrs9lovZ3u8OxQyxrWN5KrXNyMh1g2JnzWOnQ+C0NEkEqKrUd+CuNXoyTBwefKfTkQ4WnUCRA==", + "dev": true, + "requires": { + "loader-utils": "^1.2.3" + } + }, "simple-get": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", @@ -12150,6 +12159,16 @@ "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=", "dev": true }, + "string-replace-loader": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/string-replace-loader/-/string-replace-loader-2.2.0.tgz", + "integrity": "sha512-Ukt4ZC8+xVWdBRut3/iwnPJCNL1yV8AbVKXn8UcWdYrHgtuW4UDDAbBSi/J/CQDEWQBt824AJvPYahF23eJLRg==", + "dev": true, + "requires": { + "loader-utils": "^1.2.3", + "schema-utils": "^1.0.0" + } + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -14260,12 +14279,6 @@ } } }, - "webpack-node-externals": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", - "integrity": "sha512-ajerHZ+BJKeCLviLUUmnyd5B4RavLF76uv3cs6KNuO8W+HuQaEs0y0L7o40NQxdPy5w0pcv8Ew7yPUAQG0UdCg==", - "dev": true - }, "webpack-sources": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz", diff --git a/package.json b/package.json index 016081e9..b9cfabba 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "sleepDays": 7, "semverRange": "<2.0.0" }, - "main": "lib/start.js", + "main": "generated/etcher.js", "description": "Flash OS images to SD cards and USB drives, safely and easily.", "productDescription": "Etcher is a powerful OS image flasher built with web technologies to ensure flashing an SDCard or USB drive is a pleasant and safe experience. It protects you from accidentally writing to your hard-drives, ensures every byte of data was written correctly and much more.", "homepage": "https://github.com/balena-io/etcher", @@ -114,13 +114,14 @@ "pkg": "^4.3.0", "resin-lint": "^3.1.0", "sass-lint": "^1.12.1", + "simple-functional-loader": "^1.1.0", "simple-progress-webpack-plugin": "^1.1.2", "spectron": "^5.0.0", + "string-replace-loader": "^2.2.0", "ts-loader": "^6.0.4", "ts-node": "^8.3.0", "typescript": "^3.5.3", "webpack": "^4.40.2", - "webpack-cli": "^3.3.9", - "webpack-node-externals": "^1.7.2" + "webpack-cli": "^3.3.9" } } diff --git a/webpack.config.js b/webpack.config.js index 9ba522ed..f83ea111 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -16,10 +16,53 @@ 'use strict' -const _ = require('lodash') +const os = require('os') const path = require('path') +const { createLoader } = require('simple-functional-loader') const SimpleProgressWebpackPlugin = require('simple-progress-webpack-plugin') -const nodeExternals = require('webpack-node-externals') + +// eslint-disable-next-line func-style,require-jsdoc,space-before-function-paren +function platformSpecificModule(platform, module) { + // Resolves module on platform, otherwise resolves an empty file + return (context, request, callback) => { + if ((request === module) && (os.platform() !== platform)) { + callback(null, `commonjs ${path.resolve(__dirname, 'lib', 'nothing')}`) + return + } + callback() + } +} + +function fakeBindings(opts) { + // TODO: elevator.node (windows only) + const MAPPING = { + MountUtils: 'mountutils/build/Release/MountUtils.node', + bindings: 'ext2fs/build/Release/bindings.node', + drivelist: 'drivelist/build/Release/drivelist.node', + } + if (typeof opts === 'string') { + opts = { bindings: opts } + } + const { bindings, module_root } = opts; + return __non_webpack_require__(MAPPING[bindings]); +} + +const fakeNodePreGypFind = (package_json_path, opts) => { + // Return an empty string as we will replace the line that requires the native module anyway + return ''; +} + +function externalNativeModules(context, request, callback) { + if (request.toLowerCase().search('mountutils') !== -1 || context.toLowerCase().search('mountutils') !== -1) { + console.log('walala', context, request); + } + // TODO + if ((context === '/home/alexis/dev/resin.io/etcher/node_modules/xxhash/lib') && (request === '../build/Release/hash')) { + callback(null, `commonjs xxhash/build/Release/hash`) + } else { + callback() + } +} const commonConfig = { mode: 'production', @@ -27,12 +70,52 @@ const commonConfig = { // Minification breaks angular. minimize: false }, - target: 'electron-main', module: { rules: [ + { + // remove node-pre-gyp magic from lzma-native + test: /lzma\-native\/index\.js$/, + loader: "string-replace-loader", + options: { + search: 'require\(binding_path\)', + // TODO: automatically find version + replace: '__non_webpack_require__("lzma-native/binding-v4.0.5-electron-v6.0-linux-x64/lzma_native.node")', + strict: true, + }, + }, + { + // remove node-pre-gyp magic from usb + test: /usb\/usb\.js$/, + loader: "string-replace-loader", + options: { + search: 'require\(binding_path\)', + replace: '__non_webpack_require__("usb/build/Release/usb_bindings.node")', + strict: true, + }, + }, + { + // Replaces node-pre-gyp find function with a mock one + test: /node-pre-gyp\/lib\/node-pre-gyp\.js$/, + use: createLoader(function(source, map) { + return `module.exports = { find: ${fakeNodePreGypFind.toString()} }`; + }), + }, + { + test: /bindings\/bindings\.js$/, + use: createLoader(function(source, map) { + return `module.exports = ${fakeBindings.toString()}`; + }), + }, + { + // these files require aws-sdk which is not installed + test: /node-pre-gyp\/lib\/(publish|unpublish|info)\.js$/, + use: createLoader(function(source, map) { + return ''; + }), + }, { test: /\.jsx?$/, - include: [ path.resolve(__dirname, 'lib/gui') ], + include: [ path.resolve(__dirname, 'lib', 'gui') ], use: { loader: 'babel-loader', options: { @@ -47,7 +130,7 @@ const commonConfig = { }, { test: /\.html$/, - include: [ path.resolve(__dirname, 'lib/gui/app') ], + include: [ path.resolve(__dirname, 'lib', 'gui', 'app') ], use: { loader: 'html-loader' } @@ -60,100 +143,48 @@ const commonConfig = { ] }, resolve: { - extensions: [ '.js', '.jsx', '.json', '.ts', '.tsx' ] + extensions: [ '.js', '.jsx', '.json', '.ts', '.tsx', '.node' ] }, plugins: [ new SimpleProgressWebpackPlugin({ format: process.env.WEBPACK_PROGRESS || 'verbose' }) - ] + ], + externals: [ + platformSpecificModule('win32', 'winusb-driver-generator'), + externalNativeModules, + ], + output: { + path: path.join(__dirname, 'generated'), + filename: '[name].js', + }, } -const guiConfig = _.assign({ +const guiConfig = { + ...commonConfig, + target: 'electron-renderer', node: { __dirname: true, __filename: true }, - externals: [ - nodeExternals(), - (context, request, callback) => { - // eslint-disable-next-line lodash/prefer-lodash-method - const absoluteContext = path.resolve(context) - const absoluteNodeModules = path.resolve('node_modules') - - // We shouldn't rewrite any node_modules import paths - // eslint-disable-next-line lodash/prefer-lodash-method - if (!path.relative(absoluteNodeModules, absoluteContext).startsWith('..')) { - return callback() - } - - // We want to keep the SDK code outside the GUI bundle. - // This piece of code allows us to run the GUI directly - // on the tree (for testing purposes) or inside a generated - // bundle (for production purposes), by translating - // relative require paths within the bundle. - if (/\/(sdk|shared)/i.test(request) || /package\.json$/.test(request)) { - const output = path.join(__dirname, 'generated') - const dirname = path.join(context, request) - const relative = path.relative(output, dirname) - return callback(null, `commonjs ${path.join('..', '..', relative)}`) - } - - return callback() - } - ], entry: { gui: path.join(__dirname, 'lib', 'gui', 'app', 'app.js') }, - output: { - path: path.join(__dirname, 'generated'), - filename: '[name].js' - } -}, commonConfig) +} -const etcherConfig = _.assign({ +const etcherConfig = { + ...commonConfig, + target: 'electron-main', node: { __dirname: false, __filename: true }, - externals: [ - nodeExternals(), - (context, request, callback) => { - // eslint-disable-next-line lodash/prefer-lodash-method - const absoluteContext = path.resolve(context) - const absoluteNodeModules = path.resolve('node_modules') - - // We shouldn't rewrite any node_modules import paths - // eslint-disable-next-line lodash/prefer-lodash-method - if (!path.relative(absoluteNodeModules, absoluteContext).startsWith('..')) { - return callback() - } - - // We want to keep the SDK code outside the GUI bundle. - // This piece of code allows us to run the GUI directly - // on the tree (for testing purposes) or inside a generated - // bundle (for production purposes), by translating - // relative require paths within the bundle. - if (/\/shared/i.test(request) || /package\.json$/.test(request)) { - const output = path.join(__dirname, 'generated') - const dirname = path.join(context, request) - const relative = path.relative(output, dirname) - return callback(null, `commonjs ${path.join('..', 'lib', relative)}`) - } - - return callback() - } - ], entry: { - etcher: path.join(__dirname, 'lib', 'gui', 'etcher.js') + etcher: path.join(__dirname, 'lib', 'start.js') }, - output: { - path: path.join(__dirname, 'generated'), - filename: '[name].js' - } -}, commonConfig) +} module.exports = [ guiConfig, - etcherConfig + etcherConfig, ]