chore: don't remove optional dependencies in clean-shrinkwrap.js (#1551)

If we include a platform specific optional dependency in the shrinkwrap
file, then npm will insist in installing it even if the platform doesn't
match. As a solution, we figured out we can avoid putting this platform
specific optional dependencies in the npm-shrinkwrap.json file.

In order to do this, we currently have a script called
`clean-shrinkwrap.js` that runs *before* any `npm shrinkwrap` file (its
a `preshrinkwrap` npm script) that deletes all the platform specific
modules we know about using `npm rm`.

The problem with this approach is that `npm rm` will remove the module's
code from `node_modules`, which means that if we run `npm shrinkwrap`,
we will lose certain optional dependencies, that may be needed at a
later stage.

The solution is to modify the `clean-shrinkwrap.js` script to parse
`npm-shrinkwrap.json`, and manually delete the entries that we want to
omit. Also, the script needs to be run *after* `npm shrinkwrap`, so we
change the npm script name to `postshrinkwrap`.

Signed-off-by: Juan Cruz Viotti <jv@jviotti.com>

package.json

@@ -19,21 +19,17 @@
   "scripts": {
     "test": "make test",
     "start": "electron lib/start.js",
-    "preshrinkwrap": "node ./scripts/clean-shrinkwrap.js",
+    "postshrinkwrap": "node ./scripts/clean-shrinkwrap.js",
     "configure": "node-gyp configure",
     "build": "node-gyp build",
     "install": "node-gyp rebuild"
   },
   "author": "Resin Inc. <hello@etcher.io>",
   "license": "Apache-2.0",
-  "shrinkwrapIgnore": [
+  "platformSpecificDependencies": [
-    "macos-alias",
+    [ "7zip-bin-mac" ],
-    "fs-xattr",
+    [ "7zip-bin-win" ],
-    "ds-store",
+    [ "7zip-bin-linux" ]
-    "appdmg",
-    "7zip-bin-mac",
-    "7zip-bin-win",
-    "7zip-bin-linux"
   ],
   "dependencies": {
     "angular": "1.6.3",

scripts/ci/build-installers.sh

@@ -48,8 +48,8 @@ if [ "$ARGV_OPERATING_SYSTEM" == "linux" ]; then
   ./scripts/build/docker/run-command.sh \
     -r "$TARGET_ARCH" \
     -s "$(pwd)" \
-    -c 'make electron-develop installers-all'
+    -c 'make installers-all'
 else
   ./scripts/build/check-dependency.sh make
-  make electron-develop installers-all
+  make installers-all
 fi

scripts/clean-shrinkwrap.js

@@ -15,29 +15,258 @@
 
 'use strict';
 
+const _ = require('lodash');
+const fs = require('fs');
 const path = require('path');
-const os = require('os');
 const packageJSON = require('../package.json');
-const spawn = require('child_process').spawn;
+const NPM_SHRINKWRAP_FILE_PATH = path.join(__dirname, '..', 'npm-shrinkwrap.json');
-const shrinkwrapIgnore = packageJSON.shrinkwrapIgnore;
+const shrinkwrapFile = require(NPM_SHRINKWRAP_FILE_PATH);
+const platformSpecificDependencies = packageJSON.platformSpecificDependencies;
+const JSON_INDENTATION_SPACES = 2;
 
-console.log('Removing:', shrinkwrapIgnore.join(', '));
+console.log('Removing:', platformSpecificDependencies.join(', '));
 
 /**
- * Run an npm command
+ * @summary Get a shrinkwrap dependency object
- * @param {Array} command - list of arguments
+ * @function
- * @returns {ChildProcess}
+ * @private
+ *
+ * @param {Object} shrinkwrap - the shrinkwrap file contents
+ * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency
+ * @returns {Object} shrinkwrap object
+ *
+ * @example
+ * const object = getShrinkwrapDependencyObject(require('./npm-shrinkwrap.json'), [
+ *   'drivelist',
+ *   'lodash'
+ * ]);
+ *
+ * console.log(object.version);
+ * console.log(object.dependencies);
  */
-const npm = (command) => {
+const getShrinkwrapDependencyObject = (shrinkwrap, shrinkwrapPath) => {
-  const npmBinary = os.platform() === 'win32' ? 'npm.cmd' : 'npm';
+  return _.reduce(shrinkwrapPath, (accumulator, dependency) => {
-  return spawn(npmBinary, command, {
+    return _.get(accumulator, [ 'dependencies', dependency ], {});
-    cwd: path.join(__dirname, '..'),
+  }, shrinkwrap);
-    env: process.env,
-    stdio: [ process.stdin, process.stdout, process.stderr ]
-  });
 };
 
-npm([ 'rm', '--ignore-scripts' ].concat(shrinkwrapIgnore))
+/**
-  .once('close', () => {
+ * @summary Get a cleaned shrinkwrap dependency object
-    console.log('Done.');
+ * @function
+ * @private
+ *
+ * @description
+ * This function wraps `getShrinkwrapDependencyObject()` to
+ * omit unnecessary properties such as `from`, or `dependencies`.
+ *
+ * @param {Object} shrinkwrap - the shrinkwrap file contents
+ * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency
+ * @returns {Object} pretty shrinkwrap object
+ *
+ * @example
+ * const object = getPrettyShrinkwrapDependencyObject(require('./npm-shrinkwrap.json'), [
+ *   'drivelist',
+ *   'lodash'
+ * ]);
+ *
+ * console.log(object.name);
+ * console.log(object.path);
+ * console.log(object.version);
+ */
+const getPrettyShrinkwrapDependencyObject = (shrinkwrap, shrinkwrapPath) => {
+  const object = getShrinkwrapDependencyObject(shrinkwrap, shrinkwrapPath);
+
+  if (_.isEmpty(object)) {
+    return null;
+  }
+
+  return {
+    name: _.last(shrinkwrapPath),
+    path: shrinkwrapPath,
+    version: object.version,
+    development: Boolean(object.dev),
+    optional: Boolean(object.optional)
+  };
+};
+
+/**
+ * @summary Get the manifest (package.json) of a shrinkwrap dependency
+ * @function
+ * @private
+ *
+ * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency
+ * @returns {Object} dependency manifest
+ *
+ * @example
+ * const manifest = getShrinkwrapDependencyManifest([ 'bluebird' ]);
+ * console.log(manifest.devDependencies);
+ */
+const getShrinkwrapDependencyManifest = (shrinkwrapPath) => {
+  const manifestPath = _.chain(shrinkwrapPath)
+    .flatMap((dependency) => {
+      return [ 'node_modules', dependency ];
+    })
+    .concat([ 'package.json' ])
+    .reduce((accumulator, file) => {
+      return path.join(accumulator, file);
+    }, '.')
+    .value();
+
+  try {
+
+    // For example
+    // ./node_modules/drivelist/node_modules/lodash/package.json
+    return require(`.${path.sep}${manifestPath}`);
+
+  } catch (error) {
+    if (error.code === 'MODULE_NOT_FOUND') {
+      return null;
+    }
+
+    throw error;
+  }
+};
+
+/**
+ * @summary Get the top level dependencies of a shrinkwrap object
+ * @function
+ * @private
+ *
+ * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency
+ * @returns {Object} top level dependencies
+ *
+ * @example
+ * const dependencies = getTopLevelDependenciesForShrinkwrapPath([ 'debug' ]);
+ * console.log(dependencies);
+ * > {
+ * >   "lodash": "^4.0.0"
+ * > }
+ */
+const getTopLevelDependenciesForShrinkwrapPath = (shrinkwrapPath) => {
+  return _.get(getShrinkwrapDependencyManifest(shrinkwrapPath), [ 'dependencies' ], {});
+};
+
+/**
+ * @summary Get the dependency tree of a shrinkwrap dependency
+ * @function
+ * @private
+ *
+ * @param {Object} shrinkwrap - the shrinkwrap file contents
+ * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency
+ * @returns {Object[]} dependency tree
+ *
+ * @example
+ * const dependencyTree = getDependencyTree(require('./npm-shrinkwrap.json'), [ 'drivelist' ]);
+ *
+ * _.each(dependencyTree, (dependency) => {
+ *   console.log(dependency.name);
+ *   console.log(dependency.path);
+ *   console.log(dependency.version);
+ * })
+ */
+const getDependencyTree = (shrinkwrap, shrinkwrapPath) => {
+  const dependencies = getTopLevelDependenciesForShrinkwrapPath(shrinkwrapPath);
+
+  if (_.isEmpty(dependencies)) {
+    return [];
+  }
+
+  const object = getShrinkwrapDependencyObject(shrinkwrap, shrinkwrapPath);
+  const result = _.map(dependencies, (version, name) => {
+    const dependencyPath = _.has(object.dependencies, name) ? _.concat(shrinkwrapPath, [ name ]) : [ name ];
+    return getPrettyShrinkwrapDependencyObject(shrinkwrap, dependencyPath);
   });
+
+  return _.concat(result, _.flatMapDeep(result, (dependency) => {
+    return getDependencyTree(shrinkwrap, dependency.path);
+  }));
+};
+
+/**
+ * @summary Remove certain development optional dependencies from a shrinkwrap file
+ * @function
+ * @private
+ *
+ * @description
+ * A shrinkwrap object is a recursive data structure, that apart
+ * from some extra metadata, has the following structure:
+ *
+ * {
+ *   ...
+ *   "dependencies": {
+ *     "<dependency_name>": <recursive definition>,
+ *     "<dependency_name>": <recursive definition>,
+ *     "<dependency_name>": <recursive definition>,
+ *     ...
+ *   }
+ * }
+ *
+ * The purpose of this function is to remove certain dependencies
+ * that match a blacklist. In order to do so, we start from the top
+ * level object, remove the blacklisted dependencies, and recurse
+ * if possible.
+ *
+ * @param {Object} shrinkwrap - the shrinkwrap object
+ * @param {Object[]} blacklist - dependency blacklist
+ * @returns {Object} filtered shrinkwrap object
+ *
+ * @example
+ * const shrinkwrapFile = require('./npm-shrinkwrap.json');
+ * const dependencyTree = getDependencyTree(shrinkwrapFile, [ 'drivelist' ]);
+ * const filteredShrinkwrap = removeOptionalDevelopmentDependencies(shrinkwrapFile, dependencyTree);
+ */
+const removeOptionalDevelopmentDependencies = (shrinkwrap, blacklist) => {
+  if (!_.isEmpty(shrinkwrap.dependencies)) {
+    shrinkwrap.dependencies = _.chain(shrinkwrap.dependencies)
+      .omitBy((dependency, name) => {
+        return _.every([
+          _.find(blacklist, {
+            name,
+            version: dependency.version
+          }),
+          dependency.dev,
+          dependency.optional
+        ]);
+      })
+      .mapValues((dependency) => {
+        return removeOptionalDevelopmentDependencies(dependency, blacklist);
+      })
+      .value();
+  }
+
+  return shrinkwrap;
+};
+
+/**
+ * @summary Get the dependency tree of a dependency plus the dependency itself
+ * @function
+ * @private
+ *
+ * @param {Object} shrinkwrap - the shrinkwrap file contents
+ * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency
+ * @returns {Object[]} tree
+ *
+ * @example
+ * const tree = getTree(require('./npm-shrinkwrap.json'), [ 'drivelist' ]);
+ *
+ * _.each(tree, (dependency) => {
+ *   console.log(dependency.name);
+ *   console.log(dependency.path);
+ *   console.log(dependency.version);
+ * });
+ */
+const getTree = (shrinkwrap, shrinkwrapPath) => {
+  return _.compact(_.concat([
+    getPrettyShrinkwrapDependencyObject(shrinkwrap, shrinkwrapPath)
+  ], getDependencyTree(shrinkwrap, shrinkwrapPath)));
+};
+
+const blacklist = _.reduce(platformSpecificDependencies, (accumulator, dependencyPath) => {
+  return _.concat(accumulator, getTree(shrinkwrapFile, dependencyPath));
+}, []);
+
+const filteredShrinkwrap = removeOptionalDevelopmentDependencies(shrinkwrapFile, blacklist);
+const result = JSON.stringify(filteredShrinkwrap, null, JSON_INDENTATION_SPACES);
+
+fs.writeFileSync(NPM_SHRINKWRAP_FILE_PATH, `${result}\n`);
+console.log('Done');