[ci]: Made various changes for the electron app:

- Support for multiple electron targe per platform.
 - Removed packager CLI. Changed the logic we calculate the app name.
 - Fixed various OS-specific tests: stubbed `os`.
 - Restructured the final ZIP formats for Windows and Linux.
 - Added packager tests.
 - Switched from `@grpc/grpc-js` to native `grpc`.
 - Updated the version from 0.0.5 to 0.0.6.

Signed-off-by: Akos Kitta <kittaakos@typefox.io>

electron/build/scripts/arduino-pro-ide-electron-main.js

@@ -1,6 +1,9 @@
 const os = require('os');
 const path = require('path');
 
+// To be able to propagate the `process.versions.electron` to the backend main, so that we can load natives correctly.
+process.env.THEIA_ELECTRON_VERSION = process.versions.electron;
+
 process.env.THEIA_DEFAULT_PLUGINS = `local-dir:${path.resolve(__dirname, '..', 'plugins')}`;
 process.env.THEIA_PLUGINS = [
     process.env.THEIA_PLUGINS,

electron/build/scripts/patch-backend-main.js

@@ -0,0 +1,57 @@
+//@ts-check
+// Patches the `src-gen/backend/main.js` so that the forked backend process has the `process.versions.electron` in the bundled electron app.
+// https://github.com/eclipse-theia/theia/issues/7358#issue-583306096
+
+const args = process.argv.slice(2);
+if (!args.length) {
+    console.error(`Expected an argument pointing to the app folder. An app folder is where you have the package.json and src-gen folder.`);
+    process.exit(1);
+}
+if (args.length > 1) {
+    console.error(`Expected exactly one argument pointing to the app folder. Got multiple instead: ${JSON.stringify(args)}`);
+    process.exit(1);
+}
+const arg = args.shift();
+if (!arg) {
+    console.error('App path was not specified.');
+    process.exit(1);
+}
+
+const fs = require('fs');
+const path = require('path');
+const appPath = path.resolve((path.isAbsolute(arg) ? path.join(process.cwd(), arg) : arg));
+if (!fs.existsSync(appPath)) {
+    console.error(`${appPath} does not exist.`);
+    process.exit(1);
+}
+
+if (!fs.lstatSync(appPath).isDirectory()) {
+    console.error(`${appPath} is not a directory.`);
+    process.exit(1);
+}
+
+const patched = path.join(appPath, 'src-gen', 'backend', 'original-main.js');
+if (fs.existsSync(patched)) {
+    console.error(`Already patched. ${patched} already exists.`);
+    process.exit(1);
+}
+
+const toPatch = path.join(appPath, 'src-gen', 'backend', 'main.js');
+if (fs.existsSync(patched)) {
+    console.error(`Cannot patch. ${toPatch} does not exist.`);
+    process.exit(1);
+}
+
+console.log(`⏱️  >>> Patching ${toPatch}...`);
+
+const originalContent = fs.readFileSync(toPatch, { encoding: 'utf8' });
+const patchedContent = `if (typeof process.versions.electron === 'undefined' && typeof process.env.THEIA_ELECTRON_VERSION === 'string') {
+    process.versions.electron = process.env.THEIA_ELECTRON_VERSION;
+}
+require('./original-main');
+`
+
+fs.writeFileSync(patched, originalContent);
+fs.writeFileSync(toPatch, patchedContent);
+
+console.log(`👌 <<< Patched ${toPatch}. Original 'main.js' is now at ${patched}.`);

electron/build/template-package.json

@@ -12,17 +12,17 @@
   },
   "devDependencies": {
     "@theia/cli": "next",
-    "electron-builder": "^21.2.0"
+    "electron-builder": "^22.4.1"
   },
   "scripts": {
-    "build": "yarn download:plugins && theia build --mode development",
-    "build:release": "yarn download:plugins && theia build --mode development",
+    "build": "yarn download:plugins && theia build --mode development && yarn patch:main",
+    "build:release": "yarn download:plugins && theia build --mode production && yarn patch:main",
     "package": "electron-builder --publish=never",
-    "package:preview": "electron-builder --dir",
-    "download:plugins": "theia download:plugins"
+    "download:plugins": "theia download:plugins",
+    "patch:main": "node ./scripts/patch-backend-main ."
   },
   "engines": {
-    "node": ">=10.10.0"
+    "node": ">=10.11.0 <12"
   },
   "repository": {
     "type": "git",
@@ -80,9 +80,17 @@
     },
     "linux": {
       "target": [
-        "zip"
+        {
+          "target": "zip"
+        },
+        {
+          "target": "AppImage",
+          "arch": "armv7l"
+        }
       ],
-      "artifactName": "${productName}-${env.ARDUINO_VERSION}-${os}.${ext}"
+      "category": "Development",
+      "icon": "resources/icons",
+      "artifactName": "${productName}-${env.ARDUINO_VERSION}-${os}-${arch}.${ext}"
     },
     "dmg": {
       "icon": "resources/icon.icns",

electron/packager/cli

@@ -1,34 +0,0 @@
-#!/usr/bin/env node
-// @ts-check
-const { versionInfo } = require('./utils');
-const yargs = require('yargs');
-
-(() => {
-    yargs
-        .command({
-            command: 'name',
-            describe: 'Returns with the application name we build. The name includes the full application name with the version, the platform and the file extension.',
-            handler: () => {
-                const { platform } = process;
-                let ext = undefined;
-                let os = undefined;
-                if (platform === 'darwin') {
-                    ext = 'dmg';
-                    os = 'mac';
-                } else if (platform === 'win32') {
-                    ext = 'zip';
-                    os = 'win';
-                } else if (platform === 'linux') {
-                    ext = 'zip';
-                    os = 'linux';
-                } else {
-                    process.stderr.write(`Unexpected platform: ${platform}.`);
-                    process.exit(1);
-                }
-                process.stdout.write(`Arduino Pro IDE-${versionInfo().version}-${os}.${ext}`);
-                process.exit(0);
-            }
-        })
-        .demandCommand(1)
-        .argv;
-})();

electron/packager/index.js

@@ -5,6 +5,8 @@
     const fs = require('fs');
     const join = require('path').join;
     const shell = require('shelljs');
+    const glob = require('glob');
+    const isCI = require('is-ci');
     shell.env.THEIA_ELECTRON_SKIP_REPLACE_FFMPEG = '1'; // Do not run the ffmpeg validation for the packager.
     shell.env.NODE_OPTIONS = '--max_old_space_size=4096'; // Increase heap size for the CI
     const utils = require('./utils');
@@ -72,13 +74,13 @@
     // We have to do it before changing the dependencies to `local-path`.
     const unusedDependencies = await utils.collectUnusedDependencies('../working-copy/electron-app/');
 
-     //-------------------------------------------------------------------------------------------------------------+
-     // Change the regular NPM dependencies to `local-paths`, so that we can build them without any NPM registries. |
-     //-------------------------------------------------------------------------------------------------------------+
-     // @ts-ignore
-     pkg = require('../working-copy/arduino-debugger-extension/package.json');
-     pkg.dependencies['arduino-ide-extension'] = 'file:../arduino-ide-extension';
-     fs.writeFileSync(path('..', workingCopy, 'arduino-debugger-extension', 'package.json'), JSON.stringify(pkg, null, 2));
+    //-------------------------------------------------------------------------------------------------------------+
+    // Change the regular NPM dependencies to `local-paths`, so that we can build them without any NPM registries. |
+    //-------------------------------------------------------------------------------------------------------------+
+    // @ts-ignore
+    pkg = require('../working-copy/arduino-debugger-extension/package.json');
+    pkg.dependencies['arduino-ide-extension'] = 'file:../arduino-ide-extension';
+    fs.writeFileSync(path('..', workingCopy, 'arduino-debugger-extension', 'package.json'), JSON.stringify(pkg, null, 2));
 
     //------------------------------------------------------------------------------------+
     // Merge the `working-copy/package.json` with `electron/build/template-package.json`. |
@@ -138,6 +140,18 @@ ${fs.readFileSync(path('..', 'build', 'package.json')).toString()}
     // Package the electron application. |
     //-----------------------------------+
     exec(`yarn --network-timeout 1000000 --cwd ${path('..', 'build')} package`, `Packaging your Arduino Pro IDE application`);
+
+    //-----------------------------------------------------------------------------------------------------+
+    // Copy to another folder. Azure does not support wildcard for `PublishBuildArtifacts@1.pathToPublish` |
+    //-----------------------------------------------------------------------------------------------------+
+    if (isCI) {
+        try {
+            await copyFilesToBuildArtifacts();
+        } catch (e) {
+            echo(JSON.stringify(e));
+            shell.exit(1);
+        }
+    }
     echo(`🎉  Success. Your application is at: ${path('..', 'build', 'dist')}`);
 
     restore();
@@ -201,6 +215,47 @@ ${fs.readFileSync(path('..', 'build', 'package.json')).toString()}
         }
     }
 
+    async function copyFilesToBuildArtifacts() {
+        echo(`🚢  Detected CI, moving build artifacts...`);
+        const { platform } = process;
+        const cwd = path('..', 'build', 'dist');
+        const targetFolder = path('..', 'build', 'dist', 'build-artifacts');
+        mkdir('-p', targetFolder);
+        const filesToCopy = [];
+        switch (platform) {
+            case 'linux': {
+                filesToCopy.push(...glob.sync('**/Arduino Pro IDE*.{zip,AppImage}', { cwd }).map(p => join(cwd, p)));
+                break;
+            }
+            case 'win32': {
+                filesToCopy.push(...glob.sync('**/Arduino Pro IDE*.zip', { cwd }).map(p => join(cwd, p)));
+                break;
+            }
+            case 'darwin': {
+                filesToCopy.push(...glob.sync('**/Arduino Pro IDE*.dmg', { cwd }).map(p => join(cwd, p)));
+                break;
+            }
+            default: {
+                echo(`Unsupported platform: ${platform}.`);
+                shell.exit(1);
+            }
+        }
+        if (!filesToCopy.length) {
+            echo(`Could not collect any build artifacts from ${cwd}.`);
+            shell.exit(1);
+        }
+        for (const fileToCopy of filesToCopy) {
+            echo(`🚢  >>> Copying ${fileToCopy} to ${targetFolder}.`);
+            const isZip = await utils.isZip(fileToCopy);
+            if (isZip) {
+                await utils.adjustArchiveStructure(fileToCopy, targetFolder);
+            } else {
+                cp('-rf', fileToCopy, targetFolder);
+            }
+            echo(`👌  >>> Copied ${fileToCopy} to ${targetFolder}.`);
+        }
+    }
+
     /**
      * Joins tha path from `__dirname`.
      */

electron/packager/package.json

@@ -5,19 +5,36 @@
   "description": "Packager for the Arduino Pro IDE electron application",
   "main": "index.js",
   "scripts": {
+    "prepare": "yarn test",
     "package": "node index.js",
-    "cli": "./cli"
+    "test": "mocha \"./test/**/*.test.js\""
   },
   "keywords": [],
   "author": "Arduino SA",
   "license": "MIT",
   "dependencies": {
-    "deepmerge": "4.2.2",
-    "depcheck": "^0.7.1",
+    "@types/file-type": "^10.9.1",
+    "@types/temp": "^0.8.32",
+    "7zip-min": "^1.1.1",
+    "chai": "^4.2.0",
+    "deepmerge": "^4.2.2",
+    "depcheck": "^0.9.2",
+    "file-type": "^14.1.4",
+    "glob": "^7.1.6",
+    "is-ci": "^2.0.0",
+    "mocha": "^7.1.1",
+    "sinon": "^9.0.1",
     "shelljs": "^0.8.3",
+    "temp": "^0.9.1",
     "yargs": "^12.0.5"
   },
   "engines": {
-    "node": ">=8.12.0"
+    "node": ">=10.11.0 <12"
+  },
+  "mocha": {
+    "reporter": "spec",
+    "colors": true,
+    "watch-extensions": "js",
+    "timeout": 10000
   }
 }

electron/packager/test/utils.test.js

@@ -0,0 +1,119 @@
+const fs = require('fs');
+const path = require('path');
+const expect = require('chai').expect;
+const track = require('temp').track();
+const unpack = require('../utils').unpack;
+const testMe = require('../utils');
+const sinon = require('sinon');
+
+describe('utils', () => {
+
+    describe('adjustArchiveStructure', () => {
+
+        let consoleStub;
+
+        beforeEach(() => {
+            consoleStub = sinon.stub(console, 'log').value(() => { });
+        });
+
+        afterEach(() => {
+            consoleStub.reset();
+            track.cleanupSync();
+        });
+
+        it('should reject when not a zip file', async () => {
+            try {
+                const invalid = path.join(__dirname, 'resources', 'not-a-zip.dmg');
+                await testMe.adjustArchiveStructure(invalid, track.mkdirSync());
+                throw new Error('Expected a rejection');
+            } catch (e) {
+                expect(e).to.be.an.instanceOf(Error);
+                expect(e.message).to.be.equal('Expected a ZIP file.');
+            }
+        });
+
+        it('should reject when target directory does not exist', async () => {
+            try {
+                const zip = path.join(__dirname, 'resources', 'zip-with-base-folder.zip');
+                await testMe.adjustArchiveStructure(zip, path.join(__dirname, 'some', 'missing', 'path'));
+                throw new Error('Expected a rejection');
+            } catch (e) {
+                expect(e).to.be.an.instanceOf(Error);
+                expect(e.message.endsWith('does not exist.')).to.be.true;
+            }
+        });
+
+        it('should reject when target is a file', async () => {
+            try {
+                const zip = path.join(__dirname, 'resources', 'zip-with-base-folder.zip');
+                await testMe.adjustArchiveStructure(zip, path.join(__filename));
+                throw new Error('Expected a rejection');
+            } catch (e) {
+                expect(e).to.be.an.instanceOf(Error);
+                expect(e.message.endsWith('is not a directory.')).to.be.true;
+            }
+        });
+
+        it('should be a NOOP when the zip already has the desired base folder', async () => {
+            const zip = path.join(__dirname, 'resources', 'zip-with-base-folder.zip');
+            const actual = await testMe.adjustArchiveStructure(zip, track.mkdirSync());
+            expect(actual).to.be.equal(zip);
+        });
+
+        it('should handle whitespace in file path gracefully', async () => {
+            const zip = path.join(__dirname, 'resources', 'zip with whitespace.zip');
+            const out = track.mkdirSync();
+            const actual = await testMe.adjustArchiveStructure(zip, out, true);
+            expect(actual).to.be.equal(path.join(out, 'zip with whitespace.zip'));
+            console.log(actual);
+            expect(fs.existsSync(actual)).to.be.true;
+
+            const verifyOut = track.mkdirSync();
+            await unpack(actual, verifyOut);
+
+            const root = path.join(verifyOut, 'zip with whitespace');
+            expect(fs.existsSync(root)).to.be.true;
+            expect(fs.lstatSync(root).isDirectory()).to.be.true;
+            const subs = fs.readdirSync(root);
+            expect(subs).to.have.lengthOf(3);
+            expect(subs.sort()).to.be.deep.equal(['a.txt', 'b.txt', 'foo']);
+        });
+
+        it('should keep the symlinks after ZIP adjustments', async function () {
+            if (process.platform === 'win32') {
+                this.skip();
+            }
+            const zip = path.join(__dirname, 'resources', 'zip-with-symlink.zip');
+            const out = track.mkdirSync();
+            const actual = await testMe.adjustArchiveStructure(zip, out, true);
+            expect(actual).to.be.equal(path.join(out, 'zip-with-symlink.zip'));
+            console.log(actual);
+            expect(fs.existsSync(actual)).to.be.true;
+
+            const verifyOut = track.mkdirSync();
+            await unpack(actual, verifyOut);
+            expect(fs.lstatSync(path.join(verifyOut, 'zip-with-symlink', 'folder', 'symlinked-sub')).isSymbolicLink()).to.be.true;
+        });
+
+        it('should adjust the archive structure if base folder is not present', async () => {
+            const zip = path.join(__dirname, 'resources', 'zip-without-symlink.zip');
+            const out = track.mkdirSync();
+            const actual = await testMe.adjustArchiveStructure(zip, out, true);
+            expect(actual).to.be.equal(path.join(out, 'zip-without-symlink.zip'));
+            console.log(actual);
+            expect(fs.existsSync(actual)).to.be.true;
+
+            const verifyOut = track.mkdirSync();
+            await unpack(actual, verifyOut);
+
+            const root = path.join(verifyOut, 'zip-without-symlink');
+            expect(fs.existsSync(root)).to.be.true;
+            expect(fs.lstatSync(root).isDirectory()).to.be.true;
+            const subs = fs.readdirSync(root);
+            expect(subs).to.have.lengthOf(3);
+            expect(subs.sort()).to.be.deep.equal(['a.txt', 'b.txt', 'foo']);
+        });
+
+    });
+
+});

electron/packager/utils.js

@@ -2,8 +2,11 @@
 
 const fs = require('fs');
 const path = require('path');
+const temp = require('temp');
+const zip = require('7zip-min');
 const shell = require('shelljs');
 const depcheck = require('depcheck');
+const fromFile = require('file-type').fromFile;
 
 /**
  * Returns with the version info for the artifact.
@@ -67,7 +70,7 @@ function currentCommitish() {
  */
 function collectUnusedDependencies(pathToProject = process.cwd()) {
     const p = path.isAbsolute(pathToProject) ? pathToProject : path.resolve(process.cwd(), pathToProject);
-    console.log(`⏱️  >>> Collecting unused backend dependencies for ${p}.`);
+    console.log(`⏱️  >>> Collecting unused backend dependencies for ${p}...`);
     return new Promise(resolve => {
         depcheck(p, {
             ignoreDirs: [
@@ -97,4 +100,108 @@ function collectUnusedDependencies(pathToProject = process.cwd()) {
     })
 }
 
-module.exports = { versionInfo, collectUnusedDependencies };
+/**
+ * `pathToZip` is a `path/to/your/app-name.zip`.
+ * If the `pathToZip` archive does not have a root directory with name `app-name`, it creates one, and move the content from the
+ * archive's root to the new root folder. If the archive already has the desired root folder, calling this function is a NOOP.
+ * If `pathToZip` is not a ZIP, rejects. `targetFolderName` is the destination folder not the new archive location.
+ */
+function adjustArchiveStructure(pathToZip, targetFolderName, noCleanup) {
+    return new Promise(async (resolve, reject) => {
+        if (!await isZip(pathToZip)) {
+            reject(new Error(`Expected a ZIP file.`));
+            return;
+        }
+        if (!fs.existsSync(targetFolderName)) {
+            reject(new Error(`${targetFolderName} does not exist.`));
+            return;
+        }
+        if (!fs.lstatSync(targetFolderName).isDirectory()) {
+            reject(new Error(`${targetFolderName} is not a directory.`));
+            return;
+        }
+        console.log(`⏱️  >>> Adjusting ZIP structure ${pathToZip}...`);
+
+        const root = basename(pathToZip);
+        const resources = await list(pathToZip);
+        const hasBaseFolder = resources.find(name => name === root);
+        if (hasBaseFolder) {
+            if (resources.filter(name => name.indexOf(path.sep) === -1).length > 1) {
+                console.warn(`${pathToZip} ZIP has the desired root folder ${root}, however the ZIP contains other entries too: ${JSON.stringify(resources)}`);
+            }
+            console.log(`👌  <<< The ZIP already has the desired ${root} folder.`);
+            resolve(pathToZip);
+            return;
+        }
+
+        const track = temp.track();
+        try {
+            const unzipOut = path.join(track.mkdirSync(), root);
+            fs.mkdirSync(unzipOut);
+            await unpack(pathToZip, unzipOut);
+            const adjustedZip = path.join(targetFolderName, path.basename(pathToZip));
+            await pack(unzipOut, adjustedZip);
+            console.log(`👌  <<< Adjusted the ZIP structure. Moved the modified ${basename(pathToZip)} to the ${targetFolderName} folder.`);
+            resolve(adjustedZip);
+        } finally {
+            if (!noCleanup) {
+                track.cleanupSync();
+            }
+        }
+    });
+}
+
+/**
+ * Returns the `basename` of `pathToFile` without the file extension.
+ */
+function basename(pathToFile) {
+    const name = path.basename(pathToFile);
+    const ext = path.extname(pathToFile);
+    return name.substr(0, name.length - ext.length);
+}
+
+function unpack(what, where) {
+    return new Promise((resolve, reject) => {
+        zip.unpack(what, where, error => {
+            if (error) {
+                reject(error);
+                return;
+            }
+            resolve();
+        })
+    });
+}
+
+function pack(what, where) {
+    return new Promise((resolve, reject) => {
+        zip.pack(what, where, error => {
+            if (error) {
+                reject(error);
+                return;
+            }
+            resolve();
+        })
+    });
+}
+
+function list(what) {
+    return new Promise((resolve, reject) => {
+        zip.list(what, (error, result) => {
+            if (error) {
+                reject(error);
+                return;
+            }
+            resolve(result.map(({ name }) => name));
+        })
+    });
+}
+
+async function isZip(pathToFile) {
+    if (!fs.existsSync(pathToFile)) {
+        throw new Error(`${pathToFile} does not exist`);
+    }
+    const type = await fromFile(pathToFile);
+    return type && type.ext === 'zip';
+}
+
+module.exports = { versionInfo, collectUnusedDependencies, adjustArchiveStructure, isZip, unpack };