Merge pull request #114 from bcmi-labs/gh-actions

Gh actions

.github/workflows/build.yml

@@ -0,0 +1,89 @@
+name: Arduino Pro IDE
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
+  pull_request:
+    branches:
+      - master
+  schedule:
+    - cron: '0 3 * * *' # run every day at 3AM (https://docs.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events-schedule)
+
+env:
+  IS_NIGHTLY: ${{ github.event_name == 'schedule' || github.head_ref == 'gh-actions' }} # TODO: remove OR once PR is merged.
+  IS_RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }}
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        config:
+          - os: windows-2016
+          - os: ubuntu-latest
+          - os: turin-macmini # self-hosted macOS
+
+    runs-on: ${{ matrix.config.os }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install Node.js 10.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: '10.x'
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Install Python 2.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: '2.7'
+
+      - name: Build
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+        run: yarn
+
+      - name: Test
+        run: yarn test
+
+      - name: Package
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/Microsoft/vscode/issues/28434#issuecomment-346199674
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+          AC_USERNAME: ${{ secrets.AC_USERNAME }}
+          AC_PASSWORD: ${{ secrets.AC_PASSWORD }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          if [ "$RUNNER_OS" == "macOS" ]; then
+            echo ">>> Detected macOS runner. Updating keychain for the app signing..."
+            echo "${{ secrets.KEYCHAIN }}" | base64 --decode > ~/Library/Keychains/apple-developer.keychain-db
+            security list-keychains -s ~/Library/Keychains/apple-developer.keychain-db
+            security unlock-keychain -p "${{ secrets.KEYCHAIN_PASSWORD }}" ~/Library/Keychains/apple-developer.keychain-db
+            echo "<<< The keychain has been successfully updated."
+          fi
+            yarn --cwd ./electron/packager/
+            yarn --cwd ./electron/packager/ package
+
+      - name: Upload [GitHub Actions]
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ runner.os }}
+          path: electron/build/dist/build-artifacts/
+
+      - name: Upload [S3]
+        if: env.IS_NIGHTLY == 'true'
+        uses: kittaakos/upload-s3-action@v0.0.1
+        with:
+          aws_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_bucket: ${{ secrets.DOWNLOADS_BUCKET }}
+          source_dir: electron/build/dist/build-artifacts/
+          destination_dir: arduino-pro-ide/nightly/

.vscode/launch.json

@@ -13,7 +13,7 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch Electron Packager",
+      "name": "Electron Packager",
       "program": "${workspaceRoot}/electron/packager/index.js",
       "cwd": "${workspaceFolder}/electron/packager"
     },

arduino-ide-extension/scripts/download-cli.js

@@ -10,7 +10,7 @@
 
 (() => {
 
-    const DEFAULT_VERSION = '0.9.0'; // require('moment')().format('YYYYMMDD');
+    const DEFAULT_VERSION = '0.11.0'; // require('moment')().format('YYYYMMDD');
 
     const path = require('path');
     const shell = require('shelljs');

arduino-ide-extension/scripts/generate-protocol.js

@@ -35,13 +35,10 @@
         }
     }
 
-    shell.echo('Generating TS/JS API from:');
+    shell.echo('>>> Generating TS/JS API from:');
     if (shell.exec(`git -C ${repository} rev-parse --abbrev-ref HEAD`).code !== 0) {
         shell.exit(1);
     }
-    if (shell.exec(`git -C ${repository} rev-parse --short HEAD`).code !== 0) {
-        shell.exit(1);
-    }
 
     const pluginExec = shell.which('grpc_tools_node_protoc_plugin');
     if (!pluginExec || pluginExec.code !== 0) {
@@ -86,6 +83,6 @@ ${protos.join(' ')}`).code !== 0) {
         shell.exit(1);
     }
 
-    shell.echo('Done.');
+    shell.echo('<<< Generation was successful.');
 
 })();

arduino-ide-extension/src/browser/arduino-commands.ts

@@ -38,6 +38,13 @@ export namespace ArduinoCommands {
         id: 'arduino-open-file'
     }
 
+    /**
+     * Unlike `OPEN_SKETCH`, it opens all files from a sketch folder. (ino, cpp, etc...)
+     */
+    export const OPEN_SKETCH_FILES: Command = {
+        id: 'arduino-open-sketch-files'
+    }
+
     export const SAVE_SKETCH: Command = {
         id: 'arduino-save-file'
     }

arduino-ide-extension/src/browser/arduino-frontend-contribution.tsx

@@ -322,6 +322,12 @@ export class ArduinoFrontendContribution implements FrontendApplicationContribut
             }
         });
 
+        registry.registerCommand(ArduinoCommands.OPEN_SKETCH_FILES, {
+            execute: async (uri: string) => {
+                this.openSketchFiles(uri);
+            }
+        });
+
         registry.registerCommand(ArduinoCommands.SAVE_SKETCH, {
             isVisible: widget => ArduinoToolbar.is(widget) && widget.side === 'left',
             execute: (sketch: Sketch) => {
@@ -548,7 +554,7 @@ export class ArduinoFrontendContribution implements FrontendApplicationContribut
         });
     }
 
-    async openSketchFiles(uri: string): Promise<void> {
+    protected async openSketchFiles(uri: string): Promise<void> {
         const uris = await this.sketchService.getSketchFiles(uri);
         for (const uri of uris) {
             await this.editorManager.open(new URI(uri));

arduino-ide-extension/src/browser/customization/arduino-frontend-application.ts

@@ -1,9 +1,10 @@
 import { injectable, inject } from 'inversify';
 import { FileSystem } from '@theia/filesystem/lib/common/filesystem';
+import { CommandService } from '@theia/core/lib/common/command';
 import { WorkspaceService } from '@theia/workspace/lib/browser/workspace-service';
 import { FrontendApplication } from '@theia/core/lib/browser/frontend-application';
 import { EditorMode } from '../editor-mode';
-import { ArduinoFrontendContribution } from '../arduino-frontend-contribution';
+import { ArduinoCommands } from '../arduino-commands';
 
 @injectable()
 export class ArduinoFrontendApplication extends FrontendApplication {
@@ -14,8 +15,8 @@ export class ArduinoFrontendApplication extends FrontendApplication {
     @inject(WorkspaceService)
     protected readonly workspaceService: WorkspaceService;
 
-    @inject(ArduinoFrontendContribution)
+    @inject(CommandService)
-    protected readonly frontendContribution: ArduinoFrontendContribution;
+    protected readonly commandService: CommandService;
 
     @inject(EditorMode)
     protected readonly editorMode: EditorMode;
@@ -30,7 +31,7 @@ export class ArduinoFrontendApplication extends FrontendApplication {
             for (const root of roots) {
                 const exists = await this.fileSystem.exists(root.uri);
                 if (exists) {
-                    await this.frontendContribution.openSketchFiles(root.uri);
+                    await this.commandService.executeCommand(ArduinoCommands.OPEN_SKETCH_FILES.id, root.uri);
                 }
             }
         }

arduino-ide-extension/src/node/arduino-env-variables-server.ts

@@ -1,12 +1,13 @@
 import { join } from 'path';
 import { homedir } from 'os';
 import { injectable } from 'inversify';
-import { EnvVariablesServerImpl } from '@theia/core/lib/node/env-variables/env-variables-server';
 import { FileUri } from '@theia/core/lib/node/file-uri';
+import { EnvVariablesServerImpl } from '@theia/core/lib/node/env-variables/env-variables-server';
+import { BackendApplicationConfigProvider } from '@theia/core/lib/node/backend-application-config-provider';
 
 @injectable()
 export class ArduinoEnvVariablesServer extends EnvVariablesServerImpl {
 
-    protected readonly configDirUri = FileUri.create(join(homedir(), '.arduinoProIDE')).toString();
+    protected readonly configDirUri = FileUri.create(join(homedir(), BackendApplicationConfigProvider.get().configDirName)).toString();
 
 }

arduino-ide-extension/src/node/boards-service-impl.ts

@@ -220,7 +220,7 @@ export class BoardsServiceImpl implements BoardsService {
             resolve(resp);
         }));
 
-        const requiredTools = resp.getRequiredToolsList().map(t => <Tool>{
+        const requiredTools = resp.getToolsdependenciesList().map(t => <Tool>{
             name: t.getName(),
             packager: t.getPackager(),
             version: t.getVersion()

arduino-ide-extension/src/test/node/arduino-daemon-impl.test.ts

@@ -36,7 +36,7 @@ class SilentArduinoDaemonImpl extends ArduinoDaemonImpl {
         const destDir = track.mkdirSync();
         await spawnCommand(`"${cliPath}"`, ['config', 'init', '--dest-dir', destDir]);
         const content = fs.readFileSync(path.join(destDir, CLI_CONFIG), { encoding: 'utf8' });
-        const cliConfig = safeLoad(content);
+        const cliConfig = safeLoad(content) as any;
         cliConfig.daemon.port = String(this.port);
         const modifiedContent = safeDump(cliConfig);
         fs.writeFileSync(path.join(destDir, CLI_CONFIG), modifiedContent, { encoding: 'utf8' });

azure-pipelines.yml

@@ -1,81 +0,0 @@
-trigger:
-  batch: true
-  branches:
-    include:
-    - master
-
-pr:
-- master
-
-jobs:
-- job: Build
-  strategy:
-    matrix:
-      linux:
-        imageName: 'ubuntu-16.04'
-      mac:
-        imageName: 'macos-10.14'
-      windows:
-        imageName: 'vs2017-win2016'
-  pool:
-    vmImage: $(imageName)
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '2.7'
-      architecture: 'x64'
-    displayName: '[Config] Use - Python 2.7'
-  - task: NodeTool@0
-    inputs:
-      versionSpec: '10.x'
-    displayName: '[Config] Use - Node.js 10.x'
-  - script: yarn
-    env:
-      GITHUB_TOKEN: $(Personal.GitHub.Token)
-      THEIA_ELECTRON_SKIP_REPLACE_FFMPEG: 1
-    displayName: Build
-  - script: yarn test
-    displayName: Test
-  - bash: |
-      ./electron/packager/conf-node-gyp.sh
-      yarn --cwd ./electron/packager/
-      yarn --cwd ./electron/packager/ package
-    env:
-      GITHUB_TOKEN: $(Personal.GitHub.Token)
-      RELEASE_TAG: $(Release.Tag)
-    condition: or(in(variables['Agent.OS'], 'Windows_NT'), in(variables['Build.Reason'], 'Manual', 'Schedule'))
-    displayName: Package
-  - task: PublishBuildArtifacts@1
-    inputs:
-      pathtoPublish: electron/build/dist/build-artifacts
-      artifactName: 'Arduino Pro IDE - Applications'
-    condition: or(in(variables['Agent.OS'], 'Windows_NT'), in(variables['Build.Reason'], 'Manual', 'Schedule'))
-    displayName: Publish
-- job: Release
-  pool:
-    vmImage: ubuntu-16.04
-  dependsOn:
-    - Build
-  condition: and(succeeded(), and(in(variables['Build.Reason'], 'Manual', 'Schedule'), startsWith(variables['Release.Tag'], 'v')))
-  steps:
-  - task: DownloadBuildArtifacts@0
-    displayName: Download
-    inputs:
-      artifactName: 'Arduino Pro IDE - Applications'
-      downloadPath: 'gh-release' 
-  - task: GithubRelease@0
-    inputs:
-      gitHubConnection: typefox-service-account1
-      repositoryName: bcmi-labs/arduino-editor
-      assets: |
-        gh-release/Arduino Pro IDE - Applications/*.zip
-        gh-release/Arduino Pro IDE - Applications/*.dmg
-        gh-release/Arduino Pro IDE - Applications/*.AppImage
-      target: $(Build.SourceVersion)
-      action: Edit
-      tagSource: auto
-      tag: $(Release.Tag)
-      assetUploadMode: delete
-      isDraft: true
-      addChangeLog: false
-    displayName: Release

electron-app/package.json

@@ -44,6 +44,11 @@
         }
       }
     },
+    "backend": {
+      "config": {
+        "configDirName": ".arduinoProIDE"
+      }
+    },
     "generator": {
       "config": {
         "preloadTemplate": "<div class='theia-preload' style='background-color: rgb(237, 241, 242);'></div>"

electron/README.md

@@ -48,3 +48,4 @@ One has to manually publish the GitHub release.
 
  - Select `Publish release`.
  - ✨
+

electron/build/resources/entitlements.mac.plist

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>com.apple.security.cs.allow-jit</key>
+    <true/>
+    <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+    <true/>
+    <key>com.apple.security.cs.disable-library-validation</key>
+    <true/>
+    <key>com.apple.security.cs.allow-dyld-environment-variables</key>
+    <true/>
+</dict>
+</plist>

electron/build/scripts/arduino-pro-ide-electron-main.js

@@ -1,13 +0,0 @@
-const os = require('os');
-const path = require('path');
-
-// To be able to propagate the `process.versions.electron` to the backend main, so that we can load natives correctly.
-process.env.THEIA_ELECTRON_VERSION = process.versions.electron;
-
-process.env.THEIA_DEFAULT_PLUGINS = `local-dir:${path.resolve(__dirname, '..', 'plugins')}`;
-process.env.THEIA_PLUGINS = [
-    process.env.THEIA_PLUGINS,
-    `local-dir:${path.resolve(os.homedir(), '.arduinoProIDE', 'plugins')}`
-].filter(Boolean).join(',');
-
-require('../src-gen/frontend/electron-main.js');

electron/build/scripts/notarize.js

@@ -0,0 +1,24 @@
+const isCI = require('is-ci');
+const { notarize } = require('electron-notarize');
+
+exports.default = async function notarizing(context) {
+    if (!isCI) {
+        console.log('Skipping notarization: not on CI.');
+        return;
+    }
+    const { electronPlatformName, appOutDir } = context;
+    if (electronPlatformName !== 'darwin') {
+        return;
+    }
+
+    const appName = context.packager.appInfo.productFilename;
+    const appBundleId = context.packager.config.appId;
+    console.log(`>>> Notarizing ${appBundleId} at ${appOutDir}/${appName}.app...`);
+
+    return await notarize({
+        appBundleId,
+        appPath: `${appOutDir}/${appName}.app`,
+        appleId: process.env.AC_USERNAME,
+        appleIdPassword: process.env.AC_PASSWORD,
+    });
+};

electron/build/scripts/patch-backend-main.js

@@ -1,57 +0,0 @@
-//@ts-check
-// Patches the `src-gen/backend/main.js` so that the forked backend process has the `process.versions.electron` in the bundled electron app.
-// https://github.com/eclipse-theia/theia/issues/7358#issue-583306096
-
-const args = process.argv.slice(2);
-if (!args.length) {
-    console.error(`Expected an argument pointing to the app folder. An app folder is where you have the package.json and src-gen folder.`);
-    process.exit(1);
-}
-if (args.length > 1) {
-    console.error(`Expected exactly one argument pointing to the app folder. Got multiple instead: ${JSON.stringify(args)}`);
-    process.exit(1);
-}
-const arg = args.shift();
-if (!arg) {
-    console.error('App path was not specified.');
-    process.exit(1);
-}
-
-const fs = require('fs');
-const path = require('path');
-const appPath = path.resolve((path.isAbsolute(arg) ? path.join(process.cwd(), arg) : arg));
-if (!fs.existsSync(appPath)) {
-    console.error(`${appPath} does not exist.`);
-    process.exit(1);
-}
-
-if (!fs.lstatSync(appPath).isDirectory()) {
-    console.error(`${appPath} is not a directory.`);
-    process.exit(1);
-}
-
-const patched = path.join(appPath, 'src-gen', 'backend', 'original-main.js');
-if (fs.existsSync(patched)) {
-    console.error(`Already patched. ${patched} already exists.`);
-    process.exit(1);
-}
-
-const toPatch = path.join(appPath, 'src-gen', 'backend', 'main.js');
-if (fs.existsSync(patched)) {
-    console.error(`Cannot patch. ${toPatch} does not exist.`);
-    process.exit(1);
-}
-
-console.log(`⏱️  >>> Patching ${toPatch}...`);
-
-const originalContent = fs.readFileSync(toPatch, { encoding: 'utf8' });
-const patchedContent = `if (typeof process.versions.electron === 'undefined' && typeof process.env.THEIA_ELECTRON_VERSION === 'string') {
-    process.versions.electron = process.env.THEIA_ELECTRON_VERSION;
-}
-require('./original-main');
-`
-
-fs.writeFileSync(patched, originalContent);
-fs.writeFileSync(toPatch, patchedContent);
-
-console.log(`👌 <<< Patched ${toPatch}. Original 'main.js' is now at ${patched}.`);

electron/build/template-package.json

@@ -1,25 +1,23 @@
 {
-  "name": "arduino-pro-ide",
+  "main": "src-gen/frontend/electron-main.js",
-  "description": "Arduino Pro IDE",
-  "main": "scripts/arduino-pro-ide-electron-main.js",
   "author": "Arduino SA",
-  "dependencies": {
-    "arduino-ide-extension": "file:../working-copy/arduino-ide-extension",
-    "arduino-debugger-extension": "file:../working-copy/arduino-debugger-extension"
-  },
   "resolutions": {
     "**/fs-extra": "^4.0.3"
   },
   "devDependencies": {
     "@theia/cli": "next",
-    "electron-builder": "^22.4.1"
+    "cross-env": "^7.0.2",
+    "electron-builder": "^22.4.1",
+    "electron-notarize": "^0.3.0",
+    "is-ci": "^2.0.0",
+    "shelljs": "^0.8.3"
   },
   "scripts": {
-    "build": "yarn download:plugins && theia build --mode development && yarn patch:main",
+    "build": "yarn download:plugins && theia build --mode development",
-    "build:release": "yarn download:plugins && theia build --mode production && yarn patch:main",
+    "build:publish": "yarn download:plugins && theia build --mode production",
-    "package": "electron-builder --publish=never",
+    "package": "cross-env DEBUG=* && electron-builder --publish=never",
-    "download:plugins": "theia download:plugins",
+    "package:publish": "cross-env DEBUG=* && electron-builder --publish=always",
-    "patch:main": "node ./scripts/patch-backend-main ."
+    "download:plugins": "theia download:plugins"
   },
   "engines": {
     "node": ">=10.11.0 <12"
@@ -35,8 +33,15 @@
     "target": "electron",
     "frontend": {
       "config": {
+        "applicationName": "Arduino Pro IDE",
         "disallowReloadKeybinding": true
       }
+    },
+    "backend": {
+      "config": {
+        "singleInstance": true,
+        "configDirName": ".arduinoProIDE"
+      }
     }
   },
   "build": {
@@ -49,7 +54,6 @@
     "files": [
       "src-gen",
       "lib",
-      "scripts",
       "!node_modules/**/*.{ts,map}",
       "!node_modules/**/*.spec.js",
       "!node_modules/@theia/**/test/*",
@@ -68,29 +72,23 @@
     "win": {
       "target": [
         "zip"
-      ],
+      ]
-      "artifactName": "${productName}-${env.ARDUINO_VERSION}-${os}.${ext}"
     },
     "mac": {
-      "target": [
+      "darkModeSupport": true,
-        "dmg"
+      "hardenedRuntime": true,
-      ],
+      "gatekeeperAssess": false,
-      "artifactName": "${productName}-${env.ARDUINO_VERSION}-${os}.${ext}",
+      "entitlements": "resources/entitlements.mac.plist",
-      "darkModeSupport": true
+      "entitlementsInherit": "resources/entitlements.mac.plist"
     },
     "linux": {
       "target": [
         {
           "target": "zip"
-        },
-        {
-          "target": "AppImage",
-          "arch": "armv7l"
         }
       ],
       "category": "Development",
-      "icon": "resources/icons",
+      "icon": "resources/icons"
-      "artifactName": "${productName}-${env.ARDUINO_VERSION}-${os}-${arch}.${ext}"
     },
     "dmg": {
       "icon": "resources/icon.icns",
@@ -108,10 +106,19 @@
           "type": "file"
         }
       ]
-    }
+    },
+    "afterSign": "scripts/notarize.js",
+    "publish": [
+      {
+        "provider": "s3",
+        "bucket": "arduino-downloads-prod-beagle",
+        "path": "arduino-pro-ide/nightly"
+      }
+    ]
   },
   "theiaPluginsDir": "plugins",
   "theiaPlugins": {
-    "vscode-yaml": "https://github.com/redhat-developer/vscode-yaml/releases/download/0.7.2/redhat.vscode-yaml-0.7.2.vsix"
+    "vscode-builtin-cpp": "http://open-vsx.org/api/vscode/cpp/1.44.2/file/vscode.cpp-1.44.2.vsix",
+    "vscode-yaml": "https://open-vsx.org/api/redhat/vscode-yaml/0.7.2/file/redhat.vscode-yaml-0.7.2.vsix"
   }
 }

electron/packager/config.js

@@ -0,0 +1,122 @@
+//@ts-check
+
+const fs = require('fs');
+const path = require('path');
+const semver = require('semver');
+const merge = require('deepmerge');
+const dateFormat = require('dateformat');
+const { isNightly, isRelease, git } = require('./utils');
+
+function artifactName() {
+    const { platform, arch } = process;
+    const id = (() => {
+        if (isRelease) {
+            return 'latest';
+        } else if (isNightly) {
+            return `nightly-${timestamp()}`
+        } else {
+            return 'snapshot';
+        }
+    })();
+    const name = 'arduino-pro-ide';
+    switch (platform) {
+        case 'win32': {
+            if (arch === 'x64') {
+                return `${name}_${id}_Windows_64bit.\$\{ext}`;
+            }
+            throw new Error(`Unsupported platform, arch: ${platform}, ${arch}`);
+        }
+        case 'darwin': {
+            return `${name}_${id}_macOS_64bit.\$\{ext}`;
+        }
+        case 'linux': {
+            switch (arch) {
+                case 'arm': {
+                    return `${name}_${id}_Linux_ARMv7.\$\{ext}`;
+                }
+                case 'arm64': {
+                    return `${name}_${id}_Linux_ARM64.\$\{ext}`;
+                }
+                case 'x64': {
+                    return `${name}_${id}_Linux_64bit.\$\{ext}`;
+                }
+                default: {
+                    throw new Error(`Unsupported platform, arch: ${platform}, ${arch}`);
+                }
+            }
+        }
+        default: throw new Error(`Unsupported platform, arch: ${platform}, ${arch}`);
+    }
+}
+
+function electronPlatform() {
+    switch (process.platform) {
+        case 'win32': {
+            return 'win';
+        }
+        case 'darwin': {
+            return 'mac';
+        }
+        case 'linux': {
+            return 'linux';
+        }
+        default: throw new Error(`Unsupported platform: ${process.platform}.`);
+    }
+}
+
+function getVersion() {
+    const repositoryRootPath = git('rev-parse --show-toplevel');
+    let version = JSON.parse(fs.readFileSync(path.join(repositoryRootPath, 'package.json'), { encoding: 'utf8' })).version;
+    if (!semver.valid(version)) {
+        throw new Error(`Could not read version from root package.json. Version was: '${version}'.`);
+    }
+    if (!isRelease) {
+        if (isNightly) {
+            version = `${version}-nightly.${timestamp()}`;
+        } else {
+            version = `${version}-snapshot.${currentCommitish()}`;
+        }
+        if (!semver.valid(version)) {
+            throw new Error(`Invalid patched version: '${version}'.`);
+        }
+    } else {
+        version = `v${version}`;
+    }
+    return version;
+}
+
+function timestamp() {
+    return dateFormat(new Date(), 'yyyymmdd');
+}
+
+function currentCommitish() {
+    return git('rev-parse --short HEAD');
+}
+
+// function currentBranch() {
+//     return git('rev-parse --abbrev-ref HEAD');
+// }
+
+function generateTemplate() {
+    // do `export PUBLISH=true yarn package` if you want to mimic CI build locally.
+    // const electronPublish = release || (isCI && currentBranch() === 'master') || process.env.PUBLISH === 'true';
+    const version = getVersion();
+    const productName = 'Arduino Pro IDE';
+    const name = 'arduino-pro-ide';
+    const customizations = {
+        name,
+        description: productName,
+        version,
+        build: {
+            productName,
+            appId: 'arduino.ProIDE',
+            [electronPlatform()]: {
+                artifactName: artifactName()
+            }
+        }
+    };
+    const template = require('../build/template-package.json');
+    return merge(template, customizations);
+}
+
+module.exports = { generateTemplate };

electron/packager/index.js

@@ -9,11 +9,14 @@
     const isCI = require('is-ci');
     shell.env.THEIA_ELECTRON_SKIP_REPLACE_FFMPEG = '1'; // Do not run the ffmpeg validation for the packager.
     shell.env.NODE_OPTIONS = '--max_old_space_size=4096'; // Increase heap size for the CI
+    const template = require('./config').generateTemplate();
     const utils = require('./utils');
     const merge = require('deepmerge');
-    const { version, release } = utils.versionInfo();
+    const { isRelease, isElectronPublish } = utils;
+    const { version } = template;
+    const { productName } = template.build;
 
-    echo(`📦  Building ${release ? 'release ' : ''}version '${version}'...`);
+    echo(`📦  Building ${isRelease ? 'release ' : ''}version '${version}'...`);
 
     const workingCopy = 'working-copy';
 
@@ -40,17 +43,47 @@
     // Clean up the `./electron/build` folder.
     shell.exec(`git -C ${path('..', 'build')} clean -ffxdq`, { async: false });
 
+    const extensions = [
+        'arduino-ide-extension',
+        'arduino-debugger-extension',
+    ];
+    const allDependencies = [
+        ...extensions,
+        'electron-app'
+    ]
+
     //----------------------------------------------------------------------------------------------+
     // Copy the following items into the `working-copy` folder. Make sure to reuse the `yarn.lock`. |
     //----------------------------------------------------------------------------------------------+
     mkdir('-p', path('..', workingCopy));
-    for (const name of ['arduino-ide-extension', 'arduino-debugger-extension', 'electron-app', 'yarn.lock', 'package.json', 'lerna.json']) {
+    for (const name of [...allDependencies, 'yarn.lock', 'package.json', 'lerna.json']) {
         cp('-rf', path(rootPath, name), path('..', workingCopy));
     }
 
-    //---------------------------------------------+
+    //----------------------------------------------+
-    // No need to build the `browser-app` example. |
+    // Sanity check: all versions must be the same. |
-    //---------------------------------------------+
+    //----------------------------------------------+
+    verifyVersions(allDependencies);
+    //----------------------------------------------------------------------+
+    // Use the nightly patch version if not a release but requires publish. |
+    //----------------------------------------------------------------------+
+    if (!isRelease) {
+        for (const dependency of allDependencies) {
+            const pkg = require(`../working-copy/${dependency}/package.json`);
+            pkg.version = version;
+            for (const dependency in pkg.dependencies) {
+                if (allDependencies.indexOf(dependency) !== -1) {
+                    pkg.dependencies[dependency] = version;
+                }
+            }
+            fs.writeFileSync(path('..', workingCopy, dependency, 'package.json'), JSON.stringify(pkg, null, 2));
+        }
+    }
+    verifyVersions(allDependencies);
+
+    //-------------------------------------------------------------+
+    // Save some time: no need to build the `browser-app` example. |
+    //-------------------------------------------------------------+
     //@ts-ignore
     let pkg = require('../working-copy/package.json');
     const workspaces = pkg.workspaces;
@@ -68,7 +101,7 @@
     //-------------------------------------------------------------------------------------------------+
     // Rebuild the extension with the copied `yarn.lock`. It is a must to use the same Theia versions. |
     //-------------------------------------------------------------------------------------------------+
-    exec(`yarn --network-timeout 1000000 --cwd ${path('..', workingCopy)}`, 'Building the Arduino Pro IDE extensions');
+    exec(`yarn --network-timeout 1000000 --cwd ${path('..', workingCopy)}`, `Building the ${productName} application`);
     // Collect all unused dependencies by the backend. We have to remove them from the electron app.
     // The `bundle.js` already contains everything we need for the frontend.
     // We have to do it before changing the dependencies to `local-path`.
@@ -77,32 +110,45 @@
     //-------------------------------------------------------------------------------------------------------------+
     // Change the regular NPM dependencies to `local-paths`, so that we can build them without any NPM registries. |
     //-------------------------------------------------------------------------------------------------------------+
-    // @ts-ignore
+    for (const extension of extensions) {
-    pkg = require('../working-copy/arduino-debugger-extension/package.json');
+        if (extension !== 'arduino-ide-extension') { // Do not unlink self.
-    pkg.dependencies['arduino-ide-extension'] = 'file:../arduino-ide-extension';
+            // @ts-ignore
-    fs.writeFileSync(path('..', workingCopy, 'arduino-debugger-extension', 'package.json'), JSON.stringify(pkg, null, 2));
+            pkg = require(`../working-copy/${extension}/package.json`);
+            // @ts-ignore
+            pkg.dependencies['arduino-ide-extension'] = 'file:../arduino-ide-extension';
+            fs.writeFileSync(path('..', workingCopy, extension, 'package.json'), JSON.stringify(pkg, null, 2));
+        }
+    }
 
     //------------------------------------------------------------------------------------+
     // Merge the `working-copy/package.json` with `electron/build/template-package.json`. |
     //------------------------------------------------------------------------------------+
     // @ts-ignore
     pkg = require('../working-copy/electron-app/package.json');
-    // @ts-ignore
-    const template = require('../build/template-package.json');
     template.build.files = [...template.build.files, ...unusedDependencies.map(name => `!node_modules/${name}`)];
-    pkg.dependencies = { ...pkg.dependencies, ...template.dependencies };
+
+    const dependencies = {};
+    for (const extension of extensions) {
+        dependencies[extension] = `file:../working-copy/${extension}`;
+    }
+    // @ts-ignore
+    pkg.dependencies = { ...pkg.dependencies, ...dependencies };
     pkg.devDependencies = { ...pkg.devDependencies, ...template.devDependencies };
     // Deep-merging the Theia application configuration. We enable the electron window reload in dev mode but not for the final product. (arduino/arduino-pro-ide#187)
+    // @ts-ignore
     const theia = merge((pkg.theia || {}), (template.theia || {}));
-    fs.writeFileSync(path('..', 'build', 'package.json'), JSON.stringify({
+    const content = {
         ...pkg,
         ...template,
         theia,
+        // @ts-ignore
         dependencies: pkg.dependencies,
         devDependencies: pkg.devDependencies
-    }, null, 2));
+    };
+    const overwriteMerge = (destinationArray, sourceArray, options) => sourceArray;
+    fs.writeFileSync(path('..', 'build', 'package.json'), JSON.stringify(merge(content, template, { arrayMerge: overwriteMerge }), null, 2));
 
-    echo(`📜  Effective 'package.json' for the Arduino Pro IDE application is:
+    echo(`📜  Effective 'package.json' for the ${productName} application is:
 -----------------------
 ${fs.readFileSync(path('..', 'build', 'package.json')).toString()}
 -----------------------
@@ -123,7 +169,7 @@ ${fs.readFileSync(path('..', 'build', 'package.json')).toString()}
     // Install all private and public dependencies for the electron application and build Theia. |
     //-------------------------------------------------------------------------------------------+
     exec(`yarn --network-timeout 1000000 --cwd ${path('..', 'build')}`, 'Installing dependencies');
-    exec(`yarn --network-timeout 1000000 --cwd ${path('..', 'build')} build${release ? ':release' : ''}`, 'Building the Arduino Pro IDE application');
+    exec(`yarn --network-timeout 1000000 --cwd ${path('..', 'build')} build${isElectronPublish ? ':publish' : ''}`, `Building the ${productName} application`);
 
     //------------------------------------------------------------------------------+
     // Create a throw away dotenv file which we use to feed the builder with input. |
@@ -139,7 +185,7 @@ ${fs.readFileSync(path('..', 'build', 'package.json')).toString()}
     //-----------------------------------+
     // Package the electron application. |
     //-----------------------------------+
-    exec(`yarn --network-timeout 1000000 --cwd ${path('..', 'build')} package`, `Packaging your Arduino Pro IDE application`);
+    exec(`yarn --network-timeout 1000000 --cwd ${path('..', 'build')} package`, `Packaging your ${productName} application`);
 
     //-----------------------------------------------------------------------------------------------------+
     // Copy to another folder. Azure does not support wildcard for `PublishBuildArtifacts@1.pathToPublish` |
@@ -224,15 +270,15 @@ ${fs.readFileSync(path('..', 'build', 'package.json')).toString()}
         const filesToCopy = [];
         switch (platform) {
             case 'linux': {
-                filesToCopy.push(...glob.sync('**/Arduino Pro IDE*.{zip,AppImage}', { cwd }).map(p => join(cwd, p)));
+                filesToCopy.push(...glob.sync('**/arduino-pro-ide*.{zip,AppImage}', { cwd }).map(p => join(cwd, p)));
                 break;
             }
             case 'win32': {
-                filesToCopy.push(...glob.sync('**/Arduino Pro IDE*.zip', { cwd }).map(p => join(cwd, p)));
+                filesToCopy.push(...glob.sync('**/arduino-pro-ide*.zip', { cwd }).map(p => join(cwd, p)));
                 break;
             }
             case 'darwin': {
-                filesToCopy.push(...glob.sync('**/Arduino Pro IDE*.dmg', { cwd }).map(p => join(cwd, p)));
+                filesToCopy.push(...glob.sync('**/arduino-pro-ide*.{dmg,zip}', { cwd }).map(p => join(cwd, p)));
                 break;
             }
             default: {
@@ -263,4 +309,23 @@ ${fs.readFileSync(path('..', 'build', 'package.json')).toString()}
         return join(__dirname, ...paths);
     }
 
+    function verifyVersions(allDependencies, expectedVersion) {
+        const versions = new Set();
+        for (const dependency of allDependencies) {
+            versions.add(require(`../working-copy/${dependency}/package.json`).version);
+        }
+        if (versions.size !== 1) {
+            echo(`Mismatching version configuration. All dependencies must have the same version. Versions were: ${JSON.stringify(Array.from(versions), null, 2)}.`);
+            shell.exit(1);
+            process.exit(1);
+        }
+        if (expectedVersion) {
+            if (!versions.has(expectedVersion)) {
+                echo(`Mismatching version configuration. Expected version was: '${expectedVersion}' actual was: '${Array.from(versions)[0]}'.`);
+                shell.exit(1);
+                process.exit(1);
+            }
+        }
+    }
+
 })();

electron/packager/package.json

@@ -17,12 +17,14 @@
     "@types/temp": "^0.8.32",
     "7zip-min": "^1.1.1",
     "chai": "^4.2.0",
+    "dateformat": "^3.0.3",
     "deepmerge": "^4.2.2",
     "depcheck": "^0.9.2",
     "file-type": "^14.1.4",
     "glob": "^7.1.6",
     "is-ci": "^2.0.0",
     "mocha": "^7.1.1",
+    "semver": "^7.3.2",
     "sinon": "^9.0.1",
     "shelljs": "^0.8.3",
     "temp": "^0.9.1",

electron/packager/utils.js

@@ -1,70 +1,15 @@
 //@ts-check
 
 const fs = require('fs');
-const path = require('path');
-const temp = require('temp');
 const zip = require('7zip-min');
+const temp = require('temp');
+const path = require('path');
+const isCI = require('is-ci');
 const shell = require('shelljs');
+const semver = require('semver');
 const depcheck = require('depcheck');
 const fromFile = require('file-type').fromFile;
 
-/**
- * Returns with the version info for the artifact.
- * If the `RELEASE_TAG` environment variable is set, we us that.
- * Falls back to the commit SHA if the `RELEASE_TAG` is the `$(Release.Tag)` string.
- * Otherwise, we concatenate the version of the extracted from `theia-app/electron-app/package.json`
- * and append the short commit SHA.
- */
-function versionInfo() {
-    if (typeof process.env.RELEASE_TAG === 'undefined' || !process.env.RELEASE_TAG || /* Azure -> */ process.env.RELEASE_TAG === '$(Release.Tag)') {
-        return {
-            version: `${targetVersion()}-${currentCommitish()}`,
-            release: false
-        }
-    } else {
-        return {
-            version: process.env.RELEASE_TAG,
-            release: true
-        }
-    }
-}
-
-/**
- * Returns with the absolute path of the `theia-app/electron-app/`.
- */
-function arduinoExtensionPath() {
-    // TODO: be smarter and locate the extension with Git: `git rev-parse --show-toplevel`.
-    return path.join(__dirname, '..', '..', 'arduino-ide-extension');
-}
-
-/**
- * The version extracted from the `package.json` of the `arduino-ide-extension`. Falls back to `x.x.x`.
- */
-function targetVersion() {
-    return JSON.parse(fs.readFileSync(path.join(arduinoExtensionPath(), 'package.json'), { encoding: 'utf8' })).version || 'x.x.x';
-}
-
-/**
- * Returns with the trimmed result of the `git rev-parse --short HEAD` as the current commitish if `git` is on the `PATH`.
- * Otherwise, it returns with `DEV_BUILD`.
- */
-function currentCommitish() {
-    try {
-        const gitPath = shell.which('git');
-        const error = shell.error();
-        if (error) {
-            throw new Error(error);
-        }
-        const { stderr, stdout } = shell.exec(`"${gitPath}" rev-parse --short HEAD`, { silent: true });
-        if (stderr) {
-            throw new Error(stderr.toString().trim());
-        }
-        return stdout.toString().trim();
-    } catch (e) {
-        return 'DEV_BUILD';
-    }
-}
-
 /**
  * Resolves to an array of `npm` package names that are declared in the `package.json` but **not** used by the project.
  */
@@ -204,4 +149,25 @@ async function isZip(pathToFile) {
     return type && type.ext === 'zip';
 }
 
-module.exports = { versionInfo, collectUnusedDependencies, adjustArchiveStructure, isZip, unpack };
+const isElectronPublish = false; // TODO: support auto-updates
+const isNightly = process.env.IS_NIGHTLY === 'true';
+const isRelease = process.env.IS_RELEASE === 'true';
+
+function git(command) {
+    try {
+        const gitPath = shell.which('git');
+        const error = shell.error();
+        if (error) {
+            throw new Error(error);
+        }
+        const { stderr, stdout } = shell.exec(`"${gitPath}" ${command}`, { silent: true });
+        if (stderr) {
+            throw new Error(stderr.toString().trim());
+        }
+        return stdout.toString().trim();
+    } catch (e) {
+        throw e;
+    }
+}
+
+module.exports = { collectUnusedDependencies, adjustArchiveStructure, isZip, unpack, isNightly, isRelease, isElectronPublish, git };

electron/packager/yarn.lock

@@ -437,6 +437,11 @@ cross-spawn@^6.0.0:
     shebang-command "^1.2.0"
     which "^1.2.9"
 
+dateformat@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae"
+  integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==
+
 de-indent@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/de-indent/-/de-indent-1.0.2.tgz#b2038e846dc33baa5796128d0804b455b8c1e21d"
@@ -1306,6 +1311,11 @@ semver@^5.5.0, semver@^5.7.0:
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
   integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
 
+semver@^7.3.2:
+  version "7.3.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938"
+  integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==
+
 set-blocking@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"

package.json

@@ -11,10 +11,10 @@
     "lerna": "^3.20.2",
     "rimraf": "^2.6.1",
     "tslint": "^5.5.0",
-    "typescript": "3.5.1"
+    "typescript": "^3.9.2"
   },
   "scripts": {
-    "prepare": "lerna run prepare && yarn test && yarn download:plugins",
+    "prepare": "lerna run prepare && yarn download:plugins",
     "rebuild:browser": "theia rebuild:browser",
     "rebuild:electron": "theia rebuild:electron --modules \"@theia/node-pty\" nsfw native-keymap find-git-repositories grpc",
     "start": "yarn --cwd ./browser-app start",
@@ -30,7 +30,7 @@
   ],
   "theiaPluginsDir": "plugins",
   "theiaPlugins": {
-    "vscode-yaml": "https://open-vsx.org/api/redhat/vscode-yaml/0.7.2/file/redhat.vscode-yaml-0.7.2.vsix",
+    "vscode-builtin-cpp": "http://open-vsx.org/api/vscode/cpp/1.44.2/file/vscode.cpp-1.44.2.vsix",
-    "vscode-builtin-cpp": "http://open-vsx.org/api/vscode/cpp/1.39.1/file/vscode.cpp-1.39.1.vsix"
+    "vscode-yaml": "https://open-vsx.org/api/redhat/vscode-yaml/0.7.2/file/redhat.vscode-yaml-0.7.2.vsix"
   }
 }