ATL-658: IDE can use any pinned version of CLI.

- Pinned the CLI to the `20201104` nightly.
 - Updated the TS/JS API generator to fall back to forks if configured.
 - Updated the CLI JSON schema.

Signed-off-by: Akos Kitta <kittaakos@typefox.io>

arduino-ide-extension/README.md

@@ -50,3 +50,7 @@ The Config Service knows about your system, like for example the default sketch
 - [src/node/config-service-impl.ts](./src/node/config-service-impl.ts) implements the service backend:
   - getting the `arduino-cli` version and configuration
   - checking whether a file is in a data or sketch directory
+
+### `"arduino"` configuration in the `package.json`:
+ - `"cli"`:
+   - `"version"` type `string` | `{ owner: string, repo: string, commitish?: string }`: if the type is a `string` and is a valid semver, it will get the corresponding [released](https://github.com/arduino/arduino-cli/releases) CLI. If the type is `string` and is a [date in `YYYYMMDD`](https://arduino.github.io/arduino-cli/latest/installation/#nightly-builds) format, it will get a nightly CLI. If the type is an object, a CLI, build from the sources in the `owner/repo` will be used. If `commitish` is not defined, the HEAD of the default branch will be used. In any other cases an error is thrown.

arduino-ide-extension/data/cli/schema/arduino-cli.schema.json

@@ -92,6 +92,17 @@
       },
       "additionalProperties": false
     },
+    "sketch": {
+      "type": "object",
+      "description": "Sketch Configuration",
+      "properties": {
+        "always_export_binaries": {
+          "type": "boolean",
+          "description": "Controls whether the compiled binaries will be exported into the sketch's 'build' folder or not. 'false' if the binaries are not exported."
+        }
+      },
+      "additionalProperties": false
+    },
     "telemetry": {
       "type": "object",
       "description": "Telemetry Configuration",
@@ -110,8 +121,5 @@
       "additionalProperties": false
     }
   },
-  "// TODOs": [
-    "additionalProperties should be true. See the new telemetry entry"
-  ],
   "additionalProperties": false
 }

arduino-ide-extension/package.json

@@ -57,7 +57,7 @@
     "p-queue": "^5.0.0",
     "ps-tree": "^1.2.0",
     "react-select": "^3.0.4",
-    "semver": "^6.3.0",
+    "semver": "^7.3.2",
     "string-natural-compare": "^2.0.3",
     "temp": "^0.9.1",
     "tree-kill": "^1.2.1",
@@ -117,5 +117,10 @@
     {
       "electronMain": "lib/electron-main/arduino-electron-main-module"
     }
-  ]
+  ],
+  "arduino": {
+    "cli": {
+      "version": "20201102"
+    }
+  }
 }

arduino-ide-extension/scripts/download-cli.js

@@ -1,62 +1,141 @@
 // @ts-check
-// The links to the downloads as of today (02.09.) are the followings:
-// In order to get the latest nightly build for your platform use the following links replacing <DATE> with the current date, using the format YYYYMMDD (i.e for 2019/Aug/06 use 20190806 )
-// Linux 64 bit: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-<DATE>_Linux_64bit.tar.gz
-// Linux ARM 64 bit: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-<DATE>_Linux_ARM64.tar.gz
-// Windows 64 bit: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-<DATE>_Windows_64bit.zip
-// Mac OSX: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-<DATE>_macOS_64bit.tar.gz
-// [...]
-// redirecting to latest generated builds by replacing latest with the latest available build date, using the format YYYYMMDD (i.e for 2019/Aug/06 latest is replaced with 20190806 
 
-(() => {
+(async () => {
-
-    const DEFAULT_VERSION = '0.13.0'; // require('moment')().format('YYYYMMDD');
 
+    const fs = require('fs');
     const path = require('path');
+    const temp = require('temp');
     const shell = require('shelljs');
+    const semver = require('semver');
+    const moment = require('moment');
     const downloader = require('./downloader');
 
-    const yargs = require('yargs')
+    const version = (() => {
-        .option('cli-version', {
+        const pkg = require(path.join(__dirname, '..', 'package.json'));
-            alias: 'cv',
+        if (!pkg) {
-            default: DEFAULT_VERSION,
+            return undefined;
-            describe: `The version of the 'arduino-cli' to download, or 'nightly-latest'. Defaults to ${DEFAULT_VERSION}.`
-        })
-        .option('force-download', {
-            alias: 'fd',
-            default: false,
-            describe: `If set, this script force downloads the 'arduino-cli' even if it already exists on the file system.`
-        })
-        .version(false).parse();
-
-    const version = yargs['cli-version'];
-    const force = yargs['force-download'];
-    const { platform, arch } = process;
-
-    const build = path.join(__dirname, '..', 'build');
-    const cli = path.join(build, `arduino-cli${platform === 'win32' ? '.exe' : ''}`);
-
-    const suffix = (() => {
-        switch (platform) {
-            case 'darwin': return 'macOS_64bit.tar.gz';
-            case 'win32': return 'Windows_64bit.zip';
-            case 'linux': {
-                switch (arch) {
-                    case 'arm': return 'Linux_ARMv7.tar.gz';
-                    case 'arm64': return 'Linux_ARM64.tar.gz';
-                    case 'x64': return 'Linux_64bit.tar.gz';
-                    default: return undefined;
-                }
-            }
-            default: return undefined;
         }
+
+        const { arduino } = pkg;
+        if (!arduino) {
+            return undefined;
+        }
+
+        const { cli } = arduino;
+        if (!cli) {
+            return undefined;
+        }
+
+        const { version } = cli;
+        return version;
     })();
-    if (!suffix) {
+
-        shell.echo(`The CLI is not available for ${platform} ${arch}.`);
+    if (!version) {
+        shell.echo(`Could not retrieve CLI version info from the 'package.json'.`);
         shell.exit(1);
     }
 
-    const url = `https://downloads.arduino.cc/arduino-cli${version.startsWith('nightly-') ? '/nightly' : ''}/arduino-cli_${version}_${suffix}`;
+    const { platform, arch } = process;
-    downloader.downloadUnzipFile(url, cli, 'arduino-cli', force);
+    const buildFolder = path.join(__dirname, '..', 'build');
+    const cliName = `arduino-cli${platform === 'win32' ? '.exe' : ''}`;
+    const destinationPath = path.join(buildFolder, cliName);
+
+    if (typeof version === 'string') {
+        const suffix = (() => {
+            switch (platform) {
+                case 'darwin': return 'macOS_64bit.tar.gz';
+                case 'win32': return 'Windows_64bit.zip';
+                case 'linux': {
+                    switch (arch) {
+                        case 'arm': return 'Linux_ARMv7.tar.gz';
+                        case 'arm64': return 'Linux_ARM64.tar.gz';
+                        case 'x64': return 'Linux_64bit.tar.gz';
+                        default: return undefined;
+                    }
+                }
+                default: return undefined;
+            }
+        })();
+        if (!suffix) {
+            shell.echo(`The CLI is not available for ${platform} ${arch}.`);
+            shell.exit(1);
+        }
+        if (semver.valid(version)) {
+            const url = `https://downloads.arduino.cc/arduino-cli/arduino-cli_${version}_${suffix}`;
+            shell.echo(`📦  Identified released version of the CLI. Downloading version ${version} from '${url}'`);
+            await downloader.downloadUnzipFile(url, destinationPath, 'arduino-cli');
+        } else if (moment(version, 'YYYYMMDD', true).isValid()) {
+            const url = `https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-${version}_${suffix}`;
+            shell.echo(`🌙  Identified nightly version of the CLI. Downloading version ${version} from '${url}'`);
+            await downloader.downloadUnzipFile(url, destinationPath, 'arduino-cli');
+        } else {
+            shell.echo(`🔥  Could not interpret 'version': ${version}`);
+            shell.exit(1);
+        }
+    } else {
+
+        // We assume an object with `owner`, `repo`, commitish?` properties.
+        const { owner, repo, commitish } = version;
+        if (!owner) {
+            shell.echo(`Could not retrieve 'owner' from ${JSON.stringify(version)}`);
+            shell.exit(1);
+        }
+        if (!repo) {
+            shell.echo(`Could not retrieve 'repo' from ${JSON.stringify(version)}`);
+            shell.exit(1);
+        }
+        const url = `https://github.com/${owner}/${repo}.git`;
+        shell.echo(`Building CLI from ${url}. Commitish: ${commitish ? commitish : 'HEAD'}`);
+
+        if (fs.existsSync(destinationPath)) {
+            shell.echo(`Skipping the CLI build because it already exists: ${destinationPath}`);
+            return;
+        }
+
+        if (shell.mkdir('-p', buildFolder).code !== 0) {
+            shell.echo('Could not create build folder.');
+            shell.exit(1);
+        }
+
+        const tempRepoPath = temp.mkdirSync();
+        shell.echo(`>>> Cloning CLI source to ${tempRepoPath}...`);
+        if (shell.exec(`git clone ${url} ${tempRepoPath}`).code !== 0) {
+            shell.exit(1);
+        }
+        shell.echo('<<< Cloned CLI repo.')
+
+        if (commitish) {
+            shell.echo(`>>> Checking out ${commitish}...`);
+            if (shell.exec(`git -C ${tempRepoPath} checkout ${commitish}`).code !== 0) {
+                shell.exit(1);
+            }
+            shell.echo(`<<< Checked out ${commitish}.`);
+        }
+
+        shell.echo(`>>> Building the CLI...`);
+        if (shell.exec('go build', { cwd: tempRepoPath }).code !== 0) {
+            shell.exit(1);
+        }
+        shell.echo('<<< CLI build done.')
+
+        if (!fs.existsSync(path.join(tempRepoPath, cliName))) {
+            shell.echo(`Could not find the CLI at ${path.join(tempRepoPath, cliName)}.`);
+            shell.exit(1);
+        }
+
+        const builtCliPath = path.join(tempRepoPath, cliName);
+        shell.echo(`>>> Copying CLI from ${builtCliPath} to ${destinationPath}...`);
+        if (shell.cp(builtCliPath, destinationPath).code !== 0) {
+            shell.exit(1);
+        }
+        shell.echo(`<<< Copied the CLI.`);
+
+        shell.echo('<<< Verifying CLI...');
+        if (!fs.existsSync(destinationPath)) {
+            shell.exit(1);
+        }
+        shell.echo('>>> Verified CLI.');
+
+    }
 
 })();

arduino-ide-extension/scripts/downloader.js

@@ -21,9 +21,9 @@ process.on('uncaughtException', error => {
  * @param url {string}        Download URL
  * @param targetFile {string} Path to the file to copy from the decompressed archive
  * @param filePrefix {string} Prefix of the file name found in the archive
- * @param force {boolean}     Whether to download even if the target file exists
+ * @param force {boolean}     Whether to download even if the target file exists. `false` by default.
  */
-exports.downloadUnzipFile = async (url, targetFile, filePrefix, force) => {
+exports.downloadUnzipFile = async (url, targetFile, filePrefix, force = false) => {
     if (fs.existsSync(targetFile) && !force) {
         shell.echo(`Skipping download because file already exists: ${targetFile}`);
         return;

arduino-ide-extension/scripts/generate-protocol.js

@@ -16,9 +16,52 @@
         shell.exit(1);
     }
 
-    if (shell.exec(`git clone --depth 1 https://github.com/arduino/arduino-cli.git ${repository}`).code !== 0) {
+    const { owner, repo, commitish } = (() => {
+        const pkg = require(path.join(__dirname, '..', 'package.json'));
+        if (!pkg) {
+            shell.echo(`Could not parse the 'package.json'.`);
+            shell.exit(1);
+        }
+
+        const { arduino } = pkg;
+        if (!arduino) {
+            return { owner: 'arduino', repo: 'arduino-cli' };
+        }
+
+        const { cli } = arduino;
+        if (!cli) {
+            return { owner: 'arduino', repo: 'arduino-cli' };
+        }
+
+        const { version } = cli;
+        if (!version) {
+            return { owner: 'arduino', repo: 'arduino-cli' };
+        }
+
+        if (typeof version === 'string') {
+            return { owner: 'arduino', repo: 'arduino-cli' };
+        }
+
+        // We assume an object with `owner`, `repo`, commitish?` properties.
+        const { owner, repo, commitish } = version;
+        if (!owner) {
+            shell.echo(`Could not retrieve 'owner' from ${JSON.stringify(version)}`);
+            shell.exit(1);
+        }
+        if (!repo) {
+            shell.echo(`Could not retrieve 'repo' from ${JSON.stringify(version)}`);
+            shell.exit(1);
+        }
+
+        return { owner, repo, commitish };
+    })();
+
+    const url = `https://github.com/${owner}/${repo}.git`;
+    shell.echo(`>>> Cloning repository from '${url}'...`);
+    if (shell.exec(`git clone ${url} ${repository}`).code !== 0) {
         shell.exit(1);
     }
+    shell.echo(`<<< Repository cloned.`);
 
     const { platform } = process;
     const build = path.join(__dirname, '..', 'build');
@@ -30,9 +73,17 @@
     }
     const version = rawVersion.substring(rawVersion.lastIndexOf('Commit:') + 'Commit:'.length).trim();
     if (version) {
+        shell.echo(`>>> Checking out version: ${version}...`);
         if (shell.exec(`git -C ${repository} checkout ${version} -b ${version}`).code !== 0) {
             shell.exit(1);
         }
+        shell.echo(`<<< Checked out version: ${commitish}.`);
+    } else if (commitish) {
+        shell.echo(`>>> Checking out commitish: ${commitish}...`);
+        if (shell.exec(`git -C ${repository} checkout ${commitish}`).code !== 0) {
+            shell.exit(1);
+        }
+        shell.echo(`<<< Checked out commitish: ${commitish}.`);
     }
 
     shell.echo('>>> Generating TS/JS API from:');

arduino-ide-extension/src/node/config-file-validator.ts

@@ -15,6 +15,11 @@ export class ConfigFileValidator {
     protected async doValidate(object: object): Promise<boolean> {
         const valid = this.function(object);
         if (!valid) {
+            if (Array.isArray(this.function.errors)) {
+                for (const error of this.function.errors) {
+                    console.log(JSON.stringify(error));
+                }
+            }
             return false;
         }
         if (!DefaultCliConfig.is(object)) {