ATL-658: IDE can use any pinned version of CLI.

- Pinned the CLI to the `20201104` nightly.
 - Updated the TS/JS API generator to fall back to forks if configured.
 - Updated the CLI JSON schema.

Signed-off-by: Akos Kitta <kittaakos@typefox.io>

arduino-ide-extension/scripts/download-cli.js

@@ -1,62 +1,141 @@
 // @ts-check
-// The links to the downloads as of today (02.09.) are the followings:
-// In order to get the latest nightly build for your platform use the following links replacing <DATE> with the current date, using the format YYYYMMDD (i.e for 2019/Aug/06 use 20190806 )
-// Linux 64 bit: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-<DATE>_Linux_64bit.tar.gz
-// Linux ARM 64 bit: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-<DATE>_Linux_ARM64.tar.gz
-// Windows 64 bit: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-<DATE>_Windows_64bit.zip
-// Mac OSX: https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-<DATE>_macOS_64bit.tar.gz
-// [...]
-// redirecting to latest generated builds by replacing latest with the latest available build date, using the format YYYYMMDD (i.e for 2019/Aug/06 latest is replaced with 20190806 
 
-(() => {
-
-    const DEFAULT_VERSION = '0.13.0'; // require('moment')().format('YYYYMMDD');
+(async () => {
 
+    const fs = require('fs');
     const path = require('path');
+    const temp = require('temp');
     const shell = require('shelljs');
+    const semver = require('semver');
+    const moment = require('moment');
     const downloader = require('./downloader');
 
-    const yargs = require('yargs')
-        .option('cli-version', {
-            alias: 'cv',
-            default: DEFAULT_VERSION,
-            describe: `The version of the 'arduino-cli' to download, or 'nightly-latest'. Defaults to ${DEFAULT_VERSION}.`
-        })
-        .option('force-download', {
-            alias: 'fd',
-            default: false,
-            describe: `If set, this script force downloads the 'arduino-cli' even if it already exists on the file system.`
-        })
-        .version(false).parse();
-
-    const version = yargs['cli-version'];
-    const force = yargs['force-download'];
-    const { platform, arch } = process;
-
-    const build = path.join(__dirname, '..', 'build');
-    const cli = path.join(build, `arduino-cli${platform === 'win32' ? '.exe' : ''}`);
-
-    const suffix = (() => {
-        switch (platform) {
-            case 'darwin': return 'macOS_64bit.tar.gz';
-            case 'win32': return 'Windows_64bit.zip';
-            case 'linux': {
-                switch (arch) {
-                    case 'arm': return 'Linux_ARMv7.tar.gz';
-                    case 'arm64': return 'Linux_ARM64.tar.gz';
-                    case 'x64': return 'Linux_64bit.tar.gz';
-                    default: return undefined;
-                }
-            }
-            default: return undefined;
+    const version = (() => {
+        const pkg = require(path.join(__dirname, '..', 'package.json'));
+        if (!pkg) {
+            return undefined;
         }
+
+        const { arduino } = pkg;
+        if (!arduino) {
+            return undefined;
+        }
+
+        const { cli } = arduino;
+        if (!cli) {
+            return undefined;
+        }
+
+        const { version } = cli;
+        return version;
     })();
-    if (!suffix) {
-        shell.echo(`The CLI is not available for ${platform} ${arch}.`);
+
+    if (!version) {
+        shell.echo(`Could not retrieve CLI version info from the 'package.json'.`);
         shell.exit(1);
     }
 
-    const url = `https://downloads.arduino.cc/arduino-cli${version.startsWith('nightly-') ? '/nightly' : ''}/arduino-cli_${version}_${suffix}`;
-    downloader.downloadUnzipFile(url, cli, 'arduino-cli', force);
+    const { platform, arch } = process;
+    const buildFolder = path.join(__dirname, '..', 'build');
+    const cliName = `arduino-cli${platform === 'win32' ? '.exe' : ''}`;
+    const destinationPath = path.join(buildFolder, cliName);
+
+    if (typeof version === 'string') {
+        const suffix = (() => {
+            switch (platform) {
+                case 'darwin': return 'macOS_64bit.tar.gz';
+                case 'win32': return 'Windows_64bit.zip';
+                case 'linux': {
+                    switch (arch) {
+                        case 'arm': return 'Linux_ARMv7.tar.gz';
+                        case 'arm64': return 'Linux_ARM64.tar.gz';
+                        case 'x64': return 'Linux_64bit.tar.gz';
+                        default: return undefined;
+                    }
+                }
+                default: return undefined;
+            }
+        })();
+        if (!suffix) {
+            shell.echo(`The CLI is not available for ${platform} ${arch}.`);
+            shell.exit(1);
+        }
+        if (semver.valid(version)) {
+            const url = `https://downloads.arduino.cc/arduino-cli/arduino-cli_${version}_${suffix}`;
+            shell.echo(`📦  Identified released version of the CLI. Downloading version ${version} from '${url}'`);
+            await downloader.downloadUnzipFile(url, destinationPath, 'arduino-cli');
+        } else if (moment(version, 'YYYYMMDD', true).isValid()) {
+            const url = `https://downloads.arduino.cc/arduino-cli/nightly/arduino-cli_nightly-${version}_${suffix}`;
+            shell.echo(`🌙  Identified nightly version of the CLI. Downloading version ${version} from '${url}'`);
+            await downloader.downloadUnzipFile(url, destinationPath, 'arduino-cli');
+        } else {
+            shell.echo(`🔥  Could not interpret 'version': ${version}`);
+            shell.exit(1);
+        }
+    } else {
+
+        // We assume an object with `owner`, `repo`, commitish?` properties.
+        const { owner, repo, commitish } = version;
+        if (!owner) {
+            shell.echo(`Could not retrieve 'owner' from ${JSON.stringify(version)}`);
+            shell.exit(1);
+        }
+        if (!repo) {
+            shell.echo(`Could not retrieve 'repo' from ${JSON.stringify(version)}`);
+            shell.exit(1);
+        }
+        const url = `https://github.com/${owner}/${repo}.git`;
+        shell.echo(`Building CLI from ${url}. Commitish: ${commitish ? commitish : 'HEAD'}`);
+
+        if (fs.existsSync(destinationPath)) {
+            shell.echo(`Skipping the CLI build because it already exists: ${destinationPath}`);
+            return;
+        }
+
+        if (shell.mkdir('-p', buildFolder).code !== 0) {
+            shell.echo('Could not create build folder.');
+            shell.exit(1);
+        }
+
+        const tempRepoPath = temp.mkdirSync();
+        shell.echo(`>>> Cloning CLI source to ${tempRepoPath}...`);
+        if (shell.exec(`git clone ${url} ${tempRepoPath}`).code !== 0) {
+            shell.exit(1);
+        }
+        shell.echo('<<< Cloned CLI repo.')
+
+        if (commitish) {
+            shell.echo(`>>> Checking out ${commitish}...`);
+            if (shell.exec(`git -C ${tempRepoPath} checkout ${commitish}`).code !== 0) {
+                shell.exit(1);
+            }
+            shell.echo(`<<< Checked out ${commitish}.`);
+        }
+
+        shell.echo(`>>> Building the CLI...`);
+        if (shell.exec('go build', { cwd: tempRepoPath }).code !== 0) {
+            shell.exit(1);
+        }
+        shell.echo('<<< CLI build done.')
+
+        if (!fs.existsSync(path.join(tempRepoPath, cliName))) {
+            shell.echo(`Could not find the CLI at ${path.join(tempRepoPath, cliName)}.`);
+            shell.exit(1);
+        }
+
+        const builtCliPath = path.join(tempRepoPath, cliName);
+        shell.echo(`>>> Copying CLI from ${builtCliPath} to ${destinationPath}...`);
+        if (shell.cp(builtCliPath, destinationPath).code !== 0) {
+            shell.exit(1);
+        }
+        shell.echo(`<<< Copied the CLI.`);
+
+        shell.echo('<<< Verifying CLI...');
+        if (!fs.existsSync(destinationPath)) {
+            shell.exit(1);
+        }
+        shell.echo('>>> Verified CLI.');
+
+    }
 
 })();

arduino-ide-extension/scripts/downloader.js

@@ -21,9 +21,9 @@ process.on('uncaughtException', error => {
  * @param url {string}        Download URL
  * @param targetFile {string} Path to the file to copy from the decompressed archive
  * @param filePrefix {string} Prefix of the file name found in the archive
- * @param force {boolean}     Whether to download even if the target file exists
+ * @param force {boolean}     Whether to download even if the target file exists. `false` by default.
  */
-exports.downloadUnzipFile = async (url, targetFile, filePrefix, force) => {
+exports.downloadUnzipFile = async (url, targetFile, filePrefix, force = false) => {
     if (fs.existsSync(targetFile) && !force) {
         shell.echo(`Skipping download because file already exists: ${targetFile}`);
         return;

arduino-ide-extension/scripts/generate-protocol.js

@@ -16,9 +16,52 @@
         shell.exit(1);
     }
 
-    if (shell.exec(`git clone --depth 1 https://github.com/arduino/arduino-cli.git ${repository}`).code !== 0) {
+    const { owner, repo, commitish } = (() => {
+        const pkg = require(path.join(__dirname, '..', 'package.json'));
+        if (!pkg) {
+            shell.echo(`Could not parse the 'package.json'.`);
+            shell.exit(1);
+        }
+
+        const { arduino } = pkg;
+        if (!arduino) {
+            return { owner: 'arduino', repo: 'arduino-cli' };
+        }
+
+        const { cli } = arduino;
+        if (!cli) {
+            return { owner: 'arduino', repo: 'arduino-cli' };
+        }
+
+        const { version } = cli;
+        if (!version) {
+            return { owner: 'arduino', repo: 'arduino-cli' };
+        }
+
+        if (typeof version === 'string') {
+            return { owner: 'arduino', repo: 'arduino-cli' };
+        }
+
+        // We assume an object with `owner`, `repo`, commitish?` properties.
+        const { owner, repo, commitish } = version;
+        if (!owner) {
+            shell.echo(`Could not retrieve 'owner' from ${JSON.stringify(version)}`);
+            shell.exit(1);
+        }
+        if (!repo) {
+            shell.echo(`Could not retrieve 'repo' from ${JSON.stringify(version)}`);
+            shell.exit(1);
+        }
+
+        return { owner, repo, commitish };
+    })();
+
+    const url = `https://github.com/${owner}/${repo}.git`;
+    shell.echo(`>>> Cloning repository from '${url}'...`);
+    if (shell.exec(`git clone ${url} ${repository}`).code !== 0) {
         shell.exit(1);
     }
+    shell.echo(`<<< Repository cloned.`);
 
     const { platform } = process;
     const build = path.join(__dirname, '..', 'build');
@@ -30,9 +73,17 @@
     }
     const version = rawVersion.substring(rawVersion.lastIndexOf('Commit:') + 'Commit:'.length).trim();
     if (version) {
+        shell.echo(`>>> Checking out version: ${version}...`);
         if (shell.exec(`git -C ${repository} checkout ${version} -b ${version}`).code !== 0) {
             shell.exit(1);
         }
+        shell.echo(`<<< Checked out version: ${commitish}.`);
+    } else if (commitish) {
+        shell.echo(`>>> Checking out commitish: ${commitish}...`);
+        if (shell.exec(`git -C ${repository} checkout ${commitish}`).code !== 0) {
+            shell.exit(1);
+        }
+        shell.echo(`<<< Checked out commitish: ${commitish}.`);
     }
 
     shell.echo('>>> Generating TS/JS API from:');