chore(deps): Updated to Theia 1.39.0 (#2144)

- update Theia to `1.39.0`,
 - remove the application packager and fix the security vulnerabilities,
 - bundle the backed application with `webpack`, and
 - enhance the developer docs.

Co-authored-by: Akos Kitta <a.kitta@arduino.cc>
Co-authored-by: per1234 <accounts@perglass.com>

Signed-off-by: Akos Kitta <a.kitta@arduino.cc>

arduino-ide-extension/src/node/arduino-daemon-impl.ts

@@ -15,9 +15,9 @@ import { EnvVariablesServer } from '@theia/core/lib/common/env-variables';
 import { BackendApplicationContribution } from '@theia/core/lib/node/backend-application';
 import { ArduinoDaemon, NotificationServiceServer } from '../common/protocol';
 import { CLI_CONFIG } from './cli-config';
-import { getExecPath } from './exec-util';
 import { SettingsReader } from './settings-reader';
 import { ProcessUtils } from '@theia/core/lib/node/process-utils';
+import { arduinoCliPath } from './resources';
 
 @injectable()
 export class ArduinoDaemonImpl
@@ -133,7 +133,7 @@ export class ArduinoDaemonImpl
   }
 
   getExecPath(): string {
-    return getExecPath('arduino-cli');
+    return arduinoCliPath;
   }
 
   protected async getSpawnArgs(): Promise<string[]> {

arduino-ide-extension/src/node/arduino-firmware-uploader-impl.ts

@@ -5,8 +5,9 @@ import {
   ArduinoFirmwareUploader,
   FirmwareInfo,
 } from '../common/protocol/arduino-firmware-uploader';
-import { getExecPath, spawnCommand } from './exec-util';
+import { spawnCommand } from './exec-util';
 import { MonitorManager } from './monitor-manager';
+import { arduinoFirmwareUploaderPath } from './resources';
 
 @injectable()
 export class ArduinoFirmwareUploaderImpl implements ArduinoFirmwareUploader {
@@ -74,7 +75,10 @@ export class ArduinoFirmwareUploaderImpl implements ArduinoFirmwareUploader {
   }
 
   private async runCommand(args: string[]): Promise<string> {
-    const execPath = getExecPath('arduino-fwuploader');
-    return await spawnCommand(execPath, args, this.onError.bind(this));
+    return await spawnCommand(
+      arduinoFirmwareUploaderPath,
+      args,
+      this.onError.bind(this)
+    );
   }
 }

arduino-ide-extension/src/node/arduino-ide-backend-module.ts

@@ -111,7 +111,7 @@ import {
   SurveyNotificationServicePath,
 } from '../common/protocol/survey-service';
 import { IsTempSketch } from './is-temp-sketch';
-import { rebindNsfwFileSystemWatcher } from './theia/filesystem/nsfw-watcher/nsfw-bindings';
+import { rebindNsfwFileSystemWatcher } from './theia/filesystem/nsfw-bindings';
 import { MessagingContribution } from './theia/core/messaging-contribution';
 import { MessagingService } from '@theia/core/lib/node/messaging/messaging-service';
 import { HostedPluginReader } from './theia/plugin-ext/plugin-reader';

arduino-ide-extension/src/node/auth/authentication-server.ts

@@ -1,5 +1,5 @@
-import * as http from 'node:http';
-import * as url from 'node:url';
+import http from 'node:http';
+import url from 'node:url';
 import { body } from './body';
 import { authServerPort } from '../../common/protocol/authentication-service';
 

arduino-ide-extension/src/node/auth/keychain.ts

@@ -1,4 +1,4 @@
-import type * as keytarType from 'keytar';
+import type keytarType from 'keytar';
 
 export type KeychainConfig = {
   credentialsSection: string;
@@ -6,9 +6,9 @@ export type KeychainConfig = {
 };
 
 type Keytar = {
-  getPassword: typeof keytarType['getPassword'];
-  setPassword: typeof keytarType['setPassword'];
-  deletePassword: typeof keytarType['deletePassword'];
+  getPassword: (typeof keytarType)['getPassword'];
+  setPassword: (typeof keytarType)['setPassword'];
+  deletePassword: (typeof keytarType)['deletePassword'];
 };
 
 export class Keychain {

arduino-ide-extension/src/node/clang-formatter.ts

@@ -6,7 +6,9 @@ import { constants, promises as fs } from 'node:fs';
 import { join } from 'node:path';
 import { ConfigService } from '../common/protocol';
 import { Formatter, FormatterOptions } from '../common/protocol/formatter';
-import { getExecPath, spawnCommand } from './exec-util';
+import { spawnCommand } from './exec-util';
+import { clangFormatPath } from './resources';
+import defaultClangFormat from './default-formatter-config.json';
 
 @injectable()
 export class ClangFormatter implements Formatter {
@@ -37,7 +39,7 @@ export class ClangFormatter implements Formatter {
   }
 
   private execPath(): string {
-    return getExecPath('clang-format');
+    return clangFormatPath;
   }
 
   /**
@@ -129,10 +131,9 @@ function styleJson({
   TabWidth,
   UseTab,
 }: ClangFormatOptions): Record<string, unknown> {
-  // Source: https://github.com/arduino/tooling-project-assets/tree/main/other/clang-format-configuration
-  const defaultConfig = require('../../src/node/default-formatter-config.json');
   return {
-    ...defaultConfig,
+    // Source: https://github.com/arduino/tooling-project-assets/tree/main/other/clang-format-configuration
+    ...defaultClangFormat,
     TabWidth,
     UseTab,
   };

arduino-ide-extension/src/node/config-service-impl.ts

@@ -1,6 +1,6 @@
 import { promises as fs } from 'node:fs';
 import { dirname } from 'node:path';
-import * as yaml from 'js-yaml';
+import yaml from 'js-yaml';
 import * as grpc from '@grpc/grpc-js';
 import { injectable, inject, named } from '@theia/core/shared/inversify';
 import URI from '@theia/core/lib/common/uri';
@@ -131,10 +131,6 @@ export class ConfigServiceImpl
     return this.configChangeEmitter.event;
   }
 
-  async getVersion(): Promise<string> {
-    return require('../../package.json').arduino?.cli?.version || '';
-  }
-
   private async initConfig(): Promise<void> {
     this.logger.info('>>> Initializing CLI configuration...');
     try {

arduino-ide-extension/src/node/examples-service-impl.ts

@@ -4,7 +4,7 @@ import {
   postConstruct,
 } from '@theia/core/shared/inversify';
 import { join } from 'node:path';
-import * as fs from 'node:fs';
+import fs from 'node:fs';
 import { FileUri } from '@theia/core/lib/node/file-uri';
 import {
   SketchRef,
@@ -15,6 +15,7 @@ import { LibraryLocation, LibraryPackage } from '../common/protocol';
 import { URI } from '@theia/core/lib/common/uri';
 import { Path } from '@theia/core/lib/common/path';
 import { LibraryServiceImpl } from './library-service-impl';
+import { examplesPath } from './resources';
 
 interface BuiltInSketchRef {
   readonly name: string;
@@ -64,7 +65,7 @@ export class BuiltInExamplesServiceImpl {
     if (this._builtIns) {
       return this._builtIns;
     }
-    const examplesRootPath = join(__dirname, '..', '..', 'Examples');
+    const examplesRootPath = examplesPath;
     const examplesRootUri = FileUri.create(examplesRootPath);
     const rawJson = await fs.promises.readFile(
       join(examplesRootPath, 'examples.json'),

arduino-ide-extension/src/node/exec-util.ts

@@ -1,18 +1,4 @@
 import { spawn } from 'node:child_process';
-import os from 'node:os';
-import { join } from 'node:path';
-
-export type ArduinoBinaryName =
-  | 'arduino-cli'
-  | 'arduino-fwuploader'
-  | 'arduino-language-server';
-export type ClangBinaryName = 'clangd' | 'clang-format';
-export type BinaryName = ArduinoBinaryName | ClangBinaryName;
-
-export function getExecPath(binaryName: BinaryName): string {
-  const filename = `${binaryName}${os.platform() === 'win32' ? '.exe' : ''}`;
-  return join(__dirname, '..', '..', 'build', filename);
-}
 
 export function spawnCommand(
   command: string,

arduino-ide-extension/src/node/executable-service-impl.ts

@@ -1,7 +1,11 @@
 import { FileUri } from '@theia/core/lib/node/file-uri';
 import { injectable } from '@theia/core/shared/inversify';
 import { ExecutableService } from '../common/protocol/executable-service';
-import { getExecPath } from './exec-util';
+import {
+  arduinoCliPath,
+  arduinoLanguageServerPath,
+  clangdPath,
+} from './resources';
 
 @injectable()
 export class ExecutableServiceImpl implements ExecutableService {
@@ -11,9 +15,9 @@ export class ExecutableServiceImpl implements ExecutableService {
     lsUri: string;
   }> {
     return {
-      clangdUri: FileUri.create(getExecPath('clangd')).toString(),
-      cliUri: FileUri.create(getExecPath('arduino-cli')).toString(),
-      lsUri: FileUri.create(getExecPath('arduino-language-server')).toString(),
+      clangdUri: FileUri.create(clangdPath).toString(),
+      cliUri: FileUri.create(arduinoCliPath).toString(),
+      lsUri: FileUri.create(arduinoLanguageServerPath).toString(),
     };
   }
 }

arduino-ide-extension/src/node/i18n/arduino-localization-contribution.ts

@@ -3,46 +3,56 @@ import {
   LocalizationRegistry,
 } from '@theia/core/lib/node/i18n/localization-contribution';
 import { injectable } from '@theia/core/shared/inversify';
-import { join } from 'node:path';
+import bgJson from '../resources/i18n/bg.json';
+import csJson from '../resources/i18n/cs.json';
+import deJson from '../resources/i18n/de.json';
+import esJson from '../resources/i18n/es.json';
+import frJson from '../resources/i18n/fr.json';
+import huJson from '../resources/i18n/hu.json';
+import itJson from '../resources/i18n/it.json';
+import jaJson from '../resources/i18n/ja.json';
+import koJson from '../resources/i18n/ko.json';
+import nlJson from '../resources/i18n/nl.json';
+import plJson from '../resources/i18n/pl.json';
+import ptJson from '../resources/i18n/pt.json';
+import ruJson from '../resources/i18n/ru.json';
+import trJson from '../resources/i18n/tr.json';
+import uk_UAJson from '../resources/i18n/uk_UA.json';
+import zhJson from '../resources/i18n/zh.json';
+import zh_HantJson from '../resources/i18n/zh-Hant.json';
 
 @injectable()
 export class ArduinoLocalizationContribution
   implements LocalizationContribution
 {
-  // 0. index: locale
-  // 1. index: optional JSON file to `require` (if differs from the locale)
+  // keys: locales
+  // values: the required JSON modules
   // If you touch the locales, please keep the alphabetical order. Also in the `package.json` for the VS Code language packs. Thank you! ❤️
   // Note that IDE2 has more translations than available VS Code language packs. (https://github.com/arduino/arduino-ide/issues/1447)
-  private readonly locales: ReadonlyArray<[string, string?]> = [
-    ['bg'],
-    ['cs'],
-    ['de'],
-    ['es'],
-    ['fr'],
-    ['hu'],
-    // ['id'], Does not have Transifex translations, but has a VS Code language pack available on Open VSX.
-    ['it'],
-    ['ja'],
-    ['ko'],
-    ['nl'],
-    ['pl'],
-    ['pt-br', 'pt'],
-    ['ru'],
-    ['tr'],
-    ['uk', 'uk_UA'],
-    ['zh-cn', 'zh'],
-    ['zh-tw', 'zh-Hant'],
-  ];
+  private readonly locales: Readonly<Record<string, unknown>> = {
+    bg: bgJson,
+    cs: csJson,
+    de: deJson,
+    es: esJson,
+    fr: frJson,
+    hu: huJson,
+    // id: Does not have Transifex translations, but has a VS Code language pack available on Open VSX.
+    it: itJson,
+    ja: jaJson,
+    ko: koJson,
+    nl: nlJson,
+    pl: plJson,
+    'pt-br': ptJson,
+    ru: [ruJson],
+    tr: [trJson],
+    uk: uk_UAJson,
+    'zh-cn': zhJson,
+    'zh-tw': zh_HantJson,
+  };
 
   async registerLocalizations(registry: LocalizationRegistry): Promise<void> {
-    for (const [locale, jsonFilename] of this.locales) {
-      registry.registerLocalizationFromRequire(
-        locale,
-        require(join(
-          __dirname,
-          `../../../build/i18n/${jsonFilename ?? locale}.json`
-        ))
-      );
+    for (const [locale, module] of Object.entries(this.locales)) {
+      registry.registerLocalizationFromRequire(locale, module);
     }
   }
 }

arduino-ide-extension/src/node/is-temp-sketch.ts

@@ -1,4 +1,4 @@
-import * as fs from 'node:fs';
+import fs from 'node:fs';
 import tempDir from 'temp-dir';
 import { isWindows, isOSX } from '@theia/core/lib/common/os';
 import { injectable } from '@theia/core/shared/inversify';

arduino-ide-extension/src/node/monitor-settings/monitor-settings-provider-impl.ts

@@ -1,4 +1,4 @@
-import * as fs from 'node:fs';
+import fs from 'node:fs';
 import { join } from 'node:path';
 import {
   injectable,
@@ -38,19 +38,21 @@ export class MonitorSettingsProviderImpl implements MonitorSettingsProvider {
   private pluggableMonitorSettingsPath: string;
 
   @postConstruct()
-  protected async init(): Promise<void> {
-    // get the monitor settings file path
-    const configDirUri = await this.envVariablesServer.getConfigDirUri();
-    this.pluggableMonitorSettingsPath = join(
-      FileUri.fsPath(configDirUri),
-      MONITOR_SETTINGS_FILE
-    );
+  protected init(): void {
+    (async () => {
+      // get the monitor settings file path
+      const configDirUri = await this.envVariablesServer.getConfigDirUri();
+      this.pluggableMonitorSettingsPath = join(
+        FileUri.fsPath(configDirUri),
+        MONITOR_SETTINGS_FILE
+      );
 
-    // read existing settings
-    await this.readSettingsFromFS();
+      // read existing settings
+      await this.readSettingsFromFS();
 
-    // init is done, resolve the deferred and unblock any call that was waiting for it
-    this.ready.resolve();
+      // init is done, resolve the deferred and unblock any call that was waiting for it
+      this.ready.resolve();
+    })();
   }
 
   async getSettings(

arduino-ide-extension/src/node/plotter/plotter-backend-contribution.ts

@@ -1,22 +1,20 @@
-import * as path from 'node:path';
-import * as express from '@theia/core/shared/express';
-import { injectable } from '@theia/core/shared/inversify';
 import { BackendApplicationContribution } from '@theia/core/lib/node/backend-application';
+import express from '@theia/core/shared/express';
+import { injectable } from '@theia/core/shared/inversify';
+import path from 'node:path';
+import { arduinoPlotterWebAppPath } from '../resources';
 
 @injectable()
 export class PlotterBackendContribution
   implements BackendApplicationContribution
 {
   configure(app: express.Application): void {
-    const index = require.resolve(
-      'arduino-serial-plotter-webapp/build/index.html'
-    );
-    app.use(express.static(path.join(index, '..')));
+    app.use(express.static(arduinoPlotterWebAppPath));
     app.get('/plotter', (req, res) => {
       console.log(
         `Serving serial plotter on http://${req.headers.host}${req.url}`
       );
-      res.sendFile(index);
+      res.sendFile(path.join(arduinoPlotterWebAppPath, 'index.html'));
     });
   }
 }

arduino-ide-extension/src/node/resources.ts

@@ -0,0 +1,32 @@
+import path from 'node:path';
+
+// When running the tests, the JS files are not yet bundled by webpack.
+// Hence, the `resources` folder lookup is different.
+const testEnv = process.env.IDE2_TEST === 'true';
+const resourcesPath = path.join(
+  __dirname,
+  ...(testEnv ? ['..', '..', 'src', 'node', 'resources'] : ['resources'])
+);
+const exe = process.platform === 'win32' ? '.exe' : '';
+
+// binaries
+export const arduinoCliPath = path.join(resourcesPath, 'arduino-cli' + exe);
+export const arduinoFirmwareUploaderPath = path.join(
+  resourcesPath,
+  'arduino-fwuploader' + exe
+);
+export const arduinoLanguageServerPath = path.join(
+  resourcesPath,
+  'arduino-language-server' + exe
+);
+export const clangdPath = path.join(resourcesPath, 'clangd' + exe);
+export const clangFormatPath = path.join(resourcesPath, 'clang-format' + exe);
+
+// plotter
+export const arduinoPlotterWebAppPath = path.join(
+  resourcesPath,
+  'arduino-serial-plotter-webapp'
+);
+
+// examples
+export const examplesPath = path.join(resourcesPath, 'Examples');

arduino-ide-extension/src/node/theia/filesystem/nsfw-bindings.ts

@@ -8,13 +8,13 @@ import {
 import { FileSystemWatcherService } from '@theia/filesystem/lib/common/filesystem-watcher-protocol';
 import { NsfwFileSystemWatcherServerOptions } from '@theia/filesystem/lib/node/nsfw-watcher/nsfw-filesystem-service';
 import { FileSystemWatcherServiceDispatcher } from '@theia/filesystem/lib/node/filesystem-watcher-dispatcher';
-import { NoDelayDisposalTimeoutNsfwFileSystemWatcherService } from './nsfw-filesystem-service';
+import { NoDelayDisposalTimeoutNsfwFileSystemWatcherService } from './nsfw-watcher/nsfw-filesystem-service';
 
 export function rebindNsfwFileSystemWatcher(rebind: interfaces.Rebind): void {
   rebind<NsfwFileSystemWatcherServiceProcessOptions>(
     NsfwFileSystemWatcherServiceProcessOptions
   ).toConstantValue({
-    entryPoint: join(__dirname, 'index.js'),
+    entryPoint: join(__dirname, 'nsfw-watcher'),
   });
   rebind<FileSystemWatcherService>(FileSystemWatcherService)
     .toDynamicValue((context) =>

arduino-ide-extension/src/node/theia/filesystem/nsfw-watcher/index.ts

@@ -1,5 +1,5 @@
 import * as yargs from '@theia/core/shared/yargs';
-import { JsonRpcProxyFactory } from '@theia/core';
+import { JsonRpcProxyFactory } from '@theia/core/lib/common/messaging/proxy-factory';
 import { NoDelayDisposalTimeoutNsfwFileSystemWatcherService } from './nsfw-filesystem-service';
 import type { IPCEntryPoint } from '@theia/core/lib/node/messaging/ipc-protocol';
 import type { FileSystemWatcherServiceClient } from '@theia/filesystem/lib/common/filesystem-watcher-protocol';

arduino-ide-extension/src/node/web-socket/web-socket-provider-impl.ts

@@ -1,6 +1,6 @@
 import { Emitter } from '@theia/core';
 import { injectable } from '@theia/core/shared/inversify';
-import * as WebSocket from '@theia/core/shared/ws';
+import WebSocket from '@theia/core/shared/ws';
 import { WebSocketProvider } from './web-socket-provider';
 
 @injectable()

arduino-ide-extension/src/node/web-socket/web-socket-provider.ts

@@ -1,9 +1,9 @@
 import { Event } from '@theia/core/lib/common/event';
-import * as WebSocket from '@theia/core/shared/ws';
+import type { AddressInfo } from '@theia/core/shared/ws';
 
 export const WebSocketProvider = Symbol('WebSocketProvider');
 export interface WebSocketProvider {
-  getAddress(): WebSocket.AddressInfo;
+  getAddress(): AddressInfo;
   sendMessage(message: string): void;
   onMessageReceived: Event<string>;
   onClientsNumberChanged: Event<number>;