ATL-93: Added Support for .pde sketch file format.

Signed-off-by: Akos Kitta <kittaakos@typefox.io>

arduino-ide-extension/src/browser/arduino-frontend-contribution.tsx

@@ -47,6 +47,7 @@ import { FileService } from '@theia/filesystem/lib/browser/file-service';
 import { OutputService } from '../common/protocol/output-service';
 import { ArduinoPreferences } from './arduino-preferences';
 import { SketchesServiceClientImpl } from '../common/protocol/sketches-service-client-impl';
+import { SaveAsSketch } from './contributions/save-as-sketch';
 
 @injectable()
 export class ArduinoFrontendContribution implements FrontendApplicationContribution,
@@ -338,6 +339,14 @@ export class ArduinoFrontendContribution implements FrontendApplicationContribut
                 await this.ensureOpened(uri);
             }
             await this.ensureOpened(mainFileUri, true);
+            if (mainFileUri.endsWith('.pde')) {
+                const message = `The '${sketch.name}' still uses the old \`.pde\` format. Do you want to switch to the new \`.ino\` extension?`;
+                this.messageService.info(message, 'Later', 'Yes').then(async answer => {
+                    if (answer === 'Yes') {
+                        this.commandRegistry.executeCommand(SaveAsSketch.Commands.SAVE_AS_SKETCH.id, { execOnlyIfTemp: false, openAfterMove: true, wipeOriginal: false });
+                    }
+                });
+            }
         } catch (e) {
             console.error(e);
             const message = e instanceof Error ? e.message : JSON.stringify(e);

arduino-ide-extension/src/browser/contributions/open-sketch.ts

@@ -122,7 +122,7 @@ export class OpenSketch extends SketchContribution {
             filters: [
                 {
                     name: 'Sketch',
-                    extensions: ['ino']
+                    extensions: ['ino', 'pde']
                 }
             ]
         });
@@ -138,7 +138,7 @@ export class OpenSketch extends SketchContribution {
         if (sketch) {
             return sketch;
         }
-        if (sketchFileUri.endsWith('.ino')) {
+        if (Sketch.isSketchFile(sketchFileUri)) {
             const name = new URI(sketchFileUri).path.name;
             const nameWithExt = this.labelProvider.getName(new URI(sketchFileUri));
             const { response } = await remote.dialog.showMessageBox({

arduino-ide-extension/src/browser/theia/workspace/workspace-service.ts

@@ -9,7 +9,7 @@ import { FocusTracker, Widget } from '@theia/core/lib/browser';
 import { WorkspaceService as TheiaWorkspaceService } from '@theia/workspace/lib/browser/workspace-service';
 import { FrontendApplicationConfigProvider } from '@theia/core/lib/browser/frontend-application-config-provider';
 import { ConfigService } from '../../../common/protocol/config-service';
-import { SketchesService } from '../../../common/protocol/sketches-service';
+import { SketchesService, Sketch } from '../../../common/protocol/sketches-service';
 import { ArduinoWorkspaceRootResolver } from '../../arduino-workspace-resolver';
 
 @injectable()
@@ -85,7 +85,7 @@ export class WorkspaceService extends TheiaWorkspaceService {
     protected onCurrentWidgetChange({ newValue }: FocusTracker.IChangedArgs<Widget>): void {
         if (newValue instanceof EditorWidget) {
             const { uri } = newValue.editor;
-            if (uri.toString().endsWith('.ino')) {
+            if (Sketch.isSketchFile(uri.toString())) {
                 this.updateTitle();
             } else {
                 const title = this.workspaceTitle;

arduino-ide-extension/src/common/protocol/sketches-service.ts

@@ -81,4 +81,10 @@ export namespace Sketch {
         const { mainFileUri, otherSketchFileUris, additionalFileUris } = sketch;
         return [mainFileUri, ...otherSketchFileUris, ...additionalFileUris].indexOf(uri.toString()) !== -1;
     }
+    export function isSketchFile(arg: string | URI): boolean {
+        if (arg instanceof URI) {
+            return isSketchFile(arg.toString());
+        }
+        return Extensions.MAIN.some(ext => arg.endsWith(ext));
+    }
 }

arduino-ide-extension/src/node/arduino-ide-backend-module.ts

@@ -21,8 +21,6 @@ import { MonitorServiceImpl } from './monitor/monitor-service-impl';
 import { MonitorService, MonitorServicePath, MonitorServiceClient } from '../common/protocol/monitor-service';
 import { MonitorClientProvider } from './monitor/monitor-client-provider';
 import { ConfigServiceImpl } from './config-service-impl';
-import { HostedPluginReader } from './theia/plugin-ext/plugin-reader';
-import { HostedPluginReader as TheiaHostedPluginReader } from '@theia/plugin-ext/lib/hosted/node/plugin-reader';
 import { EnvVariablesServer as TheiaEnvVariablesServer } from '@theia/core/lib/common/env-variables';
 import { EnvVariablesServer } from './theia/env-variables/env-variables-server';
 import { NodeFileSystemExt } from './node-filesystem-ext';
@@ -106,9 +104,6 @@ export default new ContainerModule((bind, unbind, isBound, rebind) => {
     bind(EnvVariablesServer).toSelf().inSingletonScope();
     rebind(TheiaEnvVariablesServer).toService(EnvVariablesServer);
 
-    bind(HostedPluginReader).toSelf().inSingletonScope();
-    rebind(TheiaHostedPluginReader).toService(HostedPluginReader);
-
     // #endregion Theia customizations
 
     // Monitor client provider per connected frontend.

arduino-ide-extension/src/node/core-client-provider.ts

@@ -57,7 +57,8 @@ export class CoreClientProvider extends GrpcClientProvider<CoreClientProvider.Cl
             throw new Error(`Could not retrieve instance from the initialize response.`);
         }
 
-        await this.updateIndexes({ instance, client });
+        // No `await`. The index update event comes later. This way we do not block app startup with index update when invalid proxy is given.
+        this.updateIndexes({ instance, client });
 
         return { instance, client };
     }

arduino-ide-extension/src/node/core-service-impl.ts

@@ -1,12 +1,12 @@
 import { FileUri } from '@theia/core/lib/node/file-uri';
-import { inject, injectable, postConstruct } from 'inversify';
+import { inject, injectable } from 'inversify';
 import { dirname } from 'path';
 import { CoreService } from '../common/protocol/core-service';
 import { CompileReq, CompileResp } from './cli-protocol/commands/compile_pb';
 import { CoreClientProvider } from './core-client-provider';
 import { UploadReq, UploadResp, BurnBootloaderReq, BurnBootloaderResp, UploadUsingProgrammerReq, UploadUsingProgrammerResp } from './cli-protocol/commands/upload_pb';
 import { OutputService } from '../common/protocol/output-service';
-import { NotificationServiceServer, ConfigService } from '../common/protocol';
+import { NotificationServiceServer } from '../common/protocol';
 import { ClientReadableStream } from '@grpc/grpc-js';
 import { ArduinoCoreClient } from './cli-protocol/commands/commands_grpc_pb';
 import { firstToUpperCase, firstToLowerCase } from '../common/utils';
@@ -23,16 +23,6 @@ export class CoreServiceImpl implements CoreService {
     @inject(NotificationServiceServer)
     protected readonly notificationService: NotificationServiceServer;
 
-    @inject(ConfigService)
-    protected readonly configService: ConfigService;
-
-    @postConstruct()
-    protected init(): void {
-        this.coreClient().then(({ client, instance }) => {
-
-        });
-    }
-
     async compile(options: CoreService.Compile.Options): Promise<void> {
         this.outputService.append({ name: 'compile', chunk: 'Compile...\n' + JSON.stringify(options, null, 2) + '\n--------------------------\n' });
         const { sketchUri, fqbn } = options;

arduino-ide-extension/src/node/sketches-service-impl.ts

@@ -4,7 +4,6 @@ import * as os from 'os';
 import * as temp from 'temp';
 import * as path from 'path';
 import { ncp } from 'ncp';
-import { Stats } from 'fs';
 import { promisify } from 'util';
 import URI from '@theia/core/lib/common/uri';
 import { FileUri } from '@theia/core/lib/node';
@@ -14,22 +13,22 @@ import { SketchesService, Sketch } from '../common/protocol/sketches-service';
 import { firstToLowerCase } from '../common/utils';
 import { NotificationServiceServerImpl } from './notification-service-server';
 import { EnvVariablesServer } from '@theia/core/lib/common/env-variables';
-
+import { CoreClientProvider } from './core-client-provider';
-// As currently implemented on Linux,
+import { LoadSketchReq } from './cli-protocol/commands/commands_pb';
-// the maximum number of symbolic links that will be followed while resolving a pathname is 40
-const MAX_FILESYSTEM_DEPTH = 40;
 
 const WIN32_DRIVE_REGEXP = /^[a-zA-Z]:\\/;
 
 const prefix = '.arduinoProIDE-unsaved';
 
-// TODO: `fs`: use async API 
 @injectable()
 export class SketchesServiceImpl implements SketchesService {
 
     @inject(ConfigService)
     protected readonly configService: ConfigService;
 
+    @inject(CoreClientProvider)
+    protected readonly coreClientProvider: CoreClientProvider;
+
     @inject(NotificationServiceServerImpl)
     protected readonly notificationService: NotificationServiceServerImpl;
 
@@ -59,10 +58,10 @@ export class SketchesServiceImpl implements SketchesService {
         const filenames = await promisify(fs.readdir)(sketchbookPath);
         for (const fileName of filenames) {
             const filePath = path.join(sketchbookPath, fileName);
-            if (await this.isSketchFolder(FileUri.create(filePath).toString())) {
+            const sketch = await this._isSketchFolder(FileUri.create(filePath).toString());
+            if (sketch) {
                 try {
                     const stat = await promisify(fs.stat)(filePath);
-                    const sketch = await this.loadSketch(FileUri.create(filePath).toString());
                     sketches.push({
                         ...sketch,
                         mtimeMs: stat.mtimeMs
@@ -76,102 +75,30 @@ export class SketchesServiceImpl implements SketchesService {
         return sketches;
     }
 
-    /**
-     * This is the TS implementation of `SketchLoad` from the CLI.
-     * See: https://github.com/arduino/arduino-cli/issues/837
-     * Based on: https://github.com/arduino/arduino-cli/blob/eef3705c4afcba4317ec38b803d9ffce5dd59a28/arduino/builder/sketch.go#L100-L215
-     */
     async loadSketch(uri: string): Promise<SketchWithDetails> {
-        const sketchPath = FileUri.fsPath(uri);
+        const { client, instance } = await this.coreClient();
-        const exists = await promisify(fs.exists)(sketchPath);
+        const req = new LoadSketchReq();
-        if (!exists) {
+        req.setSketchPath(FileUri.fsPath(uri));
-            throw new Error(`${uri} does not exist.`);
+        req.setInstance(instance);
-        }
+        const sketch = await new Promise<SketchWithDetails>((resolve, reject) => {
-        const stat = await promisify(fs.lstat)(sketchPath);
+            client.loadSketch(req, async (err, resp) => {
-        let sketchFolder: string | undefined;
+                if (err) {
-        let mainSketchFile: string | undefined;
+                    reject(err);
-
+                    return;
-        // If a sketch folder was passed, save the parent and point sketchPath to the main sketch file
-        if (stat.isDirectory()) {
-            sketchFolder = sketchPath;
-            // Allowed extensions are .ino and .pde (but not both)
-            for (const extension of Sketch.Extensions.MAIN) {
-                const candidateSketchFile = path.join(sketchPath, `${path.basename(sketchPath)}${extension}`);
-                const candidateExists = await promisify(fs.exists)(candidateSketchFile);
-                if (candidateExists) {
-                    if (!mainSketchFile) {
-                        mainSketchFile = candidateSketchFile;
-                    } else {
-                        throw new Error(`Multiple main sketch files found (${path.basename(mainSketchFile)}, ${path.basename(candidateSketchFile)})`);
-                    }
                 }
-            }
+                const sketchFolderPath = resp.getLocationPath();
-
+                const { mtimeMs } = await promisify(fs.lstat)(sketchFolderPath);
-            // Check main file was found.
+                resolve({
-            if (!mainSketchFile) {
+                    name: path.basename(sketchFolderPath),
-                throw new Error(`Unable to find a sketch file in directory ${sketchFolder}`);
+                    uri: FileUri.create(sketchFolderPath).toString(),
-            }
+                    mainFileUri: FileUri.create(resp.getMainFile()).toString(),
-
+                    otherSketchFileUris: resp.getOtherSketchFilesList().map(p => FileUri.create(p).toString()),
-            // Check main file is readable.
+                    additionalFileUris: resp.getAdditionalFilesList().map(p => FileUri.create(p).toString()),
-            try {
+                    mtimeMs
-                await promisify(fs.access)(mainSketchFile, fs.constants.R_OK);
+                });
-            } catch {
+            });
-                throw new Error('Unable to open the main sketch file.');
-            }
-
-            const mainSketchFileStat = await promisify(fs.lstat)(mainSketchFile);
-            if (mainSketchFileStat.isDirectory()) {
-                throw new Error(`Sketch must not be a directory.`);
-            }
-        } else {
-            sketchFolder = path.dirname(sketchPath);
-            mainSketchFile = sketchPath;
-        }
-
-        const files: string[] = [];
-        let rootVisited = false;
-        const err = await this.simpleLocalWalk(sketchFolder, MAX_FILESYSTEM_DEPTH, async (fsPath: string, info: Stats, error: Error | undefined) => {
-            if (error) {
-                console.log(`Error during sketch processing: ${error}`);
-                return error;
-            }
-            const name = path.basename(fsPath);
-            if (info.isDirectory()) {
-                if (rootVisited) {
-                    if (name.startsWith('.') || name === 'CVS' || name === 'RCS') {
-                        return new SkipDir();
-                    }
-                } else {
-                    rootVisited = true
-                }
-                return undefined;
-            }
-
-            if (name.startsWith('.')) {
-                return undefined;
-            }
-            const ext = path.extname(fsPath);
-            const isMain = Sketch.Extensions.MAIN.indexOf(ext) !== -1;
-            const isAdditional = Sketch.Extensions.ADDITIONAL.indexOf(ext) !== -1;
-            if (!isMain && !isAdditional) {
-                return undefined;
-            }
-
-            try {
-                await promisify(fs.access)(fsPath, fs.constants.R_OK);
-                files.push(fsPath);
-            } catch { }
-
-            return undefined;
         });
-
+        return sketch;
-        if (err) {
-            console.error(`There was an error while collecting the sketch files: ${sketchPath}`)
-            throw err;
-        }
-
-        return this.newSketch(sketchFolder, mainSketchFile, files);
-
     }
 
     private get recentSketchesFsPath(): Promise<string> {
@@ -242,49 +169,6 @@ export class SketchesServiceImpl implements SketchesService {
         return sketches;
     }
 
-    private async newSketch(sketchFolderPath: string, mainFilePath: string, allFilesPaths: string[]): Promise<SketchWithDetails> {
-        let mainFile: string | undefined;
-        const paths = new Set<string>();
-        for (const p of allFilesPaths) {
-            if (p === mainFilePath) {
-                mainFile = p;
-            } else {
-                paths.add(p);
-            }
-        }
-        if (!mainFile) {
-            throw new Error('Could not locate main sketch file.');
-        }
-        const additionalFiles: string[] = [];
-        const otherSketchFiles: string[] = [];
-        for (const p of Array.from(paths)) {
-            const ext = path.extname(p);
-            if (Sketch.Extensions.MAIN.indexOf(ext) !== -1) {
-                if (path.dirname(p) === sketchFolderPath) {
-                    otherSketchFiles.push(p);
-                }
-            } else if (Sketch.Extensions.ADDITIONAL.indexOf(ext) !== -1) {
-                // XXX: this is a caveat with the CLI, we do not know the `buildPath`.
-                // https://github.com/arduino/arduino-cli/blob/0483882b4f370c288d5318913657bbaa0325f534/arduino/sketch/sketch.go#L108-L110
-                additionalFiles.push(p);
-            } else {
-                throw new Error(`Unknown sketch file extension '${ext}'.`);
-            }
-        }
-        additionalFiles.sort();
-        otherSketchFiles.sort();
-
-        const { mtimeMs } = await promisify(fs.lstat)(sketchFolderPath);
-        return {
-            uri: FileUri.create(sketchFolderPath).toString(),
-            mainFileUri: FileUri.create(mainFile).toString(),
-            name: path.basename(sketchFolderPath),
-            additionalFileUris: additionalFiles.map(p => FileUri.create(p).toString()),
-            otherSketchFileUris: otherSketchFiles.map(p => FileUri.create(p).toString()),
-            mtimeMs
-        };
-    }
-
     async cloneExample(uri: string): Promise<Sketch> {
         const sketch = await this.loadSketch(uri);
         const parentPath = await new Promise<string>((resolve, reject) => {
@@ -301,56 +185,6 @@ export class SketchesServiceImpl implements SketchesService {
         return this.loadSketch(copiedSketchUri);
     }
 
-    protected async simpleLocalWalk(
-        root: string,
-        maxDepth: number,
-        walk: (fsPath: string, info: Stats | undefined, err: Error | undefined) => Promise<Error | undefined>): Promise<Error | undefined> {
-
-        let { info, err } = await this.lstat(root);
-        if (err) {
-            return walk(root, undefined, err);
-        }
-        if (!info) {
-            return new Error(`Could not stat file: ${root}.`);
-        }
-        err = await walk(root, info, err);
-        if (err instanceof SkipDir) {
-            return undefined;
-        }
-
-        if (info.isDirectory()) {
-            if (maxDepth <= 0) {
-                return walk(root, info, new Error(`Filesystem bottom is too deep (directory recursion or filesystem really deep): ${root}`));
-            }
-            maxDepth--;
-            const files: string[] = [];
-            try {
-                files.push(...await promisify(fs.readdir)(root));
-            } catch { }
-            for (const file of files) {
-                err = await this.simpleLocalWalk(path.join(root, file), maxDepth, walk);
-                if (err instanceof SkipDir) {
-                    return undefined;
-                }
-            }
-        }
-
-        return undefined;
-    }
-
-    private async lstat(fsPath: string): Promise<{ info: Stats, err: undefined } | { info: undefined, err: Error }> {
-        const exists = await promisify(fs.exists)(fsPath);
-        if (!exists) {
-            return { info: undefined, err: new Error(`${fsPath} does not exist`) };
-        }
-        try {
-            const info = await promisify(fs.lstat)(fsPath);
-            return { info, err: undefined };
-        } catch (err) {
-            return { info: undefined, err };
-        }
-    }
-
     async createNewSketch(): Promise<Sketch> {
         const monthNames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
         const today = new Date();
@@ -404,8 +238,9 @@ void loop() {
         }
         let currentUri = new URI(uri);
         while (currentUri && !currentUri.path.isRoot) {
-            if (await this.isSketchFolder(currentUri.toString())) {
+            const sketch = await this._isSketchFolder(currentUri.toString());
-                return this.loadSketch(currentUri.toString());
+            if (sketch) {
+                return sketch;
             }
             currentUri = currentUri.parent;
         }
@@ -413,6 +248,11 @@ void loop() {
     }
 
     async isSketchFolder(uri: string): Promise<boolean> {
+        const sketch = await this._isSketchFolder(uri);
+        return !!sketch;
+    }
+
+    private async _isSketchFolder(uri: string): Promise<SketchWithDetails | undefined> {
         const fsPath = FileUri.fsPath(uri);
         let stat: fs.Stats | undefined;
         try {
@@ -422,15 +262,15 @@ void loop() {
             const basename = path.basename(fsPath);
             const files = await promisify(fs.readdir)(fsPath);
             for (let i = 0; i < files.length; i++) {
-                if (files[i] === basename + '.ino') {
+                if (files[i] === basename + '.ino' || files[i] === basename + '.pde') {
                     try {
-                        await this.loadSketch(FileUri.create(fsPath).toString());
+                        const sketch = await this.loadSketch(FileUri.create(fsPath).toString());
-                        return true;
+                        return sketch;
                     } catch { }
                 }
             }
         }
-        return false;
+        return undefined;
     }
 
     async isTemp(sketch: Sketch): Promise<boolean> {
@@ -484,16 +324,27 @@ void loop() {
         return FileUri.create(destination).toString();
     }
 
-}
+    private async coreClient(): Promise<CoreClientProvider.Client> {
-
+        const coreClient = await new Promise<CoreClientProvider.Client>(async resolve => {
-class SkipDir extends Error {
+            const client = await this.coreClientProvider.client();
-    constructor() {
+            if (client) {
-        super('skip this directory');
+                resolve(client);
-        Object.setPrototypeOf(this, SkipDir.prototype);
+                return;
+            }
+            const toDispose = this.coreClientProvider.onClientReady(async () => {
+                const client = await this.coreClientProvider.client();
+                if (client) {
+                    toDispose.dispose();
+                    resolve(client);
+                    return;
+                }
+            });
+        });
+        return coreClient;
     }
+
 }
 
 interface SketchWithDetails extends Sketch {
     readonly mtimeMs: number;
 }
-

arduino-ide-extension/src/node/theia/plugin-ext/plugin-reader.ts

@@ -1,47 +0,0 @@
-import { injectable, inject } from 'inversify';
-import { HostedPluginReader as TheiaHostedPluginReader } from '@theia/plugin-ext/lib/hosted/node/plugin-reader';
-import { PluginPackage, PluginContribution } from '@theia/plugin-ext/lib/common/plugin-protocol';
-import { CLI_CONFIG } from '../../cli-config';
-import { ConfigServiceImpl } from '../../config-service-impl';
-
-@injectable()
-export class HostedPluginReader extends TheiaHostedPluginReader {
-
-    @inject(ConfigServiceImpl)
-    protected readonly configService: ConfigServiceImpl;
-    protected cliConfigSchemaUri: string;
-
-    async onStart(): Promise<void> {
-        this.cliConfigSchemaUri = ''; // TODO: this was removed in another PR.
-    }
-
-    readContribution(plugin: PluginPackage): PluginContribution | undefined {
-        const scanner = this.scanner.getScanner(plugin);
-        const contribution = scanner.getContribution(plugin);
-        if (!contribution) {
-            return contribution;
-        }
-        if (plugin.name === 'vscode-yaml' && plugin.publisher === 'redhat' && contribution.configuration) {
-            // Use the schema for the Arduino CLI.
-            const { configuration } = contribution;
-            for (const config of configuration) {
-                if (typeof config.properties['yaml.schemas'] === 'undefined') {
-                    config.properties['yaml.schemas'] = {};
-                }
-                config.properties['yaml.schemas'].default = {
-                    [this.cliConfigSchemaUri]: [CLI_CONFIG]
-                };
-            }
-        } else if (plugin.name === 'cpp' && plugin.publisher === 'vscode' && contribution.languages) {
-            // Do not associate `.ino` files with the VS Code built-in extension for C++.
-            // https://github.com/eclipse-theia/theia/issues/7533#issuecomment-611055328
-            for (const language of contribution.languages) {
-                if (language.extensions) {
-                    language.extensions = language.extensions.filter(ext => ext !== '.ino');
-                }
-            }
-        }
-        return contribution;
-    }
-
-}