[ATL-1454] Refactor pull/push to edit files in place (#464)

* improve push/pull process

* improved diff tree performance generation

* skip some files to be synced

Co-authored-by: Francesco Stasi <f.stasi@me.com>

arduino-ide-extension/src/browser/create/create-api.ts

@@ -93,7 +93,11 @@ export class CreateApi {
 
   async readDirectory(
     posixPath: string,
-    options: { recursive?: boolean; match?: string } = {}
+    options: {
+      recursive?: boolean;
+      match?: string;
+      skipSketchCache?: boolean;
+    } = {}
   ): Promise<Create.Resource[]> {
     const url = new URL(
       `${this.domain()}/files/d/$HOME/sketches_v2${posixPath}`
@@ -106,21 +110,29 @@ export class CreateApi {
     }
     const headers = await this.headers();
 
-    return this.run<Create.RawResource[]>(url, {
+    const cachedSketch = this.sketchCache.getSketch(posixPath);
+
+    const sketchPromise = options.skipSketchCache
+      ? (cachedSketch && this.sketch(cachedSketch.id)) || Promise.resolve(null)
+      : Promise.resolve(this.sketchCache.getSketch(posixPath));
+
+    return Promise.all([
+      sketchPromise,
+      this.run<Create.RawResource[]>(url, {
         method: 'GET',
         headers,
-    })
+      }),
-      .then(async (result) => {
+    ])
-        // add arduino_secrets.h to the results, when reading a sketch main folder
+      .then(async ([sketch, result]) => {
         if (posixPath.length && posixPath !== posix.sep) {
-          const sketch = this.sketchCache.getSketch(posixPath);
-
           if (sketch && sketch.secrets && sketch.secrets.length > 0) {
             result.push(this.getSketchSecretStat(sketch));
           }
         }
 
-        return result;
+        return result.filter(
+          (res) => !Create.do_not_sync_files.includes(res.name)
+        );
       })
       .catch((reason) => {
         if (reason?.status === 404) return [] as Create.Resource[];

arduino-ide-extension/src/browser/create/create-fs-provider.ts

@@ -30,8 +30,6 @@ import { SketchesService } from '../../common/protocol';
 import { ArduinoPreferences } from '../arduino-preferences';
 import { Create } from './typings';
 
-export const REMOTE_ONLY_FILES = ['sketch.json'];
-
 @injectable()
 export class CreateFsProvider
   implements
@@ -109,14 +107,10 @@ export class CreateFsProvider
     const resources = await this.getCreateApi.readDirectory(
       uri.path.toString()
     );
-    return resources
+    return resources.map(({ name, type }) => [name, this.toFileType(type)]);
-      .filter((res) => !REMOTE_ONLY_FILES.includes(res.name))
-      .map(({ name, type }) => [name, this.toFileType(type)]);
   }
 
   async delete(uri: URI, opts: FileDeleteOptions): Promise<void> {
-    return;
-
     if (!opts.recursive) {
       throw new Error(
         'Arduino Create file-system provider does not support non-recursive deletion.'

arduino-ide-extension/src/browser/create/typings.ts

@@ -21,7 +21,7 @@ export namespace Create {
 
   export type ResourceType = 'sketch' | 'folder' | 'file';
   export const arduino_secrets_file = 'arduino_secrets.h';
-  export const do_not_sync_files = ['.theia'];
+  export const do_not_sync_files = ['.theia', 'sketch.json'];
   export interface Resource {
     readonly name: string;
     /**

arduino-ide-extension/src/browser/widgets/cloud-sketchbook/cloud-sketch-cache.ts

@@ -22,6 +22,14 @@ export class SketchCache {
     return this.fileStats[path] || null;
   }
 
+  purgeByPath(path: string): void {
+    for (const itemPath in this.fileStats) {
+      if (itemPath.indexOf(path) === 0) {
+        delete this.fileStats[itemPath];
+      }
+    }
+  }
+
   addSketch(sketch: Create.Sketch): void {
     const { path } = sketch;
     const posixPath = toPosixPath(path);

arduino-ide-extension/src/browser/widgets/cloud-sketchbook/cloud-sketchbook-tree.ts

@@ -17,7 +17,6 @@ import {
   PreferenceScope,
 } from '@theia/core/lib/browser/preferences/preference-service';
 import { MessageService } from '@theia/core/lib/common/message-service';
-import { REMOTE_ONLY_FILES } from './../../create/create-fs-provider';
 import { CreateApi } from '../../create/create-api';
 import { CreateUri } from '../../create/create-uri';
 import { CloudSketchbookTreeModel } from './cloud-sketchbook-tree-model';
@@ -33,10 +32,17 @@ import { ArduinoPreferences } from '../../arduino-preferences';
 import { SketchesServiceClientImpl } from '../../../common/protocol/sketches-service-client-impl';
 import { FileStat } from '@theia/filesystem/lib/common/files';
 import { WorkspaceNode } from '@theia/navigator/lib/browser/navigator-tree';
+import { posix, splitSketchPath } from '../../create/create-paths';
+import { Create } from '../../create/typings';
 
 const MESSAGE_TIMEOUT = 5 * 1000;
 const deepmerge = require('deepmerge').default;
 
+type FilesToWrite = { source: URI; dest: URI };
+type FilesToSync = {
+  filesToWrite: FilesToWrite[];
+  filesToDelete: URI[];
+};
 @injectable()
 export class CloudSketchbookTree extends SketchbookTree {
   @inject(FileService)
@@ -94,7 +100,7 @@ export class CloudSketchbookTree extends SketchbookTree {
 
   async pull(arg: any): Promise<void> {
     const {
-      model,
+      // model,
       node,
     }: {
       model: CloudSketchbookTreeModel;
@@ -127,47 +133,12 @@ export class CloudSketchbookTree extends SketchbookTree {
       const commandsCopy = node.commands;
       node.commands = [];
 
-      // check if the sketch dir already exist
+      const localUri = await this.fileService.toUnderlyingResource(
-      if (CloudSketchbookTree.CloudSketchTreeNode.isSynced(node)) {
+        LocalCacheUri.root.resolve(node.remoteUri.path)
-        const filesToPull = (
-          await this.createApi.readDirectory(node.remoteUri.path.toString())
-        ).filter((file: any) => !REMOTE_ONLY_FILES.includes(file.name));
-
-        await Promise.all(
-          filesToPull.map((file: any) => {
-            const uri = CreateUri.toUri(file);
-            this.fileService.copy(uri, LocalCacheUri.root.resolve(uri.path), {
-              overwrite: true,
-            });
-          })
       );
+      await this.sync(node.remoteUri, localUri);
 
-        // open the pulled files in the current workspace
+      this.sketchCache.purgeByPath(node.remoteUri.path.toString());
-        const currentSketch = await this.sketchServiceClient.currentSketch();
-
-        if (
-          !CreateUri.is(node.uri) &&
-          currentSketch &&
-          currentSketch.uri === node.uri.toString()
-        ) {
-          filesToPull.forEach(async (file) => {
-            const localUri = LocalCacheUri.root.resolve(
-              CreateUri.toUri(file).path
-            );
-            const underlying = await this.fileService.toUnderlyingResource(
-              localUri
-            );
-
-            model.open(underlying);
-          });
-        }
-      } else {
-        await this.fileService.copy(
-          node.remoteUri,
-          LocalCacheUri.root.resolve(node.uri.path),
-          { overwrite: true }
-        );
-      }
 
       node.commands = commandsCopy;
       this.messageService.info(`Done pulling ‘${node.fileStat.name}’.`, {
@@ -214,17 +185,107 @@ export class CloudSketchbookTree extends SketchbookTree {
       }
       const commandsCopy = node.commands;
       node.commands = [];
-      // delete every first level file, then push everything
+
-      const result = await this.fileService.copy(node.uri, node.remoteUri, {
+      const localUri = await this.fileService.toUnderlyingResource(
-        overwrite: true,
+        LocalCacheUri.root.resolve(node.remoteUri.path)
-      });
+      );
+      await this.sync(localUri, node.remoteUri);
+
+      this.sketchCache.purgeByPath(node.remoteUri.path.toString());
+
       node.commands = commandsCopy;
-      this.messageService.info(`Done pushing ‘${result.name}’.`, {
+      this.messageService.info(`Done pushing ‘${node.fileStat.name}’.`, {
         timeout: MESSAGE_TIMEOUT,
       });
     });
   }
 
+  async recursiveURIs(uri: URI): Promise<URI[]> {
+    // remote resources can be fetched one-shot via api
+    if (CreateUri.is(uri)) {
+      const resources = await this.createApi.readDirectory(
+        uri.path.toString(),
+        { recursive: true, skipSketchCache: true }
+      );
+      return resources.map((resource) =>
+        CreateUri.toUri(splitSketchPath(resource.path)[1])
+      );
+    }
+
+    const fileStat = await this.fileService.resolve(uri, {
+      resolveMetadata: false,
+    });
+
+    if (!fileStat.children || !fileStat.isDirectory) {
+      return [fileStat.resource];
+    }
+
+    let childrenUris: URI[] = [];
+
+    for await (const child of fileStat.children) {
+      childrenUris = [
+        ...childrenUris,
+        ...(await this.recursiveURIs(child.resource)),
+      ];
+    }
+
+    return [fileStat.resource, ...childrenUris];
+  }
+
+  private URIsToMap(uris: URI[], basepath: string): Record<string, URI> {
+    return uris.reduce((prev: Record<string, URI>, curr) => {
+      const path = curr.toString().split(basepath);
+
+      if (path.length !== 2 || path[1].length === 0) {
+        return prev;
+      }
+
+      // do not map "do_not_sync" files/directoris and their descendants
+      const segments = path[1].split(posix.sep) || [];
+      if (
+        segments.some((segment) => Create.do_not_sync_files.includes(segment))
+      ) {
+        return prev;
+      }
+
+      // skip when the filename is a hidden file (starts with `.`)
+      if (segments[segments.length - 1].indexOf('.') === 0) {
+        return prev;
+      }
+
+      return { ...prev, [path[1]]: curr };
+    }, {});
+  }
+
+  async getUrisMap(uri: URI) {
+    const basepath = uri.toString();
+    const exists = await this.fileService.exists(uri);
+    const uris =
+      (exists && this.URIsToMap(await this.recursiveURIs(uri), basepath)) || {};
+    return uris;
+  }
+
+  async treeDiff(source: URI, dest: URI): Promise<FilesToSync> {
+    const [sourceURIs, destURIs] = await Promise.all([
+      this.getUrisMap(source),
+      this.getUrisMap(dest),
+    ]);
+
+    const destBase = dest.toString();
+    const filesToWrite: FilesToWrite[] = [];
+
+    Object.keys(sourceURIs).forEach((path) => {
+      const destUri = destURIs[path] || new URI(destBase + path);
+
+      filesToWrite.push({ source: sourceURIs[path], dest: destUri });
+      delete destURIs[path];
+    });
+
+    const filesToDelete = Object.values(destURIs);
+
+    return { filesToWrite, filesToDelete };
+  }
+
   async refresh(
     node?: CompositeTreeNode
   ): Promise<CompositeTreeNode | undefined> {
@@ -266,6 +327,25 @@ export class CloudSketchbookTree extends SketchbookTree {
     }
   }
 
+  async sync(source: URI, dest: URI) {
+    const { filesToWrite, filesToDelete } = await this.treeDiff(source, dest);
+    await Promise.all(
+      filesToWrite.map(async ({ source, dest }) => {
+        if ((await this.fileService.resolve(source)).isFile) {
+          const content = await this.fileService.read(source);
+          return this.fileService.write(dest, content.value);
+        }
+        return this.fileService.createFolder(dest);
+      })
+    );
+
+    await Promise.all(
+      filesToDelete.map((file) =>
+        this.fileService.delete(file, { recursive: true })
+      )
+    );
+  }
+
   async resolveChildren(parent: CompositeTreeNode): Promise<TreeNode[]> {
     return (await super.resolveChildren(parent)).sort((a, b) => {
       if (
@@ -295,7 +375,7 @@ export class CloudSketchbookTree extends SketchbookTree {
 
   /**
    * Retrieve fileStats for the given node, merging the local and remote childrens
-   * Local children take prevedence over remote ones
+   * Local children take precedence over remote ones
    * @param node
    * @returns
    */
@@ -376,6 +456,7 @@ export class CloudSketchbookTree extends SketchbookTree {
     const node = this.getNode(id);
     if (fileStat.isDirectory) {
       if (DirNode.is(node)) {
+        node.uri = uri;
         node.fileStat = fileStat;
         return node;
       }
@@ -391,6 +472,7 @@ export class CloudSketchbookTree extends SketchbookTree {
     }
     if (FileNode.is(node)) {
       node.fileStat = fileStat;
+      node.uri = uri;
       return node;
     }
     return <FileNode>{