[ATL-1454] Refactor pull/push to edit files in place (#464)

* improve push/pull process

* improved diff tree performance generation

* skip some files to be synced

Co-authored-by: Francesco Stasi <f.stasi@me.com>
This commit is contained in:
Alberto Iannaccone 2021-07-28 14:00:54 +02:00 committed by GitHub
parent 57b9eb95bb
commit 65152731f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 162 additions and 66 deletions

View File

@ -93,7 +93,11 @@ export class CreateApi {
async readDirectory(
posixPath: string,
options: { recursive?: boolean; match?: string } = {}
options: {
recursive?: boolean;
match?: string;
skipSketchCache?: boolean;
} = {}
): Promise<Create.Resource[]> {
const url = new URL(
`${this.domain()}/files/d/$HOME/sketches_v2${posixPath}`
@ -106,21 +110,29 @@ export class CreateApi {
}
const headers = await this.headers();
return this.run<Create.RawResource[]>(url, {
method: 'GET',
headers,
})
.then(async (result) => {
// add arduino_secrets.h to the results, when reading a sketch main folder
if (posixPath.length && posixPath !== posix.sep) {
const sketch = this.sketchCache.getSketch(posixPath);
const cachedSketch = this.sketchCache.getSketch(posixPath);
const sketchPromise = options.skipSketchCache
? (cachedSketch && this.sketch(cachedSketch.id)) || Promise.resolve(null)
: Promise.resolve(this.sketchCache.getSketch(posixPath));
return Promise.all([
sketchPromise,
this.run<Create.RawResource[]>(url, {
method: 'GET',
headers,
}),
])
.then(async ([sketch, result]) => {
if (posixPath.length && posixPath !== posix.sep) {
if (sketch && sketch.secrets && sketch.secrets.length > 0) {
result.push(this.getSketchSecretStat(sketch));
}
}
return result;
return result.filter(
(res) => !Create.do_not_sync_files.includes(res.name)
);
})
.catch((reason) => {
if (reason?.status === 404) return [] as Create.Resource[];

View File

@ -30,8 +30,6 @@ import { SketchesService } from '../../common/protocol';
import { ArduinoPreferences } from '../arduino-preferences';
import { Create } from './typings';
export const REMOTE_ONLY_FILES = ['sketch.json'];
@injectable()
export class CreateFsProvider
implements
@ -109,14 +107,10 @@ export class CreateFsProvider
const resources = await this.getCreateApi.readDirectory(
uri.path.toString()
);
return resources
.filter((res) => !REMOTE_ONLY_FILES.includes(res.name))
.map(({ name, type }) => [name, this.toFileType(type)]);
return resources.map(({ name, type }) => [name, this.toFileType(type)]);
}
async delete(uri: URI, opts: FileDeleteOptions): Promise<void> {
return;
if (!opts.recursive) {
throw new Error(
'Arduino Create file-system provider does not support non-recursive deletion.'

View File

@ -21,7 +21,7 @@ export namespace Create {
export type ResourceType = 'sketch' | 'folder' | 'file';
export const arduino_secrets_file = 'arduino_secrets.h';
export const do_not_sync_files = ['.theia'];
export const do_not_sync_files = ['.theia', 'sketch.json'];
export interface Resource {
readonly name: string;
/**

View File

@ -22,6 +22,14 @@ export class SketchCache {
return this.fileStats[path] || null;
}
purgeByPath(path: string): void {
for (const itemPath in this.fileStats) {
if (itemPath.indexOf(path) === 0) {
delete this.fileStats[itemPath];
}
}
}
addSketch(sketch: Create.Sketch): void {
const { path } = sketch;
const posixPath = toPosixPath(path);

View File

@ -17,7 +17,6 @@ import {
PreferenceScope,
} from '@theia/core/lib/browser/preferences/preference-service';
import { MessageService } from '@theia/core/lib/common/message-service';
import { REMOTE_ONLY_FILES } from './../../create/create-fs-provider';
import { CreateApi } from '../../create/create-api';
import { CreateUri } from '../../create/create-uri';
import { CloudSketchbookTreeModel } from './cloud-sketchbook-tree-model';
@ -33,10 +32,17 @@ import { ArduinoPreferences } from '../../arduino-preferences';
import { SketchesServiceClientImpl } from '../../../common/protocol/sketches-service-client-impl';
import { FileStat } from '@theia/filesystem/lib/common/files';
import { WorkspaceNode } from '@theia/navigator/lib/browser/navigator-tree';
import { posix, splitSketchPath } from '../../create/create-paths';
import { Create } from '../../create/typings';
const MESSAGE_TIMEOUT = 5 * 1000;
const deepmerge = require('deepmerge').default;
type FilesToWrite = { source: URI; dest: URI };
type FilesToSync = {
filesToWrite: FilesToWrite[];
filesToDelete: URI[];
};
@injectable()
export class CloudSketchbookTree extends SketchbookTree {
@inject(FileService)
@ -94,7 +100,7 @@ export class CloudSketchbookTree extends SketchbookTree {
async pull(arg: any): Promise<void> {
const {
model,
// model,
node,
}: {
model: CloudSketchbookTreeModel;
@ -127,47 +133,12 @@ export class CloudSketchbookTree extends SketchbookTree {
const commandsCopy = node.commands;
node.commands = [];
// check if the sketch dir already exist
if (CloudSketchbookTree.CloudSketchTreeNode.isSynced(node)) {
const filesToPull = (
await this.createApi.readDirectory(node.remoteUri.path.toString())
).filter((file: any) => !REMOTE_ONLY_FILES.includes(file.name));
const localUri = await this.fileService.toUnderlyingResource(
LocalCacheUri.root.resolve(node.remoteUri.path)
);
await this.sync(node.remoteUri, localUri);
await Promise.all(
filesToPull.map((file: any) => {
const uri = CreateUri.toUri(file);
this.fileService.copy(uri, LocalCacheUri.root.resolve(uri.path), {
overwrite: true,
});
})
);
// open the pulled files in the current workspace
const currentSketch = await this.sketchServiceClient.currentSketch();
if (
!CreateUri.is(node.uri) &&
currentSketch &&
currentSketch.uri === node.uri.toString()
) {
filesToPull.forEach(async (file) => {
const localUri = LocalCacheUri.root.resolve(
CreateUri.toUri(file).path
);
const underlying = await this.fileService.toUnderlyingResource(
localUri
);
model.open(underlying);
});
}
} else {
await this.fileService.copy(
node.remoteUri,
LocalCacheUri.root.resolve(node.uri.path),
{ overwrite: true }
);
}
this.sketchCache.purgeByPath(node.remoteUri.path.toString());
node.commands = commandsCopy;
this.messageService.info(`Done pulling ${node.fileStat.name}.`, {
@ -214,17 +185,107 @@ export class CloudSketchbookTree extends SketchbookTree {
}
const commandsCopy = node.commands;
node.commands = [];
// delete every first level file, then push everything
const result = await this.fileService.copy(node.uri, node.remoteUri, {
overwrite: true,
});
const localUri = await this.fileService.toUnderlyingResource(
LocalCacheUri.root.resolve(node.remoteUri.path)
);
await this.sync(localUri, node.remoteUri);
this.sketchCache.purgeByPath(node.remoteUri.path.toString());
node.commands = commandsCopy;
this.messageService.info(`Done pushing ${result.name}.`, {
this.messageService.info(`Done pushing ${node.fileStat.name}.`, {
timeout: MESSAGE_TIMEOUT,
});
});
}
async recursiveURIs(uri: URI): Promise<URI[]> {
// remote resources can be fetched one-shot via api
if (CreateUri.is(uri)) {
const resources = await this.createApi.readDirectory(
uri.path.toString(),
{ recursive: true, skipSketchCache: true }
);
return resources.map((resource) =>
CreateUri.toUri(splitSketchPath(resource.path)[1])
);
}
const fileStat = await this.fileService.resolve(uri, {
resolveMetadata: false,
});
if (!fileStat.children || !fileStat.isDirectory) {
return [fileStat.resource];
}
let childrenUris: URI[] = [];
for await (const child of fileStat.children) {
childrenUris = [
...childrenUris,
...(await this.recursiveURIs(child.resource)),
];
}
return [fileStat.resource, ...childrenUris];
}
private URIsToMap(uris: URI[], basepath: string): Record<string, URI> {
return uris.reduce((prev: Record<string, URI>, curr) => {
const path = curr.toString().split(basepath);
if (path.length !== 2 || path[1].length === 0) {
return prev;
}
// do not map "do_not_sync" files/directoris and their descendants
const segments = path[1].split(posix.sep) || [];
if (
segments.some((segment) => Create.do_not_sync_files.includes(segment))
) {
return prev;
}
// skip when the filename is a hidden file (starts with `.`)
if (segments[segments.length - 1].indexOf('.') === 0) {
return prev;
}
return { ...prev, [path[1]]: curr };
}, {});
}
async getUrisMap(uri: URI) {
const basepath = uri.toString();
const exists = await this.fileService.exists(uri);
const uris =
(exists && this.URIsToMap(await this.recursiveURIs(uri), basepath)) || {};
return uris;
}
async treeDiff(source: URI, dest: URI): Promise<FilesToSync> {
const [sourceURIs, destURIs] = await Promise.all([
this.getUrisMap(source),
this.getUrisMap(dest),
]);
const destBase = dest.toString();
const filesToWrite: FilesToWrite[] = [];
Object.keys(sourceURIs).forEach((path) => {
const destUri = destURIs[path] || new URI(destBase + path);
filesToWrite.push({ source: sourceURIs[path], dest: destUri });
delete destURIs[path];
});
const filesToDelete = Object.values(destURIs);
return { filesToWrite, filesToDelete };
}
async refresh(
node?: CompositeTreeNode
): Promise<CompositeTreeNode | undefined> {
@ -266,6 +327,25 @@ export class CloudSketchbookTree extends SketchbookTree {
}
}
async sync(source: URI, dest: URI) {
const { filesToWrite, filesToDelete } = await this.treeDiff(source, dest);
await Promise.all(
filesToWrite.map(async ({ source, dest }) => {
if ((await this.fileService.resolve(source)).isFile) {
const content = await this.fileService.read(source);
return this.fileService.write(dest, content.value);
}
return this.fileService.createFolder(dest);
})
);
await Promise.all(
filesToDelete.map((file) =>
this.fileService.delete(file, { recursive: true })
)
);
}
async resolveChildren(parent: CompositeTreeNode): Promise<TreeNode[]> {
return (await super.resolveChildren(parent)).sort((a, b) => {
if (
@ -295,7 +375,7 @@ export class CloudSketchbookTree extends SketchbookTree {
/**
* Retrieve fileStats for the given node, merging the local and remote childrens
* Local children take prevedence over remote ones
* Local children take precedence over remote ones
* @param node
* @returns
*/
@ -376,6 +456,7 @@ export class CloudSketchbookTree extends SketchbookTree {
const node = this.getNode(id);
if (fileStat.isDirectory) {
if (DirNode.is(node)) {
node.uri = uri;
node.fileStat = fileStat;
return node;
}
@ -391,6 +472,7 @@ export class CloudSketchbookTree extends SketchbookTree {
}
if (FileNode.is(node)) {
node.fileStat = fileStat;
node.uri = uri;
return node;
}
return <FileNode>{