[atl-1217] sketchbook explorer local & remote

This commit is contained in:
Akos Kitta
2021-04-16 16:47:23 +02:00
committed by Francesco Stasi
parent e6cbefb880
commit 4c536ec8fc
75 changed files with 5559 additions and 430 deletions

View File

@@ -0,0 +1,544 @@
import { injectable } from 'inversify';
import * as createPaths from './create-paths';
import { posix, splitSketchPath } from './create-paths';
import { AuthenticationClientService } from '../auth/authentication-client-service';
import { ArduinoPreferences } from '../arduino-preferences';
export interface ResponseResultProvider {
(response: Response): Promise<any>;
}
export namespace ResponseResultProvider {
export const NOOP: ResponseResultProvider = async () => undefined;
export const TEXT: ResponseResultProvider = (response) => response.text();
export const JSON: ResponseResultProvider = (response) => response.json();
}
type ResourceType = 'f' | 'd';
export let sketchCache: Create.Sketch[] = [];
@injectable()
export class CreateApi {
protected authenticationService: AuthenticationClientService;
protected arduinoPreferences: ArduinoPreferences;
public init(
authenticationService: AuthenticationClientService,
arduinoPreferences: ArduinoPreferences
): CreateApi {
this.authenticationService = authenticationService;
this.arduinoPreferences = arduinoPreferences;
return this;
}
async findSketchByPath(
path: string,
trustCache = true
): Promise<Create.Sketch | undefined> {
const skatches = sketchCache;
const sketch = skatches.find((sketch) => {
const [, spath] = splitSketchPath(sketch.path);
return path === spath;
});
if (trustCache) {
return Promise.resolve(sketch);
}
return await this.sketch({ id: sketch?.id });
}
getSketchSecretStat(sketch: Create.Sketch): Create.Resource {
return {
href: `${sketch.href}${posix.sep}${Create.arduino_secrets_file}`,
modified_at: sketch.modified_at,
name: `${Create.arduino_secrets_file}`,
path: `${sketch.path}${posix.sep}${Create.arduino_secrets_file}`,
mimetype: 'text/x-c++src; charset=utf-8',
type: 'file',
sketchId: sketch.id,
};
}
async sketch(opt: {
id?: string;
path?: string;
}): Promise<Create.Sketch | undefined> {
let url;
if (opt.id) {
url = new URL(`${this.domain()}/sketches/byID/${opt.id}`);
} else if (opt.path) {
url = new URL(`${this.domain()}/sketches/byPath${opt.path}`);
} else {
return;
}
url.searchParams.set('user_id', 'me');
const headers = await this.headers();
const result = await this.run<Create.Sketch>(url, {
method: 'GET',
headers,
});
return result;
}
async sketches(): Promise<Create.Sketch[]> {
const url = new URL(`${this.domain()}/sketches`);
url.searchParams.set('user_id', 'me');
const headers = await this.headers();
const result = await this.run<{ sketches: Create.Sketch[] }>(url, {
method: 'GET',
headers,
});
sketchCache = result.sketches;
return result.sketches;
}
async createSketch(
posixPath: string,
content: string = CreateApi.defaultInoContent
): Promise<Create.Sketch> {
const url = new URL(`${this.domain()}/sketches`);
const headers = await this.headers();
const payload = {
ino: btoa(content),
path: posixPath,
user_id: 'me',
};
const init = {
method: 'PUT',
body: JSON.stringify(payload),
headers,
};
const result = await this.run<Create.Sketch>(url, init);
return result;
}
async readDirectory(
posixPath: string,
options: { recursive?: boolean; match?: string; secrets?: boolean } = {}
): Promise<Create.Resource[]> {
const url = new URL(
`${this.domain()}/files/d/$HOME/sketches_v2${posixPath}`
);
if (options.recursive) {
url.searchParams.set('deep', 'true');
}
if (options.match) {
url.searchParams.set('name_like', options.match);
}
const headers = await this.headers();
const sketchProm = options.secrets
? this.sketches()
: Promise.resolve(sketchCache);
return Promise.all([
this.run<Create.RawResource[]>(url, {
method: 'GET',
headers,
}),
sketchProm,
])
.then(async ([result, sketches]) => {
if (options.secrets) {
// for every sketch with secrets, create a fake arduino_secrets.h
result.forEach(async (res) => {
if (res.type !== 'sketch') {
return;
}
const [, spath] = createPaths.splitSketchPath(res.path);
const sketch = await this.findSketchByPath(spath);
if (
sketch &&
sketch.secrets &&
sketch.secrets.length > 0
) {
result.push(this.getSketchSecretStat(sketch));
}
});
if (posixPath !== posix.sep) {
const sketch = await this.findSketchByPath(posixPath);
if (
sketch &&
sketch.secrets &&
sketch.secrets.length > 0
) {
result.push(this.getSketchSecretStat(sketch));
}
}
}
const sketchesMap: Record<string, Create.Sketch> =
sketches.reduce((prev, curr) => {
return { ...prev, [curr.path]: curr };
}, {});
// add the sketch id and isPublic to the resource
return result.map((resource) => {
return {
...resource,
sketchId: sketchesMap[resource.path]?.id || '',
isPublic:
sketchesMap[resource.path]?.is_public || false,
};
});
})
.catch((reason) => {
if (reason?.status === 404) return [] as Create.Resource[];
else throw reason;
});
}
async createDirectory(posixPath: string): Promise<void> {
const url = new URL(
`${this.domain()}/files/d/$HOME/sketches_v2${posixPath}`
);
const headers = await this.headers();
await this.run(url, {
method: 'POST',
headers,
});
}
async stat(posixPath: string): Promise<Create.Resource> {
// The root is a directory read.
if (posixPath === '/') {
throw new Error('Stating the root is not supported');
}
// The RESTful API has different endpoints for files and directories.
// The RESTful API does not provide specific error codes, only HTP 500.
// We query the parent directory and look for the file with the last segment.
const parentPosixPath = createPaths.parentPosix(posixPath);
const basename = createPaths.basename(posixPath);
let resources;
if (basename === Create.arduino_secrets_file) {
const sketch = await this.findSketchByPath(parentPosixPath);
resources = sketch ? [this.getSketchSecretStat(sketch)] : [];
} else {
resources = await this.readDirectory(parentPosixPath, {
match: basename,
});
}
resources.sort((left, right) => left.path.length - right.path.length);
const resource = resources.find(({ name }) => name === basename);
if (!resource) {
throw new CreateError(`Not found: ${posixPath}.`, 404);
}
return resource;
}
async readFile(posixPath: string): Promise<string> {
const basename = createPaths.basename(posixPath);
if (basename === Create.arduino_secrets_file) {
const parentPosixPath = createPaths.parentPosix(posixPath);
const sketch = await this.findSketchByPath(parentPosixPath, false);
let file = '';
if (sketch && sketch.secrets) {
for (const item of sketch?.secrets) {
file += `#define ${item.name} "${item.value}"\r\n`;
}
}
return file;
}
const url = new URL(
`${this.domain()}/files/f/$HOME/sketches_v2${posixPath}`
);
const headers = await this.headers();
const result = await this.run<{ data: string }>(url, {
method: 'GET',
headers,
});
const { data } = result;
return atob(data);
}
async writeFile(
posixPath: string,
content: string | Uint8Array
): Promise<void> {
const basename = createPaths.basename(posixPath);
if (basename === Create.arduino_secrets_file) {
const parentPosixPath = createPaths.parentPosix(posixPath);
const sketch = await this.findSketchByPath(parentPosixPath);
if (sketch) {
const url = new URL(`${this.domain()}/sketches/${sketch.id}`);
const headers = await this.headers();
// parse the secret file
const secrets = (
typeof content === 'string'
? content
: new TextDecoder().decode(content)
)
.split(/\r?\n/)
.reduce((prev, curr) => {
// check if the line contains a secret
const secret = curr.split('SECRET_')[1] || null;
if (!secret) {
return prev;
}
const regexp = /(\S*)\s+([\S\s]*)/g;
const tokens = regexp.exec(secret) || [];
const name =
tokens[1].length > 0 ? `SECRET_${tokens[1]}` : '';
let value = '';
if (tokens[2].length > 0) {
value = JSON.parse(
JSON.stringify(
tokens[2]
.replace(/^['"]?/g, '')
.replace(/['"]?$/g, '')
)
);
}
if (name.length === 0 || value.length === 0) {
return prev;
}
return [...prev, { name, value }];
}, []);
const payload = {
id: sketch.id,
libraries: sketch.libraries,
secrets: { data: secrets },
};
// replace the sketch in the cache, so other calls will not overwrite each other
sketchCache = sketchCache.filter((skt) => skt.id !== sketch.id);
sketchCache.push({ ...sketch, secrets });
const init = {
method: 'POST',
body: JSON.stringify(payload),
headers,
};
await this.run(url, init);
}
return;
}
const url = new URL(
`${this.domain()}/files/f/$HOME/sketches_v2${posixPath}`
);
const headers = await this.headers();
const data = btoa(
typeof content === 'string'
? content
: new TextDecoder().decode(content)
);
const payload = { data };
const init = {
method: 'POST',
body: JSON.stringify(payload),
headers,
};
await this.run(url, init);
}
async deleteFile(posixPath: string): Promise<void> {
await this.delete(posixPath, 'f');
}
async deleteDirectory(posixPath: string): Promise<void> {
await this.delete(posixPath, 'd');
}
private async delete(posixPath: string, type: ResourceType): Promise<void> {
const url = new URL(
`${this.domain()}/files/${type}/$HOME/sketches_v2${posixPath}`
);
const headers = await this.headers();
await this.run(url, {
method: 'DELETE',
headers,
});
}
async rename(fromPosixPath: string, toPosixPath: string): Promise<void> {
const url = new URL(`${this.domain('v3')}/files/mv`);
const headers = await this.headers();
const payload = {
from: `$HOME/sketches_v2${fromPosixPath}`,
to: `$HOME/sketches_v2${toPosixPath}`,
};
const init = {
method: 'POST',
body: JSON.stringify(payload),
headers,
};
await this.run(url, init, ResponseResultProvider.NOOP);
}
async editSketch({
id,
params,
}: {
id: string;
params: Record<string, unknown>;
}): Promise<Create.Sketch> {
const url = new URL(`${this.domain()}/sketches/${id}`);
const headers = await this.headers();
const result = await this.run<Create.Sketch>(url, {
method: 'POST',
body: JSON.stringify({ id, ...params }),
headers,
});
return result;
}
async copy(fromPosixPath: string, toPosixPath: string): Promise<void> {
const payload = {
from: `$HOME/sketches_v2${fromPosixPath}`,
to: `$HOME/sketches_v2${toPosixPath}`,
};
const url = new URL(`${this.domain('v3')}/files/cp`);
const headers = await this.headers();
const init = {
method: 'POST',
body: JSON.stringify(payload),
headers,
};
await this.run(url, init, ResponseResultProvider.NOOP);
}
private async run<T>(
requestInfo: RequestInfo | URL,
init: RequestInit | undefined,
resultProvider: ResponseResultProvider = ResponseResultProvider.JSON
): Promise<T> {
const response = await fetch(
requestInfo instanceof URL ? requestInfo.toString() : requestInfo,
init
);
if (!response.ok) {
let details: string | undefined = undefined;
try {
details = await response.json();
} catch (e) {
console.error('Cloud not get the error details.', e);
}
const { statusText, status } = response;
throw new CreateError(statusText, status, details);
}
const result = await resultProvider(response);
return result;
}
private async headers(): Promise<Record<string, string>> {
const token = await this.token();
return {
'content-type': 'application/json',
accept: 'application/json',
authorization: `Bearer ${token}`,
};
}
private domain(apiVersion = 'v2'): string {
const endpoint =
this.arduinoPreferences['arduino.cloud.sketchSyncEnpoint'];
return `${endpoint}/${apiVersion}`;
}
private async token(): Promise<string> {
return this.authenticationService.session?.accessToken || '';
}
}
export namespace CreateApi {
export const defaultInoContent = `/*
*/
void setup() {
}
void loop() {
}
`;
}
export namespace Create {
export interface Sketch {
readonly name: string;
readonly path: string;
readonly modified_at: string;
readonly created_at: string;
readonly secrets?: { name: string; value: string }[];
readonly id: string;
readonly is_public: boolean;
// readonly board_fqbn: '',
// readonly board_name: '',
// readonly board_type: 'serial' | 'network' | 'cloud' | '',
readonly href?: string;
readonly libraries: string[];
// readonly tutorials: string[] | null;
// readonly types: string[] | null;
// readonly user_id: string;
}
export type ResourceType = 'sketch' | 'folder' | 'file';
export const arduino_secrets_file = 'arduino_secrets.h';
export interface Resource {
readonly name: string;
/**
* Note: this path is **not** the POSIX path we use. It has the leading segments with the `user_id`.
*/
readonly path: string;
readonly type: ResourceType;
readonly sketchId: string;
readonly modified_at: string; // As an ISO-8601 formatted string: `YYYY-MM-DDTHH:mm:ss.sssZ`
readonly children?: number; // For 'sketch' and 'folder' types.
readonly size?: number; // For 'sketch' type only.
readonly isPublic?: boolean; // For 'sketch' type only.
readonly mimetype?: string; // For 'file' type.
readonly href?: string;
}
export namespace Resource {
export function is(arg: any): arg is Resource {
return (
!!arg &&
'name' in arg &&
typeof arg['name'] === 'string' &&
'path' in arg &&
typeof arg['path'] === 'string' &&
'type' in arg &&
typeof arg['type'] === 'string' &&
'modified_at' in arg &&
typeof arg['modified_at'] === 'string' &&
(arg['type'] === 'sketch' ||
arg['type'] === 'folder' ||
arg['type'] === 'file')
);
}
}
export type RawResource = Omit<Resource, 'sketchId' | 'isPublic'>;
}
export class CreateError extends Error {
constructor(
message: string,
readonly status: number,
readonly details?: string
) {
super(message);
Object.setPrototypeOf(this, CreateError.prototype);
}
}

View File

@@ -0,0 +1,212 @@
import { inject, injectable } from 'inversify';
import URI from '@theia/core/lib/common/uri';
import { Event } from '@theia/core/lib/common/event';
import {
Disposable,
DisposableCollection,
} from '@theia/core/lib/common/disposable';
import { FrontendApplicationContribution } from '@theia/core/lib/browser/frontend-application';
import {
Stat,
FileType,
FileChange,
FileWriteOptions,
FileDeleteOptions,
FileOverwriteOptions,
FileSystemProvider,
FileSystemProviderError,
FileSystemProviderErrorCode,
FileSystemProviderCapabilities,
WatchOptions,
} from '@theia/filesystem/lib/common/files';
import {
FileService,
FileServiceContribution,
} from '@theia/filesystem/lib/browser/file-service';
import { AuthenticationClientService } from '../auth/authentication-client-service';
import { Create, CreateApi } from './create-api';
import { CreateUri } from './create-uri';
import { SketchesService } from '../../common/protocol';
import { ArduinoPreferences } from '../arduino-preferences';
export const REMOTE_ONLY_FILES = [
'sketch.json',
'thingsProperties.h',
'thingProperties.h',
];
@injectable()
export class CreateFsProvider
implements
FileSystemProvider,
FrontendApplicationContribution,
FileServiceContribution
{
@inject(AuthenticationClientService)
protected readonly authenticationService: AuthenticationClientService;
@inject(CreateApi)
protected readonly createApi: CreateApi;
@inject(SketchesService)
protected readonly sketchesService: SketchesService;
@inject(ArduinoPreferences)
protected readonly arduinoPreferences: ArduinoPreferences;
protected readonly toDispose = new DisposableCollection();
readonly onFileWatchError: Event<void> = Event.None;
readonly onDidChangeFile: Event<readonly FileChange[]> = Event.None;
readonly onDidChangeCapabilities: Event<void> = Event.None;
readonly capabilities: FileSystemProviderCapabilities =
FileSystemProviderCapabilities.FileReadWrite |
FileSystemProviderCapabilities.PathCaseSensitive |
FileSystemProviderCapabilities.Access;
onStop(): void {
this.toDispose.dispose();
}
registerFileSystemProviders(service: FileService): void {
service.onWillActivateFileSystemProvider((event) => {
if (event.scheme === CreateUri.scheme) {
event.waitUntil(
(async () => {
service.registerProvider(CreateUri.scheme, this);
})()
);
}
});
}
watch(uri: URI, opts: WatchOptions): Disposable {
return Disposable.NULL;
}
async stat(uri: URI): Promise<Stat> {
if (CreateUri.equals(CreateUri.root, uri)) {
this.getCreateApi; // This will throw when not logged in.
return {
type: FileType.Directory,
ctime: 0,
mtime: 0,
size: 0,
};
}
const resource = await this.getCreateApi.stat(uri.path.toString());
const mtime = Date.parse(resource.modified_at);
return {
type: this.toFileType(resource.type),
ctime: mtime,
mtime,
size: 0,
};
}
async mkdir(uri: URI): Promise<void> {
await this.getCreateApi.createDirectory(uri.path.toString());
}
async readdir(uri: URI): Promise<[string, FileType][]> {
const resources = await this.getCreateApi.readDirectory(
uri.path.toString(),
{
secrets: true,
}
);
return resources
.filter((res) => !REMOTE_ONLY_FILES.includes(res.name))
.map(({ name, type }) => [name, this.toFileType(type)]);
}
async delete(uri: URI, opts: FileDeleteOptions): Promise<void> {
return;
if (!opts.recursive) {
throw new Error(
'Arduino Create file-system provider does not support non-recursive deletion.'
);
}
const stat = await this.stat(uri);
if (!stat) {
throw new FileSystemProviderError(
'File not found.',
FileSystemProviderErrorCode.FileNotFound
);
}
switch (stat.type) {
case FileType.Directory: {
await this.getCreateApi.deleteDirectory(uri.path.toString());
break;
}
case FileType.File: {
await this.getCreateApi.deleteFile(uri.path.toString());
break;
}
default: {
throw new FileSystemProviderError(
`Unexpected file type '${
stat.type
}' for resource: ${uri.toString()}`,
FileSystemProviderErrorCode.Unknown
);
}
}
}
async rename(
oldUri: URI,
newUri: URI,
options: FileOverwriteOptions
): Promise<void> {
await this.getCreateApi.rename(
oldUri.path.toString(),
newUri.path.toString()
);
}
async readFile(uri: URI): Promise<Uint8Array> {
const content = await this.getCreateApi.readFile(uri.path.toString());
return new TextEncoder().encode(content);
}
async writeFile(
uri: URI,
content: Uint8Array,
options: FileWriteOptions
): Promise<void> {
await this.getCreateApi.writeFile(uri.path.toString(), content);
}
async access(uri: URI, mode?: number): Promise<void> {
this.getCreateApi; // Will throw if not logged in.
}
public toFileType(type: Create.ResourceType): FileType {
switch (type) {
case 'file':
return FileType.File;
case 'sketch':
case 'folder':
return FileType.Directory;
default:
return FileType.Unknown;
}
}
private get getCreateApi(): CreateApi {
const { session } = this.authenticationService;
if (!session) {
throw new FileSystemProviderError(
'Not logged in.',
FileSystemProviderErrorCode.NoPermissions
);
}
return this.createApi.init(
this.authenticationService,
this.arduinoPreferences
);
}
}

View File

@@ -0,0 +1,59 @@
export const posix = { sep: '/' };
// TODO: poor man's `path.join(path, '..')` in the browser.
export function parentPosix(path: string): string {
const segments = path.split(posix.sep) || [];
segments.pop();
let modified = segments.join(posix.sep);
if (path.charAt(path.length - 1) === posix.sep) {
modified += posix.sep;
}
return modified;
}
export function basename(path: string): string {
const segments = path.split(posix.sep) || [];
return segments.pop()!;
}
export function posixSegments(posixPath: string): string[] {
return posixPath.split(posix.sep).filter((segment) => !!segment);
}
/**
* Splits the `raw` path into two segments, a root that contains user information and the relevant POSIX path. \
* For examples:
* ```
* `29ad0829759028dde9b877343fa3b0e1:testrest/sketches_v2/xxx_folder/xxx_sub_folder/sketch_in_folder/sketch_in_folder.ino`
* ```
* will be:
* ```
* ['29ad0829759028dde9b877343fa3b0e1:testrest/sketches_v2', '/xxx_folder/xxx_sub_folder/sketch_in_folder/sketch_in_folder.ino']
* ```
*/
export function splitSketchPath(
raw: string,
sep = '/sketches_v2/'
): [string, string] {
if (!sep) {
throw new Error('Invalid separator. Cannot be zero length.');
}
const index = raw.indexOf(sep);
if (index === -1) {
throw new Error(`Invalid path pattern. Raw path was '${raw}'.`);
}
const createRoot = raw.substring(0, index + sep.length - 1); // TODO: validate the `createRoot` format.
const posixPath = raw.substr(index + sep.length - 1);
if (!posixPath) {
throw new Error(`Could not extract POSIX path from '${raw}'.`);
}
return [createRoot, posixPath];
}
export function toPosixPath(raw: string): string {
if (raw === posix.sep) {
return posix.sep; // Handles the root resource case.
}
const [, posixPath] = splitSketchPath(raw);
return posixPath;
}

View File

@@ -0,0 +1,39 @@
import { URI as Uri } from 'vscode-uri';
import URI from '@theia/core/lib/common/uri';
import { Create } from './create-api';
import { toPosixPath, parentPosix, posix } from './create-paths';
export namespace CreateUri {
export const scheme = 'arduino-create';
export const root = toUri(posix.sep);
export function toUri(posixPathOrResource: string | Create.Resource): URI {
const posixPath =
typeof posixPathOrResource === 'string'
? posixPathOrResource
: toPosixPath(posixPathOrResource.path);
return new URI(
Uri.parse(posixPath).with({ scheme, authority: 'create' })
);
}
export function is(uri: URI): boolean {
return uri.scheme === scheme;
}
export function equals(left: URI, right: URI): boolean {
return is(left) && is(right) && left.toString() === right.toString();
}
export function parent(uri: URI): URI {
if (!is(uri)) {
throw new Error(
`Invalid URI scheme. Expected '${scheme}' got '${uri.scheme}' instead.`
);
}
if (equals(uri, root)) {
return uri;
}
return toUri(parentPosix(uri.path.toString()));
}
}