Various library/platform index update fixes

- IDE2 can start if the package index download fails. Closes #1084
 - Split the lib and platform index update. Closes #1156

Signed-off-by: Akos Kitta <a.kitta@arduino.cc>

arduino-ide-extension/src/browser/arduino-ide-frontend-module.ts

@@ -332,6 +332,7 @@ import { OutputEditorFactory } from './theia/output/output-editor-factory';
 import { StartupTaskProvider } from '../electron-common/startup-task';
 import { DeleteSketch } from './contributions/delete-sketch';
 import { UserFields } from './contributions/user-fields';
+import { UpdateIndexes } from './contributions/update-indexes';
 
 const registerArduinoThemes = () => {
   const themes: MonacoThemeJson[] = [
@@ -744,6 +745,7 @@ export default new ContainerModule((bind, unbind, isBound, rebind) => {
   Contribution.configure(bind, CheckForUpdates);
   Contribution.configure(bind, UserFields);
   Contribution.configure(bind, DeleteSketch);
+  Contribution.configure(bind, UpdateIndexes);
 
   bindContributionProvider(bind, StartupTaskProvider);
   bind(StartupTaskProvider).toService(BoardsServiceProvider); // to reuse the boards config in another window

arduino-ide-extension/src/browser/boards/boards-config.tsx

@@ -132,7 +132,7 @@ export class BoardsConfig extends React.Component<
       this.props.notificationCenter.onPlatformDidUninstall(() =>
         this.updateBoards(this.state.query)
       ),
-      this.props.notificationCenter.onIndexDidUpdate(() =>
+      this.props.notificationCenter.onIndexUpdateDidComplete(() =>
         this.updateBoards(this.state.query)
       ),
       this.props.notificationCenter.onDaemonDidStart(() =>

arduino-ide-extension/src/browser/contributions/indexes-update-progress.ts

@@ -16,7 +16,7 @@ export class IndexesUpdateProgress extends Contribution {
     | undefined;
 
   override onStart(): void {
-    this.notificationCenter.onIndexWillUpdate((progressId) =>
+    this.notificationCenter.onIndexUpdateWillStart(({ progressId }) =>
       this.getOrCreateProgress(progressId)
     );
     this.notificationCenter.onIndexUpdateDidProgress((progress) => {
@@ -24,7 +24,7 @@ export class IndexesUpdateProgress extends Contribution {
         delegate.report(progress)
       );
     });
-    this.notificationCenter.onIndexDidUpdate((progressId) => {
+    this.notificationCenter.onIndexUpdateDidComplete(({ progressId }) => {
       this.cancelProgress(progressId);
     });
     this.notificationCenter.onIndexUpdateDidFail(({ progressId, message }) => {

arduino-ide-extension/src/browser/contributions/update-indexes.ts

@@ -0,0 +1,193 @@
+import { LocalStorageService } from '@theia/core/lib/browser/storage-service';
+import { nls } from '@theia/core/lib/common/nls';
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { CoreService, IndexType } from '../../common/protocol';
+import { NotificationCenter } from '../notification-center';
+import { WindowServiceExt } from '../theia/core/window-service-ext';
+import { Command, CommandRegistry, Contribution } from './contribution';
+
+@injectable()
+export class UpdateIndexes extends Contribution {
+  @inject(WindowServiceExt)
+  private readonly windowService: WindowServiceExt;
+  @inject(LocalStorageService)
+  private readonly localStorage: LocalStorageService;
+  @inject(CoreService)
+  private readonly coreService: CoreService;
+  @inject(NotificationCenter)
+  private readonly notificationCenter: NotificationCenter;
+
+  protected override init(): void {
+    super.init();
+    this.notificationCenter.onIndexUpdateDidComplete(({ summary }) =>
+      Promise.all(
+        Object.entries(summary).map(([type, updatedAt]) =>
+          this.setLastUpdateDateTime(type as IndexType, updatedAt)
+        )
+      )
+    );
+  }
+
+  override onReady(): void {
+    this.checkForUpdates();
+  }
+
+  override registerCommands(registry: CommandRegistry): void {
+    registry.registerCommand(UpdateIndexes.Commands.UPDATE_INDEXES, {
+      execute: () => this.updateIndexes(IndexType.All, true),
+    });
+    registry.registerCommand(UpdateIndexes.Commands.UPDATE_PLATFORM_INDEX, {
+      execute: () => this.updateIndexes(['platform'], true),
+    });
+    registry.registerCommand(UpdateIndexes.Commands.UPDATE_LIBRARY_INDEX, {
+      execute: () => this.updateIndexes(['library'], true),
+    });
+  }
+
+  private async checkForUpdates(): Promise<void> {
+    const checkForUpdates = this.preferences['arduino.checkForUpdates'];
+    if (!checkForUpdates) {
+      console.debug(
+        '[update-indexes]: `arduino.checkForUpdates` is `false`. Skipping updating the indexes.'
+      );
+      return;
+    }
+
+    if (await this.windowService.isFirstWindow()) {
+      const summary = await this.coreService.indexUpdateSummaryBeforeInit();
+      if (summary.message) {
+        this.messageService.error(summary.message);
+      }
+      const typesToCheck = IndexType.All.filter((type) => !(type in summary));
+      if (Object.keys(summary).length) {
+        console.debug(
+          `[update-indexes]: Detected an index update summary before the core gRPC client initialization. Updating local storage with ${JSON.stringify(
+            summary
+          )}`
+        );
+      } else {
+        console.debug(
+          '[update-indexes]: No index update summary was available before the core gRPC client initialization. Checking the status of the all the index types.'
+        );
+      }
+      await Promise.allSettled([
+        ...Object.entries(summary).map(([type, updatedAt]) =>
+          this.setLastUpdateDateTime(type as IndexType, updatedAt)
+        ),
+        this.updateIndexes(typesToCheck),
+      ]);
+    }
+  }
+
+  private async updateIndexes(
+    types: IndexType[],
+    force = false
+  ): Promise<void> {
+    const updatedAt = new Date().toISOString();
+    return Promise.all(
+      types.map((type) => this.needsIndexUpdate(type, updatedAt, force))
+    ).then((needsIndexUpdateResults) => {
+      const typesToUpdate = needsIndexUpdateResults.filter(IndexType.is);
+      if (typesToUpdate.length) {
+        console.debug(
+          `[update-indexes]: Requesting the index update of type: ${JSON.stringify(
+            typesToUpdate
+          )} with date time: ${updatedAt}.`
+        );
+        return this.coreService.updateIndex({ types: typesToUpdate });
+      }
+    });
+  }
+
+  private async needsIndexUpdate(
+    type: IndexType,
+    now: string,
+    force = false
+  ): Promise<IndexType | false> {
+    if (force) {
+      console.debug(
+        `[update-indexes]: Update for index type: '${type}' was forcefully requested.`
+      );
+      return type;
+    }
+    const lastUpdateIsoDateTime = await this.getLastUpdateDateTime(type);
+    if (!lastUpdateIsoDateTime) {
+      console.debug(
+        `[update-indexes]: No last update date time was persisted for index type: '${type}'. Index update is required.`
+      );
+      return type;
+    }
+    const lastUpdateDateTime = Date.parse(lastUpdateIsoDateTime);
+    if (Number.isNaN(lastUpdateDateTime)) {
+      console.debug(
+        `[update-indexes]: Invalid last update date time was persisted for index type: '${type}'. Last update date time was: ${lastUpdateDateTime}. Index update is required.`
+      );
+      return type;
+    }
+    const diff = new Date(now).getTime() - lastUpdateDateTime;
+    const needsIndexUpdate = diff >= this.threshold;
+    console.debug(
+      `[update-indexes]: Update for index type '${type}' is ${
+        needsIndexUpdate ? '' : 'not '
+      }required. Now: ${now}, Last index update date time: ${new Date(
+        lastUpdateDateTime
+      ).toISOString()}, diff: ${diff} ms, threshold: ${this.threshold} ms.`
+    );
+    return needsIndexUpdate ? type : false;
+  }
+
+  private async getLastUpdateDateTime(
+    type: IndexType
+  ): Promise<string | undefined> {
+    const key = this.storageKeyOf(type);
+    return this.localStorage.getData<string>(key);
+  }
+
+  private async setLastUpdateDateTime(
+    type: IndexType,
+    updatedAt: string
+  ): Promise<void> {
+    const key = this.storageKeyOf(type);
+    return this.localStorage.setData<string>(key, updatedAt).finally(() => {
+      console.debug(
+        `[update-indexes]: Updated the last index update date time of '${type}' to ${updatedAt}.`
+      );
+    });
+  }
+
+  private storageKeyOf(type: IndexType): string {
+    return `index-last-update-time--${type}`;
+  }
+
+  private get threshold(): number {
+    return 4 * 60 * 60 * 1_000; // four hours in millis
+  }
+}
+export namespace UpdateIndexes {
+  export namespace Commands {
+    export const UPDATE_INDEXES: Command & { label: string } = {
+      id: 'arduino-update-indexes',
+      label: nls.localize(
+        'arduino/updateIndexes/updateIndexes',
+        'Update Indexes'
+      ),
+      category: 'Arduino',
+    };
+    export const UPDATE_PLATFORM_INDEX: Command & { label: string } = {
+      id: 'arduino-update-package-index',
+      label: nls.localize(
+        'arduino/updateIndexes/updatePackageIndex',
+        'Update Package Index'
+      ),
+      category: 'Arduino',
+    };
+    export const UPDATE_LIBRARY_INDEX: Command & { label: string } = {
+      id: 'arduino-update-library-index',
+      label: nls.localize(
+        'arduino/updateIndexes/updateLibraryIndex',
+        'Update Library Index'
+      ),
+      category: 'Arduino',
+    };
+  }
+}

arduino-ide-extension/src/browser/notification-center.ts

@@ -8,6 +8,9 @@ import { JsonRpcProxy } from '@theia/core/lib/common/messaging/proxy-factory';
 import { DisposableCollection } from '@theia/core/lib/common/disposable';
 import { FrontendApplicationContribution } from '@theia/core/lib/browser/frontend-application';
 import {
+  IndexUpdateDidCompleteParams,
+  IndexUpdateDidFailParams,
+  IndexUpdateWillStartParams,
   NotificationServiceClient,
   NotificationServiceServer,
 } from '../common/protocol/notification-service';
@@ -29,48 +32,48 @@ export class NotificationCenter
   implements NotificationServiceClient, FrontendApplicationContribution
 {
   @inject(NotificationServiceServer)
-  protected readonly server: JsonRpcProxy<NotificationServiceServer>;
+  private readonly server: JsonRpcProxy<NotificationServiceServer>;
 
   @inject(FrontendApplicationStateService)
   private readonly appStateService: FrontendApplicationStateService;
 
-  protected readonly indexDidUpdateEmitter = new Emitter<string>();
-  protected readonly indexWillUpdateEmitter = new Emitter<string>();
-  protected readonly indexUpdateDidProgressEmitter =
+  private readonly indexUpdateDidCompleteEmitter =
+    new Emitter<IndexUpdateDidCompleteParams>();
+  private readonly indexUpdateWillStartEmitter =
+    new Emitter<IndexUpdateWillStartParams>();
+  private readonly indexUpdateDidProgressEmitter =
     new Emitter<ProgressMessage>();
-  protected readonly indexUpdateDidFailEmitter = new Emitter<{
-    progressId: string;
-    message: string;
-  }>();
-  protected readonly daemonDidStartEmitter = new Emitter<string>();
-  protected readonly daemonDidStopEmitter = new Emitter<void>();
-  protected readonly configDidChangeEmitter = new Emitter<{
+  private readonly indexUpdateDidFailEmitter =
+    new Emitter<IndexUpdateDidFailParams>();
+  private readonly daemonDidStartEmitter = new Emitter<string>();
+  private readonly daemonDidStopEmitter = new Emitter<void>();
+  private readonly configDidChangeEmitter = new Emitter<{
     config: Config | undefined;
   }>();
-  protected readonly platformDidInstallEmitter = new Emitter<{
+  private readonly platformDidInstallEmitter = new Emitter<{
     item: BoardsPackage;
   }>();
-  protected readonly platformDidUninstallEmitter = new Emitter<{
+  private readonly platformDidUninstallEmitter = new Emitter<{
     item: BoardsPackage;
   }>();
-  protected readonly libraryDidInstallEmitter = new Emitter<{
+  private readonly libraryDidInstallEmitter = new Emitter<{
     item: LibraryPackage;
   }>();
-  protected readonly libraryDidUninstallEmitter = new Emitter<{
+  private readonly libraryDidUninstallEmitter = new Emitter<{
     item: LibraryPackage;
   }>();
-  protected readonly attachedBoardsDidChangeEmitter =
+  private readonly attachedBoardsDidChangeEmitter =
     new Emitter<AttachedBoardsChangeEvent>();
-  protected readonly recentSketchesChangedEmitter = new Emitter<{
+  private readonly recentSketchesChangedEmitter = new Emitter<{
     sketches: Sketch[];
   }>();
   private readonly onAppStateDidChangeEmitter =
     new Emitter<FrontendApplicationState>();
 
-  protected readonly toDispose = new DisposableCollection(
-    this.indexWillUpdateEmitter,
+  private readonly toDispose = new DisposableCollection(
+    this.indexUpdateWillStartEmitter,
     this.indexUpdateDidProgressEmitter,
-    this.indexDidUpdateEmitter,
+    this.indexUpdateDidCompleteEmitter,
     this.indexUpdateDidFailEmitter,
     this.daemonDidStartEmitter,
     this.daemonDidStopEmitter,
@@ -82,8 +85,8 @@ export class NotificationCenter
     this.attachedBoardsDidChangeEmitter
   );
 
-  readonly onIndexDidUpdate = this.indexDidUpdateEmitter.event;
-  readonly onIndexWillUpdate = this.indexDidUpdateEmitter.event;
+  readonly onIndexUpdateDidComplete = this.indexUpdateDidCompleteEmitter.event;
+  readonly onIndexUpdateWillStart = this.indexUpdateWillStartEmitter.event;
   readonly onIndexUpdateDidProgress = this.indexUpdateDidProgressEmitter.event;
   readonly onIndexUpdateDidFail = this.indexUpdateDidFailEmitter.event;
   readonly onDaemonDidStart = this.daemonDidStartEmitter.event;
@@ -112,26 +115,20 @@ export class NotificationCenter
     this.toDispose.dispose();
   }
 
-  notifyIndexWillUpdate(progressId: string): void {
-    this.indexWillUpdateEmitter.fire(progressId);
+  notifyIndexUpdateWillStart(params: IndexUpdateWillStartParams): void {
+    this.indexUpdateWillStartEmitter.fire(params);
   }
 
   notifyIndexUpdateDidProgress(progressMessage: ProgressMessage): void {
     this.indexUpdateDidProgressEmitter.fire(progressMessage);
   }
 
-  notifyIndexDidUpdate(progressId: string): void {
-    this.indexDidUpdateEmitter.fire(progressId);
+  notifyIndexUpdateDidComplete(params: IndexUpdateDidCompleteParams): void {
+    this.indexUpdateDidCompleteEmitter.fire(params);
   }
 
-  notifyIndexUpdateDidFail({
-    progressId,
-    message,
-  }: {
-    progressId: string;
-    message: string;
-  }): void {
-    this.indexUpdateDidFailEmitter.fire({ progressId, message });
+  notifyIndexUpdateDidFail(params: IndexUpdateDidFailParams): void {
+    this.indexUpdateDidFailEmitter.fire(params);
   }
 
   notifyDaemonDidStart(port: string): void {

arduino-ide-extension/src/browser/widgets/component-list/list-widget.tsx

@@ -68,7 +68,7 @@ export abstract class ListWidget<
   @postConstruct()
   protected init(): void {
     this.toDispose.pushAll([
-      this.notificationCenter.onIndexDidUpdate(() => this.refresh(undefined)),
+      this.notificationCenter.onIndexUpdateDidComplete(() => this.refresh(undefined)),
       this.notificationCenter.onDaemonDidStart(() => this.refresh(undefined)),
       this.notificationCenter.onDaemonDidStop(() => this.refresh(undefined)),
     ]);