Use esptool-js for installation (#269)

package.json

@@ -7,7 +7,8 @@
   "author": "ESPHome maintainers",
   "license": "Apache-2.0",
   "scripts": {
-    "prepublishOnly": "script/build"
+    "prepublishOnly": "script/build",
+    "postinstall": "patch -Ntu node_modules/esptool-js/ESPLoader.js -i patches/esploader.patch || true"
   },
   "devDependencies": {
     "@rollup/plugin-json": "^4.1.0",
@@ -28,9 +29,10 @@
     "@material/mwc-formfield": "^0.26.1",
     "@material/mwc-icon-button": "^0.26.1",
     "@material/mwc-textfield": "^0.26.1",
-    "esp-web-flasher": "^5.1.4",
+    "esptool-js": "github:espressif/esptool-js#0c1b972a05d691c85da23fcc937d91dcf7e283eb",
     "improv-wifi-serial-sdk": "^2.2.2",
     "lit": "^2.0.0",
+    "pako": "^2.0.4",
     "tslib": "^2.3.1"
   }
 }

patches/esploader.patch

@@ -0,0 +1,16 @@
+--- node_modules/esptool-js/ESPLoader.js	2022-07-19 09:17:05.000000000 -0700
++++ node_modules/esptool-js/ESPLoader.fixed.js	2022-07-19 09:19:04.000000000 -0700
+@@ -1,3 +1,4 @@
++import pako from 'pako';
+ import {ESPError, TimeoutError} from "./error.js";
+ 
+ const MAGIC_TO_CHIP = {
+@@ -680,7 +681,7 @@
+ 
+         await this.run_stub();
+ 
+-        await this.change_baud();
++        // await this.change_baud();
+         return chip;
+     }
+ 

src/const.ts

@@ -38,11 +38,6 @@ export interface InitializingState extends BaseFlashState {
   details: { done: boolean };
 }
 
-export interface ManifestState extends BaseFlashState {
-  state: FlashStateType.MANIFEST;
-  details: { done: boolean };
-}
-
 export interface PreparingState extends BaseFlashState {
   state: FlashStateType.PREPARING;
   details: { done: boolean };
@@ -69,7 +64,6 @@ export interface ErrorState extends BaseFlashState {
 
 export type FlashState =
   | InitializingState
-  | ManifestState
   | PreparingState
   | ErasingState
   | WritingState
@@ -78,7 +72,6 @@ export type FlashState =
 
 export const enum FlashStateType {
   INITIALIZING = "initializing",
-  MANIFEST = "manifest",
   PREPARING = "preparing",
   ERASING = "erasing",
   WRITING = "writing",

src/flash.ts

@@ -1,4 +1,7 @@
-import { ESPLoader, Logger } from "esp-web-flasher";
+// @ts-ignore-next-line
+import { Transport } from "esptool-js/webserial.js";
+// @ts-ignore-next-line
+import { ESPLoader } from "esptool-js/esploader.js";
 import {
   Build,
   FlashError,
@@ -6,19 +9,28 @@ import {
   Manifest,
   FlashStateType,
 } from "./const";
-import { getChipFamilyName } from "./util/chip-family-name";
 import { sleep } from "./util/sleep";
 
+const resetTransport = async (transport: Transport) => {
+  await transport.device.setSignals({
+    dataTerminalReady: false,
+    requestToSend: true,
+  });
+  await transport.device.setSignals({
+    dataTerminalReady: false,
+    requestToSend: false,
+  });
+};
+
 export const flash = async (
   onEvent: (state: FlashState) => void,
   port: SerialPort,
-  logger: Logger,
   manifestPath: string,
+  manifest: Manifest,
   eraseFirst: boolean
 ) => {
-  let manifest: Manifest;
   let build: Build | undefined;
-  let chipFamily: ReturnType<typeof getChipFamilyName>;
+  let chipFamily: Build["chipFamily"];
 
   const fireStateEvent = (stateUpdate: FlashState) =>
     onEvent({
@@ -28,12 +40,8 @@ export const flash = async (
       chipFamily,
     });
 
-  const manifestURL = new URL(manifestPath, location.toString()).toString();
+  const transport = new Transport(port);
-  const manifestProm = fetch(manifestURL).then(
+  const esploader = new ESPLoader(transport, 115200);
-    (resp): Promise<Manifest> => resp.json()
-  );
-
-  const esploader = new ESPLoader(port, logger);
 
   // For debugging
   (window as any).esploader = esploader;
@@ -45,61 +53,53 @@ export const flash = async (
   });
 
   try {
-    await esploader.initialize();
+    await esploader.main_fn();
+    await esploader.flash_id();
   } catch (err: any) {
-    logger.error(err);
+    console.error(err);
     fireStateEvent({
       state: FlashStateType.ERROR,
       message:
         "Failed to initialize. Try resetting your device or holding the BOOT button while clicking INSTALL.",
       details: { error: FlashError.FAILED_INITIALIZING, details: err },
     });
-    if (esploader.connected) {
+    await resetTransport(transport);
-      await esploader.disconnect();
+    await transport.disconnect();
-    }
     return;
   }
 
-  chipFamily = getChipFamilyName(esploader);
+  chipFamily = await esploader.chip.CHIP_NAME;
+
+  if (!esploader.chip.ROM_TEXT) {
+    fireStateEvent({
+      state: FlashStateType.ERROR,
+      message: `Chip ${chipFamily} is not supported`,
+      details: {
+        error: FlashError.NOT_SUPPORTED,
+        details: `Chip ${chipFamily} is not supported`,
+      },
+    });
+    await resetTransport(transport);
+    await transport.disconnect();
+    return;
+  }
 
   fireStateEvent({
     state: FlashStateType.INITIALIZING,
     message: `Initialized. Found ${chipFamily}`,
     details: { done: true },
   });
-  fireStateEvent({
-    state: FlashStateType.MANIFEST,
-    message: "Fetching manifest...",
-    details: { done: false },
-  });
-
-  try {
-    manifest = await manifestProm;
-  } catch (err: any) {
-    fireStateEvent({
-      state: FlashStateType.ERROR,
-      message: `Unable to fetch manifest: ${err}`,
-      details: { error: FlashError.FAILED_MANIFEST_FETCH, details: err },
-    });
-    await esploader.disconnect();
-    return;
-  }
 
   build = manifest.builds.find((b) => b.chipFamily === chipFamily);
 
-  fireStateEvent({
-    state: FlashStateType.MANIFEST,
-    message: `Found manifest for ${manifest.name}`,
-    details: { done: true },
-  });
-
   if (!build) {
     fireStateEvent({
       state: FlashStateType.ERROR,
       message: `Your ${chipFamily} board is not supported.`,
       details: { error: FlashError.NOT_SUPPORTED, details: chipFamily },
     });
-    await esploader.disconnect();
+    await resetTransport(transport);
+    await transport.disconnect();
     return;
   }
 
@@ -109,6 +109,7 @@ export const flash = async (
     details: { done: false },
   });
 
+  const manifestURL = new URL(manifestPath, location.toString()).toString();
   const filePromises = build.parts.map(async (part) => {
     const url = new URL(part.path, manifestURL).toString();
     const resp = await fetch(url);
@@ -117,20 +118,24 @@ export const flash = async (
         `Downlading firmware ${part.path} failed: ${resp.status}`
       );
     }
-    return resp.arrayBuffer();
+
+    const reader = new FileReader();
+    const blob = await resp.blob();
+
+    return new Promise<string>((resolve) => {
+      reader.addEventListener("load", () => resolve(reader.result as string));
+      reader.readAsBinaryString(blob);
+    });
   });
 
-  // Run the stub while we wait for files to download
+  const fileArray: Array<{ data: string; address: number }> = [];
-  const espStub = await esploader.runStub();
-
-  const files: ArrayBuffer[] = [];
   let totalSize = 0;
 
-  for (const prom of filePromises) {
+  for (let part = 0; part < filePromises.length; part++) {
     try {
-      const data = await prom;
+      const data = await filePromises[part];
-      files.push(data);
+      fileArray.push({ data, address: build.parts[part].offset });
-      totalSize += data.byteLength;
+      totalSize += data.length;
     } catch (err: any) {
       fireStateEvent({
         state: FlashStateType.ERROR,
@@ -140,7 +145,8 @@ export const flash = async (
           details: err.message,
         },
       });
-      await esploader.disconnect();
+      await resetTransport(transport);
+      await transport.disconnect();
       return;
     }
   }
@@ -157,7 +163,7 @@ export const flash = async (
       message: "Erasing device...",
       details: { done: false },
     });
-    await espStub.eraseFlash();
+    await esploader.erase_flash();
     fireStateEvent({
       state: FlashStateType.ERASING,
       message: "Device erased",
@@ -165,57 +171,56 @@ export const flash = async (
     });
   }
 
-  let lastPct = 0;
-
   fireStateEvent({
     state: FlashStateType.WRITING,
-    message: `Writing progress: ${lastPct}%`,
+    message: `Writing progress: 0%`,
     details: {
       bytesTotal: totalSize,
       bytesWritten: 0,
-      percentage: lastPct,
+      percentage: 0,
     },
   });
 
   let totalWritten = 0;
 
-  for (const part of build.parts) {
-    const file = files.shift()!;
   try {
-      await espStub.flashData(
+    await esploader.write_flash({
-        file,
+      fileArray,
-        (bytesWritten: number) => {
+      reportProgress(fileIndex: number, written: number, total: number) {
+        const uncompressedWritten =
+          (written / total) * fileArray[fileIndex].data.length;
+
         const newPct = Math.floor(
-            ((totalWritten + bytesWritten) / totalSize) * 100
+          ((totalWritten + uncompressedWritten) / totalSize) * 100
         );
-          if (newPct === lastPct) {
+
+        // we're done with this file
+        if (written === total) {
+          totalWritten += uncompressedWritten;
           return;
         }
-          lastPct = newPct;
+
         fireStateEvent({
           state: FlashStateType.WRITING,
           message: `Writing progress: ${newPct}%`,
           details: {
             bytesTotal: totalSize,
-              bytesWritten: totalWritten + bytesWritten,
+            bytesWritten: totalWritten + written,
             percentage: newPct,
           },
         });
       },
-        part.offset,
+    });
-        true
-      );
   } catch (err: any) {
     fireStateEvent({
       state: FlashStateType.ERROR,
       message: err.message,
       details: { error: FlashError.WRITE_FAILED, details: err },
     });
-      await esploader.disconnect();
+    await resetTransport(transport);
+    await transport.disconnect();
     return;
   }
-    totalWritten += file.byteLength;
-  }
 
   fireStateEvent({
     state: FlashStateType.WRITING,
@@ -228,10 +233,10 @@ export const flash = async (
   });
 
   await sleep(100);
-  console.log("DISCONNECT");
-  await esploader.disconnect();
   console.log("HARD RESET");
-  await esploader.hardReset();
+  await resetTransport(transport);
+  console.log("DISCONNECT");
+  await transport.disconnect();
 
   fireStateEvent({
     state: FlashStateType.FINISHED,

src/install-dialog.ts

@@ -571,7 +571,6 @@ export class EwtInstallDialog extends LitElement {
     } else if (
       !this._installState ||
       this._installState.state === FlashStateType.INITIALIZING ||
-      this._installState.state === FlashStateType.MANIFEST ||
       this._installState.state === FlashStateType.PREPARING
     ) {
       heading = "Installing";
@@ -826,19 +825,27 @@ export class EwtInstallDialog extends LitElement {
     }
     this._client = undefined;
 
+    // Close port. ESPLoader likes opening it.
+    await this.port.close();
     flash(
       (state) => {
         this._installState = state;
 
         if (state.state === FlashStateType.FINISHED) {
           sleep(100)
+            // Flashing closes the port
+            .then(() => this.port.open({ baudRate: 115200 }))
             .then(() => this._initialize(true))
             .then(() => this.requestUpdate());
+        } else if (state.state === FlashStateType.ERROR) {
+          sleep(100)
+            // Flashing closes the port
+            .then(() => this.port.open({ baudRate: 115200 }));
         }
       },
       this.port,
-      this.logger,
       this.manifestPath,
+      this._manifest,
       this._installErase
     );
   }

src/util/chip-family-name.ts

@@ -1,28 +0,0 @@
-import {
-  CHIP_FAMILY_ESP32,
-  CHIP_FAMILY_ESP32S2,
-  CHIP_FAMILY_ESP32S3,
-  CHIP_FAMILY_ESP8266,
-  CHIP_FAMILY_ESP32C3,
-  ESPLoader,
-} from "esp-web-flasher";
-import type { BaseFlashState } from "../const";
-
-export const getChipFamilyName = (
-  esploader: ESPLoader
-): NonNullable<BaseFlashState["chipFamily"]> => {
-  switch (esploader.chipFamily) {
-    case CHIP_FAMILY_ESP32:
-      return "ESP32";
-    case CHIP_FAMILY_ESP8266:
-      return "ESP8266";
-    case CHIP_FAMILY_ESP32S2:
-      return "ESP32-S2";
-    case CHIP_FAMILY_ESP32S3:
-      return "ESP32-S3";
-    case CHIP_FAMILY_ESP32C3:
-      return "ESP32-C3";
-    default:
-      return "Unknown Chip";
-  }
-};