Merge pull request #10574 from Jason2866/patch-2

add ESP32 filesystem download with PIO

.gitignore

@@ -9,6 +9,8 @@
 .clang_complete
 .gcc-flags.json
 .cache
+data
+unpacked_fs
 tasmota/user_config_override.h
 build
 build_output

pio-tools/download_fs.py

@@ -1,14 +1,14 @@
 # Written by Maximilian Gerhardt <maximilian.gerhardt@rub.de>
 # 29th December 2020
 # License: Apache
-# Expanded from functionality provided by PlatformIO's espressif32 and espressif8266 platforms, credited below. 
+# Expanded from functionality provided by PlatformIO's espressif32 and espressif8266 platforms, credited below.
-# This script provides functions to download the filesystem (SPIFFS or LittleFS) from a running ESP32 / ESP8266 
+# This script provides functions to download the filesystem (SPIFFS or LittleFS) from a running ESP32 / ESP8266
 # over the serial bootloader using esptool.py, and mklittlefs / mkspiffs for extracting.
-# run by either using the VSCode task "Custom" -> "Download Filesystem" 
+# run by either using the VSCode task "Custom" -> "Download Filesystem"
-# or by doing 'pio run -t downloadfs' (with optional '-e <environment>') from the commandline. 
+# or by doing 'pio run -t downloadfs' (with optional '-e <environment>') from the commandline.
 # output will be saved, by default, in the "unpacked_fs" of the project.
-# this folder can be changed by writing 'custom_unpack_dir = some_other_dir' in the corresponding platformio.ini 
+# this folder can be changed by writing 'custom_unpack_dir = some_other_dir' in the corresponding platformio.ini
-# environment. 
+# environment.
 import re
 import sys
 from os.path import isfile, join
@@ -23,25 +23,30 @@ Import("env")
 platform = env.PioPlatform()
 board = env.BoardConfig()
 mcu = board.get("build.mcu", "esp32")
+# Hack for using mklittlefs instead of mkspiffs -> needed since littlefs is not supported with this for ESP32
+if env["PIOPLATFORM"] == "espressif32":
+    #print("Replace MKSPIFFSTOOL with mklittlefs")
+    env.Replace( MKSPIFFSTOOL=platform.get_package_dir("tool-mklittlefs") + '/mklittlefs' )
+
 # needed for later
 AutodetectUploadPort(env)
 
-class FSType(Enum): 
+class FSType(Enum):
     SPIFFS="spiffs"
     LITTLEFS="littlefs"
-    FATFS="fatfs"   
+    FATFS="fatfs"
 
 class FSInfo:
-    def __init__(self, fs_type, start, length, page_size, block_size): 
+    def __init__(self, fs_type, start, length, page_size, block_size):
         self.fs_type = fs_type
-        self.start = start 
+        self.start = start
         self.length = length
         self.page_size = page_size
         self.block_size = block_size
     def __repr__(self):
         return f"FS type {self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size}"
     # extract command supposed to be implemented by subclasses
-    def get_extract_cmd(self):
+    def get_extract_cmd(self, input_file, output_dir):
         raise NotImplementedError()
 
 class LittleFSInfo(FSInfo):
@@ -53,10 +58,10 @@ class LittleFSInfo(FSInfo):
             self.tool = env["MKFSTOOL"] # from mkspiffs package
         self.tool = join(platform.get_package_dir("tool-mklittlefs"), self.tool)
         super().__init__(FSType.LITTLEFS, start, length, page_size, block_size)
-    def __repr__(self): 
+    def __repr__(self):
         return f"FS type {self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size} Tool: {self.tool}"
     def get_extract_cmd(self, input_file, output_dir):
-        return f'"{self.tool}" -b {self.block_size} -p {self.page_size} --unpack "{output_dir}" "{input_file}"'
+        return [self.tool, "-b", str(self.block_size), "-p", str(self.page_size), "--unpack", output_dir, input_file]
 
 
 class SPIFFSInfo(FSInfo):
@@ -68,7 +73,7 @@ class SPIFFSInfo(FSInfo):
             self.tool = env["MKFSTOOL"] # from mkspiffs package
         self.tool = join(platform.get_package_dir("tool-mkspiffs"), self.tool)
         super().__init__(FSType.SPIFFS, start, length, page_size, block_size)
-    def __repr__(self): 
+    def __repr__(self):
         return f"FS type {self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size} Tool: {self.tool}"
     def get_extract_cmd(self, input_file, output_dir):
         return f'"{self.tool}" -b {self.block_size} -p {self.page_size} --unpack "{output_dir}" "{input_file}"'
@@ -258,16 +263,16 @@ def download_fs(fs_info: FSInfo):
     fs_file = join(env["PROJECT_DIR"], f"downloaded_fs_{hex(fs_info.start)}_{hex(fs_info.length)}.bin")
     esptoolpy_flags = [
             "--chip", mcu,
-            "--port", '"' + env.subst("$UPLOAD_PORT") + '"',
+            "--port", env.subst("$UPLOAD_PORT"),
             "--baud",  env.subst("$UPLOAD_SPEED"),
             "--before", "default_reset",
             "--after", "hard_reset",
-            "read_flash", 
+            "read_flash",
             hex(fs_info.start),
             hex(fs_info.length),
-            '"' + fs_file + '"'
+            fs_file
     ]
-    esptoolpy_cmd = '"' + env["PYTHONEXE"]+ '"' + ' "' + esptoolpy + '" ' + " ".join(esptoolpy_flags)
+    esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
     print("Executing flash download command.")
     print(esptoolpy_cmd)
     try:
@@ -302,7 +307,7 @@ def unpack_fs(fs_info: FSInfo, downloaded_file: str):
         return (False, "")
 
 def display_fs(extracted_dir):
-    # extract command already nicely lists all extracted files. 
+    # extract command already nicely lists all extracted files.
     # no need to display that ourselves. just display a summary
     file_count = sum([len(files) for r, d, files in os.walk(extracted_dir)])
     print("Extracted " + str(file_count) + " file(s) from filesystem.")
@@ -326,4 +331,4 @@ env.AddCustomTarget(
     ],
     title="Download Filesystem",
     description="Downloads and displays files stored in the target ESP32/ESP8266"
-)
+)

platformio.ini

@@ -67,7 +67,7 @@ default_envs                = ${build_envs.default_envs}
 framework                   = arduino
 board                       = esp01_1m
 board_build.filesystem      = littlefs
-custom_unpack_dir           = unpacked_esp8266_littlefs
+custom_unpack_dir           = unpacked_littlefs
 board_build.flash_mode      = dout
 board_build.ldscript        = eagle.flash.1m.ld
 

platformio_override_sample.ini

@@ -116,33 +116,21 @@ build_flags                 = ${esp82xx_defaults.build_flags}
                               -DWAVEFORM_LOCKED_PWM
                               -Wno-switch-unreachable
 
-
 [common32]
-platform                    = ${core32.platform}
 platform_packages           = ${core32.platform_packages}
 build_unflags               = ${core32.build_unflags}
 build_flags                 = ${core32.build_flags}
-board                       = esp32dev
+upload_port                 = COM4
-board_build.ldscript        = esp32_out.ld
-board_build.partitions      = esp32_partition_app1984k_spiffs64k.csv
-board_build.flash_mode      = ${common.board_build.flash_mode}
-board_build.f_flash         = ${common.board_build.f_flash}
-board_build.f_cpu           = ${common.board_build.f_cpu}
-monitor_speed               = ${common.monitor_speed}
-upload_port                 = ${common.upload_port}
-upload_resetmethod          = ${common.upload_resetmethod}
-upload_speed                = 921600
-extra_scripts               = ${common.extra_scripts}
 lib_extra_dirs              = ${library.lib_extra_dirs}
 ; *** ESP32 lib. ALWAYS needed for ESP32 !!!
                               lib/libesp32
+
 [core32]
 ; Activate Stage Core32 by removing ";" in next 3 lines, if you want to override the standard core32
 ;platform_packages           = ${core32_stage.platform_packages}
 ;build_unflags               = ${core32_stage.build_unflags}
 ;build_flags                 = ${core32_stage.build_flags}
 
-
 [core32_stage]
 platform_packages           = framework-arduinoespressif32 @ https://github.com/Jason2866/arduino-esp32/releases/download/1.0.5-rc6/esp32-1.0.5-rc6.zip
                               platformio/tool-mklittlefs @ ~1.203.200522

platformio_tasmota32.ini

@@ -48,6 +48,7 @@ build_unflags               = ${core32.build_unflags}
 build_flags                 = ${core32.build_flags}
 board                       = esp32dev
 board_build.filesystem      = ${common.board_build.filesystem}
+custom_unpack_dir           = ${common.custom_unpack_dir}
 board_build.ldscript        = esp32_out.ld
 board_build.partitions      = esp32_partition_app1984k_spiffs64k.csv
 board_build.flash_mode      = ${common.board_build.flash_mode}