From 64d5045ccb4db82c0c8a00c4ee3c33a965b0b3d4 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Wed, 16 Jul 2025 18:23:33 +0200 Subject: [PATCH] Simplify Pio script post_esp32.py (#23689) * simplify post_esp32.py * make sure path has correct "\" or "/" regarding OS * add os specific path separators * more path possible issues corrections * add function to normpath cmd * set board_build.variants_dir = variants/tasmota correctly for OS --- pio-tools/gzip-firmware.py | 16 +++--- pio-tools/name-firmware.py | 24 ++++---- pio-tools/override_copy.py | 34 +++++++++--- pio-tools/post_esp32.py | 109 ++++++++++++------------------------- 4 files changed, 82 insertions(+), 101 deletions(-) diff --git a/pio-tools/gzip-firmware.py b/pio-tools/gzip-firmware.py index fa10e5f46..6dc5e6c7c 100644 --- a/pio-tools/gzip-firmware.py +++ b/pio-tools/gzip-firmware.py @@ -1,6 +1,6 @@ Import("env") -import os +import pathlib import shutil import tasmotapiolib import gzip @@ -8,7 +8,7 @@ from colorama import Fore, Back, Style def map_gzip(source, target, env): # create string with location and file names based on variant - map_file = tasmotapiolib.get_final_map_path(env) + map_file = pathlib.Path(tasmotapiolib.get_final_map_path(env)) if map_file.is_file(): gzip_file = map_file.with_suffix(".map.gz") @@ -19,7 +19,7 @@ def map_gzip(source, target, env): # write gzip map file with map_file.open("rb") as fp: - with gzip.open(gzip_file, "wb", compresslevel=9) as f: + with gzip.open(str(gzip_file), "wb", compresslevel=9) as f: shutil.copyfileobj(fp, f) # remove map file @@ -39,16 +39,16 @@ if tasmotapiolib.is_env_set(tasmotapiolib.ENABLE_ESP32_GZ, env) or env["PIOPLATF def bin_gzip(source, target, env): # create string with location and file names based on variant - bin_file = tasmotapiolib.get_final_bin_path(env) + bin_file = pathlib.Path(tasmotapiolib.get_final_bin_path(env)) gzip_file = bin_file.with_suffix(".bin.gz") # check if new target files exist and remove if necessary - if os.path.isfile(gzip_file): - os.remove(gzip_file) + if gzip_file.is_file(): + gzip_file.unlink() # write gzip firmware file - with open(bin_file, "rb") as fp: - with open(gzip_file, "wb") as f: + with bin_file.open("rb") as fp: + with gzip_file.open("wb") as f: time_start = time.time() gz = tasmotapiolib.compress(fp.read(), gzip_level) time_delta = time.time() - time_start diff --git a/pio-tools/name-firmware.py b/pio-tools/name-firmware.py index 7c84351d0..3c00b9dc1 100644 --- a/pio-tools/name-firmware.py +++ b/pio-tools/name-firmware.py @@ -12,11 +12,10 @@ def bin_map_copy(source, target, env): firsttarget = pathlib.Path(target[0].path) # get locations and file names based on variant - map_file = tasmotapiolib.get_final_map_path(env) - bin_file = tasmotapiolib.get_final_bin_path(env) + map_file = os.path.normpath(str(tasmotapiolib.get_final_map_path(env))) + bin_file = os.path.normpath(str(tasmotapiolib.get_final_bin_path(env))) one_bin_file = bin_file firmware_name = env.subst("$BUILD_DIR/${PROGNAME}.bin") - if env["PIOPLATFORM"] == "espressif32": if("safeboot" in firmware_name): SAFEBOOT_SIZE = firsttarget.stat().st_size @@ -27,26 +26,27 @@ def bin_map_copy(source, target, env): ) if("safeboot" not in firmware_name): factory_tmp = pathlib.Path(firsttarget).with_suffix("") - factory = factory_tmp.with_suffix(factory_tmp.suffix + ".factory.bin") + factory = os.path.normpath(str(factory_tmp.with_suffix(factory_tmp.suffix + ".factory.bin"))) one_bin_tmp = pathlib.Path(bin_file).with_suffix("") - one_bin_file = one_bin_tmp.with_suffix(one_bin_tmp.suffix + ".factory.bin") + one_bin_file = os.path.normpath(str(one_bin_tmp.with_suffix(one_bin_tmp.suffix + ".factory.bin"))) # check if new target files exist and remove if necessary for f in [map_file, bin_file, one_bin_file]: - if f.is_file(): - f.unlink() + f_path = pathlib.Path(f) + if f_path.is_file(): + f_path.unlink() # copy firmware.bin and map to final destination - shutil.copy(firsttarget, bin_file) + shutil.copy(str(firsttarget), bin_file) if env["PIOPLATFORM"] == "espressif32": # the map file is needed later for firmware-metrics.py - shutil.copy(tasmotapiolib.get_source_map_path(env), map_file) + shutil.copy(os.path.normpath(str(tasmotapiolib.get_source_map_path(env))), map_file) if("safeboot" not in firmware_name): shutil.copy(factory, one_bin_file) else: - map_firm = join(env.subst("$BUILD_DIR")) + os.sep + "firmware.map" - shutil.copy(tasmotapiolib.get_source_map_path(env), map_firm) - shutil.move(tasmotapiolib.get_source_map_path(env), map_file) + map_firm = os.path.normpath(join(env.subst("$BUILD_DIR"), "firmware.map")) + shutil.copy(os.path.normpath(str(tasmotapiolib.get_source_map_path(env))), map_firm) + shutil.move(os.path.normpath(str(tasmotapiolib.get_source_map_path(env))), map_file) silent_action = env.Action(bin_map_copy) silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output diff --git a/pio-tools/override_copy.py b/pio-tools/override_copy.py index e1a278ae8..96d6c520c 100644 --- a/pio-tools/override_copy.py +++ b/pio-tools/override_copy.py @@ -1,28 +1,46 @@ Import('env') + import os import pathlib -from os.path import join import shutil from colorama import Fore, Back, Style -if " " in join(pathlib.Path(env["PROJECT_DIR"])): +# Ensure the variants directory is correctly formatted based on the OS +# This is necessary to avoid issues with path handling in different environments +variants_dir = env.BoardConfig().get("build.variants_dir", "") +if variants_dir: + if os.name == "nt": + variants_dir = variants_dir.replace("/", "\\") + env.BoardConfig().update("build.variants_dir", variants_dir) + else: + variants_dir = variants_dir.replace("\\", "/") + env.BoardConfig().update("build.variants_dir", variants_dir) + +project_dir = os.path.normpath(env["PROJECT_DIR"]) +if " " in project_dir: print(Fore.RED + "*** Whitespace(s) in project path, unexpected issues/errors can happen ***") # copy tasmota/user_config_override_sample.h to tasmota/user_config_override.h -if os.path.isfile("tasmota/user_config_override.h"): +uc_override = pathlib.Path(os.path.normpath("tasmota/user_config_override.h")) +uc_override_sample = pathlib.Path(os.path.normpath("tasmota/user_config_override_sample.h")) +if uc_override.is_file(): print(Fore.GREEN + "*** use provided user_config_override.h as planned ***") else: - shutil.copy("tasmota/user_config_override_sample.h", "tasmota/user_config_override.h") + shutil.copy(str(uc_override_sample), str(uc_override)) # copy platformio_override_sample.ini to platformio_override.ini -if os.path.isfile("platformio_override.ini"): +pio_override = pathlib.Path(os.path.normpath("platformio_override.ini")) +pio_override_sample = pathlib.Path(os.path.normpath("platformio_override_sample.ini")) +if pio_override.is_file(): print(Fore.GREEN + "*** use provided platformio_override.ini as planned ***") else: - shutil.copy("platformio_override_sample.ini", "platformio_override.ini") + shutil.copy(str(pio_override_sample), str(pio_override)) # copy platformio_tasmota_cenv_sample.ini to platformio_tasmota_cenv.ini -if os.path.isfile("platformio_tasmota_cenv.ini"): +pio_cenv = pathlib.Path(os.path.normpath("platformio_tasmota_cenv.ini")) +pio_cenv_sample = pathlib.Path(os.path.normpath("platformio_tasmota_cenv_sample.ini")) +if pio_cenv.is_file(): print(Fore.GREEN + "*** use provided platformio_tasmota_cenv.ini as planned ***") else: - shutil.copy("platformio_tasmota_cenv_sample.ini", "platformio_tasmota_cenv.ini") + shutil.copy(str(pio_cenv_sample), str(pio_cenv)) diff --git a/pio-tools/post_esp32.py b/pio-tools/post_esp32.py index 712ce9939..3f96cfaa2 100644 --- a/pio-tools/post_esp32.py +++ b/pio-tools/post_esp32.py @@ -20,26 +20,19 @@ # - 0xe0000 | ~\Tasmota\.pio\build\/firmware.bin # - 0x3b0000| ~\Tasmota\.pio\build\/littlefs.bin -env = DefaultEnvironment() -platform = env.PioPlatform() - from genericpath import exists import os -import sys from os.path import join, getsize import csv import requests import shutil import subprocess import codecs -from colorama import Fore, Back, Style +from colorama import Fore from SCons.Script import COMMAND_LINE_TARGETS -from platformio.project.config import ProjectConfig - -esptoolpy = os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tool-esptoolpy") -sys.path.insert(0, esptoolpy) -import esptool +env = DefaultEnvironment() +platform = env.PioPlatform() config = env.GetProjectConfig() variants_dir = env.BoardConfig().get("build.variants_dir", "") variant = env.BoardConfig().get("build.variant", "") @@ -51,49 +44,30 @@ flag_board_sdkconfig = env.BoardConfig().get("espidf.custom_sdkconfig", "") # Copy safeboots firmwares in place when running in Github github_actions = os.getenv('GITHUB_ACTIONS') -extra_flags = ''.join([element.replace("-D", " ") for element in env.BoardConfig().get("build.extra_flags", "")]) -build_flags = ''.join([element.replace("-D", " ") for element in env.GetProjectOption("build_flags")]) -if ("CORE32SOLO1" in extra_flags or "FRAMEWORK_ARDUINO_SOLO1" in build_flags) and flag_custom_sdkconfig == False and flag_board_sdkconfig == "": - FRAMEWORK_DIR = platform.get_package_dir("framework-arduino-solo1") - if github_actions and os.path.exists("./firmware/firmware"): - shutil.copytree("./firmware/firmware", "/home/runner/.platformio/packages/framework-arduino-solo1/variants/tasmota", dirs_exist_ok=True) - if variants_dir: - shutil.copytree("./firmware/firmware", variants_dir, dirs_exist_ok=True) -elif ("CORE32ITEAD" in extra_flags or "FRAMEWORK_ARDUINO_ITEAD" in build_flags) and flag_custom_sdkconfig == False and flag_board_sdkconfig == "": - FRAMEWORK_DIR = platform.get_package_dir("framework-arduino-ITEAD") - if github_actions and os.path.exists("./firmware/firmware"): - shutil.copytree("./firmware/firmware", "/home/runner/.platformio/packages/framework-arduino-ITEAD/variants/tasmota", dirs_exist_ok=True) - if variants_dir: - shutil.copytree("./firmware/firmware", variants_dir, dirs_exist_ok=True) -else: - FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") - if github_actions and os.path.exists("./firmware/firmware"): - shutil.copytree("./firmware/firmware", "/home/runner/.platformio/packages/framework-arduinoespressif32/variants/tasmota", dirs_exist_ok=True) - if variants_dir: - shutil.copytree("./firmware/firmware", variants_dir, dirs_exist_ok=True) +FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") +if github_actions and os.path.exists(os.path.normpath(os.path.join(".", "firmware", "firmware"))): + dest_dir = os.path.normpath(os.path.join(os.sep, "home", "runner", ".platformio", "packages", "framework-arduinoespressif32", "variants", "tasmota")) + shutil.copytree(os.path.normpath(os.path.join(".", "firmware", "firmware")), dest_dir, dirs_exist_ok=True) + if variants_dir: + shutil.copytree(os.path.normpath(os.path.join(".", "firmware", "firmware")), os.path.normpath(variants_dir), dirs_exist_ok=True) # Copy pins_arduino.h to variants folder if variants_dir: - mcu_build_variant_path = join(FRAMEWORK_DIR, "variants", mcu_build_variant, "pins_arduino.h") - custom_variant_build = join(env.subst("$PROJECT_DIR"), variants_dir , mcu_build_variant, "pins_arduino.h") - os.makedirs(join(env.subst("$PROJECT_DIR"), variants_dir , mcu_build_variant), exist_ok=True) + mcu_build_variant_path = os.path.normpath(join(FRAMEWORK_DIR, "variants", mcu_build_variant, "pins_arduino.h")) + custom_variant_build = os.path.normpath(join(env.subst("$PROJECT_DIR"), variants_dir , mcu_build_variant, "pins_arduino.h")) + os.makedirs(os.path.normpath(join(env.subst("$PROJECT_DIR"), variants_dir , mcu_build_variant)), exist_ok=True) shutil.copy(mcu_build_variant_path, custom_variant_build) if not variants_dir: - variants_dir = join(FRAMEWORK_DIR, "variants", "tasmota") + variants_dir = os.path.normpath(join(FRAMEWORK_DIR, "variants", "tasmota")) env.BoardConfig().update("build.variants_dir", variants_dir) -def esptool_call(cmd): - try: - esptool.main(cmd) - except SystemExit as e: - # Fetch sys.exit() without leaving the script - if e.code == 0: - return True - else: - print(f"❌ esptool failed with exit code: {e.code}") - return False +def normalize_paths(cmd): + for i, arg in enumerate(cmd): + if isinstance(arg, str) and '/' in arg: + cmd[i] = os.path.normpath(arg) + return cmd def esp32_detect_flashsize(): uploader = env.subst("$UPLOADER") @@ -103,7 +77,7 @@ def esp32_detect_flashsize(): return "4MB",False else: esptool_flags = ["flash-id"] - esptool_cmd = [env["PYTHONEXE"], env.subst("$OBJCOPY")] + esptool_flags + esptool_cmd = [env.subst("$OBJCOPY")] + esptool_flags try: output = subprocess.run(esptool_cmd, capture_output=True).stdout.splitlines() for l in output: @@ -124,7 +98,7 @@ def esp32_detect_flashsize(): flash_size_from_esp, flash_size_was_overridden = esp32_detect_flashsize() def patch_partitions_bin(size_string): - partition_bin_path = join(env.subst("$BUILD_DIR"),"partitions.bin") + partition_bin_path = os.path.normpath(join(env.subst("$BUILD_DIR"), "partitions.bin")) with open(partition_bin_path, 'r+b') as file: binary_data = file.read(0xb0) import hashlib @@ -144,12 +118,6 @@ def patch_partitions_bin(size_string): def esp32_create_chip_string(chip): tasmota_platform_org = env.subst("$BUILD_DIR").split(os.path.sep)[-1] tasmota_platform = tasmota_platform_org.split('-')[0] - if ("CORE32SOLO1" in extra_flags or "FRAMEWORK_ARDUINO_SOLO1" in build_flags) and "tasmota32-safeboot" not in tasmota_platform_org and "tasmota32solo1" not in tasmota_platform_org and flag_custom_sdkconfig == False: - print(Fore.YELLOW + "Unexpected naming convention in this build environment:" + Fore.RED, tasmota_platform_org) - print(Fore.YELLOW + "Expected build environment name like " + Fore.GREEN + "'tasmota32solo1-whatever-you-want'") - print(Fore.YELLOW + "Please correct your actual build environment, to avoid undefined behavior in build process!!") - tasmota_platform = "tasmota32solo1" - return tasmota_platform if "tasmota" + chip[3:] not in tasmota_platform: # check + fix for a valid name like 'tasmota' + '32c3' tasmota_platform = "tasmota" + chip[3:] if "-DUSE_USB_CDC_CONSOLE" not in env.BoardConfig().get("build.extra_flags"): @@ -161,7 +129,7 @@ def esp32_create_chip_string(chip): def esp32_build_filesystem(fs_size): files = env.GetProjectOption("custom_files_upload").splitlines() num_entries = len([f for f in files if f.strip()]) - filesystem_dir = join(env.subst("$BUILD_DIR"),"littlefs_data") + filesystem_dir = os.path.normpath(join(env.subst("$BUILD_DIR"), "littlefs_data")) if not os.path.exists(filesystem_dir): os.makedirs(filesystem_dir) if num_entries > 1: @@ -174,9 +142,9 @@ def esp32_build_filesystem(fs_size): if "http" and "://" in file: response = requests.get(file.split(" ")[0]) if response.ok: - target = join(filesystem_dir,file.split(os.path.sep)[-1]) + target = os.path.normpath(join(filesystem_dir, file.split(os.path.sep)[-1])) if len(file.split(" ")) > 1: - target = join(filesystem_dir,file.split(" ")[1]) + target = os.path.normpath(join(filesystem_dir, file.split(" ")[1])) print("Renaming",(file.split(os.path.sep)[-1]).split(" ")[0],"to",file.split(" ")[1]) open(target, "wb").write(response.content) else: @@ -197,7 +165,7 @@ def esp32_build_filesystem(fs_size): def esp32_fetch_safeboot_bin(tasmota_platform): safeboot_fw_url = "http://ota.tasmota.com/tasmota32/release/" + tasmota_platform + "-safeboot.bin" - safeboot_fw_name = join(variants_dir, tasmota_platform + "-safeboot.bin") + safeboot_fw_name = os.path.normpath(join(variants_dir, tasmota_platform + "-safeboot.bin")) if(exists(safeboot_fw_name)): print(Fore.GREEN + "Safeboot binary already in place") return True @@ -207,7 +175,8 @@ def esp32_fetch_safeboot_bin(tasmota_platform): try: response = requests.get(safeboot_fw_url) open(safeboot_fw_name, "wb").write(response.content) - print(Fore.GREEN + "safeboot binary written to variants dir") + print(Fore.GREEN + "Safeboot binary written to variants path:") + print(Fore.BLUE + safeboot_fw_name) return True except: print(Fore.RED + "Download of safeboot binary failed. Please check your Internet connection.") @@ -217,7 +186,7 @@ def esp32_fetch_safeboot_bin(tasmota_platform): def esp32_copy_new_safeboot_bin(tasmota_platform,new_local_safeboot_fw): print("Copy new local safeboot firmware to variants dir -> using it for further flashing operations") - safeboot_fw_name = join(variants_dir, tasmota_platform + "-safeboot.bin") + safeboot_fw_name = os.path.normpath(join(variants_dir, tasmota_platform + "-safeboot.bin")) if os.path.exists(variants_dir): try: shutil.copy(new_local_safeboot_fw, safeboot_fw_name) @@ -262,8 +231,8 @@ def esp32_create_combined_bin(source, target, env): fs_offset = int(row[3],base=16) print() - new_file_name = env.subst("$BUILD_DIR/${PROGNAME}.factory.bin") - firmware_name = env.subst("$BUILD_DIR/${PROGNAME}.bin") + new_file_name = os.path.normpath(env.subst("$BUILD_DIR/${PROGNAME}.factory.bin")) + firmware_name = os.path.normpath(env.subst("$BUILD_DIR/${PROGNAME}.bin")) tasmota_platform = esp32_create_chip_string(chip) if not os.path.exists(variants_dir): @@ -317,7 +286,7 @@ def esp32_create_combined_bin(source, target, env): upload_protocol = env.subst("$UPLOAD_PROTOCOL") if(upload_protocol == "esptool") and (fs_offset != -1): - fs_bin = join(env.subst("$BUILD_DIR"),"littlefs.bin") + fs_bin = os.path.normpath(join(env.subst("$BUILD_DIR"), "littlefs.bin")) if exists(fs_bin): before_reset = env.BoardConfig().get("upload.before_reset", "default-reset") after_reset = env.BoardConfig().get("upload.after_reset", "hard-reset") @@ -336,23 +305,17 @@ def esp32_create_combined_bin(source, target, env): "--flash-freq", "${__get_board_f_flash(__env__)}", "--flash-size", flash_size ], - UPLOADCMD='"$UPLOADER" $UPLOADERFLAGS ' + " ".join(cmd[7:]) + UPLOADCMD='"$OBJCOPY" $UPLOADERFLAGS ' + " ".join(normalize_paths(cmd[7:])) ) print(Fore.GREEN + "Will use custom upload command for flashing operation to add file system defined for this build target.") print() if("safeboot" not in firmware_name): - #print('Using esptool.py arguments: %s' % ' '.join(cmd)) - with open(os.devnull, 'w') as devnull: - old_stdout = sys.stdout - old_stderr = sys.stderr - sys.stdout = devnull - sys.stderr = devnull - try: - esptool_call(cmd) - finally: - sys.stdout = old_stdout - sys.stderr = old_stderr + cmdline = [env.subst("$OBJCOPY")] + normalize_paths(cmd) + # print('Command Line: %s' % cmdline) + result = subprocess.run(cmdline, text=True, check=False, stdout=subprocess.DEVNULL) + if result.returncode != 0: + print(Fore.RED + f"esptool create firmware failed with exit code: {result.returncode}") silent_action = env.Action(esp32_create_combined_bin) silent_action.strfunction = lambda target, source, env: '' # hack to silence scons command output