From 7998650e608c6f31dad692b3c267de5e488f32a4 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 28 Mar 2025 20:12:30 -0400 Subject: [PATCH 01/18] Fix up usermod libArchive settings The ConfigureProjectLibBuilder process will flush and reload the library settings from the on-disk manifests if any new library is installed at that stage. This has the side effect of reverting the libArchive setting applied to usermods which was performed prior to that call. Apply the setting afterwards, instead. Fixes #4597 --- pio-scripts/load_usermods.py | 39 ++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index ab3c6476a..27a4590d1 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -2,16 +2,19 @@ Import('env') import os.path from collections import deque from pathlib import Path # For OS-agnostic path manipulation +from platformio.builder.tools.piolib import LibBuilderBase from platformio.package.manager.library import LibraryPackageManager usermod_dir = Path(env["PROJECT_DIR"]) / "usermods" -all_usermods = [f for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()] +# "usermods" environment: expand list of usermods to everything in the folder if env['PIOENV'] == "usermods": # Add all usermods + all_usermods = [f for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()] env.GetProjectConfig().set(f"env:usermods", 'custom_usermods', " ".join([f.name for f in all_usermods])) -def find_usermod(mod: str): +# Utility functions +def find_usermod(mod: str) -> Path: """Locate this library in the usermods folder. We do this to avoid needing to rename a bunch of folders; this could be removed later @@ -28,6 +31,13 @@ def find_usermod(mod: str): return mp raise RuntimeError(f"Couldn't locate module {mod} in usermods directory!") +def is_wled_module(dep: LibBuilderBase) -> bool: + """Returns true if the specified library is a wled module + """ + return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-") + +## Script starts here +# Process usermod option usermods = env.GetProjectOption("custom_usermods","") if usermods: # Inject usermods in to project lib_deps @@ -82,13 +92,6 @@ old_ConfigureProjectLibBuilder = env.ConfigureProjectLibBuilder # Our new wrapper def wrapped_ConfigureProjectLibBuilder(xenv): - # Update usermod properties - # Set libArchive before build actions are added - for um in (um for um in xenv.GetLibBuilders() if usermod_dir in Path(um.src_dir).parents): - build = um._manifest.get("build", {}) - build["libArchive"] = False - um._manifest["build"] = build - # Call the wrapped function result = old_ConfigureProjectLibBuilder.clone(xenv)() @@ -102,12 +105,18 @@ def wrapped_ConfigureProjectLibBuilder(xenv): for dep in result.depbuilders: cached_add_includes(dep, processed_deps, extra_include_dirs) - for um in [dep for dep in result.depbuilders if usermod_dir in Path(dep.src_dir).parents]: - # Add the wled folder to the include path - um.env.PrependUnique(CPPPATH=wled_dir) - # Add WLED's own dependencies - for dir in extra_include_dirs: - um.env.PrependUnique(CPPPATH=dir) + for dep in result.depbuilders: + if is_wled_module(dep): + # Add the wled folder to the include path + dep.env.PrependUnique(CPPPATH=wled_dir) + # Add WLED's own dependencies + for dir in extra_include_dirs: + dep.env.PrependUnique(CPPPATH=dir) + # Enforce that libArchive is not set; we must link them directly to the executable + if dep.lib_archive: + build = dep._manifest.get("build", {}) + build["libArchive"] = False + dep._manifest["build"] = build return result From 6464c620c701bf3ad8297ab81572f23c45404353 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Wed, 30 Apr 2025 21:56:56 -0400 Subject: [PATCH 02/18] load_usermods: Enforce CPPPATH type Ensure that entries put in CPPPATH are always strings so SCons can correctlly deduplicate. --- pio-scripts/load_usermods.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index 27a4590d1..eeebcbf27 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -108,10 +108,10 @@ def wrapped_ConfigureProjectLibBuilder(xenv): for dep in result.depbuilders: if is_wled_module(dep): # Add the wled folder to the include path - dep.env.PrependUnique(CPPPATH=wled_dir) + dep.env.PrependUnique(CPPPATH=str(wled_dir)) # Add WLED's own dependencies for dir in extra_include_dirs: - dep.env.PrependUnique(CPPPATH=dir) + dep.env.PrependUnique(CPPPATH=str(dir)) # Enforce that libArchive is not set; we must link them directly to the executable if dep.lib_archive: build = dep._manifest.get("build", {}) From a8dd2435ec82b4f348b8a22f2caae0392f95c9da Mon Sep 17 00:00:00 2001 From: Will Miles Date: Tue, 6 May 2025 22:11:17 -0400 Subject: [PATCH 03/18] Revert "Usermods: Remove libArchive" This reverts commit 0d44e7ec272b130f7e3eccf1d1f20a5405a327fe. --- usermods/ADS1115_v2/library.json | 1 + usermods/AHT10_v2/library.json | 1 + usermods/Analog_Clock/library.json | 3 ++- usermods/Animated_Staircase/library.json | 3 ++- usermods/BH1750_v2/library.json | 1 + usermods/BME280_v2/library.json | 1 + usermods/Battery/library.json | 3 ++- usermods/Cronixie/library.json | 3 ++- usermods/EleksTube_IPS/library.json.disabled | 1 + usermods/Internal_Temperature_v2/library.json | 3 ++- usermods/LD2410_v2/library.json | 1 + usermods/LDR_Dusk_Dawn_v2/library.json | 3 ++- usermods/MY9291/library.json | 1 + usermods/PIR_sensor_switch/library.json | 3 ++- usermods/PWM_fan/library.json | 1 + usermods/RTC/library.json | 3 ++- usermods/SN_Photoresistor/library.json | 3 ++- usermods/ST7789_display/library.json.disabled | 3 ++- usermods/Si7021_MQTT_HA/library.json | 1 + usermods/TetrisAI_v2/library.json | 3 ++- usermods/boblight/library.json | 3 ++- usermods/buzzer/library.json | 3 ++- usermods/deep_sleep/library.json | 3 ++- usermods/multi_relay/library.json | 3 ++- usermods/pwm_outputs/library.json | 3 ++- usermods/sd_card/library.json | 3 ++- usermods/seven_segment_display/library.json | 3 ++- usermods/seven_segment_display_reloaded/library.json | 3 ++- usermods/sht/library.json | 1 + usermods/smartnest/library.json | 3 ++- usermods/stairway_wipe_basic/library.json | 3 ++- usermods/usermod_rotary_brightness_color/library.json | 3 ++- usermods/usermod_v2_HttpPullLightControl/library.json | 3 ++- usermods/usermod_v2_animartrix/library.json | 1 + usermods/usermod_v2_auto_save/library.json | 3 ++- usermods/usermod_v2_four_line_display_ALT/library.json | 1 + usermods/usermod_v2_klipper_percentage/library.json | 3 ++- usermods/usermod_v2_ping_pong_clock/library.json | 3 ++- usermods/usermod_v2_rotary_encoder_ui_ALT/library.json | 1 + usermods/usermod_v2_word_clock/library.json | 3 ++- usermods/wizlights/library.json | 3 ++- usermods/word-clock-matrix/library.json | 3 ++- 42 files changed, 71 insertions(+), 29 deletions(-) diff --git a/usermods/ADS1115_v2/library.json b/usermods/ADS1115_v2/library.json index 9f0c021ce..5e5d7e450 100644 --- a/usermods/ADS1115_v2/library.json +++ b/usermods/ADS1115_v2/library.json @@ -1,5 +1,6 @@ { "name": "ADS1115_v2", + "build": { "libArchive": false }, "dependencies": { "Adafruit BusIO": "https://github.com/adafruit/Adafruit_BusIO#1.13.2", "Adafruit ADS1X15": "https://github.com/adafruit/Adafruit_ADS1X15#2.4.0" diff --git a/usermods/AHT10_v2/library.json b/usermods/AHT10_v2/library.json index 54f8c1715..fa6c2a6fe 100644 --- a/usermods/AHT10_v2/library.json +++ b/usermods/AHT10_v2/library.json @@ -1,5 +1,6 @@ { "name": "AHT10_v2", + "build": { "libArchive": false }, "dependencies": { "enjoyneering/AHT10":"~1.1.0" } diff --git a/usermods/Analog_Clock/library.json b/usermods/Analog_Clock/library.json index 3ed596dc7..f76cf4268 100644 --- a/usermods/Analog_Clock/library.json +++ b/usermods/Analog_Clock/library.json @@ -1,3 +1,4 @@ { - "name": "Analog_Clock" + "name": "Analog_Clock", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/Animated_Staircase/library.json b/usermods/Animated_Staircase/library.json index a2c50ea4c..015b15cef 100644 --- a/usermods/Animated_Staircase/library.json +++ b/usermods/Animated_Staircase/library.json @@ -1,3 +1,4 @@ { - "name": "Animated_Staircase" + "name": "Animated_Staircase", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/BH1750_v2/library.json b/usermods/BH1750_v2/library.json index 13740e6c9..8323d1abb 100644 --- a/usermods/BH1750_v2/library.json +++ b/usermods/BH1750_v2/library.json @@ -1,5 +1,6 @@ { "name": "BH1750_v2", + "build": { "libArchive": false }, "dependencies": { "claws/BH1750":"^1.2.0" } diff --git a/usermods/BME280_v2/library.json b/usermods/BME280_v2/library.json index 626fb8b2b..cfdfe1ba1 100644 --- a/usermods/BME280_v2/library.json +++ b/usermods/BME280_v2/library.json @@ -1,5 +1,6 @@ { "name": "BME280_v2", + "build": { "libArchive": false }, "dependencies": { "finitespace/BME280":"~3.0.0" } diff --git a/usermods/Battery/library.json b/usermods/Battery/library.json index d6b8ad38a..8e71c60a7 100644 --- a/usermods/Battery/library.json +++ b/usermods/Battery/library.json @@ -1,3 +1,4 @@ { - "name": "Battery" + "name": "Battery", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/Cronixie/library.json b/usermods/Cronixie/library.json index a1454a79a..4a1b6988e 100644 --- a/usermods/Cronixie/library.json +++ b/usermods/Cronixie/library.json @@ -1,3 +1,4 @@ { - "name": "Cronixie" + "name": "Cronixie", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/EleksTube_IPS/library.json.disabled b/usermods/EleksTube_IPS/library.json.disabled index eddd12b88..d143638e4 100644 --- a/usermods/EleksTube_IPS/library.json.disabled +++ b/usermods/EleksTube_IPS/library.json.disabled @@ -1,5 +1,6 @@ { "name:": "EleksTube_IPS", + "build": { "libArchive": false }, "dependencies": { "TFT_eSPI" : "2.5.33" } diff --git a/usermods/Internal_Temperature_v2/library.json b/usermods/Internal_Temperature_v2/library.json index 571176f45..b1826ab45 100644 --- a/usermods/Internal_Temperature_v2/library.json +++ b/usermods/Internal_Temperature_v2/library.json @@ -1,3 +1,4 @@ { - "name": "Internal_Temperature_v2" + "name": "Internal_Temperature_v2", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/LD2410_v2/library.json b/usermods/LD2410_v2/library.json index 92ad54da7..757ec4047 100644 --- a/usermods/LD2410_v2/library.json +++ b/usermods/LD2410_v2/library.json @@ -1,5 +1,6 @@ { "name": "LD2410_v2", + "build": { "libArchive": false }, "dependencies": { "ncmreynolds/ld2410":"^0.1.3" } diff --git a/usermods/LDR_Dusk_Dawn_v2/library.json b/usermods/LDR_Dusk_Dawn_v2/library.json index be06c3a3a..709967ea7 100644 --- a/usermods/LDR_Dusk_Dawn_v2/library.json +++ b/usermods/LDR_Dusk_Dawn_v2/library.json @@ -1,3 +1,4 @@ { - "name": "LDR_Dusk_Dawn_v2" + "name": "LDR_Dusk_Dawn_v2", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/MY9291/library.json b/usermods/MY9291/library.json index e4c63eaf5..9c3a33d43 100644 --- a/usermods/MY9291/library.json +++ b/usermods/MY9291/library.json @@ -1,4 +1,5 @@ { "name": "MY9291", + "build": { "libArchive": false }, "platforms": ["espressif8266"] } \ No newline at end of file diff --git a/usermods/PIR_sensor_switch/library.json b/usermods/PIR_sensor_switch/library.json index d5ebb7689..b3cbcbbff 100644 --- a/usermods/PIR_sensor_switch/library.json +++ b/usermods/PIR_sensor_switch/library.json @@ -1,3 +1,4 @@ { - "name": "PIR_sensor_switch" + "name": "PIR_sensor_switch", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/PWM_fan/library.json b/usermods/PWM_fan/library.json index a8f7a9446..8ae3d7fd6 100644 --- a/usermods/PWM_fan/library.json +++ b/usermods/PWM_fan/library.json @@ -1,6 +1,7 @@ { "name": "PWM_fan", "build": { + "libArchive": false, "extraScript": "setup_deps.py" } } \ No newline at end of file diff --git a/usermods/RTC/library.json b/usermods/RTC/library.json index 8c103e06d..688dfc2d0 100644 --- a/usermods/RTC/library.json +++ b/usermods/RTC/library.json @@ -1,3 +1,4 @@ { - "name": "RTC" + "name": "RTC", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/SN_Photoresistor/library.json b/usermods/SN_Photoresistor/library.json index 45519dfa6..c896644f7 100644 --- a/usermods/SN_Photoresistor/library.json +++ b/usermods/SN_Photoresistor/library.json @@ -1,3 +1,4 @@ { - "name": "SN_Photoresistor" + "name": "SN_Photoresistor", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/ST7789_display/library.json.disabled b/usermods/ST7789_display/library.json.disabled index abcd4635c..725e20a65 100644 --- a/usermods/ST7789_display/library.json.disabled +++ b/usermods/ST7789_display/library.json.disabled @@ -1,3 +1,4 @@ { - "name:": "ST7789_display" + "name:": "ST7789_display", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/Si7021_MQTT_HA/library.json b/usermods/Si7021_MQTT_HA/library.json index cec2edfb1..e3d7635e3 100644 --- a/usermods/Si7021_MQTT_HA/library.json +++ b/usermods/Si7021_MQTT_HA/library.json @@ -1,5 +1,6 @@ { "name": "Si7021_MQTT_HA", + "build": { "libArchive": false }, "dependencies": { "finitespace/BME280":"3.0.0", "adafruit/Adafruit Si7021 Library" : "1.5.3" diff --git a/usermods/TetrisAI_v2/library.json b/usermods/TetrisAI_v2/library.json index bfff1aa4d..54aa22d35 100644 --- a/usermods/TetrisAI_v2/library.json +++ b/usermods/TetrisAI_v2/library.json @@ -1,3 +1,4 @@ { - "name": "TetrisAI_v2" + "name": "TetrisAI_v2", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/boblight/library.json b/usermods/boblight/library.json index 12debccf5..b54fb3505 100644 --- a/usermods/boblight/library.json +++ b/usermods/boblight/library.json @@ -1,3 +1,4 @@ { - "name": "boblight" + "name": "boblight", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/buzzer/library.json b/usermods/buzzer/library.json index c6af3158b..0dbb547e3 100644 --- a/usermods/buzzer/library.json +++ b/usermods/buzzer/library.json @@ -1,3 +1,4 @@ { - "name": "buzzer" + "name": "buzzer", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/deep_sleep/library.json b/usermods/deep_sleep/library.json index 8b39b2eed..82e32c994 100644 --- a/usermods/deep_sleep/library.json +++ b/usermods/deep_sleep/library.json @@ -1,3 +1,4 @@ { - "name": "deep_sleep" + "name": "deep_sleep", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/multi_relay/library.json b/usermods/multi_relay/library.json index f1caf7d42..a5e5c6934 100644 --- a/usermods/multi_relay/library.json +++ b/usermods/multi_relay/library.json @@ -1,3 +1,4 @@ { - "name": "multi_relay" + "name": "multi_relay", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/pwm_outputs/library.json b/usermods/pwm_outputs/library.json index bcdb8d5a6..a01068bd4 100644 --- a/usermods/pwm_outputs/library.json +++ b/usermods/pwm_outputs/library.json @@ -1,3 +1,4 @@ { - "name": "pwm_outputs" + "name": "pwm_outputs", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/sd_card/library.json b/usermods/sd_card/library.json index 44d3e3495..33e8f98f2 100644 --- a/usermods/sd_card/library.json +++ b/usermods/sd_card/library.json @@ -1,3 +1,4 @@ { - "name": "sd_card" + "name": "sd_card", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/seven_segment_display/library.json b/usermods/seven_segment_display/library.json index 653c3d7ff..f78aad87b 100644 --- a/usermods/seven_segment_display/library.json +++ b/usermods/seven_segment_display/library.json @@ -1,3 +1,4 @@ { - "name": "seven_segment_display" + "name": "seven_segment_display", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/seven_segment_display_reloaded/library.json b/usermods/seven_segment_display_reloaded/library.json index 4e84e38ed..1b7d0687f 100644 --- a/usermods/seven_segment_display_reloaded/library.json +++ b/usermods/seven_segment_display_reloaded/library.json @@ -1,6 +1,7 @@ { "name": "seven_segment_display_reloaded", "build": { + "libArchive": false, "extraScript": "setup_deps.py" - } + } } \ No newline at end of file diff --git a/usermods/sht/library.json b/usermods/sht/library.json index 6849628ca..0916e9a37 100644 --- a/usermods/sht/library.json +++ b/usermods/sht/library.json @@ -1,5 +1,6 @@ { "name": "sht", + "build": { "libArchive": false }, "dependencies": { "robtillaart/SHT85": "~0.3.3" } diff --git a/usermods/smartnest/library.json b/usermods/smartnest/library.json index 9b428f6b1..3e9ea63a9 100644 --- a/usermods/smartnest/library.json +++ b/usermods/smartnest/library.json @@ -1,3 +1,4 @@ { - "name": "smartnest" + "name": "smartnest", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/stairway_wipe_basic/library.json b/usermods/stairway_wipe_basic/library.json index b75baef6b..f7d353b59 100644 --- a/usermods/stairway_wipe_basic/library.json +++ b/usermods/stairway_wipe_basic/library.json @@ -1,3 +1,4 @@ { - "name": "stairway_wipe_basic" + "name": "stairway_wipe_basic", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/usermod_rotary_brightness_color/library.json b/usermods/usermod_rotary_brightness_color/library.json index ecf73c0f9..4f7a146a0 100644 --- a/usermods/usermod_rotary_brightness_color/library.json +++ b/usermods/usermod_rotary_brightness_color/library.json @@ -1,3 +1,4 @@ { - "name": "usermod_rotary_brightness_color" + "name": "usermod_rotary_brightness_color", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/usermod_v2_HttpPullLightControl/library.json b/usermods/usermod_v2_HttpPullLightControl/library.json index a9252fc0c..870753b99 100644 --- a/usermods/usermod_v2_HttpPullLightControl/library.json +++ b/usermods/usermod_v2_HttpPullLightControl/library.json @@ -1,3 +1,4 @@ { - "name": "usermod_v2_HttpPullLightControl" + "name": "usermod_v2_HttpPullLightControl", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/usermod_v2_animartrix/library.json b/usermods/usermod_v2_animartrix/library.json index 4552be330..667572bad 100644 --- a/usermods/usermod_v2_animartrix/library.json +++ b/usermods/usermod_v2_animartrix/library.json @@ -1,5 +1,6 @@ { "name": "animartrix", + "build": { "libArchive": false }, "dependencies": { "Animartrix": "https://github.com/netmindz/animartrix.git#b172586" } diff --git a/usermods/usermod_v2_auto_save/library.json b/usermods/usermod_v2_auto_save/library.json index d703487a7..127767eb0 100644 --- a/usermods/usermod_v2_auto_save/library.json +++ b/usermods/usermod_v2_auto_save/library.json @@ -1,3 +1,4 @@ { - "name": "auto_save" + "name": "auto_save", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/usermod_v2_four_line_display_ALT/library.json b/usermods/usermod_v2_four_line_display_ALT/library.json index 87a690f03..b16448223 100644 --- a/usermods/usermod_v2_four_line_display_ALT/library.json +++ b/usermods/usermod_v2_four_line_display_ALT/library.json @@ -1,5 +1,6 @@ { "name": "four_line_display_ALT", + "build": { "libArchive": false }, "dependencies": { "U8g2": "~2.34.4", "Wire": "" diff --git a/usermods/usermod_v2_klipper_percentage/library.json b/usermods/usermod_v2_klipper_percentage/library.json index 7a2df6b23..962dda14e 100644 --- a/usermods/usermod_v2_klipper_percentage/library.json +++ b/usermods/usermod_v2_klipper_percentage/library.json @@ -1,3 +1,4 @@ { - "name": "usermod_v2_klipper_percentage" + "name": "usermod_v2_klipper_percentage", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/usermod_v2_ping_pong_clock/library.json b/usermods/usermod_v2_ping_pong_clock/library.json index d6c079e58..4b272eca4 100644 --- a/usermods/usermod_v2_ping_pong_clock/library.json +++ b/usermods/usermod_v2_ping_pong_clock/library.json @@ -1,3 +1,4 @@ { - "name": "usermod_v2_ping_pong_clock" + "name": "usermod_v2_ping_pong_clock", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/usermod_v2_rotary_encoder_ui_ALT/library.json b/usermods/usermod_v2_rotary_encoder_ui_ALT/library.json index ddb6334b1..7c828d087 100644 --- a/usermods/usermod_v2_rotary_encoder_ui_ALT/library.json +++ b/usermods/usermod_v2_rotary_encoder_ui_ALT/library.json @@ -1,6 +1,7 @@ { "name": "rotary_encoder_ui_ALT", "build": { + "libArchive": false, "extraScript": "setup_deps.py" } } \ No newline at end of file diff --git a/usermods/usermod_v2_word_clock/library.json b/usermods/usermod_v2_word_clock/library.json index b0dcebc6e..0ea99d810 100644 --- a/usermods/usermod_v2_word_clock/library.json +++ b/usermods/usermod_v2_word_clock/library.json @@ -1,3 +1,4 @@ { - "name": "usermod_v2_word_clock" + "name": "usermod_v2_word_clock", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/wizlights/library.json b/usermods/wizlights/library.json index 114424e5d..0bfc097c7 100644 --- a/usermods/wizlights/library.json +++ b/usermods/wizlights/library.json @@ -1,3 +1,4 @@ { - "name": "wizlights" + "name": "wizlights", + "build": { "libArchive": false } } \ No newline at end of file diff --git a/usermods/word-clock-matrix/library.json b/usermods/word-clock-matrix/library.json index afeae5025..7bc3919de 100644 --- a/usermods/word-clock-matrix/library.json +++ b/usermods/word-clock-matrix/library.json @@ -1,3 +1,4 @@ { - "name": "word-clock-matrix" + "name": "word-clock-matrix", + "build": { "libArchive": false } } \ No newline at end of file From 849d5e6667c2f5e682281d21f46dd995606e006c Mon Sep 17 00:00:00 2001 From: Will Miles Date: Tue, 6 May 2025 22:37:57 -0400 Subject: [PATCH 04/18] Add libArchive to other usermods Not all of them were removed by the reverted commit; re-do the rest of them. --- usermods/BH1750_v2/library.json | 2 +- usermods/BME68X_v2/library.json | 1 + usermods/EXAMPLE/library.json | 1 + usermods/INA226_v2/library.json | 1 + usermods/usermod_v2_RF433/library.json | 1 + usermods/usermod_v2_brightness_follow_sun/library.json | 3 ++- 6 files changed, 7 insertions(+), 2 deletions(-) diff --git a/usermods/BH1750_v2/library.json b/usermods/BH1750_v2/library.json index 8323d1abb..4e32099b0 100644 --- a/usermods/BH1750_v2/library.json +++ b/usermods/BH1750_v2/library.json @@ -1,6 +1,6 @@ { "name": "BH1750_v2", - "build": { "libArchive": false }, + "build": { "libArchive": false }, "dependencies": { "claws/BH1750":"^1.2.0" } diff --git a/usermods/BME68X_v2/library.json b/usermods/BME68X_v2/library.json index 2f1e1a310..b315aa5d4 100644 --- a/usermods/BME68X_v2/library.json +++ b/usermods/BME68X_v2/library.json @@ -1,5 +1,6 @@ { "name": "BME68X", + "build": { "libArchive": false }, "dependencies": { "boschsensortec/BSEC Software Library":"^1.8.1492" } diff --git a/usermods/EXAMPLE/library.json b/usermods/EXAMPLE/library.json index dd8a7e5dd..d0dc2f88e 100644 --- a/usermods/EXAMPLE/library.json +++ b/usermods/EXAMPLE/library.json @@ -1,4 +1,5 @@ { "name": "EXAMPLE", + "build": { "libArchive": false }, "dependencies": {} } diff --git a/usermods/INA226_v2/library.json b/usermods/INA226_v2/library.json index ab6c81fbd..34fcd3683 100644 --- a/usermods/INA226_v2/library.json +++ b/usermods/INA226_v2/library.json @@ -1,5 +1,6 @@ { "name": "INA226_v2", + "build": { "libArchive": false }, "dependencies": { "wollewald/INA226_WE":"~1.2.9" } diff --git a/usermods/usermod_v2_RF433/library.json b/usermods/usermod_v2_RF433/library.json index d809d3a0d..d8de29b8a 100644 --- a/usermods/usermod_v2_RF433/library.json +++ b/usermods/usermod_v2_RF433/library.json @@ -1,5 +1,6 @@ { "name": "usermod_v2_RF433", + "build": { "libArchive": false }, "dependencies": { "sui77/rc-switch":"2.6.4" } diff --git a/usermods/usermod_v2_brightness_follow_sun/library.json b/usermods/usermod_v2_brightness_follow_sun/library.json index 6120d873e..dec00e55b 100644 --- a/usermods/usermod_v2_brightness_follow_sun/library.json +++ b/usermods/usermod_v2_brightness_follow_sun/library.json @@ -1,3 +1,4 @@ { - "name": "brightness_follow_sun" + "name": "brightness_follow_sun", + "build": { "libArchive": false } } \ No newline at end of file From ee3864175d90fb04030ab592cd73ae9e0b2f8673 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Tue, 6 May 2025 21:55:06 -0400 Subject: [PATCH 05/18] load_usermods: Make missing libArchive an error Rather than try and fail to add this property, abort if it's missing from any requested usermod. --- pio-scripts/load_usermods.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index eeebcbf27..8cf625ff6 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -2,6 +2,8 @@ Import('env') import os.path from collections import deque from pathlib import Path # For OS-agnostic path manipulation +from click import secho +from SCons.Script import Exit from platformio.builder.tools.piolib import LibBuilderBase from platformio.package.manager.library import LibraryPackageManager @@ -105,6 +107,7 @@ def wrapped_ConfigureProjectLibBuilder(xenv): for dep in result.depbuilders: cached_add_includes(dep, processed_deps, extra_include_dirs) + broken_usermods = [] for dep in result.depbuilders: if is_wled_module(dep): # Add the wled folder to the include path @@ -114,9 +117,15 @@ def wrapped_ConfigureProjectLibBuilder(xenv): dep.env.PrependUnique(CPPPATH=str(dir)) # Enforce that libArchive is not set; we must link them directly to the executable if dep.lib_archive: - build = dep._manifest.get("build", {}) - build["libArchive"] = False - dep._manifest["build"] = build + broken_usermods.append(dep) + + if broken_usermods: + broken_usermods = [usermod.name for usermod in broken_usermods] + secho( + f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- modules will not compile in correctly", + fg="red", + err=True) + Exit(1) return result From 999637f8adfec4fcef5ecfb41bd141c8ffba57bb Mon Sep 17 00:00:00 2001 From: Will Miles Date: Mon, 19 May 2025 16:37:07 -0400 Subject: [PATCH 06/18] Validate usermods at link time Add additional validation of the linker .map output to confirm that the correct usermods were added. --- pio-scripts/validate_usermods.py | 92 ++++++++++++++++++++++++++++++++ platformio.ini | 1 + 2 files changed, 93 insertions(+) create mode 100644 pio-scripts/validate_usermods.py diff --git a/pio-scripts/validate_usermods.py b/pio-scripts/validate_usermods.py new file mode 100644 index 000000000..50d51f99f --- /dev/null +++ b/pio-scripts/validate_usermods.py @@ -0,0 +1,92 @@ +import re +import sys +from pathlib import Path # For OS-agnostic path manipulation +from click import secho +from SCons.Script import Action, Exit +from platformio import util + +def read_lines(p: Path): + """ Read in the contents of a file for analysis """ + with p.open("r", encoding="utf-8", errors="ignore") as f: + return f.readlines() + +def check_map_file_objects(map_file: list[str], usermod_dirs: list[str]) -> set[str]: + """ Checks that an object file from each usermod_dir appears in the linked output + + Returns the (sub)set of usermod_dirs that are found in the output ELF + """ + # Pattern to match symbols in object directories + # Join directories into alternation + usermod_dir_regex = "|".join([re.escape(dir) for dir in usermod_dirs]) + # Matches nonzero address, any size, and any path in a matching directory + object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+/(" + usermod_dir_regex + r")/\S+\.o") + + found = set() + for line in map_file: + matches = object_path_regex.findall(line) + for m in matches: + found.add(m) + return found + +def count_registered_usermods(map_file: list[str]) -> int: + """ Returns the number of usermod objects in the usermod list """ + # Count the number of entries in the usermods table section + return len([x for x in map_file if ".dtors.tbl.usermods.1" in x]) + + +def validate_map_file(source, target, env): + """ Validate that all usermods appear in the output build """ + build_dir = Path(env.subst("$BUILD_DIR")) + map_file_path = build_dir / env.subst("${PROGNAME}.map") + + if not map_file_path.exists(): + secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True) + Exit(1) + + # Load project settings + usermods = env.GetProjectOption("custom_usermods","").split() + libdeps = env.GetProjectOption("lib_deps", []) + lib_builders = env.GetLibBuilders() + + secho(f"INFO: Expecting {len(usermods)} usermods: {', '.join(usermods)}") + + # Map the usermods to libdeps; every usermod should have one + usermod_dirs = [] + for mod in usermods: + modstr = f"{mod} = symlink://" + this_mod_libdeps = [libdep[len(modstr):] for libdep in libdeps if libdep.startswith(modstr)] + if not this_mod_libdeps: + secho( + f"ERROR: Usermod {mod} not found in build libdeps!", + fg="red", + err=True) + Exit(1) + # Save only the final folder name + usermod_dir = Path(this_mod_libdeps[0]).name + # Search lib_builders + this_mod_builders = [builder for builder in lib_builders if Path(builder.src_dir).name == usermod_dir] + if not this_mod_builders: + secho( + f"ERROR: Usermod {mod} not found in library builders!", + fg="red", + err=True) + Exit(1) + usermod_dirs.append(usermod_dir) + + # Now parse the map file + map_file_contents = read_lines(map_file_path) + confirmed_usermods = check_map_file_objects(map_file_contents, usermod_dirs) + usermod_object_count = count_registered_usermods(map_file_contents) + + secho(f"INFO: {len(usermod_dirs)}/{len(usermods)} libraries linked via custom_usermods, producing {usermod_object_count} usermod object entries") + missing_usermods = confirmed_usermods.difference(usermod_dirs) + if missing_usermods: + secho( + f"ERROR: No object files from {missing_usermods} found in linked output!", + fg="red", + err=True) + Exit(1) + return None + +Import("env") +env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking map file...')) diff --git a/platformio.ini b/platformio.ini index a7485244c..b713a2d39 100644 --- a/platformio.ini +++ b/platformio.ini @@ -116,6 +116,7 @@ extra_scripts = pre:pio-scripts/user_config_copy.py pre:pio-scripts/load_usermods.py pre:pio-scripts/build_ui.py + post:pio-scripts/validate_usermods.py ;; double-check the build output usermods ; post:pio-scripts/obj-dump.py ;; convenience script to create a disassembly dump of the firmware (hardcore debugging) # ------------------------------------------------------------------------------ From 24ab2952ee50dc6bd13bf96eafc3584531631cb2 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Mon, 19 May 2025 16:53:08 -0400 Subject: [PATCH 07/18] Add unambiguous usermod list to info Neither the info panel nor the settings dialog can be trusted to accurately report the usermod list: - Not all usermods necessarily add to the info panel - Not all usermods necessarily add to the config page - #4609 is required for the config page to be correct Add a short list to the info object that lists the loaded usermod IDs. This is not displayed via the UI, but can be queried with curl or web debug tools. To be removed when usermod loading is working well. --- wled00/um_manager.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/wled00/um_manager.cpp b/wled00/um_manager.cpp index 9bfb7e737..1a7cc2269 100644 --- a/wled00/um_manager.cpp +++ b/wled00/um_manager.cpp @@ -39,7 +39,13 @@ bool UsermodManager::getUMData(um_data_t **data, uint8_t mod_id) { return false; } void UsermodManager::addToJsonState(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->addToJsonState(obj); } -void UsermodManager::addToJsonInfo(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->addToJsonInfo(obj); } +void UsermodManager::addToJsonInfo(JsonObject& obj) { + auto um_id_list = obj.createNestedArray("um"); + for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) { + um_id_list.add((*mod)->getId()); + (*mod)->addToJsonInfo(obj); + } +} void UsermodManager::readFromJsonState(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->readFromJsonState(obj); } void UsermodManager::addToConfig(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->addToConfig(obj); } bool UsermodManager::readFromConfig(JsonObject& obj) { From ac61eb4b1b2ff260381dad4284ecc734cdf7479d Mon Sep 17 00:00:00 2001 From: Will Miles Date: Mon, 19 May 2025 17:41:11 -0400 Subject: [PATCH 08/18] validate_usermods: Fix inverted check Difference direction was inverted. It's tough to test when it always works correctly on your local machine! H/t @coderabbitai --- pio-scripts/validate_usermods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pio-scripts/validate_usermods.py b/pio-scripts/validate_usermods.py index 50d51f99f..82ea5e07d 100644 --- a/pio-scripts/validate_usermods.py +++ b/pio-scripts/validate_usermods.py @@ -79,7 +79,7 @@ def validate_map_file(source, target, env): usermod_object_count = count_registered_usermods(map_file_contents) secho(f"INFO: {len(usermod_dirs)}/{len(usermods)} libraries linked via custom_usermods, producing {usermod_object_count} usermod object entries") - missing_usermods = confirmed_usermods.difference(usermod_dirs) + missing_usermods = set(usermod_dirs).difference(confirmed_usermods) if missing_usermods: secho( f"ERROR: No object files from {missing_usermods} found in linked output!", From 358e38e0562b6082e26772d7407e8eed357a44a5 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 22 May 2025 08:41:59 -0400 Subject: [PATCH 09/18] validate_usermods: Ensure map file is created Not all of our platforms create one by default; ensure it's produced. --- pio-scripts/validate_usermods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pio-scripts/validate_usermods.py b/pio-scripts/validate_usermods.py index 82ea5e07d..1291a5613 100644 --- a/pio-scripts/validate_usermods.py +++ b/pio-scripts/validate_usermods.py @@ -89,4 +89,4 @@ def validate_map_file(source, target, env): return None Import("env") -env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking map file...')) +env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")]) From 242df4b04994110dd8351b85715558ccc67f5ab0 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 22 May 2025 08:43:52 -0400 Subject: [PATCH 10/18] validate_usermods: Fix old ESP32 platform The modern linker used with new platforms (ESP8266, ESP32 >v4) always produces paths in the map file with slash; however the old linker for the old ESP32 platform instead produces paths with backslash when building on Windows. Match both types as a path separator when scanning linked symbols. --- pio-scripts/validate_usermods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pio-scripts/validate_usermods.py b/pio-scripts/validate_usermods.py index 1291a5613..7b3bdf8e1 100644 --- a/pio-scripts/validate_usermods.py +++ b/pio-scripts/validate_usermods.py @@ -19,7 +19,7 @@ def check_map_file_objects(map_file: list[str], usermod_dirs: list[str]) -> set[ # Join directories into alternation usermod_dir_regex = "|".join([re.escape(dir) for dir in usermod_dirs]) # Matches nonzero address, any size, and any path in a matching directory - object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+/(" + usermod_dir_regex + r")/\S+\.o") + object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o") found = set() for line in map_file: From 7ea510e75b666f2b12ecba2b0f2c5d8d3c1722e1 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 22 May 2025 08:44:05 -0400 Subject: [PATCH 11/18] validate_usermods: Improve message --- pio-scripts/validate_usermods.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pio-scripts/validate_usermods.py b/pio-scripts/validate_usermods.py index 7b3bdf8e1..d3cf5ea8c 100644 --- a/pio-scripts/validate_usermods.py +++ b/pio-scripts/validate_usermods.py @@ -90,3 +90,4 @@ def validate_map_file(source, target, env): Import("env") env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")]) +env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked usermods in map file...')) From 792a7aa0815be4fc3a8e7779f5d0d93a49d3cc51 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Thu, 22 May 2025 09:13:54 -0400 Subject: [PATCH 12/18] load_usermods: Resolve folder paths Ensure all paths used in usermod symlinks are fully resolved, including any case correctness issues on Windows. Apparently PlatformIO does not handle symlink files correctly on Windows if there are case differences between cwd and the resolved path. --- pio-scripts/load_usermods.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index 8cf625ff6..4e0457a0c 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -7,7 +7,7 @@ from SCons.Script import Exit from platformio.builder.tools.piolib import LibBuilderBase from platformio.package.manager.library import LibraryPackageManager -usermod_dir = Path(env["PROJECT_DIR"]) / "usermods" +usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods" # "usermods" environment: expand list of usermods to everything in the folder if env['PIOENV'] == "usermods": @@ -48,7 +48,7 @@ if usermods: src_dir = proj.get("platformio", "src_dir") src_dir = src_dir.replace('\\','/') mod_paths = {mod: find_usermod(mod) for mod in usermods.split()} - usermods = [f"{mod} = symlink://{path}" for mod, path in mod_paths.items()] + usermods = [f"{mod} = symlink://{path.resolve()}" for mod, path in mod_paths.items()] proj.set("env:" + env['PIOENV'], 'lib_deps', deps + usermods) # Force usermods to be installed in to the environment build state before the LDF runs # Otherwise we won't be able to see them until it's too late to change their paths for LDF From 75cd411073ad9e6c984b0f9ffc0c4efae8bede33 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 24 May 2025 14:13:08 -0400 Subject: [PATCH 13/18] Improve all-usermod handling Use a magic custom_usermods string instead of a magic environment name; and disable the validation script as it triggers on the non- platform-compatible mods. --- pio-scripts/load_usermods.py | 42 +++++++++----------------------- pio-scripts/validate_usermods.py | 5 ++-- platformio.ini | 2 +- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index 4e0457a0c..4f986d6fe 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -9,12 +9,6 @@ from platformio.package.manager.library import LibraryPackageManager usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods" -# "usermods" environment: expand list of usermods to everything in the folder -if env['PIOENV'] == "usermods": - # Add all usermods - all_usermods = [f for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()] - env.GetProjectConfig().set(f"env:usermods", 'custom_usermods', " ".join([f.name for f in all_usermods])) - # Utility functions def find_usermod(mod: str) -> Path: """Locate this library in the usermods folder. @@ -41,38 +35,26 @@ def is_wled_module(dep: LibBuilderBase) -> bool: ## Script starts here # Process usermod option usermods = env.GetProjectOption("custom_usermods","") + +# Handle "all usermods" case +if usermods == '*': + usermods = [f.name for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()] + # Update the environment, as many modules use scripts to detect their dependencies + env.GetProjectConfig().set("env:" + env['PIOENV'], 'custom_usermods', " ".join(usermods)) + # Leave a note for the validation script + env.GetProjectConfig().set("env:" + env['PIOENV'], 'custom_all_usermods_enabled', "1") +else: + usermods = usermods.split() + if usermods: # Inject usermods in to project lib_deps proj = env.GetProjectConfig() deps = env.GetProjectOption('lib_deps') src_dir = proj.get("platformio", "src_dir") src_dir = src_dir.replace('\\','/') - mod_paths = {mod: find_usermod(mod) for mod in usermods.split()} + mod_paths = {mod: find_usermod(mod) for mod in usermods} usermods = [f"{mod} = symlink://{path.resolve()}" for mod, path in mod_paths.items()] proj.set("env:" + env['PIOENV'], 'lib_deps', deps + usermods) - # Force usermods to be installed in to the environment build state before the LDF runs - # Otherwise we won't be able to see them until it's too late to change their paths for LDF - # Logic is largely borrowed from PlaformIO internals - not_found_specs = [] - for spec in usermods: - found = False - for storage_dir in env.GetLibSourceDirs(): - #print(f"Checking {storage_dir} for {spec}") - lm = LibraryPackageManager(storage_dir) - if lm.get_package(spec): - #print("Found!") - found = True - break - if not found: - #print("Missing!") - not_found_specs.append(spec) - if not_found_specs: - lm = LibraryPackageManager( - env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV")) - ) - for spec in not_found_specs: - #print(f"LU: forcing install of {spec}") - lm.install(spec) # Utility function for assembling usermod include paths diff --git a/pio-scripts/validate_usermods.py b/pio-scripts/validate_usermods.py index d3cf5ea8c..a1a1e3c24 100644 --- a/pio-scripts/validate_usermods.py +++ b/pio-scripts/validate_usermods.py @@ -89,5 +89,6 @@ def validate_map_file(source, target, env): return None Import("env") -env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")]) -env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked usermods in map file...')) +if not env.GetProjectOption("custom_all_usermods_enabled",""): # TODO: fix handling of platform mismatches + env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")]) + env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked usermods in map file...')) diff --git a/platformio.ini b/platformio.ini index b713a2d39..e1a5014b0 100644 --- a/platformio.ini +++ b/platformio.ini @@ -660,5 +660,5 @@ build_flags = ${common.build_flags} ${esp32_idf_V4.build_flags} -D WLED_RELEASE_ lib_deps = ${esp32_idf_V4.lib_deps} monitor_filters = esp32_exception_decoder board_build.flash_mode = dio -; custom_usermods = *every folder with library.json* -- injected by pio-scripts/load_usermods.py +custom_usermods = * ; Expands to all usermods in usermods folder board_build.partitions = ${esp32.extreme_partitions} ; We're gonna need a bigger boat From 0a7d3a9d9b7437d940739c832f615481b708474a Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 24 May 2025 22:16:01 -0400 Subject: [PATCH 14/18] load_usermods: Simplify load code Remove all the unnecessary bits. --- pio-scripts/load_usermods.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index 4f986d6fe..d50bf196b 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -48,14 +48,8 @@ else: if usermods: # Inject usermods in to project lib_deps - proj = env.GetProjectConfig() - deps = env.GetProjectOption('lib_deps') - src_dir = proj.get("platformio", "src_dir") - src_dir = src_dir.replace('\\','/') - mod_paths = {mod: find_usermod(mod) for mod in usermods} - usermods = [f"{mod} = symlink://{path.resolve()}" for mod, path in mod_paths.items()] - proj.set("env:" + env['PIOENV'], 'lib_deps', deps + usermods) - + symlinks = [f"symlink://{find_usermod(mod).resolve()}" for mod in usermods] + env.GetProjectConfig().set("env:" + env['PIOENV'], 'lib_deps', env.GetProjectOption('lib_deps') + symlinks) # Utility function for assembling usermod include paths def cached_add_includes(dep, dep_cache: set, includes: deque): From 75c95d88e25698e09d5806f77249cdde1599cf3b Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 24 May 2025 22:18:22 -0400 Subject: [PATCH 15/18] usermods/*/setup_deps.py: Check lib_deps for deps Check the safest possible location for final information on what components are actually being linked in. This demonstrates a safe approach that works even for out-of-tree modules. --- usermods/PWM_fan/setup_deps.py | 9 +++++---- usermods/seven_segment_display_reloaded/setup_deps.py | 7 ++++--- usermods/usermod_v2_rotary_encoder_ui_ALT/setup_deps.py | 6 +++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/usermods/PWM_fan/setup_deps.py b/usermods/PWM_fan/setup_deps.py index b8f7276c5..11879079a 100644 --- a/usermods/PWM_fan/setup_deps.py +++ b/usermods/PWM_fan/setup_deps.py @@ -1,11 +1,12 @@ +from platformio.package.meta import PackageSpec Import('env') -usermods = env.GetProjectOption("custom_usermods","").split() +libs = [PackageSpec(lib).name for lib in env.GetProjectOption("lib_deps",[])] # Check for dependencies -if "Temperature" in usermods: +if "Temperature" in libs: env.Append(CPPDEFINES=[("USERMOD_DALLASTEMPERATURE")]) -elif "sht" in usermods: +elif "sht" in libs: env.Append(CPPDEFINES=[("USERMOD_SHT")]) -elif "PWM_fan" in usermods: # The script can be run if this module was previously selected +elif "PWM_fan" in libs: # The script can be run if this module was previously selected raise RuntimeError("PWM_fan usermod requires Temperature or sht to be enabled") diff --git a/usermods/seven_segment_display_reloaded/setup_deps.py b/usermods/seven_segment_display_reloaded/setup_deps.py index dd28f5fe9..1c51accce 100644 --- a/usermods/seven_segment_display_reloaded/setup_deps.py +++ b/usermods/seven_segment_display_reloaded/setup_deps.py @@ -1,9 +1,10 @@ +from platformio.package.meta import PackageSpec Import('env') -usermods = env.GetProjectOption("custom_usermods","").split() +libs = [PackageSpec(lib).name for lib in env.GetProjectOption("lib_deps",[])] # Check for partner usermods -if "SN_Photoresistor" in usermods: +if "SN_Photoresistor" in libs: env.Append(CPPDEFINES=[("USERMOD_SN_PHOTORESISTOR")]) -if any(mod in ("BH1750_v2", "BH1750") for mod in usermods): +if any(mod in ("BH1750_v2", "BH1750") for mod in libs): env.Append(CPPDEFINES=[("USERMOD_BH1750")]) diff --git a/usermods/usermod_v2_rotary_encoder_ui_ALT/setup_deps.py b/usermods/usermod_v2_rotary_encoder_ui_ALT/setup_deps.py index a6b565951..ed579bc12 100644 --- a/usermods/usermod_v2_rotary_encoder_ui_ALT/setup_deps.py +++ b/usermods/usermod_v2_rotary_encoder_ui_ALT/setup_deps.py @@ -1,8 +1,8 @@ +from platformio.package.meta import PackageSpec Import('env') - -usermods = env.GetProjectOption("custom_usermods","").split() +libs = [PackageSpec(lib).name for lib in env.GetProjectOption("lib_deps",[])] # Check for partner usermod # Allow both "usermod_v2" and unqualified syntax -if any(mod in ("four_line_display_ALT", "usermod_v2_four_line_display_ALT") for mod in usermods): +if any(mod in ("four_line_display_ALT", "usermod_v2_four_line_display_ALT") for mod in libs): env.Append(CPPDEFINES=[("USERMOD_FOUR_LINE_DISPLAY")]) From 309c8d67f3e66298ceb2fcc74267865ecbd3b5a6 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 24 May 2025 23:14:22 -0400 Subject: [PATCH 16/18] Generalize module link validation Perform validation for external modules, too. --- pio-scripts/load_usermods.py | 4 -- pio-scripts/validate_modules.py | 95 ++++++++++++++++++++++++++++++++ pio-scripts/validate_usermods.py | 94 ------------------------------- platformio.ini | 2 +- 4 files changed, 96 insertions(+), 99 deletions(-) create mode 100644 pio-scripts/validate_modules.py delete mode 100644 pio-scripts/validate_usermods.py diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index d50bf196b..31e211fc0 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -39,10 +39,6 @@ usermods = env.GetProjectOption("custom_usermods","") # Handle "all usermods" case if usermods == '*': usermods = [f.name for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()] - # Update the environment, as many modules use scripts to detect their dependencies - env.GetProjectConfig().set("env:" + env['PIOENV'], 'custom_usermods', " ".join(usermods)) - # Leave a note for the validation script - env.GetProjectConfig().set("env:" + env['PIOENV'], 'custom_all_usermods_enabled', "1") else: usermods = usermods.split() diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py new file mode 100644 index 000000000..eb6ebb446 --- /dev/null +++ b/pio-scripts/validate_modules.py @@ -0,0 +1,95 @@ +import re +import sys +from pathlib import Path # For OS-agnostic path manipulation +from typing import Iterable +from click import secho +from SCons.Script import Action, Exit +from platformio import util +from platformio.builder.tools.piolib import LibBuilderBase + + +def is_wled_module(env, dep: LibBuilderBase) -> bool: + """Returns true if the specified library is a wled module + """ + usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods" + return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-") + + +def read_lines(p: Path): + """ Read in the contents of a file for analysis """ + with p.open("r", encoding="utf-8", errors="ignore") as f: + return f.readlines() + + +def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]: + """ Identify which dirs contributed to the final build + + Returns the (sub)set of dirs that are found in the output ELF + """ + # Pattern to match symbols in object directories + # Join directories into alternation + usermod_dir_regex = "|".join([re.escape(dir) for dir in dirs]) + # Matches nonzero address, any size, and any path in a matching directory + object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o") + + found = set() + for line in map_file: + matches = object_path_regex.findall(line) + for m in matches: + found.add(m) + return found + + +def count_usermod_objects(map_file: list[str]) -> int: + """ Returns the number of usermod objects in the usermod list """ + # Count the number of entries in the usermods table section + return len([x for x in map_file if ".dtors.tbl.usermods.1" in x]) + + +def validate_map_file(source, target, env): + """ Validate that all modules appear in the output build """ + build_dir = Path(env.subst("$BUILD_DIR")) + map_file_path = build_dir / env.subst("${PROGNAME}.map") + + if not map_file_path.exists(): + secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True) + Exit(1) + + # Identify the WLED module source directories + module_lib_builders = [builder for builder in env.GetLibBuilders() if is_wled_module(env, builder)] + + if env.GetProjectOption("custom_usermods","") == "*": + # All usermods build; filter non-platform-OK modules + module_lib_builders = [builder for builder in module_lib_builders if env.IsCompatibleLibBuilder(builder)] + else: + incompatible_builders = [builder for builder in module_lib_builders if not env.IsCompatibleLibBuilder(builder)] + if incompatible_builders: + secho( + f"ERROR: Modules {[b.name for b in incompatible_builders]} are not compatible with this platform!", + fg="red", + err=True) + Exit(1) + pass + + # Extract the values we care about + modules = {Path(builder.build_dir).name: builder.name for builder in module_lib_builders} + secho(f"INFO: {len(modules)} libraries linked as WLED optional/user modules") + + # Now parse the map file + map_file_contents = read_lines(map_file_path) + usermod_object_count = count_usermod_objects(map_file_contents) + secho(f"INFO: {usermod_object_count} usermod object entries") + + confirmed_modules = check_map_file_objects(map_file_contents, modules.keys()) + missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules] + if missing_modules: + secho( + f"ERROR: No object files from {missing_modules} found in linked output!", + fg="red", + err=True) + Exit(1) + return None + +Import("env") +env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")]) +env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file')) diff --git a/pio-scripts/validate_usermods.py b/pio-scripts/validate_usermods.py deleted file mode 100644 index a1a1e3c24..000000000 --- a/pio-scripts/validate_usermods.py +++ /dev/null @@ -1,94 +0,0 @@ -import re -import sys -from pathlib import Path # For OS-agnostic path manipulation -from click import secho -from SCons.Script import Action, Exit -from platformio import util - -def read_lines(p: Path): - """ Read in the contents of a file for analysis """ - with p.open("r", encoding="utf-8", errors="ignore") as f: - return f.readlines() - -def check_map_file_objects(map_file: list[str], usermod_dirs: list[str]) -> set[str]: - """ Checks that an object file from each usermod_dir appears in the linked output - - Returns the (sub)set of usermod_dirs that are found in the output ELF - """ - # Pattern to match symbols in object directories - # Join directories into alternation - usermod_dir_regex = "|".join([re.escape(dir) for dir in usermod_dirs]) - # Matches nonzero address, any size, and any path in a matching directory - object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o") - - found = set() - for line in map_file: - matches = object_path_regex.findall(line) - for m in matches: - found.add(m) - return found - -def count_registered_usermods(map_file: list[str]) -> int: - """ Returns the number of usermod objects in the usermod list """ - # Count the number of entries in the usermods table section - return len([x for x in map_file if ".dtors.tbl.usermods.1" in x]) - - -def validate_map_file(source, target, env): - """ Validate that all usermods appear in the output build """ - build_dir = Path(env.subst("$BUILD_DIR")) - map_file_path = build_dir / env.subst("${PROGNAME}.map") - - if not map_file_path.exists(): - secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True) - Exit(1) - - # Load project settings - usermods = env.GetProjectOption("custom_usermods","").split() - libdeps = env.GetProjectOption("lib_deps", []) - lib_builders = env.GetLibBuilders() - - secho(f"INFO: Expecting {len(usermods)} usermods: {', '.join(usermods)}") - - # Map the usermods to libdeps; every usermod should have one - usermod_dirs = [] - for mod in usermods: - modstr = f"{mod} = symlink://" - this_mod_libdeps = [libdep[len(modstr):] for libdep in libdeps if libdep.startswith(modstr)] - if not this_mod_libdeps: - secho( - f"ERROR: Usermod {mod} not found in build libdeps!", - fg="red", - err=True) - Exit(1) - # Save only the final folder name - usermod_dir = Path(this_mod_libdeps[0]).name - # Search lib_builders - this_mod_builders = [builder for builder in lib_builders if Path(builder.src_dir).name == usermod_dir] - if not this_mod_builders: - secho( - f"ERROR: Usermod {mod} not found in library builders!", - fg="red", - err=True) - Exit(1) - usermod_dirs.append(usermod_dir) - - # Now parse the map file - map_file_contents = read_lines(map_file_path) - confirmed_usermods = check_map_file_objects(map_file_contents, usermod_dirs) - usermod_object_count = count_registered_usermods(map_file_contents) - - secho(f"INFO: {len(usermod_dirs)}/{len(usermods)} libraries linked via custom_usermods, producing {usermod_object_count} usermod object entries") - missing_usermods = set(usermod_dirs).difference(confirmed_usermods) - if missing_usermods: - secho( - f"ERROR: No object files from {missing_usermods} found in linked output!", - fg="red", - err=True) - Exit(1) - return None - -Import("env") -if not env.GetProjectOption("custom_all_usermods_enabled",""): # TODO: fix handling of platform mismatches - env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")]) - env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked usermods in map file...')) diff --git a/platformio.ini b/platformio.ini index e1a5014b0..9bdf58d34 100644 --- a/platformio.ini +++ b/platformio.ini @@ -116,7 +116,7 @@ extra_scripts = pre:pio-scripts/user_config_copy.py pre:pio-scripts/load_usermods.py pre:pio-scripts/build_ui.py - post:pio-scripts/validate_usermods.py ;; double-check the build output usermods + post:pio-scripts/validate_modules.py ;; double-check the build output usermods ; post:pio-scripts/obj-dump.py ;; convenience script to create a disassembly dump of the firmware (hardcore debugging) # ------------------------------------------------------------------------------ From e80a7c6b757a6c17b58a224c6442ce010fb41993 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 24 May 2025 23:15:36 -0400 Subject: [PATCH 17/18] usermod_v2_HttpPullLightControl: Add usermod object The module instance was missing. --- .../usermod_v2_HttpPullLightControl.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/usermods/usermod_v2_HttpPullLightControl/usermod_v2_HttpPullLightControl.cpp b/usermods/usermod_v2_HttpPullLightControl/usermod_v2_HttpPullLightControl.cpp index b908b057c..44a2726ed 100644 --- a/usermods/usermod_v2_HttpPullLightControl/usermod_v2_HttpPullLightControl.cpp +++ b/usermods/usermod_v2_HttpPullLightControl/usermod_v2_HttpPullLightControl.cpp @@ -4,6 +4,9 @@ const char HttpPullLightControl::_name[] PROGMEM = "HttpPullLightControl"; const char HttpPullLightControl::_enabled[] PROGMEM = "Enable"; +static HttpPullLightControl http_pull_usermod; +REGISTER_USERMOD(http_pull_usermod); + void HttpPullLightControl::setup() { //Serial.begin(115200); From f3623158d7051ef792bf3f3243ebee90fba794e9 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 25 May 2025 08:33:27 -0400 Subject: [PATCH 18/18] Usermod script cleanup Fix whitespace and remove unused imports --- pio-scripts/load_usermods.py | 8 +++----- pio-scripts/validate_modules.py | 11 ++++------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index 31e211fc0..146cb1f87 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -1,11 +1,9 @@ Import('env') -import os.path from collections import deque from pathlib import Path # For OS-agnostic path manipulation from click import secho from SCons.Script import Exit from platformio.builder.tools.piolib import LibBuilderBase -from platformio.package.manager.library import LibraryPackageManager usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods" @@ -21,7 +19,7 @@ def find_usermod(mod: str) -> Path: return mp mp = usermod_dir / f"{mod}_v2" if mp.exists(): - return mp + return mp mp = usermod_dir / f"usermod_v2_{mod}" if mp.exists(): return mp @@ -50,7 +48,7 @@ if usermods: # Utility function for assembling usermod include paths def cached_add_includes(dep, dep_cache: set, includes: deque): """ Add dep's include paths to includes if it's not in the cache """ - if dep not in dep_cache: + if dep not in dep_cache: dep_cache.add(dep) for include in dep.get_include_dirs(): if include not in includes: @@ -96,7 +94,7 @@ def wrapped_ConfigureProjectLibBuilder(xenv): secho( f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- modules will not compile in correctly", fg="red", - err=True) + err=True) Exit(1) return result diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py index eb6ebb446..f8c9a599d 100644 --- a/pio-scripts/validate_modules.py +++ b/pio-scripts/validate_modules.py @@ -1,10 +1,8 @@ import re -import sys from pathlib import Path # For OS-agnostic path manipulation from typing import Iterable from click import secho from SCons.Script import Action, Exit -from platformio import util from platformio.builder.tools.piolib import LibBuilderBase @@ -56,8 +54,8 @@ def validate_map_file(source, target, env): Exit(1) # Identify the WLED module source directories - module_lib_builders = [builder for builder in env.GetLibBuilders() if is_wled_module(env, builder)] - + module_lib_builders = [builder for builder in env.GetLibBuilders() if is_wled_module(env, builder)] + if env.GetProjectOption("custom_usermods","") == "*": # All usermods build; filter non-platform-OK modules module_lib_builders = [builder for builder in module_lib_builders if env.IsCompatibleLibBuilder(builder)] @@ -68,8 +66,7 @@ def validate_map_file(source, target, env): f"ERROR: Modules {[b.name for b in incompatible_builders]} are not compatible with this platform!", fg="red", err=True) - Exit(1) - pass + Exit(1) # Extract the values we care about modules = {Path(builder.build_dir).name: builder.name for builder in module_lib_builders} @@ -77,7 +74,7 @@ def validate_map_file(source, target, env): # Now parse the map file map_file_contents = read_lines(map_file_path) - usermod_object_count = count_usermod_objects(map_file_contents) + usermod_object_count = count_usermod_objects(map_file_contents) secho(f"INFO: {usermod_object_count} usermod object entries") confirmed_modules = check_map_file_objects(map_file_contents, modules.keys())