Merge pull request #4669 from willmmiles/4597-usermods-not-building

Correct issues with usermods not being linked.
- Explicitly set libArchive: false in usermod library.json files
- Fix up symlink path generation on Windows
- Add validation script to report usermod linkage in resulting binary
This commit is contained in:
Will Miles 2025-05-26 22:41:45 -04:00 committed by GitHub
commit 4a3af814bf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
55 changed files with 236 additions and 97 deletions

View File

@ -1,17 +1,14 @@
Import('env')
import os.path
from collections import deque
from pathlib import Path # For OS-agnostic path manipulation
from platformio.package.manager.library import LibraryPackageManager
from click import secho
from SCons.Script import Exit
from platformio.builder.tools.piolib import LibBuilderBase
usermod_dir = Path(env["PROJECT_DIR"]) / "usermods"
all_usermods = [f for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()]
usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods"
if env['PIOENV'] == "usermods":
# Add all usermods
env.GetProjectConfig().set(f"env:usermods", 'custom_usermods', " ".join([f.name for f in all_usermods]))
def find_usermod(mod: str):
# Utility functions
def find_usermod(mod: str) -> Path:
"""Locate this library in the usermods folder.
We do this to avoid needing to rename a bunch of folders;
this could be removed later
@ -28,40 +25,25 @@ def find_usermod(mod: str):
return mp
raise RuntimeError(f"Couldn't locate module {mod} in usermods directory!")
def is_wled_module(dep: LibBuilderBase) -> bool:
"""Returns true if the specified library is a wled module
"""
return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
## Script starts here
# Process usermod option
usermods = env.GetProjectOption("custom_usermods","")
# Handle "all usermods" case
if usermods == '*':
usermods = [f.name for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()]
else:
usermods = usermods.split()
if usermods:
# Inject usermods in to project lib_deps
proj = env.GetProjectConfig()
deps = env.GetProjectOption('lib_deps')
src_dir = proj.get("platformio", "src_dir")
src_dir = src_dir.replace('\\','/')
mod_paths = {mod: find_usermod(mod) for mod in usermods.split()}
usermods = [f"{mod} = symlink://{path}" for mod, path in mod_paths.items()]
proj.set("env:" + env['PIOENV'], 'lib_deps', deps + usermods)
# Force usermods to be installed in to the environment build state before the LDF runs
# Otherwise we won't be able to see them until it's too late to change their paths for LDF
# Logic is largely borrowed from PlaformIO internals
not_found_specs = []
for spec in usermods:
found = False
for storage_dir in env.GetLibSourceDirs():
#print(f"Checking {storage_dir} for {spec}")
lm = LibraryPackageManager(storage_dir)
if lm.get_package(spec):
#print("Found!")
found = True
break
if not found:
#print("Missing!")
not_found_specs.append(spec)
if not_found_specs:
lm = LibraryPackageManager(
env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))
)
for spec in not_found_specs:
#print(f"LU: forcing install of {spec}")
lm.install(spec)
symlinks = [f"symlink://{find_usermod(mod).resolve()}" for mod in usermods]
env.GetProjectConfig().set("env:" + env['PIOENV'], 'lib_deps', env.GetProjectOption('lib_deps') + symlinks)
# Utility function for assembling usermod include paths
def cached_add_includes(dep, dep_cache: set, includes: deque):
@ -82,13 +64,6 @@ old_ConfigureProjectLibBuilder = env.ConfigureProjectLibBuilder
# Our new wrapper
def wrapped_ConfigureProjectLibBuilder(xenv):
# Update usermod properties
# Set libArchive before build actions are added
for um in (um for um in xenv.GetLibBuilders() if usermod_dir in Path(um.src_dir).parents):
build = um._manifest.get("build", {})
build["libArchive"] = False
um._manifest["build"] = build
# Call the wrapped function
result = old_ConfigureProjectLibBuilder.clone(xenv)()
@ -102,12 +77,25 @@ def wrapped_ConfigureProjectLibBuilder(xenv):
for dep in result.depbuilders:
cached_add_includes(dep, processed_deps, extra_include_dirs)
for um in [dep for dep in result.depbuilders if usermod_dir in Path(dep.src_dir).parents]:
# Add the wled folder to the include path
um.env.PrependUnique(CPPPATH=wled_dir)
# Add WLED's own dependencies
for dir in extra_include_dirs:
um.env.PrependUnique(CPPPATH=dir)
broken_usermods = []
for dep in result.depbuilders:
if is_wled_module(dep):
# Add the wled folder to the include path
dep.env.PrependUnique(CPPPATH=str(wled_dir))
# Add WLED's own dependencies
for dir in extra_include_dirs:
dep.env.PrependUnique(CPPPATH=str(dir))
# Enforce that libArchive is not set; we must link them directly to the executable
if dep.lib_archive:
broken_usermods.append(dep)
if broken_usermods:
broken_usermods = [usermod.name for usermod in broken_usermods]
secho(
f"ERROR: libArchive=false is missing on usermod(s) {' '.join(broken_usermods)} -- modules will not compile in correctly",
fg="red",
err=True)
Exit(1)
return result

View File

@ -0,0 +1,92 @@
import re
from pathlib import Path # For OS-agnostic path manipulation
from typing import Iterable
from click import secho
from SCons.Script import Action, Exit
from platformio.builder.tools.piolib import LibBuilderBase
def is_wled_module(env, dep: LibBuilderBase) -> bool:
"""Returns true if the specified library is a wled module
"""
usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods"
return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
def read_lines(p: Path):
""" Read in the contents of a file for analysis """
with p.open("r", encoding="utf-8", errors="ignore") as f:
return f.readlines()
def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]:
""" Identify which dirs contributed to the final build
Returns the (sub)set of dirs that are found in the output ELF
"""
# Pattern to match symbols in object directories
# Join directories into alternation
usermod_dir_regex = "|".join([re.escape(dir) for dir in dirs])
# Matches nonzero address, any size, and any path in a matching directory
object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o")
found = set()
for line in map_file:
matches = object_path_regex.findall(line)
for m in matches:
found.add(m)
return found
def count_usermod_objects(map_file: list[str]) -> int:
""" Returns the number of usermod objects in the usermod list """
# Count the number of entries in the usermods table section
return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
def validate_map_file(source, target, env):
""" Validate that all modules appear in the output build """
build_dir = Path(env.subst("$BUILD_DIR"))
map_file_path = build_dir / env.subst("${PROGNAME}.map")
if not map_file_path.exists():
secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
Exit(1)
# Identify the WLED module source directories
module_lib_builders = [builder for builder in env.GetLibBuilders() if is_wled_module(env, builder)]
if env.GetProjectOption("custom_usermods","") == "*":
# All usermods build; filter non-platform-OK modules
module_lib_builders = [builder for builder in module_lib_builders if env.IsCompatibleLibBuilder(builder)]
else:
incompatible_builders = [builder for builder in module_lib_builders if not env.IsCompatibleLibBuilder(builder)]
if incompatible_builders:
secho(
f"ERROR: Modules {[b.name for b in incompatible_builders]} are not compatible with this platform!",
fg="red",
err=True)
Exit(1)
# Extract the values we care about
modules = {Path(builder.build_dir).name: builder.name for builder in module_lib_builders}
secho(f"INFO: {len(modules)} libraries linked as WLED optional/user modules")
# Now parse the map file
map_file_contents = read_lines(map_file_path)
usermod_object_count = count_usermod_objects(map_file_contents)
secho(f"INFO: {usermod_object_count} usermod object entries")
confirmed_modules = check_map_file_objects(map_file_contents, modules.keys())
missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules]
if missing_modules:
secho(
f"ERROR: No object files from {missing_modules} found in linked output!",
fg="red",
err=True)
Exit(1)
return None
Import("env")
env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))

View File

@ -116,6 +116,7 @@ extra_scripts =
pre:pio-scripts/user_config_copy.py
pre:pio-scripts/load_usermods.py
pre:pio-scripts/build_ui.py
post:pio-scripts/validate_modules.py ;; double-check the build output usermods
; post:pio-scripts/obj-dump.py ;; convenience script to create a disassembly dump of the firmware (hardcore debugging)
# ------------------------------------------------------------------------------
@ -659,5 +660,5 @@ build_flags = ${common.build_flags} ${esp32_idf_V4.build_flags} -D WLED_RELEASE_
lib_deps = ${esp32_idf_V4.lib_deps}
monitor_filters = esp32_exception_decoder
board_build.flash_mode = dio
; custom_usermods = *every folder with library.json* -- injected by pio-scripts/load_usermods.py
custom_usermods = * ; Expands to all usermods in usermods folder
board_build.partitions = ${esp32.extreme_partitions} ; We're gonna need a bigger boat

View File

@ -1,5 +1,6 @@
{
"name": "ADS1115_v2",
"build": { "libArchive": false },
"dependencies": {
"Adafruit BusIO": "https://github.com/adafruit/Adafruit_BusIO#1.13.2",
"Adafruit ADS1X15": "https://github.com/adafruit/Adafruit_ADS1X15#2.4.0"

View File

@ -1,5 +1,6 @@
{
"name": "AHT10_v2",
"build": { "libArchive": false },
"dependencies": {
"enjoyneering/AHT10":"~1.1.0"
}

View File

@ -1,3 +1,4 @@
{
"name": "Analog_Clock"
"name": "Analog_Clock",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "Animated_Staircase"
"name": "Animated_Staircase",
"build": { "libArchive": false }
}

View File

@ -1,5 +1,6 @@
{
"name": "BH1750_v2",
"build": { "libArchive": false },
"dependencies": {
"claws/BH1750":"^1.2.0"
}

View File

@ -1,5 +1,6 @@
{
"name": "BME280_v2",
"build": { "libArchive": false },
"dependencies": {
"finitespace/BME280":"~3.0.0"
}

View File

@ -1,5 +1,6 @@
{
"name": "BME68X",
"build": { "libArchive": false },
"dependencies": {
"boschsensortec/BSEC Software Library":"^1.8.1492"
}

View File

@ -1,3 +1,4 @@
{
"name": "Battery"
"name": "Battery",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "Cronixie"
"name": "Cronixie",
"build": { "libArchive": false }
}

View File

@ -1,4 +1,5 @@
{
"name": "EXAMPLE",
"build": { "libArchive": false },
"dependencies": {}
}

View File

@ -1,5 +1,6 @@
{
"name:": "EleksTube_IPS",
"build": { "libArchive": false },
"dependencies": {
"TFT_eSPI" : "2.5.33"
}

View File

@ -1,5 +1,6 @@
{
"name": "INA226_v2",
"build": { "libArchive": false },
"dependencies": {
"wollewald/INA226_WE":"~1.2.9"
}

View File

@ -1,3 +1,4 @@
{
"name": "Internal_Temperature_v2"
"name": "Internal_Temperature_v2",
"build": { "libArchive": false }
}

View File

@ -1,5 +1,6 @@
{
"name": "LD2410_v2",
"build": { "libArchive": false },
"dependencies": {
"ncmreynolds/ld2410":"^0.1.3"
}

View File

@ -1,3 +1,4 @@
{
"name": "LDR_Dusk_Dawn_v2"
"name": "LDR_Dusk_Dawn_v2",
"build": { "libArchive": false }
}

View File

@ -1,4 +1,5 @@
{
"name": "MY9291",
"build": { "libArchive": false },
"platforms": ["espressif8266"]
}

View File

@ -1,3 +1,4 @@
{
"name": "PIR_sensor_switch"
"name": "PIR_sensor_switch",
"build": { "libArchive": false }
}

View File

@ -1,6 +1,7 @@
{
"name": "PWM_fan",
"build": {
"libArchive": false,
"extraScript": "setup_deps.py"
}
}

View File

@ -1,11 +1,12 @@
from platformio.package.meta import PackageSpec
Import('env')
usermods = env.GetProjectOption("custom_usermods","").split()
libs = [PackageSpec(lib).name for lib in env.GetProjectOption("lib_deps",[])]
# Check for dependencies
if "Temperature" in usermods:
if "Temperature" in libs:
env.Append(CPPDEFINES=[("USERMOD_DALLASTEMPERATURE")])
elif "sht" in usermods:
elif "sht" in libs:
env.Append(CPPDEFINES=[("USERMOD_SHT")])
elif "PWM_fan" in usermods: # The script can be run if this module was previously selected
elif "PWM_fan" in libs: # The script can be run if this module was previously selected
raise RuntimeError("PWM_fan usermod requires Temperature or sht to be enabled")

View File

@ -1,3 +1,4 @@
{
"name": "RTC"
"name": "RTC",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "SN_Photoresistor"
"name": "SN_Photoresistor",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name:": "ST7789_display"
"name:": "ST7789_display",
"build": { "libArchive": false }
}

View File

@ -1,5 +1,6 @@
{
"name": "Si7021_MQTT_HA",
"build": { "libArchive": false },
"dependencies": {
"finitespace/BME280":"3.0.0",
"adafruit/Adafruit Si7021 Library" : "1.5.3"

View File

@ -1,3 +1,4 @@
{
"name": "TetrisAI_v2"
"name": "TetrisAI_v2",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "boblight"
"name": "boblight",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "buzzer"
"name": "buzzer",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "deep_sleep"
"name": "deep_sleep",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "multi_relay"
"name": "multi_relay",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "pwm_outputs"
"name": "pwm_outputs",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "sd_card"
"name": "sd_card",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "seven_segment_display"
"name": "seven_segment_display",
"build": { "libArchive": false }
}

View File

@ -1,6 +1,7 @@
{
"name": "seven_segment_display_reloaded",
"build": {
"libArchive": false,
"extraScript": "setup_deps.py"
}
}

View File

@ -1,9 +1,10 @@
from platformio.package.meta import PackageSpec
Import('env')
usermods = env.GetProjectOption("custom_usermods","").split()
libs = [PackageSpec(lib).name for lib in env.GetProjectOption("lib_deps",[])]
# Check for partner usermods
if "SN_Photoresistor" in usermods:
if "SN_Photoresistor" in libs:
env.Append(CPPDEFINES=[("USERMOD_SN_PHOTORESISTOR")])
if any(mod in ("BH1750_v2", "BH1750") for mod in usermods):
if any(mod in ("BH1750_v2", "BH1750") for mod in libs):
env.Append(CPPDEFINES=[("USERMOD_BH1750")])

View File

@ -1,5 +1,6 @@
{
"name": "sht",
"build": { "libArchive": false },
"dependencies": {
"robtillaart/SHT85": "~0.3.3"
}

View File

@ -1,3 +1,4 @@
{
"name": "smartnest"
"name": "smartnest",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "stairway_wipe_basic"
"name": "stairway_wipe_basic",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "usermod_rotary_brightness_color"
"name": "usermod_rotary_brightness_color",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "usermod_v2_HttpPullLightControl"
"name": "usermod_v2_HttpPullLightControl",
"build": { "libArchive": false }
}

View File

@ -4,6 +4,9 @@
const char HttpPullLightControl::_name[] PROGMEM = "HttpPullLightControl";
const char HttpPullLightControl::_enabled[] PROGMEM = "Enable";
static HttpPullLightControl http_pull_usermod;
REGISTER_USERMOD(http_pull_usermod);
void HttpPullLightControl::setup() {
//Serial.begin(115200);

View File

@ -1,5 +1,6 @@
{
"name": "usermod_v2_RF433",
"build": { "libArchive": false },
"dependencies": {
"sui77/rc-switch":"2.6.4"
}

View File

@ -1,5 +1,6 @@
{
"name": "animartrix",
"build": { "libArchive": false },
"dependencies": {
"Animartrix": "https://github.com/netmindz/animartrix.git#b172586"
}

View File

@ -1,3 +1,4 @@
{
"name": "auto_save"
"name": "auto_save",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "brightness_follow_sun"
"name": "brightness_follow_sun",
"build": { "libArchive": false }
}

View File

@ -1,5 +1,6 @@
{
"name": "four_line_display_ALT",
"build": { "libArchive": false },
"dependencies": {
"U8g2": "~2.34.4",
"Wire": ""

View File

@ -1,3 +1,4 @@
{
"name": "usermod_v2_klipper_percentage"
"name": "usermod_v2_klipper_percentage",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "usermod_v2_ping_pong_clock"
"name": "usermod_v2_ping_pong_clock",
"build": { "libArchive": false }
}

View File

@ -1,6 +1,7 @@
{
"name": "rotary_encoder_ui_ALT",
"build": {
"libArchive": false,
"extraScript": "setup_deps.py"
}
}

View File

@ -1,8 +1,8 @@
from platformio.package.meta import PackageSpec
Import('env')
usermods = env.GetProjectOption("custom_usermods","").split()
libs = [PackageSpec(lib).name for lib in env.GetProjectOption("lib_deps",[])]
# Check for partner usermod
# Allow both "usermod_v2" and unqualified syntax
if any(mod in ("four_line_display_ALT", "usermod_v2_four_line_display_ALT") for mod in usermods):
if any(mod in ("four_line_display_ALT", "usermod_v2_four_line_display_ALT") for mod in libs):
env.Append(CPPDEFINES=[("USERMOD_FOUR_LINE_DISPLAY")])

View File

@ -1,3 +1,4 @@
{
"name": "usermod_v2_word_clock"
"name": "usermod_v2_word_clock",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "wizlights"
"name": "wizlights",
"build": { "libArchive": false }
}

View File

@ -1,3 +1,4 @@
{
"name": "word-clock-matrix"
"name": "word-clock-matrix",
"build": { "libArchive": false }
}

View File

@ -39,7 +39,13 @@ bool UsermodManager::getUMData(um_data_t **data, uint8_t mod_id) {
return false;
}
void UsermodManager::addToJsonState(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->addToJsonState(obj); }
void UsermodManager::addToJsonInfo(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->addToJsonInfo(obj); }
void UsermodManager::addToJsonInfo(JsonObject& obj) {
auto um_id_list = obj.createNestedArray("um");
for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) {
um_id_list.add((*mod)->getId());
(*mod)->addToJsonInfo(obj);
}
}
void UsermodManager::readFromJsonState(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->readFromJsonState(obj); }
void UsermodManager::addToConfig(JsonObject& obj) { for (auto mod = _usermod_table_begin; mod < _usermod_table_end; ++mod) (*mod)->addToConfig(obj); }
bool UsermodManager::readFromConfig(JsonObject& obj) {