From fadc75da55bb1418f1efb63f22e7180ed6e8551d Mon Sep 17 00:00:00 2001 From: Will Miles Date: Tue, 10 Mar 2026 12:38:52 -0400 Subject: [PATCH 1/5] dynarray: Directly apply linker fix Rather than append a linker file, we edit the upstream supplied ones to add our section to the binaries. Works better on all platforms. Co-Authored-By: Claude --- pio-scripts/dynarray.py | 79 +++++++++++++++++++++++++++------ pio-scripts/validate_modules.py | 3 +- tools/dynarray_espressif32.ld | 10 ----- wled00/dynarray.h | 11 ----- 4 files changed, 67 insertions(+), 36 deletions(-) delete mode 100644 tools/dynarray_espressif32.ld diff --git a/pio-scripts/dynarray.py b/pio-scripts/dynarray.py index 2d3cfa90c..f918be960 100644 --- a/pio-scripts/dynarray.py +++ b/pio-scripts/dynarray.py @@ -4,16 +4,69 @@ Import("env") from pathlib import Path -platform = env.get("PIOPLATFORM") -script_file = Path(f"tools/dynarray_{platform}.ld") -if script_file.is_file(): - linker_script = f"-T{script_file}" - if platform == "espressif32": - # For ESP32, the script must be added at the right point in the list - linkflags = env.get("LINKFLAGS", []) - idx = linkflags.index("memory.ld") - linkflags.insert(idx+1, linker_script) - env.Replace(LINKFLAGS=linkflags) - else: - # For other platforms, put it in last - env.Append(LINKFLAGS=[linker_script]) +# Linker script fragment injected into the rodata output section of whichever +# platform we're building for. Placed just before the end-of-rodata marker so +# that the dynarray entries land in flash rodata and are correctly sorted. +DYNARRAY_INJECTION = ( + "\n /* dynarray: WLED dynamic module arrays */\n" + " . = ALIGN(0x10);\n" + " KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*)))\n" + " " +) + + +def inject_before_marker(path, marker): + """Patch a linker script file in-place, inserting DYNARRAY_INJECTION before marker.""" + original = path.read_text() + path.write_text(original.replace(marker, DYNARRAY_INJECTION + marker, 1)) + + +if env.get("PIOPLATFORM") == "espressif32": + # Find sections.ld on the linker search path (LIBPATH). + sections_ld_path = None + for ld_dir in env.get("LIBPATH", []): + candidate = Path(str(ld_dir)) / "sections.ld" + if candidate.exists(): + sections_ld_path = candidate + break + + if sections_ld_path is not None: + # Inject inside the existing .flash.rodata output section, just before + # _rodata_end. IDF v5 enforces zero gaps between adjacent output + # sections via ASSERT statements, so INSERT AFTER .flash.rodata would + # fail. Injecting inside the section creates no new output section and + # leaves the ASSERTs satisfied. + build_dir = Path(env.subst("$BUILD_DIR")) + patched_path = build_dir / "dynarray_sections.ld" + import shutil + shutil.copy(sections_ld_path, patched_path) + inject_before_marker(patched_path, "_rodata_end = ABSOLUTE(.);") + + # Replace "sections.ld" in LINKFLAGS with an absolute path to our + # patched copy. The flag may appear as a bare token, combined as + # "-Tsections.ld", or split across two tokens ("-T", "sections.ld"). + patched_str = str(patched_path) + new_flags = [] + skip_next = False + for flag in env.get("LINKFLAGS", []): + if skip_next: + new_flags.append(patched_str if flag == "sections.ld" else flag) + skip_next = False + elif flag == "-T": + new_flags.append(flag) + skip_next = True + else: + new_flags.append(flag.replace("sections.ld", patched_str)) + env.Replace(LINKFLAGS=new_flags) + +elif env.get("PIOPLATFORM") == "espressif8266": + # The ESP8266 framework preprocesses eagle.app.v6.common.ld.h into + # local.eagle.app.v6.common.ld in $BUILD_DIR/ld/ at build time. Register + # a post-action on that generated file so the injection happens after + # C-preprocessing but before linking. + build_ld = Path(env.subst("$BUILD_DIR")) / "ld" / "local.eagle.app.v6.common.ld" + + def patch_esp8266_ld(target, source, env): + inject_before_marker(build_ld, "_irom0_text_end = ABSOLUTE(.);") + + env.AddPostAction(str(build_ld), patch_esp8266_ld) diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py index ae098f43c..99f8c488f 100644 --- a/pio-scripts/validate_modules.py +++ b/pio-scripts/validate_modules.py @@ -80,8 +80,7 @@ def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]: return found -DYNARRAY_SECTION = ".dtors" if env.get("PIOPLATFORM") == "espressif8266" else ".dynarray" -USERMODS_SECTION = f"{DYNARRAY_SECTION}.usermods.1" +USERMODS_SECTION = f".dynarray.usermods.1" def count_usermod_objects(map_file: list[str]) -> int: """ Returns the number of usermod objects in the usermod list """ diff --git a/tools/dynarray_espressif32.ld b/tools/dynarray_espressif32.ld deleted file mode 100644 index 70ce51f19..000000000 --- a/tools/dynarray_espressif32.ld +++ /dev/null @@ -1,10 +0,0 @@ -/* ESP32 linker script fragment to add dynamic array section to binary */ -SECTIONS -{ - .dynarray : - { - . = ALIGN(0x10); - KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*))) - } > default_rodata_seg -} -INSERT AFTER .flash.rodata; diff --git a/wled00/dynarray.h b/wled00/dynarray.h index f9e6de19d..91fb5240f 100644 --- a/wled00/dynarray.h +++ b/wled00/dynarray.h @@ -20,15 +20,4 @@ Macros for generating a "dynamic array", a static array of objects declared in d #define DYNARRAY_END(array_name) array_name##_end #define DYNARRAY_LENGTH(array_name) (&DYNARRAY_END(array_name)[0] - &DYNARRAY_BEGIN(array_name)[0]) -#ifdef ESP8266 -// ESP8266 linker script cannot be extended with a unique section for dynamic arrays. -// We instead pack them in the ".dtors" section, as it's sorted and uploaded to the flash -// (but will never be used in the embedded system) -#define DYNARRAY_SECTION ".dtors" - -#else /* ESP8266 */ - -// Use a unique named section; the linker script must be extended to ensure it's correctly placed. #define DYNARRAY_SECTION ".dynarray" - -#endif From c789e3d18713b82e0d52ae752679ede79687cf3c Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 27 Mar 2026 15:32:40 -0400 Subject: [PATCH 2/5] Fix usermod count for LTO --- pio-scripts/validate_modules.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py index 99f8c488f..81fbce424 100644 --- a/pio-scripts/validate_modules.py +++ b/pio-scripts/validate_modules.py @@ -80,12 +80,35 @@ def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]: return found -USERMODS_SECTION = f".dynarray.usermods.1" - def count_usermod_objects(map_file: list[str]) -> int: - """ Returns the number of usermod objects in the usermod list """ - # Count the number of entries in the usermods table section - return len([x for x in map_file if USERMODS_SECTION in x]) + """ Returns the number of usermod objects in the usermod list. + + Computes the count from the address span between the .dynarray.usermods.0 + and .dynarray.usermods.99999 sentinel sections. This mirrors the + DYNARRAY_LENGTH macro and is reliable under LTO, where all entries are + merged into a single ltrans partition so counting section occurrences + always yields 1 regardless of the true count. + """ + ENTRY_SIZE = 4 # sizeof(Usermod*) on 32-bit targets + addr_begin = None + addr_end = None + + for i, line in enumerate(map_file): + stripped = line.strip() + if stripped == '.dynarray.usermods.0': + if i + 1 < len(map_file): + m = re.search(r'0x([0-9a-fA-F]+)', map_file[i + 1]) + if m: + addr_begin = int(m.group(1), 16) + elif stripped == '.dynarray.usermods.99999': + if i + 1 < len(map_file): + m = re.search(r'0x([0-9a-fA-F]+)', map_file[i + 1]) + if m: + addr_end = int(m.group(1), 16) + + if addr_begin is None or addr_end is None: + return 0 + return (addr_end - addr_begin) // ENTRY_SIZE def validate_map_file(source, target, env): From f994c5e99594339ed53ef41787afbee708d39516 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 27 Mar 2026 18:24:16 -0400 Subject: [PATCH 3/5] validate_modules: Improve performance Use readelf instead of nm for great speed. --- pio-scripts/validate_modules.py | 92 +++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py index 81fbce424..62f26c00f 100644 --- a/pio-scripts/validate_modules.py +++ b/pio-scripts/validate_modules.py @@ -13,70 +13,85 @@ def read_lines(p: Path): return f.readlines() -def _get_nm_path(env) -> str: - """ Derive the nm tool path from the build environment """ - if "NM" in env: - return env.subst("$NM") - # Derive from the C compiler: xtensa-esp32-elf-gcc → xtensa-esp32-elf-nm +def _get_readelf_path(env) -> str: + """ Derive the readelf tool path from the build environment """ + # Derive from the C compiler: xtensa-esp32-elf-gcc → xtensa-esp32-elf-readelf cc = env.subst("$CC") - nm = re.sub(r'(gcc|g\+\+)$', 'nm', os.path.basename(cc)) - return os.path.join(os.path.dirname(cc), nm) + readelf = re.sub(r'(gcc|g\+\+)$', 'readelf', os.path.basename(cc)) + return os.path.join(os.path.dirname(cc), readelf) def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]: - """ Check which modules have at least one defined symbol placed in the ELF. + """ Check which modules have at least one compilation unit in the ELF. The map file is not a reliable source for this: with LTO, original object file paths are replaced by temporary ltrans.o partitions in all output sections, making per-module attribution impossible from the map alone. - Instead we invoke nm --defined-only -l on the ELF, which uses DWARF debug - info to attribute each placed symbol to its original source file. - - Requires usermod libraries to be compiled with -g so that DWARF sections - are present in the ELF. load_usermods.py injects -g for all WLED modules - via dep.env.AppendUnique(CCFLAGS=["-g"]). + Instead we invoke readelf --debug-dump=info --dwarf-depth=1 on the ELF, + which reads only the top-level compilation-unit DIEs from .debug_info. + Each CU corresponds to one source file; matching DW_AT_comp_dir + + DW_AT_name against the module src_dirs is sufficient to confirm a module + was compiled into the ELF. The output volume is proportional to the + number of source files, not the number of symbols. Returns the set of build_dir basenames for confirmed modules. """ - nm_path = _get_nm_path(env) + readelf_path = _get_readelf_path(env) + secho(f"INFO: Checking for usermod compilation units...") + try: result = subprocess.run( - [nm_path, "--defined-only", "-l", str(elf_path)], + [readelf_path, "--debug-dump=info", "--dwarf-depth=1", str(elf_path)], capture_output=True, text=True, errors="ignore", timeout=120, ) - nm_output = result.stdout + output = result.stdout except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as e: - secho(f"WARNING: nm failed ({e}); skipping per-module validation", fg="yellow", err=True) + secho(f"WARNING: readelf failed ({e}); skipping per-module validation", fg="yellow", err=True) return {Path(b.build_dir).name for b in module_lib_builders} # conservative pass - # Match placed symbols against builders as we parse nm output, exiting early - # once all builders are accounted for. - # nm --defined-only still includes debugging symbols (type 'N') such as the - # per-CU markers GCC emits in .debug_info (e.g. "usermod_example_cpp_6734d48d"). - # These live at address 0x00000000 in their debug section — not in any load - # segment — so filtering them out leaves only genuinely placed symbols. - # nm -l appends a tab-separated "file:lineno" location to each symbol line. remaining = {Path(str(b.src_dir)): Path(b.build_dir).name for b in module_lib_builders} found = set() - for line in nm_output.splitlines(): - if not remaining: - break # all builders matched - addr, _, _ = line.partition(' ') - if not addr.lstrip('0'): - continue # zero address — skip debug-section marker - if '\t' not in line: - continue - loc = line.rsplit('\t', 1)[1] - # Strip trailing :lineno (e.g. "/path/to/foo.cpp:42" → "/path/to/foo.cpp") - src_path = Path(loc.rsplit(':', 1)[0]) - # Path.is_relative_to() handles OS-specific separators correctly without - # any regex, avoiding Windows path escaping issues. + def _flush_cu(comp_dir: str | None, name: str | None) -> None: + """Match one completed CU against remaining builders.""" + if not name or not remaining: + return + p = Path(name) + src_path = (Path(comp_dir) / p) if (comp_dir and not p.is_absolute()) else p for src_dir in list(remaining): if src_path.is_relative_to(src_dir): found.add(remaining.pop(src_dir)) break + # readelf emits one DW_TAG_compile_unit DIE per source file. Attributes + # of interest: + # DW_AT_name — source file (absolute, or relative to comp_dir) + # DW_AT_comp_dir — compile working directory + # Both appear as either a direct string or an indirect string: + # DW_AT_name : foo.cpp + # DW_AT_name : (indirect string, offset: 0x…): foo.cpp + # Taking the portion after the *last* ": " on the line handles both forms. + _CU_HEADER = re.compile(r'Compilation Unit @') + _ATTR = re.compile(r'\bDW_AT_(name|comp_dir)\b') + + comp_dir = name = None + for line in output.splitlines(): + if _CU_HEADER.search(line): + _flush_cu(comp_dir, name) + comp_dir = name = None + continue + if not remaining: + break # all builders matched + m = _ATTR.search(line) + if m: + _, _, val = line.rpartition(': ') + val = val.strip() + if m.group(1) == 'name': + name = val + else: + comp_dir = val + _flush_cu(comp_dir, name) # flush the last CU + return found @@ -133,6 +148,7 @@ def validate_map_file(source, target, env): secho(f"INFO: {usermod_object_count} usermod object entries") elf_path = build_dir / env.subst("${PROGNAME}.elf") + confirmed_modules = check_elf_modules(elf_path, env, module_lib_builders) missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules] From fd890b3d580b45e10118df14600f80498ed82bbe Mon Sep 17 00:00:00 2001 From: Will Miles Date: Fri, 27 Mar 2026 22:47:15 -0400 Subject: [PATCH 4/5] dynarray: Support ESP-IDF --- pio-scripts/dynarray.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pio-scripts/dynarray.py b/pio-scripts/dynarray.py index f918be960..7b580de08 100644 --- a/pio-scripts/dynarray.py +++ b/pio-scripts/dynarray.py @@ -58,7 +58,15 @@ if env.get("PIOPLATFORM") == "espressif32": else: new_flags.append(flag.replace("sections.ld", patched_str)) env.Replace(LINKFLAGS=new_flags) - + else: + # Assume sections.ld will be built (ESP-IDF format); add a post-action to patch it + # TODO: consider using ESP-IDF linker fragment (https://docs.espressif.com/projects/esp-idf/en/stable/esp32/api-guides/linker-script-generation.html) + # For now, patch after building + sections_ld = Path(env.subst("$BUILD_DIR")) / "sections.ld" + def patch_sections_ld(target, source, env): + inject_before_marker(sections_ld, "_rodata_end = ABSOLUTE(.);") + env.AddPostAction(str(sections_ld), patch_sections_ld) + elif env.get("PIOPLATFORM") == "espressif8266": # The ESP8266 framework preprocesses eagle.app.v6.common.ld.h into # local.eagle.app.v6.common.ld in $BUILD_DIR/ld/ at build time. Register From 48ab88e11e6ccdc73ff2cd53317d1a26c7ed605f Mon Sep 17 00:00:00 2001 From: Will Miles Date: Mon, 30 Mar 2026 19:17:17 -0400 Subject: [PATCH 5/5] Fix up usermod validation again Co-Authored-By: Claude Sonnet 4.6 --- pio-scripts/validate_modules.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py index 62f26c00f..cd2698b8e 100644 --- a/pio-scripts/validate_modules.py +++ b/pio-scripts/validate_modules.py @@ -1,11 +1,12 @@ -import os import re import subprocess -from pathlib import Path # For OS-agnostic path manipulation +from pathlib import Path from click import secho from SCons.Script import Action, Exit Import("env") +_ATTR = re.compile(r'\bDW_AT_(name|comp_dir)\b') + def read_lines(p: Path): """ Read in the contents of a file for analysis """ @@ -16,9 +17,8 @@ def read_lines(p: Path): def _get_readelf_path(env) -> str: """ Derive the readelf tool path from the build environment """ # Derive from the C compiler: xtensa-esp32-elf-gcc → xtensa-esp32-elf-readelf - cc = env.subst("$CC") - readelf = re.sub(r'(gcc|g\+\+)$', 'readelf', os.path.basename(cc)) - return os.path.join(os.path.dirname(cc), readelf) + cc = Path(env.subst("$CC")) + return str(cc.with_name(re.sub(r'(gcc|g\+\+)$', 'readelf', cc.name))) def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]: @@ -51,6 +51,7 @@ def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]: remaining = {Path(str(b.src_dir)): Path(b.build_dir).name for b in module_lib_builders} found = set() + project_dir = Path(env.subst("$PROJECT_DIR")) def _flush_cu(comp_dir: str | None, name: str | None) -> None: """Match one completed CU against remaining builders.""" @@ -58,10 +59,16 @@ def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]: return p = Path(name) src_path = (Path(comp_dir) / p) if (comp_dir and not p.is_absolute()) else p + # In arduino+espidf dual-framework builds the IDF toolchain sets DW_AT_comp_dir + # to the virtual path "/IDF_PROJECT" rather than the real project root, so + # src_path won't match. Pre-compute a fallback using $PROJECT_DIR and check + # both candidates in a single pass. + use_fallback = not p.is_absolute() and comp_dir and Path(comp_dir) != project_dir + src_path_real = project_dir / p if use_fallback else None for src_dir in list(remaining): - if src_path.is_relative_to(src_dir): + if src_path.is_relative_to(src_dir) or (src_path_real and src_path_real.is_relative_to(src_dir)): found.add(remaining.pop(src_dir)) - break + return # readelf emits one DW_TAG_compile_unit DIE per source file. Attributes # of interest: @@ -71,12 +78,10 @@ def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]: # DW_AT_name : foo.cpp # DW_AT_name : (indirect string, offset: 0x…): foo.cpp # Taking the portion after the *last* ": " on the line handles both forms. - _CU_HEADER = re.compile(r'Compilation Unit @') - _ATTR = re.compile(r'\bDW_AT_(name|comp_dir)\b') comp_dir = name = None for line in output.splitlines(): - if _CU_HEADER.search(line): + if 'Compilation Unit @' in line: _flush_cu(comp_dir, name) comp_dir = name = None continue @@ -120,6 +125,8 @@ def count_usermod_objects(map_file: list[str]) -> int: m = re.search(r'0x([0-9a-fA-F]+)', map_file[i + 1]) if m: addr_end = int(m.group(1), 16) + if addr_begin is not None and addr_end is not None: + break if addr_begin is None or addr_end is None: return 0 @@ -129,7 +136,7 @@ def count_usermod_objects(map_file: list[str]) -> int: def validate_map_file(source, target, env): """ Validate that all modules appear in the output build """ build_dir = Path(env.subst("$BUILD_DIR")) - map_file_path = build_dir / env.subst("${PROGNAME}.map") + map_file_path = build_dir / env.subst("${PROGNAME}.map") if not map_file_path.exists(): secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True) @@ -158,7 +165,6 @@ def validate_map_file(source, target, env): fg="red", err=True) Exit(1) - return None env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")]) env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))