jeans/WLED
1
0
Fork 0
mirror of https://github.com/wled/WLED.git synced 2026-04-25 16:42:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5497 from willmmiles/16_x-dynarray-fixes

Browse Source 
0.16 - Backport dynarray fixes from V5 WIP
This commit is contained in:
Will Miles
2026-04-12 11:34:41 -04:00
committed by GitHub
parent 945896ddb7 48ab88e11e
commit 4cdc23a48b
4 changed files with 168 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
pio-scripts/dynarray.py
View File

@@ -4,16 +4,77 @@
Import("env")
from pathlib import Path
platform = env.get("PIOPLATFORM")
script_file = Path(f"tools/dynarray_{platform}.ld")
if script_file.is_file():
linker_script = f"-T{script_file}"
if platform == "espressif32":
# For ESP32, the script must be added at the right point in the list
linkflags = env.get("LINKFLAGS", [])
idx = linkflags.index("memory.ld")
linkflags.insert(idx+1, linker_script)
env.Replace(LINKFLAGS=linkflags)
# Linker script fragment injected into the rodata output section of whichever
# platform we're building for. Placed just before the end-of-rodata marker so
# that the dynarray entries land in flash rodata and are correctly sorted.
DYNARRAY_INJECTION = (
"\n /* dynarray: WLED dynamic module arrays */\n"
" . = ALIGN(0x10);\n"
" KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*)))\n"
" "
)
def inject_before_marker(path, marker):
"""Patch a linker script file in-place, inserting DYNARRAY_INJECTION before marker."""
original = path.read_text()
path.write_text(original.replace(marker, DYNARRAY_INJECTION + marker, 1))
if env.get("PIOPLATFORM") == "espressif32":
# Find sections.ld on the linker search path (LIBPATH).
sections_ld_path = None
for ld_dir in env.get("LIBPATH", []):
candidate = Path(str(ld_dir)) / "sections.ld"
if candidate.exists():
sections_ld_path = candidate
break
if sections_ld_path is not None:
# Inject inside the existing .flash.rodata output section, just before
# _rodata_end. IDF v5 enforces zero gaps between adjacent output
# sections via ASSERT statements, so INSERT AFTER .flash.rodata would
# fail. Injecting inside the section creates no new output section and
# leaves the ASSERTs satisfied.
build_dir = Path(env.subst("$BUILD_DIR"))
patched_path = build_dir / "dynarray_sections.ld"
import shutil
shutil.copy(sections_ld_path, patched_path)
inject_before_marker(patched_path, "_rodata_end = ABSOLUTE(.);")
# Replace "sections.ld" in LINKFLAGS with an absolute path to our
# patched copy. The flag may appear as a bare token, combined as
# "-Tsections.ld", or split across two tokens ("-T", "sections.ld").
patched_str = str(patched_path)
new_flags = []
skip_next = False
for flag in env.get("LINKFLAGS", []):
if skip_next:
new_flags.append(patched_str if flag == "sections.ld" else flag)
skip_next = False
elif flag == "-T":
new_flags.append(flag)
skip_next = True
else:
new_flags.append(flag.replace("sections.ld", patched_str))
env.Replace(LINKFLAGS=new_flags)
else:
# For other platforms, put it in last
env.Append(LINKFLAGS=[linker_script])
# Assume sections.ld will be built (ESP-IDF format); add a post-action to patch it
# TODO: consider using ESP-IDF linker fragment (https://docs.espressif.com/projects/esp-idf/en/stable/esp32/api-guides/linker-script-generation.html)
# For now, patch after building
sections_ld = Path(env.subst("$BUILD_DIR")) / "sections.ld"
def patch_sections_ld(target, source, env):
inject_before_marker(sections_ld, "_rodata_end = ABSOLUTE(.);")
env.AddPostAction(str(sections_ld), patch_sections_ld)
elif env.get("PIOPLATFORM") == "espressif8266":
# The ESP8266 framework preprocesses eagle.app.v6.common.ld.h into
# local.eagle.app.v6.common.ld in $BUILD_DIR/ld/ at build time. Register
# a post-action on that generated file so the injection happens after
# C-preprocessing but before linking.
build_ld = Path(env.subst("$BUILD_DIR")) / "ld" / "local.eagle.app.v6.common.ld"
def patch_esp8266_ld(target, source, env):
inject_before_marker(build_ld, "_irom0_text_end = ABSOLUTE(.);")
env.AddPostAction(str(build_ld), patch_esp8266_ld)

146
pio-scripts/validate_modules.py
View File

@@ -1,11 +1,12 @@
import os
import re
import subprocess
from pathlib import Path # For OS-agnostic path manipulation
from pathlib import Path
from click import secho
from SCons.Script import Action, Exit
Import("env")
_ATTR = re.compile(r'\bDW_AT_(name|comp_dir)\b')
def read_lines(p: Path):
""" Read in the contents of a file for analysis """
@@ -13,86 +14,129 @@ def read_lines(p: Path):
return f.readlines()
def _get_nm_path(env) -> str:
""" Derive the nm tool path from the build environment """
if "NM" in env:
return env.subst("$NM")
# Derive from the C compiler: xtensa-esp32-elf-gcc → xtensa-esp32-elf-nm
cc = env.subst("$CC")
nm = re.sub(r'(gcc|g\+\+)$', 'nm', os.path.basename(cc))
return os.path.join(os.path.dirname(cc), nm)
def _get_readelf_path(env) -> str:
""" Derive the readelf tool path from the build environment """
# Derive from the C compiler: xtensa-esp32-elf-gcc → xtensa-esp32-elf-readelf
cc = Path(env.subst("$CC"))
return str(cc.with_name(re.sub(r'(gcc|g\+\+)$', 'readelf', cc.name)))
def check_elf_modules(elf_path: Path, env, module_lib_builders) -> set[str]:
""" Check which modules have at least one defined symbol placed in the ELF.
""" Check which modules have at least one compilation unit in the ELF.
The map file is not a reliable source for this: with LTO, original object
file paths are replaced by temporary ltrans.o partitions in all output
sections, making per-module attribution impossible from the map alone.
Instead we invoke nm --defined-only -l on the ELF, which uses DWARF debug
info to attribute each placed symbol to its original source file.
Requires usermod libraries to be compiled with -g so that DWARF sections
are present in the ELF. load_usermods.py injects -g for all WLED modules
via dep.env.AppendUnique(CCFLAGS=["-g"]).
Instead we invoke readelf --debug-dump=info --dwarf-depth=1 on the ELF,
which reads only the top-level compilation-unit DIEs from .debug_info.
Each CU corresponds to one source file; matching DW_AT_comp_dir +
DW_AT_name against the module src_dirs is sufficient to confirm a module
was compiled into the ELF. The output volume is proportional to the
number of source files, not the number of symbols.
Returns the set of build_dir basenames for confirmed modules.
"""
nm_path = _get_nm_path(env)
readelf_path = _get_readelf_path(env)
secho(f"INFO: Checking for usermod compilation units...")
try:
result = subprocess.run(
[nm_path, "--defined-only", "-l", str(elf_path)],
[readelf_path, "--debug-dump=info", "--dwarf-depth=1", str(elf_path)],
capture_output=True, text=True, errors="ignore", timeout=120,
)
nm_output = result.stdout
output = result.stdout
except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as e:
secho(f"WARNING: nm failed ({e}); skipping per-module validation", fg="yellow", err=True)
secho(f"WARNING: readelf failed ({e}); skipping per-module validation", fg="yellow", err=True)
return {Path(b.build_dir).name for b in module_lib_builders} # conservative pass
# Match placed symbols against builders as we parse nm output, exiting early
# once all builders are accounted for.
# nm --defined-only still includes debugging symbols (type 'N') such as the
# per-CU markers GCC emits in .debug_info (e.g. "usermod_example_cpp_6734d48d").
# These live at address 0x00000000 in their debug section — not in any load
# segment — so filtering them out leaves only genuinely placed symbols.
# nm -l appends a tab-separated "file:lineno" location to each symbol line.
remaining = {Path(str(b.src_dir)): Path(b.build_dir).name for b in module_lib_builders}
found = set()
project_dir = Path(env.subst("$PROJECT_DIR"))
for line in nm_output.splitlines():
def _flush_cu(comp_dir: str | None, name: str | None) -> None:
"""Match one completed CU against remaining builders."""
if not name or not remaining:
return
p = Path(name)
src_path = (Path(comp_dir) / p) if (comp_dir and not p.is_absolute()) else p
# In arduino+espidf dual-framework builds the IDF toolchain sets DW_AT_comp_dir
# to the virtual path "/IDF_PROJECT" rather than the real project root, so
# src_path won't match. Pre-compute a fallback using $PROJECT_DIR and check
# both candidates in a single pass.
use_fallback = not p.is_absolute() and comp_dir and Path(comp_dir) != project_dir
src_path_real = project_dir / p if use_fallback else None
for src_dir in list(remaining):
if src_path.is_relative_to(src_dir) or (src_path_real and src_path_real.is_relative_to(src_dir)):
found.add(remaining.pop(src_dir))
return
# readelf emits one DW_TAG_compile_unit DIE per source file. Attributes
# of interest:
# DW_AT_name — source file (absolute, or relative to comp_dir)
# DW_AT_comp_dir — compile working directory
# Both appear as either a direct string or an indirect string:
# DW_AT_name : foo.cpp
# DW_AT_name : (indirect string, offset: 0x…): foo.cpp
# Taking the portion after the *last* ": " on the line handles both forms.
comp_dir = name = None
for line in output.splitlines():
if 'Compilation Unit @' in line:
_flush_cu(comp_dir, name)
comp_dir = name = None
continue
if not remaining:
break # all builders matched
addr, _, _ = line.partition(' ')
if not addr.lstrip('0'):
continue # zero address — skip debug-section marker
if '\t' not in line:
continue
loc = line.rsplit('\t', 1)[1]
# Strip trailing :lineno (e.g. "/path/to/foo.cpp:42" → "/path/to/foo.cpp")
src_path = Path(loc.rsplit(':', 1)[0])
# Path.is_relative_to() handles OS-specific separators correctly without
# any regex, avoiding Windows path escaping issues.
for src_dir in list(remaining):
if src_path.is_relative_to(src_dir):
found.add(remaining.pop(src_dir))
break
m = _ATTR.search(line)
if m:
_, _, val = line.rpartition(': ')
val = val.strip()
if m.group(1) == 'name':
name = val
else:
comp_dir = val
_flush_cu(comp_dir, name) # flush the last CU
return found
DYNARRAY_SECTION = ".dtors" if env.get("PIOPLATFORM") == "espressif8266" else ".dynarray"
USERMODS_SECTION = f"{DYNARRAY_SECTION}.usermods.1"
def count_usermod_objects(map_file: list[str]) -> int:
""" Returns the number of usermod objects in the usermod list """
# Count the number of entries in the usermods table section
return len([x for x in map_file if USERMODS_SECTION in x])
""" Returns the number of usermod objects in the usermod list.
Computes the count from the address span between the .dynarray.usermods.0
and .dynarray.usermods.99999 sentinel sections. This mirrors the
DYNARRAY_LENGTH macro and is reliable under LTO, where all entries are
merged into a single ltrans partition so counting section occurrences
always yields 1 regardless of the true count.
"""
ENTRY_SIZE = 4 # sizeof(Usermod*) on 32-bit targets
addr_begin = None
addr_end = None
for i, line in enumerate(map_file):
stripped = line.strip()
if stripped == '.dynarray.usermods.0':
if i + 1 < len(map_file):
m = re.search(r'0x([0-9a-fA-F]+)', map_file[i + 1])
if m:
addr_begin = int(m.group(1), 16)
elif stripped == '.dynarray.usermods.99999':
if i + 1 < len(map_file):
m = re.search(r'0x([0-9a-fA-F]+)', map_file[i + 1])
if m:
addr_end = int(m.group(1), 16)
if addr_begin is not None and addr_end is not None:
break
if addr_begin is None or addr_end is None:
return 0
return (addr_end - addr_begin) // ENTRY_SIZE
def validate_map_file(source, target, env):
""" Validate that all modules appear in the output build """
build_dir = Path(env.subst("$BUILD_DIR"))
map_file_path = build_dir / env.subst("${PROGNAME}.map")
map_file_path = build_dir / env.subst("${PROGNAME}.map")
if not map_file_path.exists():
secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
@@ -111,6 +155,7 @@ def validate_map_file(source, target, env):
secho(f"INFO: {usermod_object_count} usermod object entries")
elf_path = build_dir / env.subst("${PROGNAME}.elf")
confirmed_modules = check_elf_modules(elf_path, env, module_lib_builders)
missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules]
@@ -120,7 +165,6 @@ def validate_map_file(source, target, env):
fg="red",
err=True)
Exit(1)
return None
env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))

10
tools/dynarray_espressif32.ld
View File

@@ -1,10 +0,0 @@
/* ESP32 linker script fragment to add dynamic array section to binary */
SECTIONS
{
.dynarray :
{
. = ALIGN(0x10);
KEEP(*(SORT_BY_INIT_PRIORITY(.dynarray.*)))
} > default_rodata_seg
}
INSERT AFTER .flash.rodata;

11
wled00/dynarray.h
View File

@@ -20,15 +20,4 @@ Macros for generating a "dynamic array", a static array of objects declared in d
#define DYNARRAY_END(array_name) array_name##_end
#define DYNARRAY_LENGTH(array_name) (&DYNARRAY_END(array_name)[0] - &DYNARRAY_BEGIN(array_name)[0])
#ifdef ESP8266
// ESP8266 linker script cannot be extended with a unique section for dynamic arrays.
// We instead pack them in the ".dtors" section, as it's sorted and uploaded to the flash
// (but will never be used in the embedded system)
#define DYNARRAY_SECTION ".dtors"
#else /* ESP8266 */
// Use a unique named section; the linker script must be extended to ensure it's correctly placed.
#define DYNARRAY_SECTION ".dynarray"
#endif