diff --git a/pio-scripts/load_usermods.py b/pio-scripts/load_usermods.py index d50bf196b..31e211fc0 100644 --- a/pio-scripts/load_usermods.py +++ b/pio-scripts/load_usermods.py @@ -39,10 +39,6 @@ usermods = env.GetProjectOption("custom_usermods","") # Handle "all usermods" case if usermods == '*': usermods = [f.name for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()] - # Update the environment, as many modules use scripts to detect their dependencies - env.GetProjectConfig().set("env:" + env['PIOENV'], 'custom_usermods', " ".join(usermods)) - # Leave a note for the validation script - env.GetProjectConfig().set("env:" + env['PIOENV'], 'custom_all_usermods_enabled', "1") else: usermods = usermods.split() diff --git a/pio-scripts/validate_modules.py b/pio-scripts/validate_modules.py new file mode 100644 index 000000000..eb6ebb446 --- /dev/null +++ b/pio-scripts/validate_modules.py @@ -0,0 +1,95 @@ +import re +import sys +from pathlib import Path # For OS-agnostic path manipulation +from typing import Iterable +from click import secho +from SCons.Script import Action, Exit +from platformio import util +from platformio.builder.tools.piolib import LibBuilderBase + + +def is_wled_module(env, dep: LibBuilderBase) -> bool: + """Returns true if the specified library is a wled module + """ + usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods" + return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-") + + +def read_lines(p: Path): + """ Read in the contents of a file for analysis """ + with p.open("r", encoding="utf-8", errors="ignore") as f: + return f.readlines() + + +def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]: + """ Identify which dirs contributed to the final build + + Returns the (sub)set of dirs that are found in the output ELF + """ + # Pattern to match symbols in object directories + # Join directories into alternation + usermod_dir_regex = "|".join([re.escape(dir) for dir in dirs]) + # Matches nonzero address, any size, and any path in a matching directory + object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o") + + found = set() + for line in map_file: + matches = object_path_regex.findall(line) + for m in matches: + found.add(m) + return found + + +def count_usermod_objects(map_file: list[str]) -> int: + """ Returns the number of usermod objects in the usermod list """ + # Count the number of entries in the usermods table section + return len([x for x in map_file if ".dtors.tbl.usermods.1" in x]) + + +def validate_map_file(source, target, env): + """ Validate that all modules appear in the output build """ + build_dir = Path(env.subst("$BUILD_DIR")) + map_file_path = build_dir / env.subst("${PROGNAME}.map") + + if not map_file_path.exists(): + secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True) + Exit(1) + + # Identify the WLED module source directories + module_lib_builders = [builder for builder in env.GetLibBuilders() if is_wled_module(env, builder)] + + if env.GetProjectOption("custom_usermods","") == "*": + # All usermods build; filter non-platform-OK modules + module_lib_builders = [builder for builder in module_lib_builders if env.IsCompatibleLibBuilder(builder)] + else: + incompatible_builders = [builder for builder in module_lib_builders if not env.IsCompatibleLibBuilder(builder)] + if incompatible_builders: + secho( + f"ERROR: Modules {[b.name for b in incompatible_builders]} are not compatible with this platform!", + fg="red", + err=True) + Exit(1) + pass + + # Extract the values we care about + modules = {Path(builder.build_dir).name: builder.name for builder in module_lib_builders} + secho(f"INFO: {len(modules)} libraries linked as WLED optional/user modules") + + # Now parse the map file + map_file_contents = read_lines(map_file_path) + usermod_object_count = count_usermod_objects(map_file_contents) + secho(f"INFO: {usermod_object_count} usermod object entries") + + confirmed_modules = check_map_file_objects(map_file_contents, modules.keys()) + missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules] + if missing_modules: + secho( + f"ERROR: No object files from {missing_modules} found in linked output!", + fg="red", + err=True) + Exit(1) + return None + +Import("env") +env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")]) +env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file')) diff --git a/pio-scripts/validate_usermods.py b/pio-scripts/validate_usermods.py deleted file mode 100644 index a1a1e3c24..000000000 --- a/pio-scripts/validate_usermods.py +++ /dev/null @@ -1,94 +0,0 @@ -import re -import sys -from pathlib import Path # For OS-agnostic path manipulation -from click import secho -from SCons.Script import Action, Exit -from platformio import util - -def read_lines(p: Path): - """ Read in the contents of a file for analysis """ - with p.open("r", encoding="utf-8", errors="ignore") as f: - return f.readlines() - -def check_map_file_objects(map_file: list[str], usermod_dirs: list[str]) -> set[str]: - """ Checks that an object file from each usermod_dir appears in the linked output - - Returns the (sub)set of usermod_dirs that are found in the output ELF - """ - # Pattern to match symbols in object directories - # Join directories into alternation - usermod_dir_regex = "|".join([re.escape(dir) for dir in usermod_dirs]) - # Matches nonzero address, any size, and any path in a matching directory - object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o") - - found = set() - for line in map_file: - matches = object_path_regex.findall(line) - for m in matches: - found.add(m) - return found - -def count_registered_usermods(map_file: list[str]) -> int: - """ Returns the number of usermod objects in the usermod list """ - # Count the number of entries in the usermods table section - return len([x for x in map_file if ".dtors.tbl.usermods.1" in x]) - - -def validate_map_file(source, target, env): - """ Validate that all usermods appear in the output build """ - build_dir = Path(env.subst("$BUILD_DIR")) - map_file_path = build_dir / env.subst("${PROGNAME}.map") - - if not map_file_path.exists(): - secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True) - Exit(1) - - # Load project settings - usermods = env.GetProjectOption("custom_usermods","").split() - libdeps = env.GetProjectOption("lib_deps", []) - lib_builders = env.GetLibBuilders() - - secho(f"INFO: Expecting {len(usermods)} usermods: {', '.join(usermods)}") - - # Map the usermods to libdeps; every usermod should have one - usermod_dirs = [] - for mod in usermods: - modstr = f"{mod} = symlink://" - this_mod_libdeps = [libdep[len(modstr):] for libdep in libdeps if libdep.startswith(modstr)] - if not this_mod_libdeps: - secho( - f"ERROR: Usermod {mod} not found in build libdeps!", - fg="red", - err=True) - Exit(1) - # Save only the final folder name - usermod_dir = Path(this_mod_libdeps[0]).name - # Search lib_builders - this_mod_builders = [builder for builder in lib_builders if Path(builder.src_dir).name == usermod_dir] - if not this_mod_builders: - secho( - f"ERROR: Usermod {mod} not found in library builders!", - fg="red", - err=True) - Exit(1) - usermod_dirs.append(usermod_dir) - - # Now parse the map file - map_file_contents = read_lines(map_file_path) - confirmed_usermods = check_map_file_objects(map_file_contents, usermod_dirs) - usermod_object_count = count_registered_usermods(map_file_contents) - - secho(f"INFO: {len(usermod_dirs)}/{len(usermods)} libraries linked via custom_usermods, producing {usermod_object_count} usermod object entries") - missing_usermods = set(usermod_dirs).difference(confirmed_usermods) - if missing_usermods: - secho( - f"ERROR: No object files from {missing_usermods} found in linked output!", - fg="red", - err=True) - Exit(1) - return None - -Import("env") -if not env.GetProjectOption("custom_all_usermods_enabled",""): # TODO: fix handling of platform mismatches - env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")]) - env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked usermods in map file...')) diff --git a/platformio.ini b/platformio.ini index e1a5014b0..9bdf58d34 100644 --- a/platformio.ini +++ b/platformio.ini @@ -116,7 +116,7 @@ extra_scripts = pre:pio-scripts/user_config_copy.py pre:pio-scripts/load_usermods.py pre:pio-scripts/build_ui.py - post:pio-scripts/validate_usermods.py ;; double-check the build output usermods + post:pio-scripts/validate_modules.py ;; double-check the build output usermods ; post:pio-scripts/obj-dump.py ;; convenience script to create a disassembly dump of the firmware (hardcore debugging) # ------------------------------------------------------------------------------