mirror of
https://github.com/wled/WLED.git
synced 2025-07-24 11:16:37 +00:00
Generalize module link validation
Perform validation for external modules, too.
This commit is contained in:
Will Miles
parent
75c95d88e2
commit
309c8d67f3
@ -39,10 +39,6 @@ usermods = env.GetProjectOption("custom_usermods","")
|
||||
# Handle "all usermods" case
|
||||
if usermods == '*':
|
||||
usermods = [f.name for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()]
|
||||
# Update the environment, as many modules use scripts to detect their dependencies
|
||||
env.GetProjectConfig().set("env:" + env['PIOENV'], 'custom_usermods', " ".join(usermods))
|
||||
# Leave a note for the validation script
|
||||
env.GetProjectConfig().set("env:" + env['PIOENV'], 'custom_all_usermods_enabled', "1")
|
||||
else:
|
||||
usermods = usermods.split()
|
||||
|
||||
|
||||
95
pio-scripts/validate_modules.py
Normal file
95
pio-scripts/validate_modules.py
Normal file
@ -0,0 +1,95 @@
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path # For OS-agnostic path manipulation
|
||||
from typing import Iterable
|
||||
from click import secho
|
||||
from SCons.Script import Action, Exit
|
||||
from platformio import util
|
||||
from platformio.builder.tools.piolib import LibBuilderBase
|
||||
|
||||
|
||||
def is_wled_module(env, dep: LibBuilderBase) -> bool:
|
||||
"""Returns true if the specified library is a wled module
|
||||
"""
|
||||
usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods"
|
||||
return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
|
||||
|
||||
|
||||
def read_lines(p: Path):
|
||||
""" Read in the contents of a file for analysis """
|
||||
with p.open("r", encoding="utf-8", errors="ignore") as f:
|
||||
return f.readlines()
|
||||
|
||||
|
||||
def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]:
|
||||
""" Identify which dirs contributed to the final build
|
||||
|
||||
Returns the (sub)set of dirs that are found in the output ELF
|
||||
"""
|
||||
# Pattern to match symbols in object directories
|
||||
# Join directories into alternation
|
||||
usermod_dir_regex = "|".join([re.escape(dir) for dir in dirs])
|
||||
# Matches nonzero address, any size, and any path in a matching directory
|
||||
object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o")
|
||||
|
||||
found = set()
|
||||
for line in map_file:
|
||||
matches = object_path_regex.findall(line)
|
||||
for m in matches:
|
||||
found.add(m)
|
||||
return found
|
||||
|
||||
|
||||
def count_usermod_objects(map_file: list[str]) -> int:
|
||||
""" Returns the number of usermod objects in the usermod list """
|
||||
# Count the number of entries in the usermods table section
|
||||
return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
|
||||
|
||||
|
||||
def validate_map_file(source, target, env):
|
||||
""" Validate that all modules appear in the output build """
|
||||
build_dir = Path(env.subst("$BUILD_DIR"))
|
||||
map_file_path = build_dir / env.subst("${PROGNAME}.map")
|
||||
|
||||
if not map_file_path.exists():
|
||||
secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
|
||||
Exit(1)
|
||||
|
||||
# Identify the WLED module source directories
|
||||
module_lib_builders = [builder for builder in env.GetLibBuilders() if is_wled_module(env, builder)]
|
||||
|
||||
if env.GetProjectOption("custom_usermods","") == "*":
|
||||
# All usermods build; filter non-platform-OK modules
|
||||
module_lib_builders = [builder for builder in module_lib_builders if env.IsCompatibleLibBuilder(builder)]
|
||||
else:
|
||||
incompatible_builders = [builder for builder in module_lib_builders if not env.IsCompatibleLibBuilder(builder)]
|
||||
if incompatible_builders:
|
||||
secho(
|
||||
f"ERROR: Modules {[b.name for b in incompatible_builders]} are not compatible with this platform!",
|
||||
fg="red",
|
||||
err=True)
|
||||
Exit(1)
|
||||
pass
|
||||
|
||||
# Extract the values we care about
|
||||
modules = {Path(builder.build_dir).name: builder.name for builder in module_lib_builders}
|
||||
secho(f"INFO: {len(modules)} libraries linked as WLED optional/user modules")
|
||||
|
||||
# Now parse the map file
|
||||
map_file_contents = read_lines(map_file_path)
|
||||
usermod_object_count = count_usermod_objects(map_file_contents)
|
||||
secho(f"INFO: {usermod_object_count} usermod object entries")
|
||||
|
||||
confirmed_modules = check_map_file_objects(map_file_contents, modules.keys())
|
||||
missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules]
|
||||
if missing_modules:
|
||||
secho(
|
||||
f"ERROR: No object files from {missing_modules} found in linked output!",
|
||||
fg="red",
|
||||
err=True)
|
||||
Exit(1)
|
||||
return None
|
||||
|
||||
Import("env")
|
||||
env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
|
||||
env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))
|
||||
@ -1,94 +0,0 @@
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path # For OS-agnostic path manipulation
|
||||
from click import secho
|
||||
from SCons.Script import Action, Exit
|
||||
from platformio import util
|
||||
|
||||
def read_lines(p: Path):
|
||||
""" Read in the contents of a file for analysis """
|
||||
with p.open("r", encoding="utf-8", errors="ignore") as f:
|
||||
return f.readlines()
|
||||
|
||||
def check_map_file_objects(map_file: list[str], usermod_dirs: list[str]) -> set[str]:
|
||||
""" Checks that an object file from each usermod_dir appears in the linked output
|
||||
|
||||
Returns the (sub)set of usermod_dirs that are found in the output ELF
|
||||
"""
|
||||
# Pattern to match symbols in object directories
|
||||
# Join directories into alternation
|
||||
usermod_dir_regex = "|".join([re.escape(dir) for dir in usermod_dirs])
|
||||
# Matches nonzero address, any size, and any path in a matching directory
|
||||
object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o")
|
||||
|
||||
found = set()
|
||||
for line in map_file:
|
||||
matches = object_path_regex.findall(line)
|
||||
for m in matches:
|
||||
found.add(m)
|
||||
return found
|
||||
|
||||
def count_registered_usermods(map_file: list[str]) -> int:
|
||||
""" Returns the number of usermod objects in the usermod list """
|
||||
# Count the number of entries in the usermods table section
|
||||
return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
|
||||
|
||||
|
||||
def validate_map_file(source, target, env):
|
||||
""" Validate that all usermods appear in the output build """
|
||||
build_dir = Path(env.subst("$BUILD_DIR"))
|
||||
map_file_path = build_dir / env.subst("${PROGNAME}.map")
|
||||
|
||||
if not map_file_path.exists():
|
||||
secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
|
||||
Exit(1)
|
||||
|
||||
# Load project settings
|
||||
usermods = env.GetProjectOption("custom_usermods","").split()
|
||||
libdeps = env.GetProjectOption("lib_deps", [])
|
||||
lib_builders = env.GetLibBuilders()
|
||||
|
||||
secho(f"INFO: Expecting {len(usermods)} usermods: {', '.join(usermods)}")
|
||||
|
||||
# Map the usermods to libdeps; every usermod should have one
|
||||
usermod_dirs = []
|
||||
for mod in usermods:
|
||||
modstr = f"{mod} = symlink://"
|
||||
this_mod_libdeps = [libdep[len(modstr):] for libdep in libdeps if libdep.startswith(modstr)]
|
||||
if not this_mod_libdeps:
|
||||
secho(
|
||||
f"ERROR: Usermod {mod} not found in build libdeps!",
|
||||
fg="red",
|
||||
err=True)
|
||||
Exit(1)
|
||||
# Save only the final folder name
|
||||
usermod_dir = Path(this_mod_libdeps[0]).name
|
||||
# Search lib_builders
|
||||
this_mod_builders = [builder for builder in lib_builders if Path(builder.src_dir).name == usermod_dir]
|
||||
if not this_mod_builders:
|
||||
secho(
|
||||
f"ERROR: Usermod {mod} not found in library builders!",
|
||||
fg="red",
|
||||
err=True)
|
||||
Exit(1)
|
||||
usermod_dirs.append(usermod_dir)
|
||||
|
||||
# Now parse the map file
|
||||
map_file_contents = read_lines(map_file_path)
|
||||
confirmed_usermods = check_map_file_objects(map_file_contents, usermod_dirs)
|
||||
usermod_object_count = count_registered_usermods(map_file_contents)
|
||||
|
||||
secho(f"INFO: {len(usermod_dirs)}/{len(usermods)} libraries linked via custom_usermods, producing {usermod_object_count} usermod object entries")
|
||||
missing_usermods = set(usermod_dirs).difference(confirmed_usermods)
|
||||
if missing_usermods:
|
||||
secho(
|
||||
f"ERROR: No object files from {missing_usermods} found in linked output!",
|
||||
fg="red",
|
||||
err=True)
|
||||
Exit(1)
|
||||
return None
|
||||
|
||||
Import("env")
|
||||
if not env.GetProjectOption("custom_all_usermods_enabled",""): # TODO: fix handling of platform mismatches
|
||||
env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
|
||||
env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked usermods in map file...'))
|
||||
@ -116,7 +116,7 @@ extra_scripts =
|
||||
pre:pio-scripts/user_config_copy.py
|
||||
pre:pio-scripts/load_usermods.py
|
||||
pre:pio-scripts/build_ui.py
|
||||
post:pio-scripts/validate_usermods.py ;; double-check the build output usermods
|
||||
post:pio-scripts/validate_modules.py ;; double-check the build output usermods
|
||||
; post:pio-scripts/obj-dump.py ;; convenience script to create a disassembly dump of the firmware (hardcore debugging)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user