Generalize module link validation

Perform validation for external modules, too.

pio-scripts/load_usermods.py

@@ -39,10 +39,6 @@ usermods = env.GetProjectOption("custom_usermods","")
 # Handle "all usermods" case
 if usermods == '*':
   usermods = [f.name for f in usermod_dir.iterdir() if f.is_dir() and f.joinpath('library.json').exists()]
-  # Update the environment, as many modules use scripts to detect their dependencies
-  env.GetProjectConfig().set("env:" + env['PIOENV'], 'custom_usermods', " ".join(usermods))
-  # Leave a note for the validation script
-  env.GetProjectConfig().set("env:" + env['PIOENV'], 'custom_all_usermods_enabled', "1")
 else:
   usermods = usermods.split()
 

pio-scripts/validate_modules.py

@@ -0,0 +1,95 @@
+import re
+import sys
+from pathlib import Path   # For OS-agnostic path manipulation
+from typing import Iterable
+from click import secho
+from SCons.Script import Action, Exit
+from platformio import util
+from platformio.builder.tools.piolib import LibBuilderBase
+
+
+def is_wled_module(env, dep: LibBuilderBase) -> bool:
+  """Returns true if the specified library is a wled module
+  """
+  usermod_dir = Path(env["PROJECT_DIR"]).resolve() / "usermods"
+  return usermod_dir in Path(dep.src_dir).parents or str(dep.name).startswith("wled-")
+
+
+def read_lines(p: Path):
+    """ Read in the contents of a file for analysis """
+    with p.open("r", encoding="utf-8", errors="ignore") as f:
+        return f.readlines()
+
+
+def check_map_file_objects(map_file: list[str], dirs: Iterable[str]) -> set[str]:
+    """ Identify which dirs contributed to the final build
+
+        Returns the (sub)set of dirs that are found in the output ELF
+    """
+    # Pattern to match symbols in object directories
+    # Join directories into alternation
+    usermod_dir_regex = "|".join([re.escape(dir) for dir in dirs])
+    # Matches nonzero address, any size, and any path in a matching directory
+    object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o")
+
+    found = set()
+    for line in map_file:
+        matches = object_path_regex.findall(line)
+        for m in matches:
+            found.add(m)
+    return found
+
+
+def count_usermod_objects(map_file: list[str]) -> int:
+    """ Returns the number of usermod objects in the usermod list """
+    # Count the number of entries in the usermods table section
+    return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
+
+
+def validate_map_file(source, target, env):
+    """ Validate that all modules appear in the output build """
+    build_dir = Path(env.subst("$BUILD_DIR"))
+    map_file_path = build_dir /  env.subst("${PROGNAME}.map")
+
+    if not map_file_path.exists():
+        secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
+        Exit(1)
+
+    # Identify the WLED module source directories
+    module_lib_builders = [builder for builder in env.GetLibBuilders() if is_wled_module(env, builder)]    
+    
+    if env.GetProjectOption("custom_usermods","") == "*":
+        # All usermods build; filter non-platform-OK modules
+        module_lib_builders = [builder for builder in module_lib_builders if env.IsCompatibleLibBuilder(builder)]
+    else:
+        incompatible_builders = [builder for builder in module_lib_builders if not env.IsCompatibleLibBuilder(builder)]
+        if incompatible_builders:
+            secho(
+                f"ERROR: Modules {[b.name for b in incompatible_builders]} are not compatible with this platform!",
+                fg="red",
+                err=True)
+            Exit(1)            
+        pass
+
+    # Extract the values we care about
+    modules = {Path(builder.build_dir).name: builder.name for builder in module_lib_builders}
+    secho(f"INFO: {len(modules)} libraries linked as WLED optional/user modules")
+
+    # Now parse the map file
+    map_file_contents = read_lines(map_file_path)
+    usermod_object_count = count_usermod_objects(map_file_contents)   
+    secho(f"INFO: {usermod_object_count} usermod object entries")
+
+    confirmed_modules = check_map_file_objects(map_file_contents, modules.keys())
+    missing_modules = [modname for mdir, modname in modules.items() if mdir not in confirmed_modules]
+    if missing_modules:
+        secho(
+            f"ERROR: No object files from {missing_modules} found in linked output!",
+            fg="red",
+            err=True)
+        Exit(1)
+    return None
+
+Import("env")
+env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked optional modules (usermods) in map file'))

pio-scripts/validate_usermods.py

@@ -1,94 +0,0 @@
-import re
-import sys
-from pathlib import Path   # For OS-agnostic path manipulation
-from click import secho
-from SCons.Script import Action, Exit
-from platformio import util
-
-def read_lines(p: Path):
-    """ Read in the contents of a file for analysis """
-    with p.open("r", encoding="utf-8", errors="ignore") as f:
-        return f.readlines()
-
-def check_map_file_objects(map_file: list[str], usermod_dirs: list[str]) -> set[str]:
-    """ Checks that an object file from each usermod_dir appears in the linked output
-
-        Returns the (sub)set of usermod_dirs that are found in the output ELF
-    """
-    # Pattern to match symbols in object directories
-    # Join directories into alternation
-    usermod_dir_regex = "|".join([re.escape(dir) for dir in usermod_dirs])
-    # Matches nonzero address, any size, and any path in a matching directory
-    object_path_regex = re.compile(r"0x0*[1-9a-f][0-9a-f]*\s+0x[0-9a-f]+\s+\S+[/\\](" + usermod_dir_regex + r")[/\\]\S+\.o")
-
-    found = set()
-    for line in map_file:
-        matches = object_path_regex.findall(line)
-        for m in matches:
-            found.add(m)
-    return found
-
-def count_registered_usermods(map_file: list[str]) -> int:
-    """ Returns the number of usermod objects in the usermod list """
-    # Count the number of entries in the usermods table section
-    return len([x for x in map_file if ".dtors.tbl.usermods.1" in x])
-
-
-def validate_map_file(source, target, env):
-    """ Validate that all usermods appear in the output build """
-    build_dir = Path(env.subst("$BUILD_DIR"))
-    map_file_path = build_dir /  env.subst("${PROGNAME}.map")
-
-    if not map_file_path.exists():
-        secho(f"ERROR: Map file not found: {map_file_path}", fg="red", err=True)
-        Exit(1)
-
-    # Load project settings
-    usermods = env.GetProjectOption("custom_usermods","").split()
-    libdeps = env.GetProjectOption("lib_deps", [])
-    lib_builders = env.GetLibBuilders()
-
-    secho(f"INFO: Expecting {len(usermods)} usermods: {', '.join(usermods)}")
-
-    # Map the usermods to libdeps; every usermod should have one
-    usermod_dirs = []
-    for mod in usermods:
-        modstr = f"{mod} = symlink://"
-        this_mod_libdeps = [libdep[len(modstr):] for libdep in libdeps if libdep.startswith(modstr)]
-        if not this_mod_libdeps:
-            secho(
-                f"ERROR: Usermod {mod} not found in build libdeps!",
-                fg="red",
-                err=True)
-            Exit(1)
-        # Save only the final folder name
-        usermod_dir = Path(this_mod_libdeps[0]).name
-        # Search lib_builders
-        this_mod_builders = [builder for builder in lib_builders if Path(builder.src_dir).name == usermod_dir]
-        if not this_mod_builders:
-            secho(
-                f"ERROR: Usermod {mod} not found in library builders!",
-                fg="red",
-                err=True)
-            Exit(1)
-        usermod_dirs.append(usermod_dir)
-
-    # Now parse the map file
-    map_file_contents = read_lines(map_file_path)
-    confirmed_usermods = check_map_file_objects(map_file_contents, usermod_dirs)
-    usermod_object_count = count_registered_usermods(map_file_contents)
-
-    secho(f"INFO: {len(usermod_dirs)}/{len(usermods)} libraries linked via custom_usermods, producing {usermod_object_count} usermod object entries")
-    missing_usermods = set(usermod_dirs).difference(confirmed_usermods)
-    if missing_usermods:
-        secho(
-            f"ERROR: No object files from {missing_usermods} found in linked output!",
-            fg="red",
-            err=True)
-        Exit(1)
-    return None
-
-Import("env")
-if not env.GetProjectOption("custom_all_usermods_enabled",""):   # TODO: fix handling of platform mismatches
-    env.Append(LINKFLAGS=[env.subst("-Wl,--Map=${BUILD_DIR}/${PROGNAME}.map")])
-    env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", Action(validate_map_file, cmdstr='Checking linked usermods in map file...'))

platformio.ini

@@ -116,7 +116,7 @@ extra_scripts =
   pre:pio-scripts/user_config_copy.py
   pre:pio-scripts/load_usermods.py
   pre:pio-scripts/build_ui.py
-  post:pio-scripts/validate_usermods.py  ;; double-check the build output usermods
+  post:pio-scripts/validate_modules.py  ;; double-check the build output usermods
   ; post:pio-scripts/obj-dump.py  ;; convenience script to create a disassembly dump of the firmware (hardcore debugging)
 
 # ------------------------------------------------------------------------------