diff --git a/pio-tools/ldf_cache.py b/pio-tools/ldf_cache.py new file mode 100644 index 000000000..ac8db29b0 --- /dev/null +++ b/pio-tools/ldf_cache.py @@ -0,0 +1,1512 @@ +""" +PlatformIO Advanced Script for intelligent LDF caching. + +This script implements a two-phase caching system: +1. First run: Performs verbose build, collects dependencies, creates cache +2. Second run: Applies cached dependencies with lib_ldf_mode=off for faster builds + +Features: +- Intelligent cache invalidation based on file hashes +- Build order preservation for correct symbol resolution + +Copyright: Jason2866 +""" + +Import("env") + +import os +import hashlib +import datetime +import re +import pprint +import json +import shutil +import subprocess +import sys +import shlex +import atexit +from pathlib import Path +from platformio.builder.tools.piolib import LibBuilderBase +from platformio.builder.tools.piobuild import SRC_HEADER_EXT, SRC_C_EXT, SRC_CXX_EXT, SRC_ASM_EXT, SRC_BUILD_EXT +from SCons.Script import COMMAND_LINE_TARGETS +from dataclasses import dataclass +from typing import Optional + +# INFO PlatformIO Core constants +# SRC_HEADER_EXT = ["h", "hpp", "hxx", "h++", "hh", "inc", "tpp", "tcc"] +# SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"] +# SRC_C_EXT = ["c"] +# SRC_CXX_EXT = ["cc", "cpp", "cxx", "c++"] +# SRC_BUILD_EXT = SRC_C_EXT + SRC_CXX_EXT + SRC_ASM_EXT + +github_actions = os.getenv('GITHUB_ACTIONS') +project_dir = env.subst("$PROJECT_DIR") +env_name = env.subst("$PIOENV") +compiledb_path = Path(project_dir) / ".pio" / "compiledb" / f"compile_commands_{env_name}.json" +logfile_path = Path(project_dir) / ".pio" / "compiledb" / f"compile_commands_{env_name}.log" +cache_base = Path(project_dir) / ".pio" / "ldf_cache" +cache_file = cache_base / f"ldf_cache_{env_name}.py" +build_dir = Path(env.subst("$BUILD_DIR")) +src_dir = Path(env.subst("$PROJECT_SRC_DIR")) +config = env.GetProjectConfig() +flag_custom_sdkconfig = False +if config.has_option("env:"+env["PIOENV"], "custom_sdkconfig") or env.BoardConfig().get("espidf.custom_sdkconfig", ""): + flag_custom_sdkconfig = True + +# Ensure log directory exists +logfile_path.parent.mkdir(parents=True, exist_ok=True) + +def is_first_run_needed(): + """ + Determines if the first run (full verbose build) is needed based on file dependencies. + + Checks for essential build artifacts and compile database existence. + + Returns: + bool: True if first run is needed, False if cache can be used + """ + if not compiledb_path.exists() or compiledb_path.stat().st_size == 0: + return True + + lib_dirs = list(build_dir.glob("lib*")) + if not lib_dirs: + return True + + return False + +def is_build_environment_ready(): + """ + Checks if the build environment is complete and ready for cache application. + + Validates that all necessary build artifacts exist for second run. + + Returns: + bool: True if build environment is ready for cache application + """ + # Compile database must exist + if not compiledb_path.exists(): + return False + + # At least one library directory must exist + lib_dirs = list(build_dir.glob("lib*")) + if not lib_dirs: + return False + + return True + +def should_trigger_verbose_build(): + """ + Determines if a verbose build should be triggered for first run. + + Considers environment variables, cache existence, and build targets. + + Returns: + bool: True if verbose build should be triggered + """ + # Prevent recursive calls + if os.environ.get('_PIO_RECURSIVE_CALL') == 'true': + return False + if os.environ.get('PLATFORMIO_SETTING_FORCE_VERBOSE') == 'true': + return False + + # Check for return code from previous recursive call + if os.environ.get('_PIO_REC_CALL_RETURN_CODE') is not None: + return False + + # If cache exists, no need for verbose build + if cache_file.exists(): + return False + + # Only trigger for build-related targets + current_targets = COMMAND_LINE_TARGETS[:] + is_build_target = ( + not current_targets or + any(target in ["build", "buildprog"] for target in current_targets) + ) + if not is_build_target: + return False + + return is_first_run_needed() + +# Integrated log2compdb components for compile_commands.json generation +DIRCHANGE_PATTERN = re.compile(r"(?P\w+) directory '(?P.+)'") +INFILE_PATTERN = re.compile(r"(?P.+\.(cpp|cxx|cc|c|hpp|hxx|h))", re.IGNORECASE) + +@dataclass +class CompileCommand: + """ + Represents a single compile command extracted from build logs. + + Attributes: + file: Source file path + output: Output object file path + directory: Working directory for compilation + arguments: Complete compiler command line arguments + """ + file: str + output: str + directory: str + arguments: list + + @classmethod + def from_cmdline(cls, cc_cmd: Path, cmd_args: list[str], directory=None) -> Optional["CompileCommand"]: + """ + Create a CompileCommand from a command line. + + Parses compiler command line to extract source file, output file, and arguments. + + Args: + cc_cmd: Path to the compiler executable + cmd_args: List of command line arguments + directory: Optional working directory + + Returns: + CompileCommand or None if no valid input file found + """ + if cc_cmd.name not in cmd_args[0]: + return None + + cmd_args = cmd_args[:] + cmd_args[0] = str(cc_cmd) + + if directory is None: + directory = Path.cwd() + else: + directory = Path(directory) + + input_path = None + + # Try to find output file (-o flag) + try: + output_index = cmd_args.index("-o") + output_arg = cmd_args[output_index + 1] + if output_arg == "/dev/null": + output_path = None + else: + output_path = directory / Path(output_arg) + except (ValueError, IndexError): + output_index = None + output_path = None + + # Find input file based on output file stem or pattern matching + if output_index is not None and output_path is not None: + stem_matches = [item for item in cmd_args if Path(item).stem == output_path.stem] + for item in stem_matches: + if input_file_match := INFILE_PATTERN.search(item): + input_path = input_file_match.group("path") + break + if not input_path and stem_matches: + input_path = stem_matches[0] + if not input_path: + return None + input_path = directory / Path(input_path) + else: + # Fallback: search for source file patterns + match = None + for item in cmd_args: + match = INFILE_PATTERN.search(item) + if match: + break + if not match: + return None + input_path = Path(match.group("path")) + output_path = None + + return cls( + file=str(input_path), + arguments=cmd_args, + directory=str(directory), + output=str(output_path) if output_path else "", + ) + +@dataclass +class Compiler: + """ + Represents a compiler toolchain. + + Attributes: + name: Compiler name (e.g., 'gcc', 'g++') + path: Path to compiler executable + """ + name: str + path: Path + + @classmethod + def from_name(cls, compiler_name: str) -> "Compiler": + """ + Create Compiler from name string. + + Args: + compiler_name: Name of the compiler + + Returns: + Compiler instance + """ + path = Path(compiler_name) + return cls(name=compiler_name, path=path) + + def find_invocation_start(self, cmd_args: list[str]) -> int: + """ + Find compiler invocation in argument list. + + Args: + cmd_args: List of command line arguments + + Returns: + int: Index of compiler invocation + + Raises: + ValueError: If compiler invocation not found + """ + for index, arg in enumerate(cmd_args): + if self.name in arg or Path(arg).stem == self.name: + return index + raise ValueError(f"compiler invocation for {self.name} not found") + +def parse_build_log_to_compile_commands(logfile_path: Path, compiler_names: list[str]) -> list[CompileCommand]: + """ + Parse build log to extract compile commands for compile_commands.json generation. + + Processes verbose build log to extract compiler invocations and create + compile database entries for IDE integration. + + Args: + logfile_path: Path to build log file + compiler_names: List of compiler names to look for + + Returns: + List of CompileCommand objects + """ + if not logfile_path.exists(): + return [] + + compilers = [Compiler.from_name(name) for name in compiler_names] + entries = [] + file_entries = {} + dirstack = [os.getcwd()] + + try: + with logfile_path.open('r', encoding='utf-8', errors='ignore') as logfile: + for line in logfile: + line = line.strip() + if not line: + continue + + # Handle directory changes (make-style output) + if dirchange_match := DIRCHANGE_PATTERN.search(line): + action = dirchange_match.group("action") + path = dirchange_match.group("path") + if action == "Leaving": + if len(dirstack) > 1: + dirstack.pop() + elif action == "Entering": + dirstack.append(path) + continue + + # Parse command line + try: + cmd_args = shlex.split(line) + except ValueError: + continue + + if not cmd_args: + continue + + # Try to match against known compilers + for compiler in compilers: + try: + compiler_invocation_start = compiler.find_invocation_start(cmd_args) + entry = CompileCommand.from_cmdline( + compiler.path, + cmd_args[compiler_invocation_start:], + dirstack[-1] + ) + + # Avoid duplicate entries for the same file + if entry is not None and entry.file not in file_entries: + entries.append(entry) + file_entries[entry.file] = entry + break + except ValueError: + continue + + except Exception as e: + print(f"⚠ Error parsing build log: {e}") + + return entries + +class LDFCacheOptimizer: + """ + PlatformIO LDF cache optimizer to avoid unnecessary LDF runs. + + This class implements intelligent caching of Library Dependency Finder (LDF) + results to significantly speed up subsequent builds. + + The optimizer works in two phases: + 1. First run: Collects dependencies, creates cache, modifies platformio.ini + 2. Second run: Applies cached dependencies with lib_ldf_mode=off + + Attributes: + env: PlatformIO SCons environment + env_name: Current environment name + project_dir: Project root directory + src_dir: Source directory path + build_dir: Build output directory + cache_file: Path to cache file + _cache_applied_successfully: Flag indicating successful cache application + """ + + # File extensions relevant for LDF processing + HEADER_EXTENSIONS = set(SRC_HEADER_EXT) + SOURCE_EXTENSIONS = set(SRC_BUILD_EXT + SRC_C_EXT + SRC_CXX_EXT + SRC_ASM_EXT) + CONFIG_EXTENSIONS = {'.json', '.properties', '.txt', '.ini'} + ALL_RELEVANT_EXTENSIONS = HEADER_EXTENSIONS | SOURCE_EXTENSIONS | CONFIG_EXTENSIONS + + # Directories to ignore during file scanning (optimized for ESP/Tasmota projects) + IGNORE_DIRS = frozenset([ + '.git', '.github', '.cache', '.vscode', '.pio', 'boards', + 'data', 'build', 'pio-tools', 'tools', '__pycache__', 'variants', + 'berry', 'berry_tasmota', 'berry_matter', 'berry_custom', + 'berry_animate', 'berry_mapping', 'berry_int64', 'displaydesc', + 'html_compressed', 'html_uncompressed', 'language', 'energy_modbus_configs' + ]) + + def __init__(self, environment): + """ + Initialize the LDF cache optimizer with lazy initialization. + + Sets up paths, initializes PlatformIO integration, and determines + whether to execute second run logic based on build environment state. + + Args: + environment: PlatformIO SCons environment + """ + self.env = environment + self.env_name = self.env.get("PIOENV") + self.project_dir = Path(self.env.subst("$PROJECT_DIR")).resolve() + self.src_dir = Path(self.env.subst("$PROJECT_SRC_DIR")).resolve() + self.build_dir = Path(self.env.subst("$BUILD_DIR")).resolve() + + # Setup cache and backup file paths + cache_base = Path(self.project_dir) / ".pio" / "ldf_cache" + self.cache_file = cache_base / f"ldf_cache_{self.env_name}.py" + self.ldf_cache_ini = Path(self.project_dir) / "ldf_cache.ini" + self.platformio_ini = Path(self.project_dir) / "platformio.ini" + self.platformio_ini_backup = Path(self.project_dir) / ".pio" / "platformio_backup.ini" + + # Setup compile database paths + compiledb_base = Path(self.project_dir) / ".pio" / "compiledb" + self.compiledb_dir = compiledb_base + self.compile_commands_file = compiledb_base / f"compile_commands_{self.env_name}.json" + self.compile_commands_log_file = compiledb_base / f"compile_commands_{self.env_name}.log" + + self.lib_build_dir = Path(self.project_dir) / ".pio" / "build" / self.env_name + self.ALL_RELEVANT_EXTENSIONS = self.HEADER_EXTENSIONS | self.SOURCE_EXTENSIONS | self.CONFIG_EXTENSIONS + + # Cache application status tracking + self._cache_applied_successfully = False + + # Determine if second run should be executed (lazy initialization) + if is_build_environment_ready() and not is_first_run_needed(): + print("πŸ”„ Second run: Cache application mode") + self.execute_second_run() + + def execute_second_run(self): + """ + Execute second run logic: Apply cached dependencies with LDF disabled. + + Loads and validates cache, applies cached dependencies to SCons environment, + and handles fallback to normal build if cache application fails. + """ + self._cache_applied_successfully = False + + # Register exit handler for cleanup + self.register_exit_handler() + + try: + # Load and validate cache data + cache_data = self.load_cache() + if cache_data and self.validate_cache(cache_data): + # Apply cached dependencies to build environment + success = self.apply_ldf_cache_with_build_order(cache_data) + if success: + self._cache_applied_successfully = True + print("βœ… Cache applied successfully - lib_ldf_mode=off") + #print(f" CPPPATH: {len(self.env.get('CPPPATH', []))} entries") + #print(f" LIBS: {len(self.env.get('LIBS', []))} entries") + #print(f" OBJECTS: {len(self.env.get('OBJECTS', []))} objects") + else: + print("❌ Cache application failed") + else: + print("⚠ No valid cache found, falling back to normal build") + + except Exception as e: + print(f"❌ Error in second run: {e}") + self._cache_applied_successfully = False + + finally: + # Handle cleanup based on success/failure + if not self._cache_applied_successfully: + print("πŸ”„ Restoring original platformio.ini due to cache failure") + self.restore_platformio_ini() + + def apply_ldf_cache_with_build_order(self, cache_data): + """ + Apply cached dependencies with correct build order preservation. + + Coordinates application of build order and SCons variables to ensure + correct dependency resolution and linking order. + + Args: + cache_data: Dictionary containing cached build data + + Returns: + bool: True if cache application succeeded + """ + try: + self._current_cache_data = cache_data + + artifacts = cache_data.get('artifacts', {}) + build_order = artifacts + + if not build_order: + print("❌ No build order data in cache") + return False + # Apply build order (OBJECTS, linker configuration) + build_order_success = self.apply_build_order_to_environment(build_order) + # Apply SCons variables (include paths, libraries) + scons_vars_success = self.apply_cache_to_scons_vars(cache_data) + if build_order_success and scons_vars_success: + return True + else: + print("❌ Partial cache application failure") + return False + + except Exception as e: + print(f"βœ— Error applying LDF cache: {e}") + import traceback + traceback.print_exc() + return False + + def register_exit_handler(self): + """ + Register exit handler for conditional platformio.ini restoration. + + Ensures platformio.ini is restored to original state if cache + application fails or script exits unexpectedly. + """ + def cleanup_on_exit(): + """ + Cleanup function called on script exit. + + Restores platformio.ini if cache was not successfully applied. + """ + try: + if hasattr(self, '_cache_applied_successfully'): + if not self._cache_applied_successfully: + self.restore_platformio_ini() + else: + self.restore_platformio_ini() + except: + # Silently handle cleanup errors to avoid exit issues + pass + + atexit.register(cleanup_on_exit) + + def modify_platformio_ini_for_second_run(self): + """ + Modify platformio.ini for second run by setting lib_ldf_mode = off. + + Creates backup and modifies platformio.ini to disable LDF for subsequent + builds that will use cached dependencies. + + Returns: + bool: True if modification was successful or not needed + """ + try: + # Check if platformio.ini exists + if not self.platformio_ini.exists(): + print("❌ platformio.ini not found") + return False + + # Create backup if it doesn't exist + if not self.platformio_ini_backup.exists(): + shutil.copy2(self.platformio_ini, self.platformio_ini_backup) + print(f"βœ… Configuration backup created: {self.platformio_ini_backup.name}") + + # Read current platformio.ini content + with self.platformio_ini.open('r', encoding='utf-8') as f: + lines = f.readlines() + + # Find and modify lib_ldf_mode line + modified = False + for i, line in enumerate(lines): + stripped_line = line.strip() + if stripped_line.startswith('lib_ldf_mode'): + lines[i] = 'lib_ldf_mode = off ; LDF cache modified. To restore change to: lib_ldf_mode = chain\n' + modified = True + break + + # Write modified content or report no changes needed + if modified: + with self.platformio_ini.open('w', encoding='utf-8') as f: + f.writelines(lines) + print("βœ… platformio.ini successfully modified") + return True + else: + print("β„Ή No lib_ldf_mode entry found, no changes made") + return True + + except Exception as e: + print(f"❌ Error modifying platformio.ini: {e}") + return False + + @staticmethod + def restore_platformio_ini(project_dir=None): + """ + Restore original platformio.ini from backup. + + Restores platformio.ini to its original state using the backup + created during first run. + + Returns: + bool: True if restoration was successful + """ + if project_dir is None: + project_dir = env.subst("$PROJECT_DIR") if 'env' in globals() else "." + + platformio_ini = Path(project_dir) / "platformio.ini" + platformio_ini_backup = Path(project_dir) / ".pio" / "platformio_backup.ini" + + try: + if platformio_ini_backup.exists(): + platformio_ini_backup.rename(platformio_ini) + print("βœ… platformio.ini restored from backup") + else: + print("πŸ’‘ No backup found - platformio.ini not modified") + except Exception as e: + print(f"⚠ Error restoring platformio.ini: {e}") + + def validate_ldf_mode_compatibility(self): + """ + Validate that the current LDF mode is compatible with caching. + + Uses PlatformIO Core's native LDF mode validation to ensure + compatibility with the caching system. + + Returns: + bool: True if LDF mode is compatible with caching + """ + try: + current_mode = self.env.GetProjectOption("lib_ldf_mode", "chain") + validated_mode = LibBuilderBase.validate_ldf_mode(current_mode) + + # Check against supported modes + compatible_modes = ["chain", "off"] + if validated_mode.lower() in compatible_modes: + print(f"βœ… LDF mode '{validated_mode}' is compatible with caching") + return True + else: + print(f"⚠ LDF mode '{validated_mode}' not optimal for caching") + print(f"πŸ’‘ Recommended modes: {', '.join(compatible_modes)}") + return False + except Exception as e: + print(f"⚠ Could not determine LDF mode: {e}") + return True + + def create_compiledb_integrated(self): + """ + Create compile_commands.json using integrated log parsing functionality. + + Generates compile database from verbose build log for IDE integration + and IntelliSense support. + + Returns: + bool: True if compile_commands.json was created successfully + """ + # Check if compile_commands.json already exists + if self.compile_commands_file.exists(): + print(f"βœ… {self.compile_commands_file} exists") + return True + + # Search for build log files + build_log = None + possible_logs = [ + self.compile_commands_log_file, + Path(self.project_dir) / f"build_{self.env_name}.log", + Path(self.build_dir) / f"build_{self.env_name}.log", + Path(self.build_dir) / "build.log" + ] + + for log_path in possible_logs: + if log_path.exists() and log_path.stat().st_size > 0: + build_log = log_path + break + + if not build_log: + print("⚠ No build log found for compile_commands.json generation") + return False + + # Define supported compiler toolchains + compiler_names = [ + "xtensa-esp32-elf-gcc", "xtensa-esp32-elf-g++", + "xtensa-esp32s2-elf-gcc", "xtensa-esp32s2-elf-g++", + "xtensa-esp32s3-elf-gcc", "xtensa-esp32s3-elf-g++", + "riscv32-esp-elf-gcc", "riscv32-esp-elf-g++", + "xtensa-lx106-elf-gcc", "xtensa-lx106-elf-g++", + "arm-none-eabi-gcc", "arm-none-eabi-g++" + ] + + try: + # Parse build log to extract compile commands + compile_commands = parse_build_log_to_compile_commands(build_log, compiler_names) + if not compile_commands: + print("❌ No compiler commands found in build log") + return False + + # Create output directory + self.compiledb_dir.mkdir(parents=True, exist_ok=True) + + # Convert to JSON format + json_entries = [] + for cmd in compile_commands: + json_entries.append({ + 'file': cmd.file, + 'output': cmd.output, + 'directory': cmd.directory, + 'arguments': cmd.arguments + }) + + # Write compile_commands.json + with self.compile_commands_file.open('w') as f: + json.dump(json_entries, f, indent=2) + + file_size = self.compile_commands_file.stat().st_size + return True + + except Exception as e: + print(f"❌ Error creating compile_commands.json: {e}") + return False + + def get_correct_build_order(self): + """ + Extract build order from compile_commands.json with build artifacts. + + Combines compile_commands.json (which preserves compilation order) + with build artifact paths to create comprehensive build order data. + + Returns: + dict: Build order data with sources, objects, and include paths + None: If compile_commands.json doesn't exist or is invalid + """ + if not self.compile_commands_file.exists(): + print(f"⚠ compile_commands_{self.env_name}.json not found") + return None + + try: + # Load compile database + with self.compile_commands_file.open("r", encoding='utf-8') as f: + compile_db = json.load(f) + except Exception as e: + print(f"βœ— Error reading compile_commands.json: {e}") + return None + + # Initialize data structures for build order extraction + object_paths = [] + include_paths = set() + + # Process each compile command entry + for i, entry in enumerate(compile_db, 1): + source_file = entry.get('file', '') + + if source_file.endswith(('.elf', '.bin', '.hex', '.map')): + # not a source file, skip + continue + + if not source_file.endswith(tuple(SRC_BUILD_EXT + SRC_HEADER_EXT)): + # not a recognized source file, skip + continue + + # Handle both 'arguments' and 'command' formats + if 'arguments' in entry: + command = ' '.join(entry['arguments']) + elif 'command' in entry: + command = entry['command'] + else: + print(f"⚠ Unsupported entry format in compile_commands.json (index {i})") + continue + + # Extract object file from -o flag + obj_match = re.search(r'-o\s+(\S+\.o)', command) + if obj_match: + obj_file = obj_match.group(1) + object_paths.append({ + 'order': i, + 'source': source_file, + 'object': obj_file, + }) + + # Extract include paths from -I flags + include_matches = re.findall(r'-I\s*([^\s]+)', command) + for inc_path in include_matches: + inc_path = inc_path.strip('"\'') + if Path(inc_path).exists(): + include_paths.add(str(Path(inc_path))) + + return { + 'object_paths': object_paths, + 'include_paths': sorted(include_paths) + } + + def collect_build_artifacts_paths(self): + """ + Collect paths to build artifacts without copying them. + + Scans build directory for library (.a) and object (.o) files, + collecting their paths for cache storage and later reuse. + + Returns: + dict: Artifact paths organized by type with metadata + """ + if not self.lib_build_dir.exists(): + print(f"⚠ Build directory not found: {self.lib_build_dir}") + return {} + + library_paths = [] + object_paths = [] + + # Walk through build directory to find artifacts + for root, dirs, files in os.walk(self.lib_build_dir): + root_path = Path(root) + for file in files: + if file.endswith('.a'): + file_path = root_path / file + library_paths.append(str(file_path)) + elif file.endswith('.o'): + file_path = root_path / file + object_paths.append(str(file_path)) + + total_count = len(library_paths) + len(object_paths) + + #print(f"πŸ“¦ Collected {len(library_paths)} library paths (*.a)") + #print(f"πŸ“¦ Collected {len(object_paths)} object paths (*.o)") + #print(f"πŸ“¦ Total: {total_count} artifact paths collected") + + return { + 'library_paths': library_paths, + 'object_paths': object_paths, + 'total_count': total_count + } + + + def get_project_hash_with_details(self): + """ + Calculate comprehensive project hash for cache invalidation. + + Computes hash based on all LDF-relevant files to detect changes + that would require cache invalidation. Only includes files that + can affect dependency resolution. + + Returns: + dict: Hash details including file hashes and final combined hash + """ + file_hashes = {} + src_path = Path(self.src_dir) + + # Process all files in source directory + for file_path in src_path.rglob('*'): + # Skip directories and ignored directories + if file_path.is_dir() or self._is_ignored_directory(file_path.parent): + continue + + # Only process LDF-relevant file extensions + if file_path.suffix in self.ALL_RELEVANT_EXTENSIONS: + try: + rel_path = self._get_relative_path_from_project(file_path) + + # Hash source files based on their include dependencies + if file_path.suffix in self.SOURCE_EXTENSIONS: + includes = self._extract_includes(file_path) + include_hash = hashlib.md5(str(sorted(includes)).encode()).hexdigest() + file_hashes[rel_path] = include_hash + # Hash header files based on content + elif file_path.suffix in self.HEADER_EXTENSIONS: + file_content = file_path.read_bytes() + file_hash = hashlib.md5(file_content).hexdigest() + file_hashes[rel_path] = file_hash + # Hash config files based on content + elif file_path.suffix in self.CONFIG_EXTENSIONS: + file_content = file_path.read_bytes() + file_hash = hashlib.md5(file_content).hexdigest() + file_hashes[rel_path] = file_hash + except (IOError, OSError) as e: + print(f"⚠ Could not hash {file_path}: {e}") + continue + + # Process platformio.ini files + project_path = Path(self.project_dir) + for ini_path in project_path.glob('platformio*.ini'): + if ini_path.exists() and ini_path.is_file(): + try: + platformio_hash = self._hash_platformio_ini_selective(ini_path) + if platformio_hash: + rel_ini_path = self._get_relative_path_from_project(ini_path) + file_hashes[rel_ini_path] = platformio_hash + except (IOError, OSError) as e: + print(f"⚠ Could not hash {ini_path}: {e}") + + # Compute final combined hash + combined_content = json.dumps(file_hashes, sort_keys=True) + final_hash = hashlib.sha256(combined_content.encode()).hexdigest() + + return { + 'file_hashes': file_hashes, + 'final_hash': final_hash, + 'file_count': len(file_hashes) + } + + def create_comprehensive_cache(self): + """ + Create comprehensive cache data including all build information. + + Combines project hash, build order, and artifact information into + a complete cache that can be used for subsequent builds. + + Returns: + dict: Complete cache data with signature + None: If cache creation failed + """ + try: + print("πŸ”§ Creating comprehensive cache...") + + # Collect all cache components + project_hash = self.get_project_hash_with_details() + build_order = self.get_correct_build_order() + artifacts = self.collect_build_artifacts_paths() + + if not build_order: + print("⚠ No build order data available") + return None + + # Create cache data structure + cache_data = { + 'version': '1.0', + 'env_name': self.env_name, + 'timestamp': datetime.datetime.now().isoformat(), + 'project_hash': project_hash['final_hash'], + 'file_hashes': project_hash['file_hashes'], + 'build_order': build_order, + 'artifacts': artifacts + } + + # Add signature for integrity verification + cache_data['signature'] = self.compute_signature(cache_data) + return cache_data + + except Exception as e: + print(f"❌ Error creating cache: {e}") + return None + + def save_cache(self, cache_data): + """ + Save cache data to file in Python format. + + Saves cache as executable Python code for easy loading and + human-readable format for debugging. + + Args: + cache_data: Dictionary containing cache data + + Returns: + bool: True if cache was saved successfully + """ + try: + # Ensure cache directory exists + self.cache_file.parent.mkdir(parents=True, exist_ok=True) + + # Write cache as Python code + with self.cache_file.open('w') as f: + f.write("# LDF Cache Data - Auto-generated\n") + f.write("# Do not edit manually, this will break the Hash checksum\n\n") + f.write("cache_data = ") + f.write(pprint.pformat(cache_data, width=120, depth=None)) + f.write("\n") + + return True + + except Exception as e: + print(f"❌ Error saving cache: {e}") + return False + + def compute_signature(self, data): + """ + Compute SHA256 signature for cache data validation. + + Creates tamper-evident signature to ensure cache integrity + and detect corruption or manual modifications. + + Args: + data: Dictionary to sign + + Returns: + str: SHA256 signature hex string + """ + # Create copy without signature field to avoid circular reference + data_copy = data.copy() + data_copy.pop('signature', None) + + # Create deterministic JSON representation + data_str = json.dumps(data_copy, sort_keys=True) + return hashlib.sha256(data_str.encode()).hexdigest() + + def load_cache(self): + """ + Load cache data from file. + + Executes Python cache file to load cache data into memory + for validation and application. + + Returns: + dict: Cache data if loaded successfully + None: If cache file doesn't exist or loading failed + """ + if not self.cache_file.exists(): + return None + + try: + # Execute Python cache file in isolated namespace + cache_globals = {} + with self.cache_file.open('r') as f: + exec(f.read(), cache_globals) + + # Extract cache data from executed namespace + cache_data = cache_globals.get('cache_data') + if cache_data: + return cache_data + else: + print("⚠ No cache_data found in cache file") + return None + + except Exception as e: + print(f"❌ Error loading cache: {e}") + return None + + def validate_cache(self, cache_data): + """ + Validate cache integrity and freshness. + + Performs comprehensive validation including signature verification + and project hash comparison to ensure cache is valid and current. + + Args: + cache_data: Cache data to validate + + Returns: + bool: True if cache is valid and current + """ + if not cache_data: + return False + + try: + # Verify signature integrity + stored_signature = cache_data.get('signature') + if not stored_signature: + print("⚠ Cache missing signature") + return False + + computed_signature = self.compute_signature(cache_data) + if stored_signature != computed_signature: + print("⚠ Cache signature mismatch") + return False + + # Verify project hasn't changed + current_hash = self.get_project_hash_with_details() + if cache_data.get('project_hash') != current_hash['final_hash']: + print("⚠ Project files changed, cache invalid") + return False + + print("βœ… Cache validation successful") + return True + + except Exception as e: + print(f"⚠ Cache validation error: {e}") + return False + + def apply_build_order_to_environment(self, build_order_data): + """ + Apply correct build order to SCons environment. + + Set OBJECTS variable in correct order and configures + linker for optimal symbol resolution. + + Args: + build_order_data: Dictionary containing build order information + + Returns: + bool: True if build order was applied successfully + """ + if not build_order_data: + return False + + try: + # Apply object file order + object_paths = build_order_data.get('object_paths', []) + if object_paths: + # Use ALL cached object files + object_file_paths = [] + for obj_entry in object_paths: + if isinstance(obj_entry, dict): + obj_path = obj_entry.get('object', obj_entry.get('path', '')) + else: + obj_path = str(obj_entry) + + if obj_path and Path(obj_path).exists(): + object_file_paths.append(obj_path) + + if object_file_paths: + self.env.Replace(OBJECTS=object_file_paths) + self._apply_correct_linker_order(object_file_paths) + else: + print("⚠ No valid object files found") + + return True + + except Exception as e: + print(f"βœ— Error applying build order: {e}") + return False + + def apply_cache_to_scons_vars(self, cache_data): + """ + Apply cache data to SCons variables. + + Uses PlatformIO's ParseFlagsExtended for robust flag processing + and applies cached include paths and library paths to build environment. + + Args: + cache_ Dictionary containing cached build data + + Returns: + bool: True if cache variables were applied successfully + """ + try: + build_order = cache_data.get('build_order', {}) + + # Apply include paths using PlatformIO's native flag processing + if 'include_paths' in build_order: + include_flags = [f"-I{path}" for path in build_order['include_paths']] + parsed_flags = self.env.ParseFlagsExtended(include_flags) + self.env.Append(**parsed_flags) + + # Apply library paths and convert to proper LIBS/LIBPATH format + artifacts = cache_data.get('artifacts', {}) + if 'library_paths' in artifacts: + library_paths = artifacts['library_paths'] + if library_paths: + lib_flags = [] + lib_directories = set() + + for lib_path in library_paths: + lib_path_obj = Path(lib_path) + filename = lib_path_obj.name + + # Extrahiere Verzeichnis fΓΌr LIBPATH + directory = str(lib_path_obj.parent) + if directory and directory != '.': + lib_directories.add(directory) + + # Convert to -l Flag for LIBS + if filename.startswith('lib') and filename.endswith('.a'): + lib_name = filename[3:-2] # Remove 'lib' und '.a' + lib_flags.append(f"-l{lib_name}") + elif filename.endswith('.a'): + lib_name = filename[:-2] # Remove only '.a' + lib_flags.append(f"-l{lib_name}") + else: + lib_flags.append(f"-l{filename}") + + if lib_directories: + current_libpath = self.env.get('LIBPATH', []) + if isinstance(current_libpath, str): + current_libpath = [current_libpath] + + updated_libpath = list(current_libpath) + for path in lib_directories: + if path not in updated_libpath: + updated_libpath.append(path) + + self.env['LIBPATH'] = updated_libpath + + if lib_flags: + current_libs = self.env.get('LIBS', []) + if isinstance(current_libs, str): + current_libs = [current_libs] + + updated_libs = list(current_libs) + lib_flags + self.env['LIBS'] = updated_libs + + return True + + except Exception as e: + print(f"❌ Error applying cache to SCons vars: {e}") + import traceback + traceback.print_exc() + return False + + def _apply_correct_linker_order(self, object_files): + """ + Apply correct linker order for symbol resolution. + + Configures linker with optimized flags and proper grouping + to ensure correct symbol resolution and garbage collection. + + Args: + object_files: List of object file paths + """ + try: + current_linkflags = self.env.get('LINKFLAGS', []) + + # Configure custom linker command for better control + custom_linkcom = ( + "$LINK -o $TARGET " + "${_long_sources_hook(__env__, SOURCES)} " + "$LINKFLAGS " + "$__RPATH $_LIBDIRFLAGS $_LIBFLAGS" + ) + + self.env.Replace(LINKCOM=custom_linkcom) + + # Build optimized linker flags + optimized_linkflags = [] + optimized_linkflags.extend(["-Wl,--start-group"]) + + # Add existing flags without duplicates + for flag in current_linkflags: + if flag not in optimized_linkflags: + optimized_linkflags.append(flag) + + # Add optimization flags + optimized_linkflags.extend([ + "-Wl,--end-group", + "-Wl,--gc-sections", + ]) + + self.env.Replace(LINKFLAGS=optimized_linkflags) + + except Exception as e: + print(f"⚠ Warning during linker optimization: {e}") + + def is_file_cached(self, file_path): + """ + Check if file is present in cache. + + Args: + file_path: Path to file to check + + Returns: + bool: True if file is in cache + """ + cache_data = self.load_cache() + if not cache_data: + return False + + rel_path = self._get_relative_path_from_project(file_path) + return rel_path in cache_data.get('file_hashes', {}) + + def _get_relative_path_from_project(self, file_path): + """ + Calculate relative path from project root with consistent path handling. + + Normalizes paths to use forward slashes for cross-platform compatibility. + + Args: + file_path: File path to make relative + + Returns: + str: Relative path from project root with forward slashes + """ + try: + file_path = Path(file_path).resolve() + project_dir = Path(self.project_dir).resolve() + rel_path = file_path.relative_to(project_dir) + return str(rel_path).replace(os.sep, '/') + except (ValueError, OSError): + return str(Path(file_path)).replace(os.sep, '/') + + def _extract_includes(self, file_path): + """ + Extract #include directives from source files. + + Parses source files to extract include dependencies for + hash calculation and dependency tracking. + + Args: + file_path: Source file to analyze + + Returns: + set: Set of normalized include paths + """ + includes = set() + try: + file_path = Path(file_path) + with file_path.open('r', encoding='utf-8', errors='ignore') as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if line.startswith('#include'): + include_match = re.search(r'#include\s*[<"]([^>"]+)[>"]', line) + if include_match: + include_path = include_match.group(1) + normalized_include = str(Path(include_path)).replace(os.sep, '/') + includes.add(normalized_include) + except (IOError, OSError, UnicodeDecodeError) as e: + print(f"⚠ Could not read {file_path}: {e}") + return includes + + def _hash_platformio_ini_selective(self, ini_path=None): + """ + Hash platformio.ini excluding LDF-related lines modified by script. + + Creates hash of platformio.ini while excluding lines that are + modified by this script to avoid cache invalidation loops. + + Args: + ini_path: Path to ini file (defaults to self.platformio_ini) + + Returns: + str: MD5 hash of relevant platformio.ini content + """ + if ini_path is None: + ini_path = self.platformio_ini + if not ini_path.exists(): + return "" + + # Lines to exclude from hashing (modified by this script) + excluded_patterns = ['lib_ldf_mode'] + + try: + relevant_lines = [] + with ini_path.open('r', encoding='utf-8') as f: + for line in f: + line_stripped = line.strip() + + # Skip empty lines and comments + if not line_stripped or line_stripped.startswith(';') or line_stripped.startswith('#'): + continue + + # Skip excluded patterns + should_exclude = any( + pattern.lower() in line_stripped.lower() + for pattern in excluded_patterns + ) + if not should_exclude: + relevant_lines.append(line_stripped) + + # Create deterministic hash from sorted lines + relevant_content = '\n'.join(sorted(relevant_lines)) + return hashlib.md5(relevant_content.encode()).hexdigest() + except (IOError, OSError) as e: + print(f"⚠ Could not read {ini_path}: {e}") + return "" + + def _is_ignored_directory(self, dir_path): + """ + Check if a directory should be ignored during file scanning. + + Uses predefined ignore patterns optimized for ESP/Tasmota projects + to skip irrelevant directories during cache creation. + + Args: + dir_path: Directory path to check + + Returns: + bool: True if directory should be ignored + """ + if not dir_path: + return False + + path_obj = Path(dir_path) + + # Check if directory name is in ignore list + if path_obj.name in self.IGNORE_DIRS: + return True + + # Check if any parent directory is in ignore list + for part in path_obj.parts: + if part in self.IGNORE_DIRS: + return True + return False + +def modify_platformio_ini_for_second_run(self): + """ + Einfache Modifikation: Suche lib_ldf_mode und ersetze durch lib_ldf_mode = off + Wenn kein Eintrag vorhanden, nichts Γ€ndern. + Returns: + bool: True if modification was successful or not needed + """ + try: + if not self.platformio_ini.exists(): + print("❌ platformio.ini not found") + return False + + if not self.platformio_ini_backup.exists(): + shutil.copy2(self.platformio_ini, self.platformio_ini_backup) + + with self.platformio_ini.open('r', encoding='utf-8') as f: + lines = f.readlines() + + modified = False + for i, line in enumerate(lines): + if line.strip().startswith('lib_ldf_mode'): + lines[i] = 'lib_ldf_mode = off\n' + modified = True + break + + if modified: + with self.platformio_ini.open('w', encoding='utf-8') as f: + f.writelines(lines) + return True + else: + print("β„Ή No lib_ldf_mode entry found, no changes made") + return True + + except Exception as e: + print(f"❌ Error modifying platformio.ini: {e}") + return False + +def execute_first_run_post_actions(): + """ + Execute post-build actions after successful first run. + + Creates cache data, generates compile_commands.json, validates LDF mode, + and modifies platformio.ini for subsequent cached builds. + + Returns: + bool: True if all post-actions completed successfully + """ + print("🎯 First run completed successfully - executing post-build actions...") + + try: + # Initialize optimizer for first run tasks + optimizer = LDFCacheOptimizer(env) + + # Create compile_commands.json if needed + if not compiledb_path.exists() or compiledb_path.stat().st_size == 0: + success_compiledb = optimizer.create_compiledb_integrated() + if not success_compiledb: + print("❌ Failed to create compile_commands.json") + return False + else: + print(f"βœ… compile_commands.json already exists: {compiledb_path}") + + # Create cache if it doesn't exist + if not cache_file.exists(): + cache_data = optimizer.create_comprehensive_cache() + + if not cache_data: + print("❌ Failed to create cache data") + return False + + # Save cache to file + success_save = optimizer.save_cache(cache_data) + if not success_save: + print("❌ Failed to save cache") + return False + + print(f"βœ… LDF cache created: {cache_file}") + else: + print(f"βœ… LDF cache already exists: {cache_file}") + + # Validate LDF mode compatibility + optimizer.validate_ldf_mode_compatibility() + + # Check current LDF mode and modify platformio.ini if needed + current_ldf_mode = optimizer.env.GetProjectOption("lib_ldf_mode", "chain") + print(f"πŸ” Current lib_ldf_mode: {current_ldf_mode}") + + if current_ldf_mode.lower() in ["chain", "off"]: + print("πŸ”§ Modifying platformio.ini for second run...") + success_ini_mod = optimizer.modify_platformio_ini_for_second_run() + if success_ini_mod: + print("πŸŽ‰ First run post-build actions completed successfully!") + print("πŸ”„ platformio.ini configured for cached build (lib_ldf_mode = off)") + print("πŸš€ Next build will be using cached dependencies") + else: + print("⚠ Cache created but platformio.ini modification failed") + print("πŸ’‘ Manual setting: lib_ldf_mode = off recommended for next build") + return False + else: + print(f"⚠ lib_ldf_mode '{current_ldf_mode}' not supported for caching") + print("πŸ’‘ Supported modes: chain, off") + return False + + return True + + except Exception as e: + print(f"❌ Error in first run post-build actions: {e}") + import traceback + traceback.print_exc() + return False + +# FIRST RUN LOGIC - Execute verbose build and create cache +if should_trigger_verbose_build() and not github_actions and not flag_custom_sdkconfig: + print("πŸ”„ Starting LDF Cache Optimizer...") + print(f"πŸ”„ First run needed - starting verbose build for {env_name}...") + print("πŸ“‹ Reasons:") + + # Report reasons for first run + if not compiledb_path.exists(): + print(" - compile_commands.json missing") + elif compiledb_path.stat().st_size == 0: + print(" - compile_commands.json is empty") + + if not is_build_environment_ready(): + print(" - Build environment incomplete") + + # Setup environment for verbose build + env_vars = os.environ.copy() + env_vars['PLATFORMIO_SETTING_FORCE_VERBOSE'] = 'true' + env_vars['_PIO_RECURSIVE_CALL'] = 'true' + + # Handle recursive call return codes + if os.environ.get('_PIO_REC_CALL_RETURN_CODE') is not None: + sys.exit(int(os.environ.get('_PIO_REC_CALL_RETURN_CODE'))) + + # Execute verbose build with output capture + with open(logfile_path, "w") as logfile: + process = subprocess.Popen( + ['pio', 'run', '-e', env_name, '--disable-auto-clean'], + env=env_vars, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + bufsize=1 + ) + print(f"πŸ”„ Running verbose build... full log in {logfile_path}") + for line in process.stdout: + print("πŸ”„ " + line[:120].splitlines()[0], end='\r') + sys.stdout.write("\033[F") + logfile.write(line) + logfile.flush() + print("\nβœ… Build process completed, waiting for process to finish...") + process.wait() + + if process.returncode == 0: + post_actions_success = execute_first_run_post_actions() + if post_actions_success: + print("βœ… All first run actions completed successfully") + else: + print("⚠ Some first run actions failed") + else: + print(f"❌ First run failed with return code: {process.returncode}") + LDFCacheOptimizer.restore_platformio_ini() + + sys.exit(process.returncode) + +# SECOND RUN LOGIC (wird nur erreicht wenn First Run nicht stattfand) +try: + if (not should_trigger_verbose_build() and + is_build_environment_ready()): + optimizer = LDFCacheOptimizer(env) +except Exception as e: + print(f"❌ Error initializing LDF Cache Optimizer: {e}") + import traceback + traceback.print_exc()