From cb0ef0b54afd1424e400c8af0da793c6424314f4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 16 Jul 2025 10:56:29 -1000 Subject: [PATCH 01/38] Fix lwIP thread safety assertion failures on ESP32 --- esphome/components/e131/e131_packet.cpp | 8 ++++- esphome/components/esp32/helpers.cpp | 23 ++++++++++++++ .../ethernet/ethernet_component.cpp | 11 +++++-- esphome/components/mqtt/mqtt_client.cpp | 12 ++++--- .../wifi/wifi_component_esp32_arduino.cpp | 31 ++++++------------- esphome/core/helpers.h | 18 +++++++++++ 6 files changed, 74 insertions(+), 29 deletions(-) diff --git a/esphome/components/e131/e131_packet.cpp b/esphome/components/e131/e131_packet.cpp index b8fa73b707..e663a3d0fc 100644 --- a/esphome/components/e131/e131_packet.cpp +++ b/esphome/components/e131/e131_packet.cpp @@ -4,6 +4,7 @@ #include "esphome/components/network/ip_address.h" #include "esphome/core/log.h" #include "esphome/core/util.h" +#include "esphome/core/helpers.h" #include #include @@ -71,7 +72,11 @@ bool E131Component::join_igmp_groups_() { ip4_addr_t multicast_addr = network::IPAddress(239, 255, ((universe.first >> 8) & 0xff), ((universe.first >> 0) & 0xff)); - auto err = igmp_joingroup(IP4_ADDR_ANY4, &multicast_addr); + err_t err; + { + LwIPLock lock; + err = igmp_joingroup(IP4_ADDR_ANY4, &multicast_addr); + } if (err) { ESP_LOGW(TAG, "IGMP join for %d universe of E1.31 failed. Multicast might not work.", universe.first); @@ -104,6 +109,7 @@ void E131Component::leave_(int universe) { if (listen_method_ == E131_MULTICAST) { ip4_addr_t multicast_addr = network::IPAddress(239, 255, ((universe >> 8) & 0xff), ((universe >> 0) & 0xff)); + LwIPLock lock; igmp_leavegroup(IP4_ADDR_ANY4, &multicast_addr); } diff --git a/esphome/components/esp32/helpers.cpp b/esphome/components/esp32/helpers.cpp index 310e7bd94a..cfc648e1d1 100644 --- a/esphome/components/esp32/helpers.cpp +++ b/esphome/components/esp32/helpers.cpp @@ -30,6 +30,29 @@ void Mutex::unlock() { xSemaphoreGive(this->handle_); } IRAM_ATTR InterruptLock::InterruptLock() { portDISABLE_INTERRUPTS(); } IRAM_ATTR InterruptLock::~InterruptLock() { portENABLE_INTERRUPTS(); } +#ifdef CONFIG_LWIP_TCPIP_CORE_LOCKING +#include "lwip/priv/tcpip_priv.h" +#endif + +LwIPLock::LwIPLock() { +#ifdef CONFIG_LWIP_TCPIP_CORE_LOCKING + // Only lock if we're not already in the TCPIP thread + if (!sys_thread_tcpip(LWIP_CORE_LOCK_QUERY_HOLDER)) { + LOCK_TCPIP_CORE(); + locked_ = true; + } +#endif +} + +LwIPLock::~LwIPLock() { +#ifdef CONFIG_LWIP_TCPIP_CORE_LOCKING + // Only unlock if we locked it + if (locked_ && sys_thread_tcpip(LWIP_CORE_LOCK_QUERY_HOLDER)) { + UNLOCK_TCPIP_CORE(); + } +#endif +} + void get_mac_address_raw(uint8_t *mac) { // NOLINT(readability-non-const-parameter) #if defined(CONFIG_SOC_IEEE802154_SUPPORTED) // When CONFIG_SOC_IEEE802154_SUPPORTED is defined, esp_efuse_mac_get_default diff --git a/esphome/components/ethernet/ethernet_component.cpp b/esphome/components/ethernet/ethernet_component.cpp index f8c2f3a72e..ff37dcfdd1 100644 --- a/esphome/components/ethernet/ethernet_component.cpp +++ b/esphome/components/ethernet/ethernet_component.cpp @@ -420,6 +420,7 @@ network::IPAddresses EthernetComponent::get_ip_addresses() { } network::IPAddress EthernetComponent::get_dns_address(uint8_t num) { + LwIPLock lock; const ip_addr_t *dns_ip = dns_getserver(num); return dns_ip; } @@ -527,6 +528,7 @@ void EthernetComponent::start_connect_() { ESPHL_ERROR_CHECK(err, "DHCPC set IP info error"); if (this->manual_ip_.has_value()) { + LwIPLock lock; if (this->manual_ip_->dns1.is_set()) { ip_addr_t d; d = this->manual_ip_->dns1; @@ -559,8 +561,13 @@ bool EthernetComponent::is_connected() { return this->state_ == EthernetComponen void EthernetComponent::dump_connect_params_() { esp_netif_ip_info_t ip; esp_netif_get_ip_info(this->eth_netif_, &ip); - const ip_addr_t *dns_ip1 = dns_getserver(0); - const ip_addr_t *dns_ip2 = dns_getserver(1); + const ip_addr_t *dns_ip1; + const ip_addr_t *dns_ip2; + { + LwIPLock lock; + dns_ip1 = dns_getserver(0); + dns_ip2 = dns_getserver(1); + } ESP_LOGCONFIG(TAG, " IP Address: %s\n" diff --git a/esphome/components/mqtt/mqtt_client.cpp b/esphome/components/mqtt/mqtt_client.cpp index 5b93789447..f3e57a66be 100644 --- a/esphome/components/mqtt/mqtt_client.cpp +++ b/esphome/components/mqtt/mqtt_client.cpp @@ -193,13 +193,17 @@ void MQTTClientComponent::start_dnslookup_() { this->dns_resolve_error_ = false; this->dns_resolved_ = false; ip_addr_t addr; + err_t err; + { + LwIPLock lock; #if USE_NETWORK_IPV6 - err_t err = dns_gethostbyname_addrtype(this->credentials_.address.c_str(), &addr, - MQTTClientComponent::dns_found_callback, this, LWIP_DNS_ADDRTYPE_IPV6_IPV4); + err = dns_gethostbyname_addrtype(this->credentials_.address.c_str(), &addr, MQTTClientComponent::dns_found_callback, + this, LWIP_DNS_ADDRTYPE_IPV6_IPV4); #else - err_t err = dns_gethostbyname_addrtype(this->credentials_.address.c_str(), &addr, - MQTTClientComponent::dns_found_callback, this, LWIP_DNS_ADDRTYPE_IPV4); + err = dns_gethostbyname_addrtype(this->credentials_.address.c_str(), &addr, MQTTClientComponent::dns_found_callback, + this, LWIP_DNS_ADDRTYPE_IPV4); #endif /* USE_NETWORK_IPV6 */ + } switch (err) { case ERR_OK: { // Got IP immediately diff --git a/esphome/components/wifi/wifi_component_esp32_arduino.cpp b/esphome/components/wifi/wifi_component_esp32_arduino.cpp index a7877eb90b..b3167c5696 100644 --- a/esphome/components/wifi/wifi_component_esp32_arduino.cpp +++ b/esphome/components/wifi/wifi_component_esp32_arduino.cpp @@ -20,10 +20,6 @@ #include "lwip/dns.h" #include "lwip/err.h" -#ifdef CONFIG_LWIP_TCPIP_CORE_LOCKING -#include "lwip/priv/tcpip_priv.h" -#endif - #include "esphome/core/application.h" #include "esphome/core/hal.h" #include "esphome/core/helpers.h" @@ -295,25 +291,16 @@ bool WiFiComponent::wifi_sta_ip_config_(optional manual_ip) { } if (!manual_ip.has_value()) { -// sntp_servermode_dhcp lwip/sntp.c (Required to lock TCPIP core functionality!) -// https://github.com/esphome/issues/issues/6591 -// https://github.com/espressif/arduino-esp32/issues/10526 -#ifdef CONFIG_LWIP_TCPIP_CORE_LOCKING - if (!sys_thread_tcpip(LWIP_CORE_LOCK_QUERY_HOLDER)) { - LOCK_TCPIP_CORE(); + // sntp_servermode_dhcp lwip/sntp.c (Required to lock TCPIP core functionality!) + // https://github.com/esphome/issues/issues/6591 + // https://github.com/espressif/arduino-esp32/issues/10526 + { + LwIPLock lock; + // lwIP starts the SNTP client if it gets an SNTP server from DHCP. We don't need the time, and more importantly, + // the built-in SNTP client has a memory leak in certain situations. Disable this feature. + // https://github.com/esphome/issues/issues/2299 + sntp_servermode_dhcp(false); } -#endif - - // lwIP starts the SNTP client if it gets an SNTP server from DHCP. We don't need the time, and more importantly, - // the built-in SNTP client has a memory leak in certain situations. Disable this feature. - // https://github.com/esphome/issues/issues/2299 - sntp_servermode_dhcp(false); - -#ifdef CONFIG_LWIP_TCPIP_CORE_LOCKING - if (sys_thread_tcpip(LWIP_CORE_LOCK_QUERY_HOLDER)) { - UNLOCK_TCPIP_CORE(); - } -#endif // No manual IP is set; use DHCP client if (dhcp_status != ESP_NETIF_DHCP_STARTED) { diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 488ea3cdb3..fd1fbc3d05 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -684,6 +684,24 @@ class InterruptLock { #endif }; +/** Helper class to lock the lwIP TCPIP core when making lwIP API calls from non-TCPIP threads. + * + * This is needed on multi-threaded platforms (ESP32) when CONFIG_LWIP_TCPIP_CORE_LOCKING is enabled. + * It ensures thread-safe access to lwIP APIs. + * + * @note This follows the same pattern as InterruptLock - platform-specific implementations in helpers.cpp + */ +class LwIPLock { + public: + LwIPLock(); + ~LwIPLock(); + + protected: +#if defined(USE_ESP32) + bool locked_; +#endif +}; + /** Helper class to request `loop()` to be called as fast as possible. * * Usually the ESPHome main loop runs at 60 Hz, sleeping in between invocations of `loop()` if necessary. When a higher From a399e90ed663869d3bff34be31beb99b98b95abe Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 16 Jul 2025 11:10:36 -1000 Subject: [PATCH 02/38] fix missing init --- esphome/core/helpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index fd1fbc3d05..745b3e5e8e 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -698,7 +698,7 @@ class LwIPLock { protected: #if defined(USE_ESP32) - bool locked_; + bool locked_{false}; #endif }; From f1d2300153c0aa26484874f54909c9d1b2b966e2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 16 Jul 2025 13:10:00 -1000 Subject: [PATCH 03/38] simplify --- esphome/components/esp32/helpers.cpp | 5 ++--- esphome/core/helpers.h | 5 ----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/esphome/components/esp32/helpers.cpp b/esphome/components/esp32/helpers.cpp index cfc648e1d1..13b12157c4 100644 --- a/esphome/components/esp32/helpers.cpp +++ b/esphome/components/esp32/helpers.cpp @@ -39,15 +39,14 @@ LwIPLock::LwIPLock() { // Only lock if we're not already in the TCPIP thread if (!sys_thread_tcpip(LWIP_CORE_LOCK_QUERY_HOLDER)) { LOCK_TCPIP_CORE(); - locked_ = true; } #endif } LwIPLock::~LwIPLock() { #ifdef CONFIG_LWIP_TCPIP_CORE_LOCKING - // Only unlock if we locked it - if (locked_ && sys_thread_tcpip(LWIP_CORE_LOCK_QUERY_HOLDER)) { + // Only unlock if we hold the lock + if (sys_thread_tcpip(LWIP_CORE_LOCK_QUERY_HOLDER)) { UNLOCK_TCPIP_CORE(); } #endif diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 745b3e5e8e..6650a1c4d5 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -695,11 +695,6 @@ class LwIPLock { public: LwIPLock(); ~LwIPLock(); - - protected: -#if defined(USE_ESP32) - bool locked_{false}; -#endif }; /** Helper class to request `loop()` to be called as fast as possible. From 88323bcca0f7e854f49b5f3db001c453aa3bfc9b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 16 Jul 2025 15:42:48 -1000 Subject: [PATCH 04/38] Allow disabling OTA for web_server while keeping it enabled for captive_portal --- esphome/components/web_server/__init__.py | 23 +++++++++++-------- .../web_server/ota/ota_web_server.cpp | 20 +++++++++++++++- esphome/components/web_server/web_server.cpp | 6 ++--- .../components/web_server/web_server_v1.cpp | 4 +++- .../web_server/test_ota_migration.py | 12 +++++----- 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py index 6890f60014..572b75a8f1 100644 --- a/esphome/components/web_server/__init__.py +++ b/esphome/components/web_server/__init__.py @@ -74,13 +74,14 @@ def validate_local(config: ConfigType) -> ConfigType: return config -def validate_ota_removed(config: ConfigType) -> ConfigType: - # Only raise error if OTA is explicitly enabled (True) - # If it's False or not specified, we can safely ignore it - if config.get(CONF_OTA): +def validate_ota(config: ConfigType) -> ConfigType: + # The OTA option only accepts False to explicitly disable OTA for web_server + # IMPORTANT: Setting ota: false ONLY affects the web_server component + # The captive_portal component will still be able to perform OTA updates + if CONF_OTA in config and config[CONF_OTA] is not False: raise cv.Invalid( - f"The '{CONF_OTA}' option has been removed from 'web_server'. " - f"Please use the new OTA platform structure instead:\n\n" + f"The '{CONF_OTA}' option in 'web_server' only accepts 'false' to disable OTA. " + f"To enable OTA, please use the new OTA platform structure instead:\n\n" f"ota:\n" f" - platform: web_server\n\n" f"See https://esphome.io/components/ota for more information." @@ -185,7 +186,7 @@ CONFIG_SCHEMA = cv.All( web_server_base.WebServerBase ), cv.Optional(CONF_INCLUDE_INTERNAL, default=False): cv.boolean, - cv.Optional(CONF_OTA, default=False): cv.boolean, + cv.Optional(CONF_OTA): cv.boolean, cv.Optional(CONF_LOG, default=True): cv.boolean, cv.Optional(CONF_LOCAL): cv.boolean, cv.Optional(CONF_SORTING_GROUPS): cv.ensure_list(sorting_group), @@ -203,7 +204,7 @@ CONFIG_SCHEMA = cv.All( default_url, validate_local, validate_sorting_groups, - validate_ota_removed, + validate_ota, ) @@ -288,7 +289,11 @@ async def to_code(config): cg.add(var.set_css_url(config[CONF_CSS_URL])) cg.add(var.set_js_url(config[CONF_JS_URL])) # OTA is now handled by the web_server OTA platform - # The CONF_OTA option is kept only for backwards compatibility validation + # The CONF_OTA option is kept to allow explicitly disabling OTA for web_server + # IMPORTANT: This ONLY affects the web_server component, NOT captive_portal + # Captive portal will still be able to perform OTA updates even when this is set + if config.get(CONF_OTA) is False: + cg.add_define("USE_WEBSERVER_OTA_DISABLED") cg.add(var.set_expose_log(config[CONF_LOG])) if config[CONF_ENABLE_PRIVATE_NETWORK_ACCESS]: cg.add_define("USE_WEBSERVER_PRIVATE_NETWORK_ACCESS") diff --git a/esphome/components/web_server/ota/ota_web_server.cpp b/esphome/components/web_server/ota/ota_web_server.cpp index adac05cbe5..26d86ac3cf 100644 --- a/esphome/components/web_server/ota/ota_web_server.cpp +++ b/esphome/components/web_server/ota/ota_web_server.cpp @@ -5,6 +5,10 @@ #include "esphome/core/application.h" #include "esphome/core/log.h" +#ifdef USE_CAPTIVE_PORTAL +#include "esphome/components/captive_portal/captive_portal.h" +#endif + #ifdef USE_ARDUINO #ifdef USE_ESP8266 #include @@ -25,7 +29,21 @@ class OTARequestHandler : public AsyncWebHandler { void handleUpload(AsyncWebServerRequest *request, const String &filename, size_t index, uint8_t *data, size_t len, bool final) override; bool canHandle(AsyncWebServerRequest *request) const override { - return request->url() == "/update" && request->method() == HTTP_POST; + if (request->url() != "/update" || request->method() != HTTP_POST) { + return false; + } + +#if defined(USE_WEBSERVER_OTA_DISABLED) && defined(USE_CAPTIVE_PORTAL) + // IMPORTANT: USE_WEBSERVER_OTA_DISABLED only disables OTA for the web_server component + // Captive portal can still perform OTA updates - check if request is from active captive portal + return captive_portal::global_captive_portal != nullptr && captive_portal::global_captive_portal->is_active(); +#elif defined(USE_WEBSERVER_OTA_DISABLED) + // OTA disabled for web_server and no captive portal compiled in + return false; +#else + // OTA enabled for web_server + return true; +#endif } // NOLINTNEXTLINE(readability-identifier-naming) diff --git a/esphome/components/web_server/web_server.cpp b/esphome/components/web_server/web_server.cpp index 9ec667dbc5..14791071e6 100644 --- a/esphome/components/web_server/web_server.cpp +++ b/esphome/components/web_server/web_server.cpp @@ -268,10 +268,10 @@ std::string WebServer::get_config_json() { return json::build_json([this](JsonObject root) { root["title"] = App.get_friendly_name().empty() ? App.get_name() : App.get_friendly_name(); root["comment"] = App.get_comment(); -#ifdef USE_WEBSERVER_OTA - root["ota"] = true; // web_server OTA platform is configured +#if defined(USE_WEBSERVER_OTA_DISABLED) || !defined(USE_WEBSERVER_OTA) + root["ota"] = false; // Note: USE_WEBSERVER_OTA_DISABLED only affects web_server, not captive_portal #else - root["ota"] = false; + root["ota"] = true; #endif root["log"] = this->expose_log_; root["lang"] = "en"; diff --git a/esphome/components/web_server/web_server_v1.cpp b/esphome/components/web_server/web_server_v1.cpp index 5db0f1cae9..0f558f6d81 100644 --- a/esphome/components/web_server/web_server_v1.cpp +++ b/esphome/components/web_server/web_server_v1.cpp @@ -192,7 +192,9 @@ void WebServer::handle_index_request(AsyncWebServerRequest *request) { stream->print(F("

See ESPHome Web API for " "REST API documentation.

")); -#ifdef USE_WEBSERVER_OTA +#if defined(USE_WEBSERVER_OTA) && !defined(USE_WEBSERVER_OTA_DISABLED) + // Show OTA form only if web_server OTA is not explicitly disabled + // Note: USE_WEBSERVER_OTA_DISABLED only affects web_server, not captive_portal stream->print(F("

OTA Update

")); #endif diff --git a/tests/component_tests/web_server/test_ota_migration.py b/tests/component_tests/web_server/test_ota_migration.py index 7f34ec75f6..da25bab0e8 100644 --- a/tests/component_tests/web_server/test_ota_migration.py +++ b/tests/component_tests/web_server/test_ota_migration.py @@ -8,31 +8,31 @@ from esphome.types import ConfigType def test_web_server_ota_true_fails_validation() -> None: """Test that web_server with ota: true fails validation with helpful message.""" - from esphome.components.web_server import validate_ota_removed + from esphome.components.web_server import validate_ota # Config with ota: true should fail config: ConfigType = {"ota": True} with pytest.raises(cv.Invalid) as exc_info: - validate_ota_removed(config) + validate_ota(config) # Check error message contains migration instructions error_msg = str(exc_info.value) - assert "has been removed from 'web_server'" in error_msg + assert "only accepts 'false' to disable OTA" in error_msg assert "platform: web_server" in error_msg assert "ota:" in error_msg def test_web_server_ota_false_passes_validation() -> None: """Test that web_server with ota: false passes validation.""" - from esphome.components.web_server import validate_ota_removed + from esphome.components.web_server import validate_ota # Config with ota: false should pass config: ConfigType = {"ota": False} - result = validate_ota_removed(config) + result = validate_ota(config) assert result == config # Config without ota should also pass config: ConfigType = {} - result = validate_ota_removed(config) + result = validate_ota(config) assert result == config From ce21b992e3f09409803df89ce05373ea68c24c56 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 16 Jul 2025 15:49:30 -1000 Subject: [PATCH 05/38] tidy happy --- esphome/components/web_server/ota/ota_web_server.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esphome/components/web_server/ota/ota_web_server.cpp b/esphome/components/web_server/ota/ota_web_server.cpp index 26d86ac3cf..3e566f14bf 100644 --- a/esphome/components/web_server/ota/ota_web_server.cpp +++ b/esphome/components/web_server/ota/ota_web_server.cpp @@ -29,20 +29,20 @@ class OTARequestHandler : public AsyncWebHandler { void handleUpload(AsyncWebServerRequest *request, const String &filename, size_t index, uint8_t *data, size_t len, bool final) override; bool canHandle(AsyncWebServerRequest *request) const override { - if (request->url() != "/update" || request->method() != HTTP_POST) { - return false; - } + // Check if this is an OTA update request + bool is_ota_request = request->url() == "/update" && request->method() == HTTP_POST; #if defined(USE_WEBSERVER_OTA_DISABLED) && defined(USE_CAPTIVE_PORTAL) // IMPORTANT: USE_WEBSERVER_OTA_DISABLED only disables OTA for the web_server component // Captive portal can still perform OTA updates - check if request is from active captive portal - return captive_portal::global_captive_portal != nullptr && captive_portal::global_captive_portal->is_active(); + return is_ota_request && captive_portal::global_captive_portal != nullptr && + captive_portal::global_captive_portal->is_active(); #elif defined(USE_WEBSERVER_OTA_DISABLED) // OTA disabled for web_server and no captive portal compiled in return false; #else // OTA enabled for web_server - return true; + return is_ota_request; #endif } From c3da5b7a3f2376e297865f7cc130fa7bce5d44b9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 16 Jul 2025 15:50:42 -1000 Subject: [PATCH 06/38] tell the bot --- esphome/components/web_server/ota/ota_web_server.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/components/web_server/ota/ota_web_server.cpp b/esphome/components/web_server/ota/ota_web_server.cpp index 3e566f14bf..966c1c1024 100644 --- a/esphome/components/web_server/ota/ota_web_server.cpp +++ b/esphome/components/web_server/ota/ota_web_server.cpp @@ -35,6 +35,7 @@ class OTARequestHandler : public AsyncWebHandler { #if defined(USE_WEBSERVER_OTA_DISABLED) && defined(USE_CAPTIVE_PORTAL) // IMPORTANT: USE_WEBSERVER_OTA_DISABLED only disables OTA for the web_server component // Captive portal can still perform OTA updates - check if request is from active captive portal + // Note: global_captive_portal is the standard way components communicate in ESPHome return is_ota_request && captive_portal::global_captive_portal != nullptr && captive_portal::global_captive_portal->is_active(); #elif defined(USE_WEBSERVER_OTA_DISABLED) From 984601f0b20fc7cc9642ec7977cca563181c6028 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 16 Jul 2025 20:39:15 -1000 Subject: [PATCH 07/38] ble churn fix --- esphome/components/api/api.proto | 2 +- esphome/components/api/api_options.proto | 1 + esphome/components/api/api_pb2.cpp | 5 +- esphome/components/api/api_pb2.h | 3 +- esphome/components/api/api_pb2_dump.cpp | 2 +- .../bluetooth_proxy/bluetooth_proxy.cpp | 89 +++++++++----- .../bluetooth_proxy/bluetooth_proxy.h | 7 +- script/api_protobuf/api_protobuf.py | 116 ++++++++++++++++-- 8 files changed, 177 insertions(+), 48 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index c8b046c1e2..b0ce21b1ce 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1381,7 +1381,7 @@ message BluetoothLERawAdvertisement { sint32 rssi = 2; uint32 address_type = 3; - bytes data = 4; + bytes data = 4 [(fixed_array_size) = 62]; } message BluetoothLERawAdvertisementsResponse { diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index 022cd8b3d2..bb3947e8a3 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -26,4 +26,5 @@ extend google.protobuf.MessageOptions { extend google.protobuf.FieldOptions { optional string field_ifdef = 1042; + optional uint32 fixed_array_size = 50007; } diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index b7a69a5d95..64a6fae1a3 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -3,6 +3,7 @@ #include "api_pb2.h" #include "esphome/core/log.h" #include "esphome/core/helpers.h" +#include namespace esphome { namespace api { @@ -1916,13 +1917,13 @@ void BluetoothLERawAdvertisement::encode(ProtoWriteBuffer buffer) const { buffer.encode_uint64(1, this->address); buffer.encode_sint32(2, this->rssi); buffer.encode_uint32(3, this->address_type); - buffer.encode_bytes(4, reinterpret_cast(this->data.data()), this->data.size()); + buffer.encode_bytes(4, this->data, this->data_len); } void BluetoothLERawAdvertisement::calculate_size(uint32_t &total_size) const { ProtoSize::add_uint64_field(total_size, 1, this->address); ProtoSize::add_sint32_field(total_size, 1, this->rssi); ProtoSize::add_uint32_field(total_size, 1, this->address_type); - ProtoSize::add_string_field(total_size, 1, this->data); + total_size += 1 + ProtoSize::varint(static_cast(this->data_len)) + this->data_len; } void BluetoothLERawAdvertisementsResponse::encode(ProtoWriteBuffer buffer) const { for (auto &it : this->advertisements) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 99486f57d7..39f00b4adc 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1768,7 +1768,8 @@ class BluetoothLERawAdvertisement : public ProtoMessage { uint64_t address{0}; int32_t rssi{0}; uint32_t address_type{0}; - std::string data{}; + uint8_t data[62]{}; + uint8_t data_len{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(uint32_t &total_size) const override; #ifdef HAS_PROTO_MESSAGE_DUMP diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 7d4150a857..56aa4683ba 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -3132,7 +3132,7 @@ void BluetoothLERawAdvertisement::dump_to(std::string &out) const { out.append("\n"); out.append(" data: "); - out.append(format_hex_pretty(this->data)); + out.append(format_hex_pretty(reinterpret_cast(this->data), this->data_len)); out.append("\n"); out.append("}"); } diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index 1c856b8d93..33def12027 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -3,6 +3,7 @@ #include "esphome/core/log.h" #include "esphome/core/macros.h" #include "esphome/core/application.h" +#include #ifdef USE_ESP32 @@ -27,6 +28,15 @@ std::vector get_128bit_uuid_vec(esp_bt_uuid_t uuid_source) { BluetoothProxy::BluetoothProxy() { global_bluetooth_proxy = this; } void BluetoothProxy::setup() { + // Pre-allocate response object + this->response_ = std::make_unique(); + + // Reserve capacity but start with size 0 + this->response_->advertisements.reserve(FLUSH_BATCH_SIZE); + + // Pre-allocate pool for overflow + this->advertisement_pool_.reserve(FLUSH_BATCH_SIZE); + this->parent_->add_scanner_state_callback([this](esp32_ble_tracker::ScannerState state) { if (this->api_connection_ != nullptr) { this->send_bluetooth_scanner_state_(state); @@ -57,61 +67,78 @@ bool BluetoothProxy::parse_device(const esp32_ble_tracker::ESPBTDevice &device) // This achieves ~97% WiFi MTU utilization while staying under the limit static constexpr size_t FLUSH_BATCH_SIZE = 16; -namespace { -// Batch buffer in anonymous namespace to avoid guard variable (saves 8 bytes) -// This is initialized at program startup before any threads -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -std::vector batch_buffer; -} // namespace - -static std::vector &get_batch_buffer() { return batch_buffer; } - bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, size_t count) { if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr) return false; - // Get the batch buffer reference - auto &batch_buffer = get_batch_buffer(); + auto &advertisements = this->response_->advertisements; - // Reserve additional capacity if needed - size_t new_size = batch_buffer.size() + count; - if (batch_buffer.capacity() < new_size) { - batch_buffer.reserve(new_size); - } - - // Add new advertisements to the batch buffer for (size_t i = 0; i < count; i++) { auto &result = scan_results[i]; uint8_t length = result.adv_data_len + result.scan_rsp_len; - batch_buffer.emplace_back(); - auto &adv = batch_buffer.back(); + // Validate length + if (length > 62) { + ESP_LOGW(TAG, "BLE advertisement too large: %d bytes (max 62)", length); + length = 62; + } + + // Check if we need to expand the vector + if (this->advertisement_count_ >= advertisements.size()) { + if (this->advertisement_pool_.empty()) { + // No room in pool, need to allocate + advertisements.emplace_back(); + } else { + // Pull from pool + advertisements.push_back(std::move(this->advertisement_pool_.back())); + this->advertisement_pool_.pop_back(); + } + } + + // Fill in the data directly at current position + auto &adv = advertisements[this->advertisement_count_]; adv.address = esp32_ble::ble_addr_to_uint64(result.bda); adv.rssi = result.rssi; adv.address_type = result.ble_addr_type; - adv.data.assign(&result.ble_adv[0], &result.ble_adv[length]); + adv.data_len = length; + std::memcpy(adv.data, result.ble_adv, length); + + this->advertisement_count_++; ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d. RSSI: %d dB", result.bda[0], result.bda[1], result.bda[2], result.bda[3], result.bda[4], result.bda[5], length, result.rssi); - } - // Only send if we've accumulated a good batch size to maximize batching efficiency - // https://github.com/esphome/backlog/issues/21 - if (batch_buffer.size() >= FLUSH_BATCH_SIZE) { - this->flush_pending_advertisements(); + // Flush if we have reached FLUSH_BATCH_SIZE + if (this->advertisement_count_ >= FLUSH_BATCH_SIZE) { + this->flush_pending_advertisements(); + } } return true; } void BluetoothProxy::flush_pending_advertisements() { - auto &batch_buffer = get_batch_buffer(); - if (batch_buffer.empty() || !api::global_api_server->is_connected() || this->api_connection_ == nullptr) + if (this->advertisement_count_ == 0 || !api::global_api_server->is_connected() || this->api_connection_ == nullptr) return; - api::BluetoothLERawAdvertisementsResponse resp; - resp.advertisements.swap(batch_buffer); - this->api_connection_->send_message(resp); + auto &advertisements = this->response_->advertisements; + + // Return any items beyond advertisement_count_ to the pool + if (advertisements.size() > this->advertisement_count_) { + // Move unused items back to pool + this->advertisement_pool_.insert(this->advertisement_pool_.end(), + std::make_move_iterator(advertisements.begin() + this->advertisement_count_), + std::make_move_iterator(advertisements.end())); + + // Resize to actual count + advertisements.resize(this->advertisement_count_); + } + + // Send the message + this->api_connection_->send_message(*this->response_); + + // Reset count - existing items will be overwritten in next batch + this->advertisement_count_ = 0; } #ifdef USE_ESP32_BLE_DEVICE diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.h b/esphome/components/bluetooth_proxy/bluetooth_proxy.h index 3ccf0706a7..52f1d0f88a 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.h +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.h @@ -145,9 +145,14 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com // Group 2: Container types (typically 12 bytes on 32-bit) std::vector connections_{}; + // BLE advertisement batching + std::vector advertisement_pool_; + std::unique_ptr response_; + // Group 3: 1-byte types grouped together bool active_; - // 1 byte used, 3 bytes padding + uint8_t advertisement_count_{0}; + // 2 bytes used, 2 bytes padding }; extern BluetoothProxy *global_bluetooth_proxy; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index e441d4c6e9..e245ad4739 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -313,13 +313,18 @@ def validate_field_type(field_type: int, field_name: str = "") -> None: ) -def get_type_info_for_field(field: descriptor.FieldDescriptorProto) -> TypeInfo: - """Get the appropriate TypeInfo for a field, handling repeated fields. - - Also validates that the field type is supported. - """ +def create_field_type_info(field: descriptor.FieldDescriptorProto) -> TypeInfo: + """Create the appropriate TypeInfo instance for a field, handling repeated fields and custom options.""" if field.label == 3: # repeated return RepeatedTypeInfo(field) + + # Check for fixed_array_size option on bytes fields + if ( + field.type == 12 + and (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None + ): + return FixedArrayBytesType(field, fixed_size) + validate_field_type(field.type, field.name) return TYPE_INFO[field.type](field) @@ -603,6 +608,76 @@ class BytesType(TypeInfo): return self.calculate_field_id_size() + 8 # field ID + 8 bytes typical bytes +class FixedArrayBytesType(TypeInfo): + """Special type for fixed-size byte arrays.""" + + def __init__(self, field: descriptor.FieldDescriptorProto, size: int) -> None: + super().__init__(field) + self.array_size = size + + @property + def cpp_type(self) -> str: + return "uint8_t" + + @property + def default_value(self) -> str: + return "{}" + + @property + def reference_type(self) -> str: + return f"uint8_t (&)[{self.array_size}]" + + @property + def const_reference_type(self) -> str: + return f"const uint8_t (&)[{self.array_size}]" + + @property + def public_content(self) -> list[str]: + # Add both the array and length fields + return [ + f"uint8_t {self.field_name}[{self.array_size}]{{}};", + f"uint8_t {self.field_name}_len{{0}};", + ] + + @property + def decode_length_content(self) -> str: + o = f"case {self.number}: {{\n" + o += " const std::string &data_str = value.as_string();\n" + o += f" this->{self.field_name}_len = data_str.size();\n" + o += f" if (this->{self.field_name}_len > {self.array_size}) {{\n" + o += f" this->{self.field_name}_len = {self.array_size};\n" + o += " }\n" + o += f" memcpy(this->{self.field_name}, data_str.data(), this->{self.field_name}_len);\n" + o += " break;\n" + o += "}" + return o + + @property + def encode_content(self) -> str: + return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);" + + def dump(self, name: str) -> str: + o = f"out.append(format_hex_pretty(reinterpret_cast({name}), {name}_len));" + return o + + def get_size_calculation(self, name: str, force: bool = False) -> str: + # Use the actual length stored in the _len field + length_field = f"this->{self.field_name}_len" + # Size = field_id_size + varint(length) + actual_data_bytes + field_id_size = self.calculate_field_id_size() + return f" total_size += {field_id_size} + ProtoSize::varint(static_cast({length_field})) + {length_field};\n" + + def get_estimated_size(self) -> int: + # Estimate based on typical BLE advertisement size + return ( + self.calculate_field_id_size() + 1 + 31 + ) # field ID + length byte + typical 31 bytes + + @property + def wire_type(self) -> WireType: + return WireType.LENGTH_DELIMITED + + @register_type(13) class UInt32Type(TypeInfo): cpp_type = "uint32_t" @@ -748,6 +823,16 @@ class SInt64Type(TypeInfo): class RepeatedTypeInfo(TypeInfo): def __init__(self, field: descriptor.FieldDescriptorProto) -> None: super().__init__(field) + # For repeated fields, we need to get the base type info + # but we can't call create_field_type_info as it would cause recursion + # So we extract just the type creation logic + if ( + field.type == 12 + and (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None + ): + self._ti: TypeInfo = FixedArrayBytesType(field, fixed_size) + return + validate_field_type(field.type, field.name) self._ti: TypeInfo = TYPE_INFO[field.type](field) @@ -1051,7 +1136,7 @@ def calculate_message_estimated_size(desc: descriptor.DescriptorProto) -> int: total_size = 0 for field in desc.field: - ti = get_type_info_for_field(field) + ti = create_field_type_info(field) # Add estimated size for this field total_size += ti.get_estimated_size() @@ -1119,10 +1204,7 @@ def build_message_type( public_content.append("#endif") for field in desc.field: - if field.label == 3: - ti = RepeatedTypeInfo(field) - else: - ti = TYPE_INFO[field.type](field) + ti = create_field_type_info(field) # Skip field declarations for fields that are in the base class # but include their encode/decode logic @@ -1327,6 +1409,17 @@ def get_opt( return desc.options.Extensions[opt] +def get_field_opt( + field: descriptor.FieldDescriptorProto, + opt: descriptor.FieldOptions, + default: Any = None, +) -> Any: + """Get the option from a field descriptor.""" + if not field.options.HasExtension(opt): + return default + return field.options.Extensions[opt] + + def get_base_class(desc: descriptor.DescriptorProto) -> str | None: """Get the base_class option from a message descriptor.""" if not desc.options.HasExtension(pb.base_class): @@ -1401,7 +1494,7 @@ def build_base_class( # For base classes, we only declare the fields but don't handle encode/decode # The derived classes will handle encoding/decoding with their specific field numbers for field in common_fields: - ti = get_type_info_for_field(field) + ti = create_field_type_info(field) # Only add field declarations, not encode/decode logic protected_content.extend(ti.protected_content) @@ -1543,6 +1636,7 @@ namespace api { #include "api_pb2.h" #include "esphome/core/log.h" #include "esphome/core/helpers.h" + #include namespace esphome { namespace api { From 7c45afa338928d38528e2bd3bc575061bbcdadfb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 16 Jul 2025 20:40:46 -1000 Subject: [PATCH 08/38] ble churn fix --- esphome/components/bluetooth_proxy/bluetooth_proxy.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index 33def12027..8e5eb64417 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -25,6 +25,13 @@ std::vector get_128bit_uuid_vec(esp_bt_uuid_t uuid_source) { ((uint64_t) uuid.uuid.uuid128[1] << 8) | ((uint64_t) uuid.uuid.uuid128[0])}; } +// Batch size for BLE advertisements to maximize WiFi efficiency +// Each advertisement is up to 80 bytes when packaged (including protocol overhead) +// Most advertisements are 20-30 bytes, allowing even more to fit per packet +// 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload +// This achieves ~97% WiFi MTU utilization while staying under the limit +static constexpr size_t FLUSH_BATCH_SIZE = 16; + BluetoothProxy::BluetoothProxy() { global_bluetooth_proxy = this; } void BluetoothProxy::setup() { From dbbcbc09986482b9bfea9d3ff988aaafc7fabb25 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 16 Jul 2025 20:41:01 -1000 Subject: [PATCH 09/38] ble churn fix --- esphome/components/bluetooth_proxy/bluetooth_proxy.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index 8e5eb64417..ffaadd8149 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -67,13 +67,6 @@ bool BluetoothProxy::parse_device(const esp32_ble_tracker::ESPBTDevice &device) } #endif -// Batch size for BLE advertisements to maximize WiFi efficiency -// Each advertisement is up to 80 bytes when packaged (including protocol overhead) -// Most advertisements are 20-30 bytes, allowing even more to fit per packet -// 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload -// This achieves ~97% WiFi MTU utilization while staying under the limit -static constexpr size_t FLUSH_BATCH_SIZE = 16; - bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, size_t count) { if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr) return false; From 72419eb540353f167595dd9266f002404c64f5b1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Jul 2025 07:21:32 -1000 Subject: [PATCH 10/38] fix --- esphome/components/api/api_pb2.cpp | 4 +++- script/api_protobuf/api_protobuf.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 64a6fae1a3..437c9ece1d 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -1923,7 +1923,9 @@ void BluetoothLERawAdvertisement::calculate_size(uint32_t &total_size) const { ProtoSize::add_uint64_field(total_size, 1, this->address); ProtoSize::add_sint32_field(total_size, 1, this->rssi); ProtoSize::add_uint32_field(total_size, 1, this->address_type); - total_size += 1 + ProtoSize::varint(static_cast(this->data_len)) + this->data_len; + if (this->data_len != 0) { + total_size += 1 + ProtoSize::varint(static_cast(this->data_len)) + this->data_len; + } } void BluetoothLERawAdvertisementsResponse::encode(ProtoWriteBuffer buffer) const { for (auto &it : this->advertisements) { diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index e245ad4739..f6612be14a 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -663,9 +663,18 @@ class FixedArrayBytesType(TypeInfo): def get_size_calculation(self, name: str, force: bool = False) -> str: # Use the actual length stored in the _len field length_field = f"this->{self.field_name}_len" - # Size = field_id_size + varint(length) + actual_data_bytes field_id_size = self.calculate_field_id_size() - return f" total_size += {field_id_size} + ProtoSize::varint(static_cast({length_field})) + {length_field};\n" + + if force: + # For repeated fields, always calculate size + return f"total_size += {field_id_size} + ProtoSize::varint(static_cast({length_field})) + {length_field};" + else: + # For non-repeated fields, skip if length is 0 (matching encode_string behavior) + return ( + f"if ({length_field} != 0) {{\n" + f" total_size += {field_id_size} + ProtoSize::varint(static_cast({length_field})) + {length_field};\n" + f"}}" + ) def get_estimated_size(self) -> int: # Estimate based on typical BLE advertisement size From 1f0958e824f50909addcb5eb07201fdcbbe62224 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Jul 2025 09:57:29 -1000 Subject: [PATCH 11/38] safer --- esphome/components/bluetooth_proxy/bluetooth_proxy.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index ffaadd8149..bb789347af 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -32,6 +32,10 @@ std::vector get_128bit_uuid_vec(esp_bt_uuid_t uuid_source) { // This achieves ~97% WiFi MTU utilization while staying under the limit static constexpr size_t FLUSH_BATCH_SIZE = 16; +// Verify BLE advertisement data array size matches the BLE specification (31 bytes adv + 31 bytes scan response) +static_assert(sizeof(((api::BluetoothLERawAdvertisement *) nullptr)->data) == 62, + "BLE advertisement data array size mismatch"); + BluetoothProxy::BluetoothProxy() { global_bluetooth_proxy = this; } void BluetoothProxy::setup() { @@ -77,12 +81,6 @@ bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, auto &result = scan_results[i]; uint8_t length = result.adv_data_len + result.scan_rsp_len; - // Validate length - if (length > 62) { - ESP_LOGW(TAG, "BLE advertisement too large: %d bytes (max 62)", length); - length = 62; - } - // Check if we need to expand the vector if (this->advertisement_count_ >= advertisements.size()) { if (this->advertisement_pool_.empty()) { From c17fdd91dee5475f80a0b8140dd0dd4e723b4e82 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Jul 2025 10:23:00 -1000 Subject: [PATCH 12/38] commit overreserve fix --- esphome/components/bluetooth_proxy/bluetooth_proxy.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index bb789347af..c22788de66 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -43,10 +43,11 @@ void BluetoothProxy::setup() { this->response_ = std::make_unique(); // Reserve capacity but start with size 0 - this->response_->advertisements.reserve(FLUSH_BATCH_SIZE); + // Reserve 50% since we'll grow naturally and flush at FLUSH_BATCH_SIZE + this->response_->advertisements.reserve(FLUSH_BATCH_SIZE / 2); - // Pre-allocate pool for overflow - this->advertisement_pool_.reserve(FLUSH_BATCH_SIZE); + // Don't pre-allocate pool - let it grow only if needed in busy environments + // Many devices in quiet areas will never need the overflow pool this->parent_->add_scanner_state_callback([this](esp32_ble_tracker::ScannerState state) { if (this->api_connection_ != nullptr) { From d40fcb324ca2568931b8ab5a3c3255f090fc3132 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Jul 2025 12:24:36 -1000 Subject: [PATCH 13/38] Revert "missed one" This reverts commit 8ba14d1f548b38d41ae85da5d0e8da8eda1da06a. --- esphome/components/api/api_server.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 78c04f79c2..f7020b829e 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -428,8 +428,7 @@ bool APIServer::save_noise_psk(psk_t psk, bool make_active) { ESP_LOGW(TAG, "Disconnecting all clients to reset PSK"); this->set_noise_psk(psk); for (auto &c : this->clients_) { - DisconnectRequest req; - c->send_message(req, DisconnectRequest::MESSAGE_TYPE); + c->send_message(DisconnectRequest()); } }); } From ee7bda74c06900fdfb84f64e6a5b3aa886c9e91f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Jul 2025 12:24:57 -1000 Subject: [PATCH 14/38] Revert "Revert "missed one"" This reverts commit d40fcb324ca2568931b8ab5a3c3255f090fc3132. --- esphome/components/api/api_server.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index f7020b829e..78c04f79c2 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -428,7 +428,8 @@ bool APIServer::save_noise_psk(psk_t psk, bool make_active) { ESP_LOGW(TAG, "Disconnecting all clients to reset PSK"); this->set_noise_psk(psk); for (auto &c : this->clients_) { - c->send_message(DisconnectRequest()); + DisconnectRequest req; + c->send_message(req, DisconnectRequest::MESSAGE_TYPE); } }); } From 732370effcef01cfaf198f1a1ed8766e33eface8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Jul 2025 13:34:00 -1000 Subject: [PATCH 15/38] remove unneeded cast --- esphome/components/api/api_pb2_dump.cpp | 2 +- script/api_protobuf/api_protobuf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 56aa4683ba..ad5a5fdcaa 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -3132,7 +3132,7 @@ void BluetoothLERawAdvertisement::dump_to(std::string &out) const { out.append("\n"); out.append(" data: "); - out.append(format_hex_pretty(reinterpret_cast(this->data), this->data_len)); + out.append(format_hex_pretty(this->data, this->data_len)); out.append("\n"); out.append("}"); } diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 5d78da6a58..23d8a53b70 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -657,7 +657,7 @@ class FixedArrayBytesType(TypeInfo): return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);" def dump(self, name: str) -> str: - o = f"out.append(format_hex_pretty(reinterpret_cast({name}), {name}_len));" + o = f"out.append(format_hex_pretty({name}, {name}_len));" return o def get_size_calculation(self, name: str, force: bool = False) -> str: From 6740561bd77d46a0df2d655d300fdb81de75c9dc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Jul 2025 14:24:31 -1000 Subject: [PATCH 16/38] Fix scheduler with libretiny --- esphome/core/scheduler.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 1ab2c3838b..193c2a967a 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -509,7 +509,11 @@ uint64_t Scheduler::millis_64_(uint32_t now) { #if !defined(USE_ESP8266) && !defined(USE_RP2040) // Multi-threaded platforms: Need to handle rollover carefully +#ifdef USE_LIBRETINY + uint32_t last = this->last_millis_; +#else uint32_t last = this->last_millis_.load(std::memory_order_relaxed); +#endif // USE_LIBRETINY // If we might be near a rollover (large backwards jump), take the lock for the entire operation // This ensures rollover detection and last_millis_ update are atomic together @@ -517,7 +521,11 @@ uint64_t Scheduler::millis_64_(uint32_t now) { // Potential rollover - need lock for atomic rollover detection + update LockGuard guard{this->lock_}; // Re-read with lock held +#ifdef USE_LIBRETINY + last = this->last_millis_; +#else last = this->last_millis_.load(std::memory_order_relaxed); +#endif if (now < last && (last - now) > HALF_MAX_UINT32) { // True rollover detected (happens every ~49.7 days) @@ -527,8 +535,21 @@ uint64_t Scheduler::millis_64_(uint32_t now) { #endif } // Update last_millis_ while holding lock to prevent races +#ifdef USE_LIBRETINY + this->last_millis_ = now; +#else this->last_millis_.store(now, std::memory_order_relaxed); +#endif } else { +#ifdef USE_LIBRETINY + // LibreTiny does not support atomics, so we use a simple lock-free update + // This is not completely safe, but without atomics we don't have a choice + // and in practice we don't have a lot of task on libretiny so it should be fine. + if (now > last && (now - last) < HALF_MAX_UINT32) { + // Normal case: Update last_millis_ if time moved forward + this->last_millis_ = now; + } +#else // Normal case: Try lock-free update, but only allow forward movement within same epoch // This prevents accidentally moving backwards across a rollover boundary while (now > last && (now - last) < HALF_MAX_UINT32) { @@ -537,6 +558,7 @@ uint64_t Scheduler::millis_64_(uint32_t now) { } // last is automatically updated by compare_exchange_weak if it fails } +#endif // USE_LIBRETINY } #else From e26c20910d2f24a7f52fe0649bfea90243c6df37 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Jul 2025 14:42:35 -1000 Subject: [PATCH 17/38] [scheduler] Fix LibreTiny compilation error due to missing atomic operations --- esphome/core/scheduler.cpp | 67 ++++++++++++++++++++++++-------------- esphome/core/scheduler.h | 8 ++--- 2 files changed, 47 insertions(+), 28 deletions(-) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 193c2a967a..ddf11e5b16 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -273,7 +273,7 @@ void HOT Scheduler::call(uint32_t now) { if (now_64 - last_print > 2000) { last_print = now_64; std::vector> old_items; -#if !defined(USE_ESP8266) && !defined(USE_RP2040) +#if !defined(USE_ESP8266) && !defined(USE_RP2040) && !defined(USE_LIBRETINY) ESP_LOGD(TAG, "Items: count=%zu, now=%" PRIu64 " (%u, %" PRIu32 ")", this->items_.size(), now_64, this->millis_major_, this->last_millis_.load(std::memory_order_relaxed)); #else @@ -507,13 +507,50 @@ uint64_t Scheduler::millis_64_(uint32_t now) { // This prevents race conditions at the rollover boundary without requiring // 64-bit atomics or locking on every call. -#if !defined(USE_ESP8266) && !defined(USE_RP2040) - // Multi-threaded platforms: Need to handle rollover carefully #ifdef USE_LIBRETINY + // LibreTiny: Multi-threaded but lacks atomic operation support + // TODO: If LibreTiny ever adds atomic support, remove this entire block and + // let it fall through to the atomic-based implementation below + // We need to use a lock when near the rollover boundary to prevent races uint32_t last = this->last_millis_; -#else + + // Define a safe window around the rollover point (10 seconds) + // This covers any reasonable scheduler delays or thread preemption + static const uint32_t ROLLOVER_WINDOW = 10000; // 10 seconds in milliseconds + + // Check if we're near the rollover boundary (close to 0xFFFFFFFF or just past 0) + bool near_rollover = (last > (0xFFFFFFFF - ROLLOVER_WINDOW)) || (now < ROLLOVER_WINDOW); + + if (near_rollover || (now < last && (last - now) > HALF_MAX_UINT32)) { + // Near rollover or detected a rollover - need lock for safety + LockGuard guard{this->lock_}; + // Re-read with lock held + last = this->last_millis_; + + if (now < last && (last - now) > HALF_MAX_UINT32) { + // True rollover detected (happens every ~49.7 days) + this->millis_major_++; +#ifdef ESPHOME_DEBUG_SCHEDULER + ESP_LOGD(TAG, "Detected true 32-bit rollover at %" PRIu32 "ms (was %" PRIu32 ")", now, last); +#endif + } + // Update last_millis_ while holding lock + this->last_millis_ = now; + } else if (now > last) { + // Normal case: Not near rollover and time moved forward + // Update without lock. While this may cause minor races (microseconds of + // backwards time movement), they're acceptable because: + // 1. The scheduler operates at millisecond resolution, not microsecond + // 2. We've already prevented the critical rollover race condition + // 3. Any backwards movement is orders of magnitude smaller than scheduler delays + this->last_millis_ = now; + } + // If now <= last and we're not near rollover, don't update + // This minimizes backwards time movement + +#elif !defined(USE_ESP8266) && !defined(USE_RP2040) + // Multi-threaded platforms with atomic support (ESP32) uint32_t last = this->last_millis_.load(std::memory_order_relaxed); -#endif // USE_LIBRETINY // If we might be near a rollover (large backwards jump), take the lock for the entire operation // This ensures rollover detection and last_millis_ update are atomic together @@ -521,11 +558,7 @@ uint64_t Scheduler::millis_64_(uint32_t now) { // Potential rollover - need lock for atomic rollover detection + update LockGuard guard{this->lock_}; // Re-read with lock held -#ifdef USE_LIBRETINY - last = this->last_millis_; -#else last = this->last_millis_.load(std::memory_order_relaxed); -#endif if (now < last && (last - now) > HALF_MAX_UINT32) { // True rollover detected (happens every ~49.7 days) @@ -535,21 +568,8 @@ uint64_t Scheduler::millis_64_(uint32_t now) { #endif } // Update last_millis_ while holding lock to prevent races -#ifdef USE_LIBRETINY - this->last_millis_ = now; -#else this->last_millis_.store(now, std::memory_order_relaxed); -#endif } else { -#ifdef USE_LIBRETINY - // LibreTiny does not support atomics, so we use a simple lock-free update - // This is not completely safe, but without atomics we don't have a choice - // and in practice we don't have a lot of task on libretiny so it should be fine. - if (now > last && (now - last) < HALF_MAX_UINT32) { - // Normal case: Update last_millis_ if time moved forward - this->last_millis_ = now; - } -#else // Normal case: Try lock-free update, but only allow forward movement within same epoch // This prevents accidentally moving backwards across a rollover boundary while (now > last && (now - last) < HALF_MAX_UINT32) { @@ -558,11 +578,10 @@ uint64_t Scheduler::millis_64_(uint32_t now) { } // last is automatically updated by compare_exchange_weak if it fails } -#endif // USE_LIBRETINY } #else - // Single-threaded platforms: No atomics needed + // Single-threaded platforms (ESP8266, RP2040): No atomics needed uint32_t last = this->last_millis_; // Check for rollover diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 0546d3694c..1fc2006697 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -4,7 +4,7 @@ #include #include #include -#if !defined(USE_ESP8266) && !defined(USE_RP2040) +#if !defined(USE_ESP8266) && !defined(USE_RP2040) && !defined(USE_LIBRETINY) #include #endif @@ -210,11 +210,11 @@ class Scheduler { // Both platforms save 40 bytes of RAM by excluding this std::deque> defer_queue_; // FIFO queue for defer() calls #endif -#if !defined(USE_ESP8266) && !defined(USE_RP2040) - // Multi-threaded platforms: last_millis_ needs atomic for lock-free updates +#if !defined(USE_ESP8266) && !defined(USE_RP2040) && !defined(USE_LIBRETINY) + // Multi-threaded platforms with atomic support: last_millis_ needs atomic for lock-free updates std::atomic last_millis_{0}; #else - // Single-threaded platforms: no atomics needed + // Platforms without atomic support or single-threaded platforms uint32_t last_millis_{0}; #endif // millis_major_ is protected by lock when incrementing, volatile ensures From 759fe53fd4c025fda9e47acbbbef83e6ff6b718a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Jul 2025 20:18:40 -1000 Subject: [PATCH 18/38] [libretiny] Remove unsupported lock-free queue and event pool implementations --- esphome/core/event_pool.h | 4 ++-- esphome/core/lock_free_queue.h | 9 ++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/core/event_pool.h b/esphome/core/event_pool.h index 69e03bafac..928a4e7dee 100644 --- a/esphome/core/event_pool.h +++ b/esphome/core/event_pool.h @@ -1,6 +1,6 @@ #pragma once -#if defined(USE_ESP32) || defined(USE_LIBRETINY) +#if defined(USE_ESP32) #include #include @@ -78,4 +78,4 @@ template class EventPool { } // namespace esphome -#endif // defined(USE_ESP32) || defined(USE_LIBRETINY) +#endif // defined(USE_ESP32) diff --git a/esphome/core/lock_free_queue.h b/esphome/core/lock_free_queue.h index f35cfa5af9..de07b0ebba 100644 --- a/esphome/core/lock_free_queue.h +++ b/esphome/core/lock_free_queue.h @@ -1,17 +1,12 @@ #pragma once -#if defined(USE_ESP32) || defined(USE_LIBRETINY) +#if defined(USE_ESP32) #include #include -#if defined(USE_ESP32) #include #include -#elif defined(USE_LIBRETINY) -#include -#include -#endif /* * Lock-free queue for single-producer single-consumer scenarios. @@ -148,4 +143,4 @@ template class NotifyingLockFreeQueue : public LockFreeQu } // namespace esphome -#endif // defined(USE_ESP32) || defined(USE_LIBRETINY) +#endif // defined(USE_ESP32) From e189add8a391e914380c5aab968c4287656874f2 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Fri, 18 Jul 2025 22:57:25 +1200 Subject: [PATCH 19/38] [CI] New workflow to mention codeowners on issues (#9658) --- .github/workflows/issue-codeowner-notify.yml | 119 +++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 .github/workflows/issue-codeowner-notify.yml diff --git a/.github/workflows/issue-codeowner-notify.yml b/.github/workflows/issue-codeowner-notify.yml new file mode 100644 index 0000000000..3ff9c58510 --- /dev/null +++ b/.github/workflows/issue-codeowner-notify.yml @@ -0,0 +1,119 @@ +# This workflow automatically notifies codeowners when an issue is labeled with component labels. +# It reads the CODEOWNERS file to find the maintainers for the labeled components +# and posts a comment mentioning them to ensure they're aware of the issue. + +name: Notify Issue Codeowners + +on: + issues: + types: [labeled] + +permissions: + issues: write + contents: read + +jobs: + notify-codeowners: + name: Run + if: ${{ startsWith(github.event.label.name, format('component{0} ', ':')) }} + runs-on: ubuntu-latest + steps: + - name: Notify codeowners for component issues + uses: actions/github-script@v7.0.1 + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const issue_number = context.payload.issue.number; + const labelName = context.payload.label.name; + + console.log(`Processing issue #${issue_number} with label: ${labelName}`); + + // Extract component name from label + const componentName = labelName.replace('component: ', ''); + console.log(`Component: ${componentName}`); + + try { + // Fetch CODEOWNERS file from root + const { data: codeownersFile } = await github.rest.repos.getContent({ + owner, + repo, + path: 'CODEOWNERS' + }); + const codeownersContent = Buffer.from(codeownersFile.content, 'base64').toString('utf8'); + + // Parse CODEOWNERS file to extract component mappings + const codeownersLines = codeownersContent.split('\n') + .map(line => line.trim()) + .filter(line => line && !line.startsWith('#')); + + let componentOwners = null; + + for (const line of codeownersLines) { + const parts = line.split(/\s+/); + if (parts.length < 2) continue; + + const pattern = parts[0]; + const owners = parts.slice(1); + + // Look for component patterns: esphome/components/{component}/* + const componentMatch = pattern.match(/^esphome\/components\/([^\/]+)\/\*$/); + if (componentMatch && componentMatch[1] === componentName) { + componentOwners = owners; + break; + } + } + + if (!componentOwners) { + console.log(`No codeowners found for component: ${componentName}`); + return; + } + + console.log(`Found codeowners for '${componentName}': ${componentOwners.join(', ')}`); + + // Separate users and teams + const userOwners = []; + const teamOwners = []; + + for (const owner of componentOwners) { + const cleanOwner = owner.startsWith('@') ? owner.slice(1) : owner; + if (cleanOwner.includes('/')) { + // Team mention (org/team-name) + teamOwners.push(`@${cleanOwner}`); + } else { + // Individual user + userOwners.push(`@${cleanOwner}`); + } + } + + // Remove issue author from mentions to avoid self-notification + const issueAuthor = context.payload.issue.user.login; + const filteredUserOwners = userOwners.filter(mention => + mention !== `@${issueAuthor}` + ); + + const allMentions = [...filteredUserOwners, ...teamOwners]; + + if (allMentions.length === 0) { + console.log('No codeowners to notify (issue author is the only codeowner)'); + return; + } + + // Create comment body + const mentionString = allMentions.join(', '); + const commentBody = `👋 Hey ${mentionString}!\n\nThis issue has been labeled with \`component: ${componentName}\` and you've been identified as a codeowner of this component. Please take a look when you have a chance!\n\nThanks for maintaining this component! 🙏`; + + // Post comment + await github.rest.issues.createComment({ + owner, + repo, + issue_number: issue_number, + body: commentBody + }); + + console.log(`Successfully notified codeowners: ${mentionString}`); + + } catch (error) { + console.log('Failed to process codeowner notifications:', error.message); + console.error(error); + } From afc48812fa425596e7045d25cb541442c4a30785 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Fri, 18 Jul 2025 23:21:38 +1200 Subject: [PATCH 20/38] [CI] Add codeowners mention workflow (#9651) --- .../workflows/codeowner-review-request.yml | 264 ++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 .github/workflows/codeowner-review-request.yml diff --git a/.github/workflows/codeowner-review-request.yml b/.github/workflows/codeowner-review-request.yml new file mode 100644 index 0000000000..ddf5698211 --- /dev/null +++ b/.github/workflows/codeowner-review-request.yml @@ -0,0 +1,264 @@ +# This workflow automatically requests reviews from codeowners when: +# 1. A PR is opened, reopened, or synchronized (updated) +# 2. A PR is marked as ready for review +# +# It reads the CODEOWNERS file and matches all changed files in the PR against +# the codeowner patterns, then requests reviews from the appropriate owners +# while avoiding duplicate requests for users who have already been requested +# or have already reviewed the PR. + +name: Request Codeowner Reviews + +on: + # Needs to be pull_request_target to get write permissions + pull_request_target: + types: [opened, reopened, synchronize, ready_for_review] + +permissions: + pull-requests: write + contents: read + +jobs: + request-codeowner-reviews: + name: Run + if: ${{ !github.event.pull_request.draft }} + runs-on: ubuntu-latest + steps: + - name: Request reviews from component codeowners + uses: actions/github-script@v7.0.1 + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const pr_number = context.payload.pull_request.number; + + console.log(`Processing PR #${pr_number} for codeowner review requests`); + + try { + // Get the list of changed files in this PR + const { data: files } = await github.rest.pulls.listFiles({ + owner, + repo, + pull_number: pr_number + }); + + const changedFiles = files.map(file => file.filename); + console.log(`Found ${changedFiles.length} changed files`); + + if (changedFiles.length === 0) { + console.log('No changed files found, skipping codeowner review requests'); + return; + } + + // Fetch CODEOWNERS file from root + const { data: codeownersFile } = await github.rest.repos.getContent({ + owner, + repo, + path: 'CODEOWNERS', + ref: context.payload.pull_request.base.sha + }); + const codeownersContent = Buffer.from(codeownersFile.content, 'base64').toString('utf8'); + + // Parse CODEOWNERS file to extract all patterns and their owners + const codeownersLines = codeownersContent.split('\n') + .map(line => line.trim()) + .filter(line => line && !line.startsWith('#')); + + const codeownersPatterns = []; + + // Convert CODEOWNERS pattern to regex (robust glob handling) + function globToRegex(pattern) { + // Escape regex special characters except for glob wildcards + let regexStr = pattern + .replace(/([.+^=!:${}()|[\]\\])/g, '\\$1') // escape regex chars + .replace(/\*\*/g, '.*') // globstar + .replace(/\*/g, '[^/]*') // single star + .replace(/\?/g, '.'); // question mark + return new RegExp('^' + regexStr + '$'); + } + + // Helper function to create comment body + function createCommentBody(reviewersList, teamsList, matchedFileCount, isSuccessful = true) { + const reviewerMentions = reviewersList.map(r => `@${r}`); + const teamMentions = teamsList.map(t => `@${owner}/${t}`); + const allMentions = [...reviewerMentions, ...teamMentions].join(', '); + + if (isSuccessful) { + return `👋 Hi there! I've automatically requested reviews from codeowners based on the files changed in this PR.\n\n${allMentions} - You've been requested to review this PR as codeowner(s) of ${matchedFileCount} file(s) that were modified. Thanks for your time! 🙏`; + } else { + return `👋 Hi there! This PR modifies ${matchedFileCount} file(s) with codeowners.\n\n${allMentions} - As codeowner(s) of the affected files, your review would be appreciated! 🙏\n\n_Note: Automatic review request may have failed, but you're still welcome to review._`; + } + } + + for (const line of codeownersLines) { + const parts = line.split(/\s+/); + if (parts.length < 2) continue; + + const pattern = parts[0]; + const owners = parts.slice(1); + + // Use robust glob-to-regex conversion + const regex = globToRegex(pattern); + codeownersPatterns.push({ pattern, regex, owners }); + } + + console.log(`Parsed ${codeownersPatterns.length} codeowner patterns`); + + // Match changed files against CODEOWNERS patterns + const matchedOwners = new Set(); + const matchedTeams = new Set(); + const fileMatches = new Map(); // Track which files matched which patterns + + for (const file of changedFiles) { + for (const { pattern, regex, owners } of codeownersPatterns) { + if (regex.test(file)) { + console.log(`File '${file}' matches pattern '${pattern}' with owners: ${owners.join(', ')}`); + + if (!fileMatches.has(file)) { + fileMatches.set(file, []); + } + fileMatches.get(file).push({ pattern, owners }); + + // Add owners to the appropriate set (remove @ prefix) + for (const owner of owners) { + const cleanOwner = owner.startsWith('@') ? owner.slice(1) : owner; + if (cleanOwner.includes('/')) { + // Team mention (org/team-name) + const teamName = cleanOwner.split('/')[1]; + matchedTeams.add(teamName); + } else { + // Individual user + matchedOwners.add(cleanOwner); + } + } + } + } + } + + if (matchedOwners.size === 0 && matchedTeams.size === 0) { + console.log('No codeowners found for any changed files'); + return; + } + + // Remove the PR author from reviewers + const prAuthor = context.payload.pull_request.user.login; + matchedOwners.delete(prAuthor); + + // Get current reviewers to avoid duplicate requests (but still mention them) + const { data: prData } = await github.rest.pulls.get({ + owner, + repo, + pull_number: pr_number + }); + + const currentReviewers = new Set(); + const currentTeams = new Set(); + + if (prData.requested_reviewers) { + prData.requested_reviewers.forEach(reviewer => { + currentReviewers.add(reviewer.login); + }); + } + + if (prData.requested_teams) { + prData.requested_teams.forEach(team => { + currentTeams.add(team.slug); + }); + } + + // Check for completed reviews to avoid re-requesting users who have already reviewed + const { data: reviews } = await github.rest.pulls.listReviews({ + owner, + repo, + pull_number: pr_number + }); + + const reviewedUsers = new Set(); + reviews.forEach(review => { + reviewedUsers.add(review.user.login); + }); + + // Remove only users who have already submitted reviews (not just requested reviewers) + reviewedUsers.forEach(reviewer => { + matchedOwners.delete(reviewer); + }); + + // For teams, we'll still remove already requested teams to avoid API errors + currentTeams.forEach(team => { + matchedTeams.delete(team); + }); + + const reviewersList = Array.from(matchedOwners); + const teamsList = Array.from(matchedTeams); + + if (reviewersList.length === 0 && teamsList.length === 0) { + console.log('No eligible reviewers found (all may already be requested or reviewed)'); + return; + } + + const totalReviewers = reviewersList.length + teamsList.length; + console.log(`Requesting reviews from ${reviewersList.length} users and ${teamsList.length} teams for ${fileMatches.size} matched files`); + + // Request reviews + try { + const requestParams = { + owner, + repo, + pull_number: pr_number + }; + + // Filter out users who are already requested reviewers for the API call + const newReviewers = reviewersList.filter(reviewer => !currentReviewers.has(reviewer)); + const newTeams = teamsList.filter(team => !currentTeams.has(team)); + + if (newReviewers.length > 0) { + requestParams.reviewers = newReviewers; + } + + if (newTeams.length > 0) { + requestParams.team_reviewers = newTeams; + } + + // Only make the API call if there are new reviewers to request + if (newReviewers.length > 0 || newTeams.length > 0) { + await github.rest.pulls.requestReviewers(requestParams); + console.log(`Successfully requested reviews from ${newReviewers.length} new users and ${newTeams.length} new teams`); + } else { + console.log('All codeowners are already requested reviewers or have reviewed'); + } + + // Add a comment to the PR mentioning what happened (include all matched codeowners) + const commentBody = createCommentBody(reviewersList, teamsList, fileMatches.size, true); + + await github.rest.issues.createComment({ + owner, + repo, + issue_number: pr_number, + body: commentBody + }); + } catch (error) { + if (error.status === 422) { + console.log('Some reviewers may already be requested or unavailable:', error.message); + + // Try to add a comment even if review request failed + const commentBody = createCommentBody(reviewersList, teamsList, fileMatches.size, false); + + try { + await github.rest.issues.createComment({ + owner, + repo, + issue_number: pr_number, + body: commentBody + }); + } catch (commentError) { + console.log('Failed to add comment:', commentError.message); + } + } else { + throw error; + } + } + + } catch (error) { + console.log('Failed to process codeowner review requests:', error.message); + console.error(error); + } From b5b301f93529ee146231ad1ac341a01021253ef8 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Fri, 18 Jul 2025 23:24:06 +1200 Subject: [PATCH 21/38] [CI] Fix by-code-owner labelling (#9661) --- .github/workflows/auto-label-pr.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/auto-label-pr.yml b/.github/workflows/auto-label-pr.yml index 7c602d7056..c3e1c641ce 100644 --- a/.github/workflows/auto-label-pr.yml +++ b/.github/workflows/auto-label-pr.yml @@ -305,8 +305,7 @@ jobs: const { data: codeownersFile } = await github.rest.repos.getContent({ owner, repo, - path: '.github/CODEOWNERS', - ref: context.payload.pull_request.head.sha + path: 'CODEOWNERS', }); const codeownersContent = Buffer.from(codeownersFile.content, 'base64').toString('utf8'); From 72905f5f42f69cdee6904fa01596cc8f8348e997 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 01:40:14 -1000 Subject: [PATCH 22/38] [libretiny] Remove unsupported lock-free queue and event pool implementations (#9653) --- esphome/core/event_pool.h | 4 ++-- esphome/core/lock_free_queue.h | 9 ++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/core/event_pool.h b/esphome/core/event_pool.h index 69e03bafac..928a4e7dee 100644 --- a/esphome/core/event_pool.h +++ b/esphome/core/event_pool.h @@ -1,6 +1,6 @@ #pragma once -#if defined(USE_ESP32) || defined(USE_LIBRETINY) +#if defined(USE_ESP32) #include #include @@ -78,4 +78,4 @@ template class EventPool { } // namespace esphome -#endif // defined(USE_ESP32) || defined(USE_LIBRETINY) +#endif // defined(USE_ESP32) diff --git a/esphome/core/lock_free_queue.h b/esphome/core/lock_free_queue.h index f35cfa5af9..de07b0ebba 100644 --- a/esphome/core/lock_free_queue.h +++ b/esphome/core/lock_free_queue.h @@ -1,17 +1,12 @@ #pragma once -#if defined(USE_ESP32) || defined(USE_LIBRETINY) +#if defined(USE_ESP32) #include #include -#if defined(USE_ESP32) #include #include -#elif defined(USE_LIBRETINY) -#include -#include -#endif /* * Lock-free queue for single-producer single-consumer scenarios. @@ -148,4 +143,4 @@ template class NotifyingLockFreeQueue : public LockFreeQu } // namespace esphome -#endif // defined(USE_ESP32) || defined(USE_LIBRETINY) +#endif // defined(USE_ESP32) From ce3a16f03caa2d55385ae722729f4cf6ff61f99d Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Fri, 18 Jul 2025 23:49:34 +1200 Subject: [PATCH 23/38] [lvgl] Prevent keyerror on min/max value widgets with no default (#9660) --- esphome/components/lvgl/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/lvgl/types.py b/esphome/components/lvgl/types.py index 40e69119f0..10b6f63528 100644 --- a/esphome/components/lvgl/types.py +++ b/esphome/components/lvgl/types.py @@ -192,7 +192,7 @@ class WidgetType: class NumberType(WidgetType): def get_max(self, config: dict): - return int(config[CONF_MAX_VALUE] or 100) + return int(config.get(CONF_MAX_VALUE, 100)) def get_min(self, config: dict): - return int(config[CONF_MIN_VALUE] or 0) + return int(config.get(CONF_MIN_VALUE, 0)) From 0d422bd74ff3a12c507d9a25c1deffa1129ef8f0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 02:26:54 -1000 Subject: [PATCH 24/38] [scheduler] Add integration tests for set_retry functionality (#9644) --- .../fixtures/scheduler_retry_test.yaml | 207 ++++++++++++++++ .../integration/test_scheduler_retry_test.py | 234 ++++++++++++++++++ 2 files changed, 441 insertions(+) create mode 100644 tests/integration/fixtures/scheduler_retry_test.yaml create mode 100644 tests/integration/test_scheduler_retry_test.py diff --git a/tests/integration/fixtures/scheduler_retry_test.yaml b/tests/integration/fixtures/scheduler_retry_test.yaml new file mode 100644 index 0000000000..bae50e9ed7 --- /dev/null +++ b/tests/integration/fixtures/scheduler_retry_test.yaml @@ -0,0 +1,207 @@ +esphome: + name: scheduler-retry-test + on_boot: + priority: -100 + then: + - logger.log: "Starting scheduler retry tests" + # Run all tests sequentially with delays + - script.execute: run_all_tests + +host: +api: +logger: + level: VERBOSE + +globals: + - id: simple_retry_counter + type: int + initial_value: '0' + - id: backoff_retry_counter + type: int + initial_value: '0' + - id: immediate_done_counter + type: int + initial_value: '0' + - id: cancel_retry_counter + type: int + initial_value: '0' + - id: empty_name_retry_counter + type: int + initial_value: '0' + - id: script_retry_counter + type: int + initial_value: '0' + - id: multiple_same_name_counter + type: int + initial_value: '0' + +sensor: + - platform: template + name: Test Sensor + id: test_sensor + lambda: return 1.0; + update_interval: never + +script: + - id: run_all_tests + then: + # Test 1: Simple retry + - logger.log: "=== Test 1: Simple retry ===" + - lambda: |- + auto *component = id(test_sensor); + App.scheduler.set_retry(component, "simple_retry", 50, 3, + [](uint8_t retry_countdown) { + id(simple_retry_counter)++; + ESP_LOGI("test", "Simple retry attempt %d (countdown=%d)", + id(simple_retry_counter), retry_countdown); + + if (id(simple_retry_counter) >= 2) { + ESP_LOGI("test", "Simple retry succeeded on attempt %d", id(simple_retry_counter)); + return RetryResult::DONE; + } + return RetryResult::RETRY; + }); + + # Test 2: Backoff retry + - logger.log: "=== Test 2: Retry with backoff ===" + - lambda: |- + auto *component = id(test_sensor); + static uint32_t backoff_start_time = 0; + static uint32_t last_attempt_time = 0; + + backoff_start_time = millis(); + last_attempt_time = backoff_start_time; + + App.scheduler.set_retry(component, "backoff_retry", 50, 4, + [](uint8_t retry_countdown) { + id(backoff_retry_counter)++; + uint32_t now = millis(); + uint32_t interval = now - last_attempt_time; + last_attempt_time = now; + + ESP_LOGI("test", "Backoff retry attempt %d (countdown=%d, interval=%dms)", + id(backoff_retry_counter), retry_countdown, interval); + + if (id(backoff_retry_counter) == 1) { + ESP_LOGI("test", "First call was immediate"); + } else if (id(backoff_retry_counter) == 2) { + ESP_LOGI("test", "Second call interval: %dms (expected ~50ms)", interval); + } else if (id(backoff_retry_counter) == 3) { + ESP_LOGI("test", "Third call interval: %dms (expected ~100ms)", interval); + } else if (id(backoff_retry_counter) == 4) { + ESP_LOGI("test", "Fourth call interval: %dms (expected ~200ms)", interval); + ESP_LOGI("test", "Backoff retry completed"); + return RetryResult::DONE; + } + + return RetryResult::RETRY; + }, 2.0f); + + # Test 3: Immediate done + - logger.log: "=== Test 3: Immediate done ===" + - lambda: |- + auto *component = id(test_sensor); + App.scheduler.set_retry(component, "immediate_done", 50, 5, + [](uint8_t retry_countdown) { + id(immediate_done_counter)++; + ESP_LOGI("test", "Immediate done retry called (countdown=%d)", retry_countdown); + return RetryResult::DONE; + }); + + # Test 4: Cancel retry + - logger.log: "=== Test 4: Cancel retry ===" + - lambda: |- + auto *component = id(test_sensor); + App.scheduler.set_retry(component, "cancel_test", 25, 10, + [](uint8_t retry_countdown) { + id(cancel_retry_counter)++; + ESP_LOGI("test", "Cancel test retry attempt %d", id(cancel_retry_counter)); + return RetryResult::RETRY; + }); + + // Cancel it after 100ms + App.scheduler.set_timeout(component, "cancel_timer", 100, []() { + bool cancelled = App.scheduler.cancel_retry(id(test_sensor), "cancel_test"); + ESP_LOGI("test", "Retry cancellation result: %s", cancelled ? "true" : "false"); + ESP_LOGI("test", "Cancel retry ran %d times before cancellation", id(cancel_retry_counter)); + }); + + # Test 5: Empty name retry + - logger.log: "=== Test 5: Empty name retry ===" + - lambda: |- + auto *component = id(test_sensor); + App.scheduler.set_retry(component, "", 50, 5, + [](uint8_t retry_countdown) { + id(empty_name_retry_counter)++; + ESP_LOGI("test", "Empty name retry attempt %d", id(empty_name_retry_counter)); + return RetryResult::RETRY; + }); + + // Try to cancel after 75ms + App.scheduler.set_timeout(component, "empty_cancel_timer", 75, []() { + bool cancelled = App.scheduler.cancel_retry(id(test_sensor), ""); + ESP_LOGI("test", "Empty name retry cancel result: %s", + cancelled ? "true" : "false"); + ESP_LOGI("test", "Empty name retry ran %d times", id(empty_name_retry_counter)); + }); + + # Test 6: Component method + - logger.log: "=== Test 6: Component::set_retry method ===" + - lambda: |- + class TestRetryComponent : public Component { + public: + void test_retry() { + this->set_retry(50, 3, + [](uint8_t retry_countdown) { + id(script_retry_counter)++; + ESP_LOGI("test", "Component retry attempt %d", id(script_retry_counter)); + if (id(script_retry_counter) >= 2) { + return RetryResult::DONE; + } + return RetryResult::RETRY; + }, 1.5f); + } + }; + + static TestRetryComponent test_component; + test_component.test_retry(); + + # Test 7: Multiple same name + - logger.log: "=== Test 7: Multiple retries with same name ===" + - lambda: |- + auto *component = id(test_sensor); + + // Set first retry + App.scheduler.set_retry(component, "duplicate_retry", 100, 5, + [](uint8_t retry_countdown) { + id(multiple_same_name_counter) += 1; + ESP_LOGI("test", "First duplicate retry - should not run"); + return RetryResult::RETRY; + }); + + // Set second retry with same name (should cancel first) + App.scheduler.set_retry(component, "duplicate_retry", 50, 3, + [](uint8_t retry_countdown) { + id(multiple_same_name_counter) += 10; + ESP_LOGI("test", "Second duplicate retry attempt (counter=%d)", + id(multiple_same_name_counter)); + if (id(multiple_same_name_counter) >= 20) { + return RetryResult::DONE; + } + return RetryResult::RETRY; + }); + + # Wait for all tests to complete before reporting + - delay: 500ms + + # Final report + - logger.log: "=== Retry Test Results ===" + - lambda: |- + ESP_LOGI("test", "Simple retry counter: %d (expected 2)", id(simple_retry_counter)); + ESP_LOGI("test", "Backoff retry counter: %d (expected 4)", id(backoff_retry_counter)); + ESP_LOGI("test", "Immediate done counter: %d (expected 1)", id(immediate_done_counter)); + ESP_LOGI("test", "Cancel retry counter: %d (expected ~3-4)", id(cancel_retry_counter)); + ESP_LOGI("test", "Empty name retry counter: %d (expected 1-2)", id(empty_name_retry_counter)); + ESP_LOGI("test", "Component retry counter: %d (expected 2)", id(script_retry_counter)); + ESP_LOGI("test", "Multiple same name counter: %d (expected 20+)", id(multiple_same_name_counter)); + ESP_LOGI("test", "All retry tests completed"); diff --git a/tests/integration/test_scheduler_retry_test.py b/tests/integration/test_scheduler_retry_test.py new file mode 100644 index 0000000000..0c4d573c1b --- /dev/null +++ b/tests/integration/test_scheduler_retry_test.py @@ -0,0 +1,234 @@ +"""Test scheduler retry functionality.""" + +import asyncio +import re + +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_scheduler_retry_test( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test that scheduler retry functionality works correctly.""" + # Track test progress + simple_retry_done = asyncio.Event() + backoff_retry_done = asyncio.Event() + immediate_done_done = asyncio.Event() + cancel_retry_done = asyncio.Event() + empty_name_retry_done = asyncio.Event() + component_retry_done = asyncio.Event() + multiple_name_done = asyncio.Event() + test_complete = asyncio.Event() + + # Track retry counts + simple_retry_count = 0 + backoff_retry_count = 0 + immediate_done_count = 0 + cancel_retry_count = 0 + empty_name_retry_count = 0 + component_retry_count = 0 + multiple_name_count = 0 + + # Track specific test results + cancel_result = None + empty_cancel_result = None + backoff_intervals = [] + + def on_log_line(line: str) -> None: + nonlocal simple_retry_count, backoff_retry_count, immediate_done_count + nonlocal cancel_retry_count, empty_name_retry_count, component_retry_count + nonlocal multiple_name_count, cancel_result, empty_cancel_result + + # Strip ANSI color codes + clean_line = re.sub(r"\x1b\[[0-9;]*m", "", line) + + # Simple retry test + if "Simple retry attempt" in clean_line: + if match := re.search(r"Simple retry attempt (\d+)", clean_line): + simple_retry_count = int(match.group(1)) + + elif "Simple retry succeeded on attempt" in clean_line: + simple_retry_done.set() + + # Backoff retry test + elif "Backoff retry attempt" in clean_line: + if match := re.search( + r"Backoff retry attempt (\d+).*interval=(\d+)ms", clean_line + ): + backoff_retry_count = int(match.group(1)) + interval = int(match.group(2)) + if backoff_retry_count > 1: # Skip first (immediate) call + backoff_intervals.append(interval) + + elif "Backoff retry completed" in clean_line: + backoff_retry_done.set() + + # Immediate done test + elif "Immediate done retry called" in clean_line: + immediate_done_count += 1 + immediate_done_done.set() + + # Cancel retry test + elif "Cancel test retry attempt" in clean_line: + cancel_retry_count += 1 + + elif "Retry cancellation result:" in clean_line: + cancel_result = "true" in clean_line + cancel_retry_done.set() + + # Empty name retry test + elif "Empty name retry attempt" in clean_line: + if match := re.search(r"Empty name retry attempt (\d+)", clean_line): + empty_name_retry_count = int(match.group(1)) + + elif "Empty name retry cancel result:" in clean_line: + empty_cancel_result = "true" in clean_line + + elif "Empty name retry ran" in clean_line: + empty_name_retry_done.set() + + # Component retry test + elif "Component retry attempt" in clean_line: + if match := re.search(r"Component retry attempt (\d+)", clean_line): + component_retry_count = int(match.group(1)) + if component_retry_count >= 2: + component_retry_done.set() + + # Multiple same name test + elif "Second duplicate retry attempt" in clean_line: + if match := re.search(r"counter=(\d+)", clean_line): + multiple_name_count = int(match.group(1)) + if multiple_name_count >= 20: + multiple_name_done.set() + + # Test completion + elif "All retry tests completed" in clean_line: + test_complete.set() + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + # Verify we can connect + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "scheduler-retry-test" + + # Wait for simple retry test + try: + await asyncio.wait_for(simple_retry_done.wait(), timeout=1.0) + except TimeoutError: + pytest.fail( + f"Simple retry test did not complete. Count: {simple_retry_count}" + ) + + assert simple_retry_count == 2, ( + f"Expected 2 simple retry attempts, got {simple_retry_count}" + ) + + # Wait for backoff retry test + try: + await asyncio.wait_for(backoff_retry_done.wait(), timeout=3.0) + except TimeoutError: + pytest.fail( + f"Backoff retry test did not complete. Count: {backoff_retry_count}" + ) + + assert backoff_retry_count == 4, ( + f"Expected 4 backoff retry attempts, got {backoff_retry_count}" + ) + + # Verify backoff intervals (allowing for timing variations) + assert len(backoff_intervals) >= 2, ( + f"Expected at least 2 intervals, got {len(backoff_intervals)}" + ) + if len(backoff_intervals) >= 3: + # First interval should be ~50ms + assert 30 <= backoff_intervals[0] <= 70, ( + f"First interval {backoff_intervals[0]}ms not ~50ms" + ) + # Second interval should be ~100ms (50ms * 2.0) + assert 80 <= backoff_intervals[1] <= 120, ( + f"Second interval {backoff_intervals[1]}ms not ~100ms" + ) + # Third interval should be ~200ms (100ms * 2.0) + assert 180 <= backoff_intervals[2] <= 220, ( + f"Third interval {backoff_intervals[2]}ms not ~200ms" + ) + + # Wait for immediate done test + try: + await asyncio.wait_for(immediate_done_done.wait(), timeout=3.0) + except TimeoutError: + pytest.fail( + f"Immediate done test did not complete. Count: {immediate_done_count}" + ) + + assert immediate_done_count == 1, ( + f"Expected 1 immediate done call, got {immediate_done_count}" + ) + + # Wait for cancel retry test + try: + await asyncio.wait_for(cancel_retry_done.wait(), timeout=2.0) + except TimeoutError: + pytest.fail( + f"Cancel retry test did not complete. Count: {cancel_retry_count}" + ) + + assert cancel_result is True, "Retry cancellation should have succeeded" + assert 2 <= cancel_retry_count <= 5, ( + f"Expected 2-5 cancel retry attempts before cancellation, got {cancel_retry_count}" + ) + + # Wait for empty name retry test + try: + await asyncio.wait_for(empty_name_retry_done.wait(), timeout=1.0) + except TimeoutError: + pytest.fail( + f"Empty name retry test did not complete. Count: {empty_name_retry_count}" + ) + + # Empty name retry should run at least once before being cancelled + assert 1 <= empty_name_retry_count <= 2, ( + f"Expected 1-2 empty name retry attempts, got {empty_name_retry_count}" + ) + assert empty_cancel_result is True, ( + "Empty name retry cancel should have succeeded" + ) + + # Wait for component retry test + try: + await asyncio.wait_for(component_retry_done.wait(), timeout=1.0) + except TimeoutError: + pytest.fail( + f"Component retry test did not complete. Count: {component_retry_count}" + ) + + assert component_retry_count >= 2, ( + f"Expected at least 2 component retry attempts, got {component_retry_count}" + ) + + # Wait for multiple same name test + try: + await asyncio.wait_for(multiple_name_done.wait(), timeout=1.0) + except TimeoutError: + pytest.fail( + f"Multiple same name test did not complete. Count: {multiple_name_count}" + ) + + # Should be 20+ (only second retry should run) + assert multiple_name_count >= 20, ( + f"Expected multiple name count >= 20 (second retry only), got {multiple_name_count}" + ) + + # Wait for test completion + try: + await asyncio.wait_for(test_complete.wait(), timeout=1.0) + except TimeoutError: + pytest.fail("Test did not complete within timeout") From 71cc298363f0c27d0eb89c7c6ba97a9205ba4ba5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 02:28:08 -1000 Subject: [PATCH 25/38] Use message_source_map consistently in proto generation (#9542) --- script/api_protobuf/api_protobuf.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 23d8a53b70..4df7692167 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -1495,6 +1495,7 @@ def build_base_class( base_class_name: str, common_fields: list[descriptor.FieldDescriptorProto], messages: list[descriptor.DescriptorProto], + message_source_map: dict[str, int], ) -> tuple[str, str, str]: """Build the base class definition and implementation.""" public_content = [] @@ -1511,7 +1512,7 @@ def build_base_class( # Determine if any message using this base class needs decoding needs_decode = any( - get_opt(msg, pb.source, SOURCE_BOTH) in (SOURCE_BOTH, SOURCE_CLIENT) + message_source_map.get(msg.name, SOURCE_BOTH) in (SOURCE_BOTH, SOURCE_CLIENT) for msg in messages ) @@ -1543,6 +1544,7 @@ def build_base_class( def generate_base_classes( base_class_groups: dict[str, list[descriptor.DescriptorProto]], + message_source_map: dict[str, int], ) -> tuple[str, str, str]: """Generate all base classes.""" all_headers = [] @@ -1556,7 +1558,7 @@ def generate_base_classes( if common_fields: # Generate base class header, cpp, dump_cpp = build_base_class( - base_class_name, common_fields, messages + base_class_name, common_fields, messages, message_source_map ) all_headers.append(header) all_cpp.append(cpp) @@ -1567,6 +1569,7 @@ def generate_base_classes( def build_service_message_type( mt: descriptor.DescriptorProto, + message_source_map: dict[str, int], ) -> tuple[str, str] | None: """Builds the service message type.""" snake = camel_to_snake(mt.name) @@ -1574,7 +1577,7 @@ def build_service_message_type( if id_ is None: return None - source: int = get_opt(mt, pb.source, 0) + source: int = message_source_map.get(mt.name, SOURCE_BOTH) ifdef: str | None = get_opt(mt, pb.ifdef) log: bool = get_opt(mt, pb.log, True) @@ -1714,7 +1717,9 @@ namespace api { # Generate base classes if base_class_fields: - base_headers, base_cpp, base_dump_cpp = generate_base_classes(base_class_groups) + base_headers, base_cpp, base_dump_cpp = generate_base_classes( + base_class_groups, message_source_map + ) content += base_headers cpp += base_cpp dump_cpp += base_dump_cpp @@ -1832,7 +1837,7 @@ static const char *const TAG = "api.service"; cpp += "#endif\n\n" for mt in file.message_type: - obj = build_service_message_type(mt) + obj = build_service_message_type(mt, message_source_map) if obj is None: continue hout, cout = obj From 5f9331b1129dfed01d602554a2c5f004a5dc830a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 07:50:28 -1000 Subject: [PATCH 26/38] Fix AsyncTCP version mismatch between platformio.ini and async_tcp component --- .clang-tidy.hash | 2 +- platformio.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.clang-tidy.hash b/.clang-tidy.hash index 18be8d78a9..50a7fa9709 100644 --- a/.clang-tidy.hash +++ b/.clang-tidy.hash @@ -1 +1 @@ -07f621354fe1350ba51953c80273cd44a04aa44f15cc30bd7b8fe2a641427b7a +0c2acbc16bfb7d63571dbe7042f94f683be25e4ca8a0f158a960a94adac4b931 diff --git a/platformio.ini b/platformio.ini index 8fcc578103..7fb301c08b 100644 --- a/platformio.ini +++ b/platformio.ini @@ -138,7 +138,7 @@ lib_deps = WiFi ; wifi,web_server_base,ethernet (Arduino built-in) Update ; ota,web_server_base (Arduino built-in) ${common:arduino.lib_deps} - ESP32Async/AsyncTCP@3.4.4 ; async_tcp + ESP32Async/AsyncTCP@3.4.5 ; async_tcp NetworkClientSecure ; http_request,nextion (Arduino built-in) HTTPClient ; http_request,nextion (Arduino built-in) ESPmDNS ; mdns (Arduino built-in) From a11c39bdc98e5730abc3e6dad39e8a2a75db8295 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Jul 2025 17:57:40 +0000 Subject: [PATCH 27/38] Bump aioesphomeapi from 36.0.1 to 37.0.0 (#9677) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 38bbc2d94c..acfa31ddca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ platformio==6.1.18 # When updating platformio, also update /docker/Dockerfile esptool==4.9.0 click==8.1.7 esphome-dashboard==20250514.0 -aioesphomeapi==36.0.1 +aioesphomeapi==37.0.0 zeroconf==0.147.0 puremagic==1.30 ruamel.yaml==0.18.14 # dashboard_import From 0a450143305d6ba91040b4687f9384257cc3cdde Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 08:13:33 -1000 Subject: [PATCH 28/38] Remove deprecated protobuf fields to reduce flash usage --- esphome/components/api/api.proto | 49 ++++++++++----- esphome/components/api/api_pb2.cpp | 56 ----------------- esphome/components/api/api_pb2.h | 42 +++---------- esphome/components/api/api_pb2_dump.cpp | 81 ------------------------- script/api_protobuf/api_protobuf.py | 17 +++++- 5 files changed, 60 insertions(+), 185 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index b0ce21b1ce..2e8c863b87 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -230,14 +230,16 @@ message DeviceInfoResponse { uint32 webserver_port = 10 [(field_ifdef) = "USE_WEBSERVER"]; - uint32 legacy_bluetooth_proxy_version = 11 [(field_ifdef) = "USE_BLUETOOTH_PROXY"]; + // Deprecated in API version 1.9 + uint32 legacy_bluetooth_proxy_version = 11 [deprecated=true, (field_ifdef) = "USE_BLUETOOTH_PROXY"]; uint32 bluetooth_proxy_feature_flags = 15 [(field_ifdef) = "USE_BLUETOOTH_PROXY"]; string manufacturer = 12; string friendly_name = 13; - uint32 legacy_voice_assistant_version = 14 [(field_ifdef) = "USE_VOICE_ASSISTANT"]; + // Deprecated in API version 1.10 + uint32 legacy_voice_assistant_version = 14 [deprecated=true, (field_ifdef) = "USE_VOICE_ASSISTANT"]; uint32 voice_assistant_feature_flags = 17 [(field_ifdef) = "USE_VOICE_ASSISTANT"]; string suggested_area = 16 [(field_ifdef) = "USE_AREAS"]; @@ -337,6 +339,7 @@ message ListEntitiesCoverResponse { uint32 device_id = 13 [(field_ifdef) = "USE_DEVICES"]; } +// Deprecated in API version 1.1 enum LegacyCoverState { LEGACY_COVER_STATE_OPEN = 0; LEGACY_COVER_STATE_CLOSED = 1; @@ -356,7 +359,8 @@ message CoverStateResponse { fixed32 key = 1; // legacy: state has been removed in 1.13 // clients/servers must still send/accept it until the next protocol change - LegacyCoverState legacy_state = 2; + // Deprecated in API version 1.1 + LegacyCoverState legacy_state = 2 [deprecated=true]; float position = 3; float tilt = 4; @@ -364,6 +368,7 @@ message CoverStateResponse { uint32 device_id = 6 [(field_ifdef) = "USE_DEVICES"]; } +// Deprecated in API version 1.1 enum LegacyCoverCommand { LEGACY_COVER_COMMAND_OPEN = 0; LEGACY_COVER_COMMAND_CLOSE = 1; @@ -380,8 +385,10 @@ message CoverCommandRequest { // legacy: command has been removed in 1.13 // clients/servers must still send/accept it until the next protocol change - bool has_legacy_command = 2; - LegacyCoverCommand legacy_command = 3; + // Deprecated in API version 1.1 + bool has_legacy_command = 2 [deprecated=true]; + // Deprecated in API version 1.1 + LegacyCoverCommand legacy_command = 3 [deprecated=true]; bool has_position = 4; float position = 5; @@ -432,7 +439,8 @@ message FanStateResponse { fixed32 key = 1; bool state = 2; bool oscillating = 3; - FanSpeed speed = 4 [deprecated = true]; + // Deprecated in API version 1.6 + FanSpeed speed = 4 [deprecated=true]; FanDirection direction = 5; int32 speed_level = 6; string preset_mode = 7; @@ -448,8 +456,10 @@ message FanCommandRequest { fixed32 key = 1; bool has_state = 2; bool state = 3; - bool has_speed = 4 [deprecated = true]; - FanSpeed speed = 5 [deprecated = true]; + // Deprecated in API version 1.6 + bool has_speed = 4 [deprecated=true]; + // Deprecated in API version 1.6 + FanSpeed speed = 5 [deprecated=true]; bool has_oscillating = 6; bool oscillating = 7; bool has_direction = 8; @@ -488,9 +498,13 @@ message ListEntitiesLightResponse { repeated ColorMode supported_color_modes = 12; // next four supports_* are for legacy clients, newer clients should use color modes + // Deprecated in API version 1.6 bool legacy_supports_brightness = 5 [deprecated=true]; + // Deprecated in API version 1.6 bool legacy_supports_rgb = 6 [deprecated=true]; + // Deprecated in API version 1.6 bool legacy_supports_white_value = 7 [deprecated=true]; + // Deprecated in API version 1.6 bool legacy_supports_color_temperature = 8 [deprecated=true]; float min_mireds = 9; float max_mireds = 10; @@ -567,6 +581,7 @@ enum SensorStateClass { STATE_CLASS_TOTAL = 3; } +// Deprecated in API version 1.5 enum SensorLastResetType { LAST_RESET_NONE = 0; LAST_RESET_NEVER = 1; @@ -591,7 +606,8 @@ message ListEntitiesSensorResponse { string device_class = 9; SensorStateClass state_class = 10; // Last reset type removed in 2021.9.0 - SensorLastResetType legacy_last_reset_type = 11; + // Deprecated in API version 1.5 + SensorLastResetType legacy_last_reset_type = 11 [deprecated=true]; bool disabled_by_default = 12; EntityCategory entity_category = 13; uint32 device_id = 14 [(field_ifdef) = "USE_DEVICES"]; @@ -947,7 +963,8 @@ message ListEntitiesClimateResponse { float visual_target_temperature_step = 10; // for older peer versions - in new system this // is if CLIMATE_PRESET_AWAY exists is supported_presets - bool legacy_supports_away = 11; + // Deprecated in API version 1.5 + bool legacy_supports_away = 11 [deprecated=true]; bool supports_action = 12; repeated ClimateFanMode supported_fan_modes = 13; repeated ClimateSwingMode supported_swing_modes = 14; @@ -978,7 +995,8 @@ message ClimateStateResponse { float target_temperature_low = 5; float target_temperature_high = 6; // For older peers, equal to preset == CLIMATE_PRESET_AWAY - bool unused_legacy_away = 7; + // Deprecated in API version 1.5 + bool unused_legacy_away = 7 [deprecated=true]; ClimateAction action = 8; ClimateFanMode fan_mode = 9; ClimateSwingMode swing_mode = 10; @@ -1006,8 +1024,10 @@ message ClimateCommandRequest { bool has_target_temperature_high = 8; float target_temperature_high = 9; // legacy, for older peers, newer ones should use CLIMATE_PRESET_AWAY in preset - bool unused_has_legacy_away = 10; - bool unused_legacy_away = 11; + // Deprecated in API version 1.5 + bool unused_has_legacy_away = 10 [deprecated=true]; + // Deprecated in API version 1.5 + bool unused_legacy_away = 11 [deprecated=true]; bool has_fan_mode = 12; ClimateFanMode fan_mode = 13; bool has_swing_mode = 14; @@ -1356,7 +1376,8 @@ message SubscribeBluetoothLEAdvertisementsRequest { message BluetoothServiceData { string uuid = 1; - repeated uint32 legacy_data = 2 [deprecated = true]; // Removed in api version 1.7 + // Deprecated in API version 1.7 + repeated uint32 legacy_data = 2 [deprecated=true]; // Removed in api version 1.7 bytes data = 3; // Added in api version 1.7 } message BluetoothLEAdvertisementResponse { diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 437c9ece1d..44e3a3205b 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -94,17 +94,11 @@ void DeviceInfoResponse::encode(ProtoWriteBuffer buffer) const { #ifdef USE_WEBSERVER buffer.encode_uint32(10, this->webserver_port); #endif -#ifdef USE_BLUETOOTH_PROXY - buffer.encode_uint32(11, this->legacy_bluetooth_proxy_version); -#endif #ifdef USE_BLUETOOTH_PROXY buffer.encode_uint32(15, this->bluetooth_proxy_feature_flags); #endif buffer.encode_string(12, this->manufacturer); buffer.encode_string(13, this->friendly_name); -#ifdef USE_VOICE_ASSISTANT - buffer.encode_uint32(14, this->legacy_voice_assistant_version); -#endif #ifdef USE_VOICE_ASSISTANT buffer.encode_uint32(17, this->voice_assistant_feature_flags); #endif @@ -150,17 +144,11 @@ void DeviceInfoResponse::calculate_size(uint32_t &total_size) const { #ifdef USE_WEBSERVER ProtoSize::add_uint32_field(total_size, 1, this->webserver_port); #endif -#ifdef USE_BLUETOOTH_PROXY - ProtoSize::add_uint32_field(total_size, 1, this->legacy_bluetooth_proxy_version); -#endif #ifdef USE_BLUETOOTH_PROXY ProtoSize::add_uint32_field(total_size, 1, this->bluetooth_proxy_feature_flags); #endif ProtoSize::add_string_field(total_size, 1, this->manufacturer); ProtoSize::add_string_field(total_size, 1, this->friendly_name); -#ifdef USE_VOICE_ASSISTANT - ProtoSize::add_uint32_field(total_size, 1, this->legacy_voice_assistant_version); -#endif #ifdef USE_VOICE_ASSISTANT ProtoSize::add_uint32_field(total_size, 2, this->voice_assistant_feature_flags); #endif @@ -270,7 +258,6 @@ void ListEntitiesCoverResponse::calculate_size(uint32_t &total_size) const { } void CoverStateResponse::encode(ProtoWriteBuffer buffer) const { buffer.encode_fixed32(1, this->key); - buffer.encode_uint32(2, static_cast(this->legacy_state)); buffer.encode_float(3, this->position); buffer.encode_float(4, this->tilt); buffer.encode_uint32(5, static_cast(this->current_operation)); @@ -280,7 +267,6 @@ void CoverStateResponse::encode(ProtoWriteBuffer buffer) const { } void CoverStateResponse::calculate_size(uint32_t &total_size) const { ProtoSize::add_fixed32_field(total_size, 1, this->key); - ProtoSize::add_enum_field(total_size, 1, static_cast(this->legacy_state)); ProtoSize::add_float_field(total_size, 1, this->position); ProtoSize::add_float_field(total_size, 1, this->tilt); ProtoSize::add_enum_field(total_size, 1, static_cast(this->current_operation)); @@ -290,12 +276,6 @@ void CoverStateResponse::calculate_size(uint32_t &total_size) const { } bool CoverCommandRequest::decode_varint(uint32_t field_id, ProtoVarInt value) { switch (field_id) { - case 2: - this->has_legacy_command = value.as_bool(); - break; - case 3: - this->legacy_command = static_cast(value.as_uint32()); - break; case 4: this->has_position = value.as_bool(); break; @@ -379,7 +359,6 @@ void FanStateResponse::encode(ProtoWriteBuffer buffer) const { buffer.encode_fixed32(1, this->key); buffer.encode_bool(2, this->state); buffer.encode_bool(3, this->oscillating); - buffer.encode_uint32(4, static_cast(this->speed)); buffer.encode_uint32(5, static_cast(this->direction)); buffer.encode_int32(6, this->speed_level); buffer.encode_string(7, this->preset_mode); @@ -391,7 +370,6 @@ void FanStateResponse::calculate_size(uint32_t &total_size) const { ProtoSize::add_fixed32_field(total_size, 1, this->key); ProtoSize::add_bool_field(total_size, 1, this->state); ProtoSize::add_bool_field(total_size, 1, this->oscillating); - ProtoSize::add_enum_field(total_size, 1, static_cast(this->speed)); ProtoSize::add_enum_field(total_size, 1, static_cast(this->direction)); ProtoSize::add_int32_field(total_size, 1, this->speed_level); ProtoSize::add_string_field(total_size, 1, this->preset_mode); @@ -407,12 +385,6 @@ bool FanCommandRequest::decode_varint(uint32_t field_id, ProtoVarInt value) { case 3: this->state = value.as_bool(); break; - case 4: - this->has_speed = value.as_bool(); - break; - case 5: - this->speed = static_cast(value.as_uint32()); - break; case 6: this->has_oscillating = value.as_bool(); break; @@ -473,10 +445,6 @@ void ListEntitiesLightResponse::encode(ProtoWriteBuffer buffer) const { for (auto &it : this->supported_color_modes) { buffer.encode_uint32(12, static_cast(it), true); } - buffer.encode_bool(5, this->legacy_supports_brightness); - buffer.encode_bool(6, this->legacy_supports_rgb); - buffer.encode_bool(7, this->legacy_supports_white_value); - buffer.encode_bool(8, this->legacy_supports_color_temperature); buffer.encode_float(9, this->min_mireds); buffer.encode_float(10, this->max_mireds); for (auto &it : this->effects) { @@ -500,10 +468,6 @@ void ListEntitiesLightResponse::calculate_size(uint32_t &total_size) const { ProtoSize::add_enum_field_repeated(total_size, 1, static_cast(it)); } } - ProtoSize::add_bool_field(total_size, 1, this->legacy_supports_brightness); - ProtoSize::add_bool_field(total_size, 1, this->legacy_supports_rgb); - ProtoSize::add_bool_field(total_size, 1, this->legacy_supports_white_value); - ProtoSize::add_bool_field(total_size, 1, this->legacy_supports_color_temperature); ProtoSize::add_float_field(total_size, 1, this->min_mireds); ProtoSize::add_float_field(total_size, 1, this->max_mireds); if (!this->effects.empty()) { @@ -677,7 +641,6 @@ void ListEntitiesSensorResponse::encode(ProtoWriteBuffer buffer) const { buffer.encode_bool(8, this->force_update); buffer.encode_string(9, this->device_class); buffer.encode_uint32(10, static_cast(this->state_class)); - buffer.encode_uint32(11, static_cast(this->legacy_last_reset_type)); buffer.encode_bool(12, this->disabled_by_default); buffer.encode_uint32(13, static_cast(this->entity_category)); #ifdef USE_DEVICES @@ -696,7 +659,6 @@ void ListEntitiesSensorResponse::calculate_size(uint32_t &total_size) const { ProtoSize::add_bool_field(total_size, 1, this->force_update); ProtoSize::add_string_field(total_size, 1, this->device_class); ProtoSize::add_enum_field(total_size, 1, static_cast(this->state_class)); - ProtoSize::add_enum_field(total_size, 1, static_cast(this->legacy_last_reset_type)); ProtoSize::add_bool_field(total_size, 1, this->disabled_by_default); ProtoSize::add_enum_field(total_size, 1, static_cast(this->entity_category)); #ifdef USE_DEVICES @@ -1105,7 +1067,6 @@ void ListEntitiesClimateResponse::encode(ProtoWriteBuffer buffer) const { buffer.encode_float(8, this->visual_min_temperature); buffer.encode_float(9, this->visual_max_temperature); buffer.encode_float(10, this->visual_target_temperature_step); - buffer.encode_bool(11, this->legacy_supports_away); buffer.encode_bool(12, this->supports_action); for (auto &it : this->supported_fan_modes) { buffer.encode_uint32(13, static_cast(it), true); @@ -1150,7 +1111,6 @@ void ListEntitiesClimateResponse::calculate_size(uint32_t &total_size) const { ProtoSize::add_float_field(total_size, 1, this->visual_min_temperature); ProtoSize::add_float_field(total_size, 1, this->visual_max_temperature); ProtoSize::add_float_field(total_size, 1, this->visual_target_temperature_step); - ProtoSize::add_bool_field(total_size, 1, this->legacy_supports_away); ProtoSize::add_bool_field(total_size, 1, this->supports_action); if (!this->supported_fan_modes.empty()) { for (const auto &it : this->supported_fan_modes) { @@ -1198,7 +1158,6 @@ void ClimateStateResponse::encode(ProtoWriteBuffer buffer) const { buffer.encode_float(4, this->target_temperature); buffer.encode_float(5, this->target_temperature_low); buffer.encode_float(6, this->target_temperature_high); - buffer.encode_bool(7, this->unused_legacy_away); buffer.encode_uint32(8, static_cast(this->action)); buffer.encode_uint32(9, static_cast(this->fan_mode)); buffer.encode_uint32(10, static_cast(this->swing_mode)); @@ -1218,7 +1177,6 @@ void ClimateStateResponse::calculate_size(uint32_t &total_size) const { ProtoSize::add_float_field(total_size, 1, this->target_temperature); ProtoSize::add_float_field(total_size, 1, this->target_temperature_low); ProtoSize::add_float_field(total_size, 1, this->target_temperature_high); - ProtoSize::add_bool_field(total_size, 1, this->unused_legacy_away); ProtoSize::add_enum_field(total_size, 1, static_cast(this->action)); ProtoSize::add_enum_field(total_size, 1, static_cast(this->fan_mode)); ProtoSize::add_enum_field(total_size, 1, static_cast(this->swing_mode)); @@ -1248,12 +1206,6 @@ bool ClimateCommandRequest::decode_varint(uint32_t field_id, ProtoVarInt value) case 8: this->has_target_temperature_high = value.as_bool(); break; - case 10: - this->unused_has_legacy_away = value.as_bool(); - break; - case 11: - this->unused_legacy_away = value.as_bool(); - break; case 12: this->has_fan_mode = value.as_bool(); break; @@ -1871,18 +1823,10 @@ bool SubscribeBluetoothLEAdvertisementsRequest::decode_varint(uint32_t field_id, } void BluetoothServiceData::encode(ProtoWriteBuffer buffer) const { buffer.encode_string(1, this->uuid); - for (auto &it : this->legacy_data) { - buffer.encode_uint32(2, it, true); - } buffer.encode_bytes(3, reinterpret_cast(this->data.data()), this->data.size()); } void BluetoothServiceData::calculate_size(uint32_t &total_size) const { ProtoSize::add_string_field(total_size, 1, this->uuid); - if (!this->legacy_data.empty()) { - for (const auto &it : this->legacy_data) { - ProtoSize::add_uint32_field_repeated(total_size, 1, it); - } - } ProtoSize::add_string_field(total_size, 1, this->data); } void BluetoothLEAdvertisementResponse::encode(ProtoWriteBuffer buffer) const { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 39f00b4adc..a9fe3cb538 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -477,7 +477,7 @@ class DeviceInfo : public ProtoMessage { class DeviceInfoResponse : public ProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 10; - static constexpr uint8_t ESTIMATED_SIZE = 219; + static constexpr uint8_t ESTIMATED_SIZE = 211; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "device_info_response"; } #endif @@ -499,17 +499,11 @@ class DeviceInfoResponse : public ProtoMessage { #ifdef USE_WEBSERVER uint32_t webserver_port{0}; #endif -#ifdef USE_BLUETOOTH_PROXY - uint32_t legacy_bluetooth_proxy_version{0}; -#endif #ifdef USE_BLUETOOTH_PROXY uint32_t bluetooth_proxy_feature_flags{0}; #endif std::string manufacturer{}; std::string friendly_name{}; -#ifdef USE_VOICE_ASSISTANT - uint32_t legacy_voice_assistant_version{0}; -#endif #ifdef USE_VOICE_ASSISTANT uint32_t voice_assistant_feature_flags{0}; #endif @@ -638,11 +632,10 @@ class ListEntitiesCoverResponse : public InfoResponseProtoMessage { class CoverStateResponse : public StateResponseProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 22; - static constexpr uint8_t ESTIMATED_SIZE = 23; + static constexpr uint8_t ESTIMATED_SIZE = 21; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "cover_state_response"; } #endif - enums::LegacyCoverState legacy_state{}; float position{0.0f}; float tilt{0.0f}; enums::CoverOperation current_operation{}; @@ -657,12 +650,10 @@ class CoverStateResponse : public StateResponseProtoMessage { class CoverCommandRequest : public CommandProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 30; - static constexpr uint8_t ESTIMATED_SIZE = 29; + static constexpr uint8_t ESTIMATED_SIZE = 25; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "cover_command_request"; } #endif - bool has_legacy_command{false}; - enums::LegacyCoverCommand legacy_command{}; bool has_position{false}; float position{0.0f}; bool has_tilt{false}; @@ -701,13 +692,12 @@ class ListEntitiesFanResponse : public InfoResponseProtoMessage { class FanStateResponse : public StateResponseProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 23; - static constexpr uint8_t ESTIMATED_SIZE = 30; + static constexpr uint8_t ESTIMATED_SIZE = 28; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "fan_state_response"; } #endif bool state{false}; bool oscillating{false}; - enums::FanSpeed speed{}; enums::FanDirection direction{}; int32_t speed_level{0}; std::string preset_mode{}; @@ -722,14 +712,12 @@ class FanStateResponse : public StateResponseProtoMessage { class FanCommandRequest : public CommandProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 31; - static constexpr uint8_t ESTIMATED_SIZE = 42; + static constexpr uint8_t ESTIMATED_SIZE = 38; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "fan_command_request"; } #endif bool has_state{false}; bool state{false}; - bool has_speed{false}; - enums::FanSpeed speed{}; bool has_oscillating{false}; bool oscillating{false}; bool has_direction{false}; @@ -752,15 +740,11 @@ class FanCommandRequest : public CommandProtoMessage { class ListEntitiesLightResponse : public InfoResponseProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 15; - static constexpr uint8_t ESTIMATED_SIZE = 81; + static constexpr uint8_t ESTIMATED_SIZE = 73; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "list_entities_light_response"; } #endif std::vector supported_color_modes{}; - bool legacy_supports_brightness{false}; - bool legacy_supports_rgb{false}; - bool legacy_supports_white_value{false}; - bool legacy_supports_color_temperature{false}; float min_mireds{0.0f}; float max_mireds{0.0f}; std::vector effects{}; @@ -846,7 +830,7 @@ class LightCommandRequest : public CommandProtoMessage { class ListEntitiesSensorResponse : public InfoResponseProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 16; - static constexpr uint8_t ESTIMATED_SIZE = 68; + static constexpr uint8_t ESTIMATED_SIZE = 66; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "list_entities_sensor_response"; } #endif @@ -855,7 +839,6 @@ class ListEntitiesSensorResponse : public InfoResponseProtoMessage { bool force_update{false}; std::string device_class{}; enums::SensorStateClass state_class{}; - enums::SensorLastResetType legacy_last_reset_type{}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(uint32_t &total_size) const override; #ifdef HAS_PROTO_MESSAGE_DUMP @@ -1281,7 +1264,7 @@ class CameraImageRequest : public ProtoDecodableMessage { class ListEntitiesClimateResponse : public InfoResponseProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 46; - static constexpr uint8_t ESTIMATED_SIZE = 147; + static constexpr uint8_t ESTIMATED_SIZE = 145; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "list_entities_climate_response"; } #endif @@ -1291,7 +1274,6 @@ class ListEntitiesClimateResponse : public InfoResponseProtoMessage { float visual_min_temperature{0.0f}; float visual_max_temperature{0.0f}; float visual_target_temperature_step{0.0f}; - bool legacy_supports_away{false}; bool supports_action{false}; std::vector supported_fan_modes{}; std::vector supported_swing_modes{}; @@ -1314,7 +1296,7 @@ class ListEntitiesClimateResponse : public InfoResponseProtoMessage { class ClimateStateResponse : public StateResponseProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 47; - static constexpr uint8_t ESTIMATED_SIZE = 70; + static constexpr uint8_t ESTIMATED_SIZE = 68; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "climate_state_response"; } #endif @@ -1323,7 +1305,6 @@ class ClimateStateResponse : public StateResponseProtoMessage { float target_temperature{0.0f}; float target_temperature_low{0.0f}; float target_temperature_high{0.0f}; - bool unused_legacy_away{false}; enums::ClimateAction action{}; enums::ClimateFanMode fan_mode{}; enums::ClimateSwingMode swing_mode{}; @@ -1343,7 +1324,7 @@ class ClimateStateResponse : public StateResponseProtoMessage { class ClimateCommandRequest : public CommandProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 48; - static constexpr uint8_t ESTIMATED_SIZE = 88; + static constexpr uint8_t ESTIMATED_SIZE = 84; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "climate_command_request"; } #endif @@ -1355,8 +1336,6 @@ class ClimateCommandRequest : public CommandProtoMessage { float target_temperature_low{0.0f}; bool has_target_temperature_high{false}; float target_temperature_high{0.0f}; - bool unused_has_legacy_away{false}; - bool unused_legacy_away{false}; bool has_fan_mode{false}; enums::ClimateFanMode fan_mode{}; bool has_swing_mode{false}; @@ -1731,7 +1710,6 @@ class SubscribeBluetoothLEAdvertisementsRequest : public ProtoDecodableMessage { class BluetoothServiceData : public ProtoMessage { public: std::string uuid{}; - std::vector legacy_data{}; std::string data{}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(uint32_t &total_size) const override; diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index ad5a5fdcaa..35d5e2e91c 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -737,13 +737,6 @@ void DeviceInfoResponse::dump_to(std::string &out) const { out.append(buffer); out.append("\n"); -#endif -#ifdef USE_BLUETOOTH_PROXY - out.append(" legacy_bluetooth_proxy_version: "); - snprintf(buffer, sizeof(buffer), "%" PRIu32, this->legacy_bluetooth_proxy_version); - out.append(buffer); - out.append("\n"); - #endif #ifdef USE_BLUETOOTH_PROXY out.append(" bluetooth_proxy_feature_flags: "); @@ -760,13 +753,6 @@ void DeviceInfoResponse::dump_to(std::string &out) const { out.append("'").append(this->friendly_name).append("'"); out.append("\n"); -#ifdef USE_VOICE_ASSISTANT - out.append(" legacy_voice_assistant_version: "); - snprintf(buffer, sizeof(buffer), "%" PRIu32, this->legacy_voice_assistant_version); - out.append(buffer); - out.append("\n"); - -#endif #ifdef USE_VOICE_ASSISTANT out.append(" voice_assistant_feature_flags: "); snprintf(buffer, sizeof(buffer), "%" PRIu32, this->voice_assistant_feature_flags); @@ -961,10 +947,6 @@ void CoverStateResponse::dump_to(std::string &out) const { out.append(buffer); out.append("\n"); - out.append(" legacy_state: "); - out.append(proto_enum_to_string(this->legacy_state)); - out.append("\n"); - out.append(" position: "); snprintf(buffer, sizeof(buffer), "%g", this->position); out.append(buffer); @@ -996,14 +978,6 @@ void CoverCommandRequest::dump_to(std::string &out) const { out.append(buffer); out.append("\n"); - out.append(" has_legacy_command: "); - out.append(YESNO(this->has_legacy_command)); - out.append("\n"); - - out.append(" legacy_command: "); - out.append(proto_enum_to_string(this->legacy_command)); - out.append("\n"); - out.append(" has_position: "); out.append(YESNO(this->has_position)); out.append("\n"); @@ -1115,10 +1089,6 @@ void FanStateResponse::dump_to(std::string &out) const { out.append(YESNO(this->oscillating)); out.append("\n"); - out.append(" speed: "); - out.append(proto_enum_to_string(this->speed)); - out.append("\n"); - out.append(" direction: "); out.append(proto_enum_to_string(this->direction)); out.append("\n"); @@ -1157,14 +1127,6 @@ void FanCommandRequest::dump_to(std::string &out) const { out.append(YESNO(this->state)); out.append("\n"); - out.append(" has_speed: "); - out.append(YESNO(this->has_speed)); - out.append("\n"); - - out.append(" speed: "); - out.append(proto_enum_to_string(this->speed)); - out.append("\n"); - out.append(" has_oscillating: "); out.append(YESNO(this->has_oscillating)); out.append("\n"); @@ -1231,22 +1193,6 @@ void ListEntitiesLightResponse::dump_to(std::string &out) const { out.append("\n"); } - out.append(" legacy_supports_brightness: "); - out.append(YESNO(this->legacy_supports_brightness)); - out.append("\n"); - - out.append(" legacy_supports_rgb: "); - out.append(YESNO(this->legacy_supports_rgb)); - out.append("\n"); - - out.append(" legacy_supports_white_value: "); - out.append(YESNO(this->legacy_supports_white_value)); - out.append("\n"); - - out.append(" legacy_supports_color_temperature: "); - out.append(YESNO(this->legacy_supports_color_temperature)); - out.append("\n"); - out.append(" min_mireds: "); snprintf(buffer, sizeof(buffer), "%g", this->min_mireds); out.append(buffer); @@ -1537,10 +1483,6 @@ void ListEntitiesSensorResponse::dump_to(std::string &out) const { out.append(proto_enum_to_string(this->state_class)); out.append("\n"); - out.append(" legacy_last_reset_type: "); - out.append(proto_enum_to_string(this->legacy_last_reset_type)); - out.append("\n"); - out.append(" disabled_by_default: "); out.append(YESNO(this->disabled_by_default)); out.append("\n"); @@ -2107,10 +2049,6 @@ void ListEntitiesClimateResponse::dump_to(std::string &out) const { out.append(buffer); out.append("\n"); - out.append(" legacy_supports_away: "); - out.append(YESNO(this->legacy_supports_away)); - out.append("\n"); - out.append(" supports_action: "); out.append(YESNO(this->supports_action)); out.append("\n"); @@ -2223,10 +2161,6 @@ void ClimateStateResponse::dump_to(std::string &out) const { out.append(buffer); out.append("\n"); - out.append(" unused_legacy_away: "); - out.append(YESNO(this->unused_legacy_away)); - out.append("\n"); - out.append(" action: "); out.append(proto_enum_to_string(this->action)); out.append("\n"); @@ -2313,14 +2247,6 @@ void ClimateCommandRequest::dump_to(std::string &out) const { out.append(buffer); out.append("\n"); - out.append(" unused_has_legacy_away: "); - out.append(YESNO(this->unused_has_legacy_away)); - out.append("\n"); - - out.append(" unused_legacy_away: "); - out.append(YESNO(this->unused_legacy_away)); - out.append("\n"); - out.append(" has_fan_mode: "); out.append(YESNO(this->has_fan_mode)); out.append("\n"); @@ -3060,13 +2986,6 @@ void BluetoothServiceData::dump_to(std::string &out) const { out.append("'").append(this->uuid).append("'"); out.append("\n"); - for (const auto &it : this->legacy_data) { - out.append(" legacy_data: "); - snprintf(buffer, sizeof(buffer), "%" PRIu32, it); - out.append(buffer); - out.append("\n"); - } - out.append(" data: "); out.append(format_hex_pretty(this->data)); out.append("\n"); diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 4df7692167..eddaa1b9b2 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -1145,6 +1145,10 @@ def calculate_message_estimated_size(desc: descriptor.DescriptorProto) -> int: total_size = 0 for field in desc.field: + # Skip deprecated fields + if field.options.deprecated: + continue + ti = create_field_type_info(field) # Add estimated size for this field @@ -1213,6 +1217,10 @@ def build_message_type( public_content.append("#endif") for field in desc.field: + # Skip deprecated fields completely + if field.options.deprecated: + continue + ti = create_field_type_info(field) # Skip field declarations for fields that are in the base class @@ -1459,8 +1467,10 @@ def find_common_fields( if not messages: return [] - # Start with fields from the first message - first_msg_fields = {field.name: field for field in messages[0].field} + # Start with fields from the first message (excluding deprecated fields) + first_msg_fields = { + field.name: field for field in messages[0].field if not field.options.deprecated + } common_fields = [] # Check each field to see if it exists in all messages with same type @@ -1471,6 +1481,9 @@ def find_common_fields( for msg in messages[1:]: found = False for other_field in msg.field: + # Skip deprecated fields + if other_field.options.deprecated: + continue if ( other_field.name == field_name and other_field.type == field.type From 8a2599b7c26963ab1012474e417463c3ebbfac2e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 08:36:15 -1000 Subject: [PATCH 29/38] preen --- esphome/components/api/api_connection.cpp | 29 ++--------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 2ac3303691..07c2b27c80 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -362,8 +362,6 @@ uint16_t APIConnection::try_send_cover_state(EntityBase *entity, APIConnection * auto *cover = static_cast(entity); CoverStateResponse msg; auto traits = cover->get_traits(); - msg.legacy_state = - (cover->position == cover::COVER_OPEN) ? enums::LEGACY_COVER_STATE_OPEN : enums::LEGACY_COVER_STATE_CLOSED; msg.position = cover->position; if (traits.get_supports_tilt()) msg.tilt = cover->tilt; @@ -385,19 +383,6 @@ uint16_t APIConnection::try_send_cover_info(EntityBase *entity, APIConnection *c } void APIConnection::cover_command(const CoverCommandRequest &msg) { ENTITY_COMMAND_MAKE_CALL(cover::Cover, cover, cover) - if (msg.has_legacy_command) { - switch (msg.legacy_command) { - case enums::LEGACY_COVER_COMMAND_OPEN: - call.set_command_open(); - break; - case enums::LEGACY_COVER_COMMAND_CLOSE: - call.set_command_close(); - break; - case enums::LEGACY_COVER_COMMAND_STOP: - call.set_command_stop(); - break; - } - } if (msg.has_position) call.set_position(msg.position); if (msg.has_tilt) @@ -495,17 +480,8 @@ uint16_t APIConnection::try_send_light_info(EntityBase *entity, APIConnection *c auto traits = light->get_traits(); for (auto mode : traits.get_supported_color_modes()) msg.supported_color_modes.push_back(static_cast(mode)); - msg.legacy_supports_brightness = traits.supports_color_capability(light::ColorCapability::BRIGHTNESS); - msg.legacy_supports_rgb = traits.supports_color_capability(light::ColorCapability::RGB); - msg.legacy_supports_white_value = - msg.legacy_supports_rgb && (traits.supports_color_capability(light::ColorCapability::WHITE) || - traits.supports_color_capability(light::ColorCapability::COLD_WARM_WHITE)); - msg.legacy_supports_color_temperature = traits.supports_color_capability(light::ColorCapability::COLOR_TEMPERATURE) || - traits.supports_color_capability(light::ColorCapability::COLD_WARM_WHITE); - if (msg.legacy_supports_color_temperature) { - msg.min_mireds = traits.get_min_mireds(); - msg.max_mireds = traits.get_max_mireds(); - } + msg.min_mireds = traits.get_min_mireds(); + msg.max_mireds = traits.get_max_mireds(); if (light->supports_effects()) { msg.effects.emplace_back("None"); for (auto *effect : light->get_effects()) { @@ -692,7 +668,6 @@ uint16_t APIConnection::try_send_climate_info(EntityBase *entity, APIConnection msg.visual_current_temperature_step = traits.get_visual_current_temperature_step(); msg.visual_min_humidity = traits.get_visual_min_humidity(); msg.visual_max_humidity = traits.get_visual_max_humidity(); - msg.legacy_supports_away = traits.supports_preset(climate::CLIMATE_PRESET_AWAY); msg.supports_action = traits.get_supports_action(); for (auto fan_mode : traits.get_supported_fan_modes()) msg.supported_fan_modes.push_back(static_cast(fan_mode)); From 19ab40e5c24b31add3544356591db0613c4ed271 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 08:38:22 -1000 Subject: [PATCH 30/38] preen --- esphome/components/api/api.proto | 1 + esphome/components/api/api_connection.cpp | 9 +++++---- script/api_protobuf/api_protobuf.py | 4 ++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 2e8c863b87..6c0ce045d8 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -341,6 +341,7 @@ message ListEntitiesCoverResponse { // Deprecated in API version 1.1 enum LegacyCoverState { + option deprecated = true; LEGACY_COVER_STATE_OPEN = 0; LEGACY_COVER_STATE_CLOSED = 1; } diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 07c2b27c80..109ed7229d 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -480,8 +480,11 @@ uint16_t APIConnection::try_send_light_info(EntityBase *entity, APIConnection *c auto traits = light->get_traits(); for (auto mode : traits.get_supported_color_modes()) msg.supported_color_modes.push_back(static_cast(mode)); - msg.min_mireds = traits.get_min_mireds(); - msg.max_mireds = traits.get_max_mireds(); + if (traits.supports_color_capability(light::ColorCapability::COLOR_TEMPERATURE) || + traits.supports_color_capability(light::ColorCapability::COLD_WARM_WHITE)) { + msg.min_mireds = traits.get_min_mireds(); + msg.max_mireds = traits.get_max_mireds(); + } if (light->supports_effects()) { msg.effects.emplace_back("None"); for (auto *effect : light->get_effects()) { @@ -1474,12 +1477,10 @@ DeviceInfoResponse APIConnection::device_info(const DeviceInfoRequest &msg) { resp.webserver_port = USE_WEBSERVER_PORT; #endif #ifdef USE_BLUETOOTH_PROXY - resp.legacy_bluetooth_proxy_version = bluetooth_proxy::global_bluetooth_proxy->get_legacy_version(); resp.bluetooth_proxy_feature_flags = bluetooth_proxy::global_bluetooth_proxy->get_feature_flags(); resp.bluetooth_mac_address = bluetooth_proxy::global_bluetooth_proxy->get_bluetooth_mac_address_pretty(); #endif #ifdef USE_VOICE_ASSISTANT - resp.legacy_voice_assistant_version = voice_assistant::global_voice_assistant->get_legacy_version(); resp.voice_assistant_feature_flags = voice_assistant::global_voice_assistant->get_feature_flags(); #endif #ifdef USE_API_NOISE diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index eddaa1b9b2..8104c747ef 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -1692,6 +1692,10 @@ namespace api { current_ifdef = None for enum in file.enum_type: + # Skip deprecated enums + if enum.options.deprecated: + continue + s, c, dc = build_enum_type(enum, enum_ifdef_map) enum_ifdef = enum_ifdef_map.get(enum.name) From dc7b39722d6e66bdd1e278d8697cb6472b709596 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 08:39:31 -1000 Subject: [PATCH 31/38] preen --- esphome/components/api/api.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 6c0ce045d8..dafb42193b 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -371,6 +371,7 @@ message CoverStateResponse { // Deprecated in API version 1.1 enum LegacyCoverCommand { + option deprecated = true; LEGACY_COVER_COMMAND_OPEN = 0; LEGACY_COVER_COMMAND_CLOSE = 1; LEGACY_COVER_COMMAND_STOP = 2; From db59f3ae8892bbfce0e6bad49170eb721aada2a1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 08:42:18 -1000 Subject: [PATCH 32/38] preen --- esphome/components/api/api_pb2.h | 28 ---------- esphome/components/api/api_pb2_dump.cpp | 69 ------------------------- script/api_protobuf/api_protobuf.py | 24 +++++++-- 3 files changed, 19 insertions(+), 102 deletions(-) diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index a9fe3cb538..570e7fab17 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -17,27 +17,13 @@ enum EntityCategory : uint32_t { ENTITY_CATEGORY_DIAGNOSTIC = 2, }; #ifdef USE_COVER -enum LegacyCoverState : uint32_t { - LEGACY_COVER_STATE_OPEN = 0, - LEGACY_COVER_STATE_CLOSED = 1, -}; enum CoverOperation : uint32_t { COVER_OPERATION_IDLE = 0, COVER_OPERATION_IS_OPENING = 1, COVER_OPERATION_IS_CLOSING = 2, }; -enum LegacyCoverCommand : uint32_t { - LEGACY_COVER_COMMAND_OPEN = 0, - LEGACY_COVER_COMMAND_CLOSE = 1, - LEGACY_COVER_COMMAND_STOP = 2, -}; #endif #ifdef USE_FAN -enum FanSpeed : uint32_t { - FAN_SPEED_LOW = 0, - FAN_SPEED_MEDIUM = 1, - FAN_SPEED_HIGH = 2, -}; enum FanDirection : uint32_t { FAN_DIRECTION_FORWARD = 0, FAN_DIRECTION_REVERSE = 1, @@ -65,11 +51,6 @@ enum SensorStateClass : uint32_t { STATE_CLASS_TOTAL_INCREASING = 2, STATE_CLASS_TOTAL = 3, }; -enum SensorLastResetType : uint32_t { - LAST_RESET_NONE = 0, - LAST_RESET_NEVER = 1, - LAST_RESET_AUTO = 2, -}; #endif enum LogLevel : uint32_t { LOG_LEVEL_NONE = 0, @@ -204,15 +185,6 @@ enum BluetoothScannerMode : uint32_t { BLUETOOTH_SCANNER_MODE_ACTIVE = 1, }; #endif -enum VoiceAssistantSubscribeFlag : uint32_t { - VOICE_ASSISTANT_SUBSCRIBE_NONE = 0, - VOICE_ASSISTANT_SUBSCRIBE_API_AUDIO = 1, -}; -enum VoiceAssistantRequestFlag : uint32_t { - VOICE_ASSISTANT_REQUEST_NONE = 0, - VOICE_ASSISTANT_REQUEST_USE_VAD = 1, - VOICE_ASSISTANT_REQUEST_USE_WAKE_WORD = 2, -}; #ifdef USE_VOICE_ASSISTANT enum VoiceAssistantEvent : uint32_t { VOICE_ASSISTANT_ERROR = 0, diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 35d5e2e91c..09b3d3ae8c 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -23,16 +23,6 @@ template<> const char *proto_enum_to_string(enums::Entity } } #ifdef USE_COVER -template<> const char *proto_enum_to_string(enums::LegacyCoverState value) { - switch (value) { - case enums::LEGACY_COVER_STATE_OPEN: - return "LEGACY_COVER_STATE_OPEN"; - case enums::LEGACY_COVER_STATE_CLOSED: - return "LEGACY_COVER_STATE_CLOSED"; - default: - return "UNKNOWN"; - } -} template<> const char *proto_enum_to_string(enums::CoverOperation value) { switch (value) { case enums::COVER_OPERATION_IDLE: @@ -45,32 +35,8 @@ template<> const char *proto_enum_to_string(enums::CoverO return "UNKNOWN"; } } -template<> const char *proto_enum_to_string(enums::LegacyCoverCommand value) { - switch (value) { - case enums::LEGACY_COVER_COMMAND_OPEN: - return "LEGACY_COVER_COMMAND_OPEN"; - case enums::LEGACY_COVER_COMMAND_CLOSE: - return "LEGACY_COVER_COMMAND_CLOSE"; - case enums::LEGACY_COVER_COMMAND_STOP: - return "LEGACY_COVER_COMMAND_STOP"; - default: - return "UNKNOWN"; - } -} #endif #ifdef USE_FAN -template<> const char *proto_enum_to_string(enums::FanSpeed value) { - switch (value) { - case enums::FAN_SPEED_LOW: - return "FAN_SPEED_LOW"; - case enums::FAN_SPEED_MEDIUM: - return "FAN_SPEED_MEDIUM"; - case enums::FAN_SPEED_HIGH: - return "FAN_SPEED_HIGH"; - default: - return "UNKNOWN"; - } -} template<> const char *proto_enum_to_string(enums::FanDirection value) { switch (value) { case enums::FAN_DIRECTION_FORWARD: @@ -127,18 +93,6 @@ template<> const char *proto_enum_to_string(enums::Sens return "UNKNOWN"; } } -template<> const char *proto_enum_to_string(enums::SensorLastResetType value) { - switch (value) { - case enums::LAST_RESET_NONE: - return "LAST_RESET_NONE"; - case enums::LAST_RESET_NEVER: - return "LAST_RESET_NEVER"; - case enums::LAST_RESET_AUTO: - return "LAST_RESET_AUTO"; - default: - return "UNKNOWN"; - } -} #endif template<> const char *proto_enum_to_string(enums::LogLevel value) { switch (value) { @@ -427,29 +381,6 @@ template<> const char *proto_enum_to_string(enums:: } } #endif -template<> -const char *proto_enum_to_string(enums::VoiceAssistantSubscribeFlag value) { - switch (value) { - case enums::VOICE_ASSISTANT_SUBSCRIBE_NONE: - return "VOICE_ASSISTANT_SUBSCRIBE_NONE"; - case enums::VOICE_ASSISTANT_SUBSCRIBE_API_AUDIO: - return "VOICE_ASSISTANT_SUBSCRIBE_API_AUDIO"; - default: - return "UNKNOWN"; - } -} -template<> const char *proto_enum_to_string(enums::VoiceAssistantRequestFlag value) { - switch (value) { - case enums::VOICE_ASSISTANT_REQUEST_NONE: - return "VOICE_ASSISTANT_REQUEST_NONE"; - case enums::VOICE_ASSISTANT_REQUEST_USE_VAD: - return "VOICE_ASSISTANT_REQUEST_USE_VAD"; - case enums::VOICE_ASSISTANT_REQUEST_USE_WAKE_WORD: - return "VOICE_ASSISTANT_REQUEST_USE_WAKE_WORD"; - default: - return "UNKNOWN"; - } -} #ifdef USE_VOICE_ASSISTANT template<> const char *proto_enum_to_string(enums::VoiceAssistantEvent value) { switch (value) { diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 8104c747ef..dca92279b5 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -971,11 +971,11 @@ class RepeatedTypeInfo(TypeInfo): def build_type_usage_map( file_desc: descriptor.FileDescriptorProto, -) -> tuple[dict[str, str | None], dict[str, str | None], dict[str, int]]: +) -> tuple[dict[str, str | None], dict[str, str | None], dict[str, int], set[str]]: """Build mappings for both enums and messages to their ifdefs based on usage. Returns: - tuple: (enum_ifdef_map, message_ifdef_map, message_source_map) + tuple: (enum_ifdef_map, message_ifdef_map, message_source_map, used_enums) """ enum_ifdef_map: dict[str, str | None] = {} message_ifdef_map: dict[str, str | None] = {} @@ -988,6 +988,9 @@ def build_type_usage_map( message_usage: dict[ str, set[str] ] = {} # message_name -> set of message names that use it + used_enums: set[str] = ( + set() + ) # Track which enums are actually used by non-deprecated fields # Build message name to ifdef mapping for quick lookup message_to_ifdef: dict[str, str | None] = { @@ -997,13 +1000,18 @@ def build_type_usage_map( # Analyze field usage for message in file_desc.message_type: for field in message.field: + # Skip deprecated fields when tracking enum usage + if field.options.deprecated: + continue + type_name = field.type_name.split(".")[-1] if field.type_name else None if not type_name: continue - # Track enum usage + # Track enum usage (only from non-deprecated fields) if field.type == 14: # TYPE_ENUM enum_usage.setdefault(type_name, set()).add(message.name) + used_enums.add(type_name) # Track message usage elif field.type == 11: # TYPE_MESSAGE message_usage.setdefault(type_name, set()).add(message.name) @@ -1103,7 +1111,7 @@ def build_type_usage_map( # Not used by any message and no explicit source - default to encode-only message_source_map[msg.name] = SOURCE_SERVER - return enum_ifdef_map, message_ifdef_map, message_source_map + return enum_ifdef_map, message_ifdef_map, message_source_map, used_enums def build_enum_type(desc, enum_ifdef_map) -> tuple[str, str, str]: @@ -1686,7 +1694,9 @@ namespace api { content += "namespace enums {\n\n" # Build dynamic ifdef mappings for both enums and messages - enum_ifdef_map, message_ifdef_map, message_source_map = build_type_usage_map(file) + enum_ifdef_map, message_ifdef_map, message_source_map, used_enums = ( + build_type_usage_map(file) + ) # Simple grouping of enums by ifdef current_ifdef = None @@ -1696,6 +1706,10 @@ namespace api { if enum.options.deprecated: continue + # Skip enums that aren't used by any non-deprecated fields + if enum.name not in used_enums: + continue + s, c, dc = build_enum_type(enum, enum_ifdef_map) enum_ifdef = enum_ifdef_map.get(enum.name) From 7566d859414fb9df5480f3e80e5353a239240331 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 08:47:27 -1000 Subject: [PATCH 33/38] preen --- esphome/components/api/api_connection.cpp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 109ed7229d..84d51210f3 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1092,18 +1092,6 @@ void APIConnection::unsubscribe_bluetooth_le_advertisements(const UnsubscribeBlu bluetooth_proxy::global_bluetooth_proxy->unsubscribe_api_connection(this); } bool APIConnection::send_bluetooth_le_advertisement(const BluetoothLEAdvertisementResponse &msg) { - if (this->client_api_version_major_ < 1 || this->client_api_version_minor_ < 7) { - BluetoothLEAdvertisementResponse resp = msg; - for (auto &service : resp.service_data) { - service.legacy_data.assign(service.data.begin(), service.data.end()); - service.data.clear(); - } - for (auto &manufacturer_data : resp.manufacturer_data) { - manufacturer_data.legacy_data.assign(manufacturer_data.data.begin(), manufacturer_data.data.end()); - manufacturer_data.data.clear(); - } - return this->send_message(resp, BluetoothLEAdvertisementResponse::MESSAGE_TYPE); - } return this->send_message(msg, BluetoothLEAdvertisementResponse::MESSAGE_TYPE); } void APIConnection::bluetooth_device_request(const BluetoothDeviceRequest &msg) { From 1aab2f5a7f878630c31d8da940618fccfc7378c6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 08:57:30 -1000 Subject: [PATCH 34/38] missed one --- esphome/components/api/api.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index dafb42193b..b1ad674d39 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1382,7 +1382,9 @@ message BluetoothServiceData { repeated uint32 legacy_data = 2 [deprecated=true]; // Removed in api version 1.7 bytes data = 3; // Added in api version 1.7 } +// Removed in ESPHome 2025.8.0 - use BluetoothLERawAdvertisementsResponse instead message BluetoothLEAdvertisementResponse { + option deprecated = true; option (id) = 67; option (source) = SOURCE_SERVER; option (ifdef) = "USE_BLUETOOTH_PROXY"; From 7f5eefed109d5196d3d59b87f551de1f13fb0994 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 09:01:18 -1000 Subject: [PATCH 35/38] remove dead code --- esphome/components/api/api_connection.cpp | 3 -- esphome/components/api/api_connection.h | 1 - esphome/components/api/api_pb2.cpp | 30 +------------ esphome/components/api/api_pb2.h | 24 +---------- esphome/components/api/api_pb2_dump.cpp | 43 +------------------ .../bluetooth_proxy/bluetooth_proxy.cpp | 40 ----------------- .../bluetooth_proxy/bluetooth_proxy.h | 3 -- script/api_protobuf/api_protobuf.py | 12 ++++++ 8 files changed, 18 insertions(+), 138 deletions(-) diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 84d51210f3..ef0b95c30e 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1091,9 +1091,6 @@ void APIConnection::subscribe_bluetooth_le_advertisements(const SubscribeBluetoo void APIConnection::unsubscribe_bluetooth_le_advertisements(const UnsubscribeBluetoothLEAdvertisementsRequest &msg) { bluetooth_proxy::global_bluetooth_proxy->unsubscribe_api_connection(this); } -bool APIConnection::send_bluetooth_le_advertisement(const BluetoothLEAdvertisementResponse &msg) { - return this->send_message(msg, BluetoothLEAdvertisementResponse::MESSAGE_TYPE); -} void APIConnection::bluetooth_device_request(const BluetoothDeviceRequest &msg) { bluetooth_proxy::global_bluetooth_proxy->bluetooth_device_request(msg); } diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index 3873c7fcac..3df25840cd 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -116,7 +116,6 @@ class APIConnection : public APIServerConnection { #ifdef USE_BLUETOOTH_PROXY void subscribe_bluetooth_le_advertisements(const SubscribeBluetoothLEAdvertisementsRequest &msg) override; void unsubscribe_bluetooth_le_advertisements(const UnsubscribeBluetoothLEAdvertisementsRequest &msg) override; - bool send_bluetooth_le_advertisement(const BluetoothLEAdvertisementResponse &msg); void bluetooth_device_request(const BluetoothDeviceRequest &msg) override; void bluetooth_gatt_read(const BluetoothGATTReadRequest &msg) override; diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 44e3a3205b..348ca382d5 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -1821,6 +1821,7 @@ bool SubscribeBluetoothLEAdvertisementsRequest::decode_varint(uint32_t field_id, } return true; } +#endif void BluetoothServiceData::encode(ProtoWriteBuffer buffer) const { buffer.encode_string(1, this->uuid); buffer.encode_bytes(3, reinterpret_cast(this->data.data()), this->data.size()); @@ -1829,34 +1830,7 @@ void BluetoothServiceData::calculate_size(uint32_t &total_size) const { ProtoSize::add_string_field(total_size, 1, this->uuid); ProtoSize::add_string_field(total_size, 1, this->data); } -void BluetoothLEAdvertisementResponse::encode(ProtoWriteBuffer buffer) const { - buffer.encode_uint64(1, this->address); - buffer.encode_bytes(2, reinterpret_cast(this->name.data()), this->name.size()); - buffer.encode_sint32(3, this->rssi); - for (auto &it : this->service_uuids) { - buffer.encode_string(4, it, true); - } - for (auto &it : this->service_data) { - buffer.encode_message(5, it, true); - } - for (auto &it : this->manufacturer_data) { - buffer.encode_message(6, it, true); - } - buffer.encode_uint32(7, this->address_type); -} -void BluetoothLEAdvertisementResponse::calculate_size(uint32_t &total_size) const { - ProtoSize::add_uint64_field(total_size, 1, this->address); - ProtoSize::add_string_field(total_size, 1, this->name); - ProtoSize::add_sint32_field(total_size, 1, this->rssi); - if (!this->service_uuids.empty()) { - for (const auto &it : this->service_uuids) { - ProtoSize::add_string_field_repeated(total_size, 1, it); - } - } - ProtoSize::add_repeated_message(total_size, 1, this->service_data); - ProtoSize::add_repeated_message(total_size, 1, this->manufacturer_data); - ProtoSize::add_uint32_field(total_size, 1, this->address_type); -} +#ifdef USE_BLUETOOTH_PROXY void BluetoothLERawAdvertisement::encode(ProtoWriteBuffer buffer) const { buffer.encode_uint64(1, this->address); buffer.encode_sint32(2, this->rssi); diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 570e7fab17..a378a1f57c 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1679,6 +1679,7 @@ class SubscribeBluetoothLEAdvertisementsRequest : public ProtoDecodableMessage { protected: bool decode_varint(uint32_t field_id, ProtoVarInt value) override; }; +#endif class BluetoothServiceData : public ProtoMessage { public: std::string uuid{}; @@ -1691,28 +1692,7 @@ class BluetoothServiceData : public ProtoMessage { protected: }; -class BluetoothLEAdvertisementResponse : public ProtoMessage { - public: - static constexpr uint8_t MESSAGE_TYPE = 67; - static constexpr uint8_t ESTIMATED_SIZE = 107; -#ifdef HAS_PROTO_MESSAGE_DUMP - const char *message_name() const override { return "bluetooth_le_advertisement_response"; } -#endif - uint64_t address{0}; - std::string name{}; - int32_t rssi{0}; - std::vector service_uuids{}; - std::vector service_data{}; - std::vector manufacturer_data{}; - uint32_t address_type{0}; - void encode(ProtoWriteBuffer buffer) const override; - void calculate_size(uint32_t &total_size) const override; -#ifdef HAS_PROTO_MESSAGE_DUMP - void dump_to(std::string &out) const override; -#endif - - protected: -}; +#ifdef USE_BLUETOOTH_PROXY class BluetoothLERawAdvertisement : public ProtoMessage { public: uint64_t address{0}; diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 09b3d3ae8c..246d9b3114 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -2910,6 +2910,7 @@ void SubscribeBluetoothLEAdvertisementsRequest::dump_to(std::string &out) const out.append("\n"); out.append("}"); } +#endif void BluetoothServiceData::dump_to(std::string &out) const { __attribute__((unused)) char buffer[64]; out.append("BluetoothServiceData {\n"); @@ -2922,47 +2923,7 @@ void BluetoothServiceData::dump_to(std::string &out) const { out.append("\n"); out.append("}"); } -void BluetoothLEAdvertisementResponse::dump_to(std::string &out) const { - __attribute__((unused)) char buffer[64]; - out.append("BluetoothLEAdvertisementResponse {\n"); - out.append(" address: "); - snprintf(buffer, sizeof(buffer), "%llu", this->address); - out.append(buffer); - out.append("\n"); - - out.append(" name: "); - out.append(format_hex_pretty(this->name)); - out.append("\n"); - - out.append(" rssi: "); - snprintf(buffer, sizeof(buffer), "%" PRId32, this->rssi); - out.append(buffer); - out.append("\n"); - - for (const auto &it : this->service_uuids) { - out.append(" service_uuids: "); - out.append("'").append(it).append("'"); - out.append("\n"); - } - - for (const auto &it : this->service_data) { - out.append(" service_data: "); - it.dump_to(out); - out.append("\n"); - } - - for (const auto &it : this->manufacturer_data) { - out.append(" manufacturer_data: "); - it.dump_to(out); - out.append("\n"); - } - - out.append(" address_type: "); - snprintf(buffer, sizeof(buffer), "%" PRIu32, this->address_type); - out.append(buffer); - out.append("\n"); - out.append("}"); -} +#ifdef USE_BLUETOOTH_PROXY void BluetoothLERawAdvertisement::dump_to(std::string &out) const { __attribute__((unused)) char buffer[64]; out.append("BluetoothLERawAdvertisement {\n"); diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index 7d12842a24..569b3f6565 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -140,46 +140,6 @@ void BluetoothProxy::flush_pending_advertisements() { this->advertisement_count_ = 0; } -#ifdef USE_ESP32_BLE_DEVICE -void BluetoothProxy::send_api_packet_(const esp32_ble_tracker::ESPBTDevice &device) { - api::BluetoothLEAdvertisementResponse resp; - resp.address = device.address_uint64(); - resp.address_type = device.get_address_type(); - if (!device.get_name().empty()) - resp.name = device.get_name(); - resp.rssi = device.get_rssi(); - - // Pre-allocate vectors based on known sizes - auto service_uuids = device.get_service_uuids(); - resp.service_uuids.reserve(service_uuids.size()); - for (auto &uuid : service_uuids) { - resp.service_uuids.emplace_back(uuid.to_string()); - } - - // Pre-allocate service data vector - auto service_datas = device.get_service_datas(); - resp.service_data.reserve(service_datas.size()); - for (auto &data : service_datas) { - resp.service_data.emplace_back(); - auto &service_data = resp.service_data.back(); - service_data.uuid = data.uuid.to_string(); - service_data.data.assign(data.data.begin(), data.data.end()); - } - - // Pre-allocate manufacturer data vector - auto manufacturer_datas = device.get_manufacturer_datas(); - resp.manufacturer_data.reserve(manufacturer_datas.size()); - for (auto &data : manufacturer_datas) { - resp.manufacturer_data.emplace_back(); - auto &manufacturer_data = resp.manufacturer_data.back(); - manufacturer_data.uuid = data.uuid.to_string(); - manufacturer_data.data.assign(data.data.begin(), data.data.end()); - } - - this->api_connection_->send_message(resp, api::BluetoothLEAdvertisementResponse::MESSAGE_TYPE); -} -#endif // USE_ESP32_BLE_DEVICE - void BluetoothProxy::dump_config() { ESP_LOGCONFIG(TAG, "Bluetooth Proxy:"); ESP_LOGCONFIG(TAG, diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.h b/esphome/components/bluetooth_proxy/bluetooth_proxy.h index 52f1d0f88a..d43e167ed3 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.h +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.h @@ -131,9 +131,6 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com } protected: -#ifdef USE_ESP32_BLE_DEVICE - void send_api_packet_(const esp32_ble_tracker::ESPBTDevice &device); -#endif void send_bluetooth_scanner_state_(esp32_ble_tracker::ScannerState state); BluetoothConnection *get_connection_(uint64_t address, bool reserve); diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index dca92279b5..8a57d453fd 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -999,6 +999,10 @@ def build_type_usage_map( # Analyze field usage for message in file_desc.message_type: + # Skip deprecated messages entirely + if message.options.deprecated: + continue + for field in message.field: # Skip deprecated fields when tracking enum usage if field.options.deprecated: @@ -1593,6 +1597,10 @@ def build_service_message_type( message_source_map: dict[str, int], ) -> tuple[str, str] | None: """Builds the service message type.""" + # Skip deprecated messages + if mt.options.deprecated: + return None + snake = camel_to_snake(mt.name) id_: int | None = get_opt(mt, pb.id) if id_ is None: @@ -1760,6 +1768,10 @@ namespace api { current_ifdef = None for m in mt: + # Skip deprecated messages + if m.options.deprecated: + continue + s, c, dc = build_message_type(m, base_class_fields, message_source_map) msg_ifdef = message_ifdef_map.get(m.name) From cde4fc0609007c6694fd1d89152655a856eebd9d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 09:07:13 -1000 Subject: [PATCH 36/38] missed some more --- esphome/components/api/api_pb2.cpp | 10 --------- esphome/components/api/api_pb2.h | 14 ------------- esphome/components/api/api_pb2_dump.cpp | 14 ------------- script/api_protobuf/api_protobuf.py | 28 +++++++++++++++++++++---- 4 files changed, 24 insertions(+), 42 deletions(-) diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 348ca382d5..c32a15760a 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -1821,16 +1821,6 @@ bool SubscribeBluetoothLEAdvertisementsRequest::decode_varint(uint32_t field_id, } return true; } -#endif -void BluetoothServiceData::encode(ProtoWriteBuffer buffer) const { - buffer.encode_string(1, this->uuid); - buffer.encode_bytes(3, reinterpret_cast(this->data.data()), this->data.size()); -} -void BluetoothServiceData::calculate_size(uint32_t &total_size) const { - ProtoSize::add_string_field(total_size, 1, this->uuid); - ProtoSize::add_string_field(total_size, 1, this->data); -} -#ifdef USE_BLUETOOTH_PROXY void BluetoothLERawAdvertisement::encode(ProtoWriteBuffer buffer) const { buffer.encode_uint64(1, this->address); buffer.encode_sint32(2, this->rssi); diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index a378a1f57c..9788545e33 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1679,20 +1679,6 @@ class SubscribeBluetoothLEAdvertisementsRequest : public ProtoDecodableMessage { protected: bool decode_varint(uint32_t field_id, ProtoVarInt value) override; }; -#endif -class BluetoothServiceData : public ProtoMessage { - public: - std::string uuid{}; - std::string data{}; - void encode(ProtoWriteBuffer buffer) const override; - void calculate_size(uint32_t &total_size) const override; -#ifdef HAS_PROTO_MESSAGE_DUMP - void dump_to(std::string &out) const override; -#endif - - protected: -}; -#ifdef USE_BLUETOOTH_PROXY class BluetoothLERawAdvertisement : public ProtoMessage { public: uint64_t address{0}; diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 246d9b3114..9e4a7e91fa 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -2910,20 +2910,6 @@ void SubscribeBluetoothLEAdvertisementsRequest::dump_to(std::string &out) const out.append("\n"); out.append("}"); } -#endif -void BluetoothServiceData::dump_to(std::string &out) const { - __attribute__((unused)) char buffer[64]; - out.append("BluetoothServiceData {\n"); - out.append(" uuid: "); - out.append("'").append(this->uuid).append("'"); - out.append("\n"); - - out.append(" data: "); - out.append(format_hex_pretty(this->data)); - out.append("\n"); - out.append("}"); -} -#ifdef USE_BLUETOOTH_PROXY void BluetoothLERawAdvertisement::dump_to(std::string &out) const { __attribute__((unused)) char buffer[64]; out.append("BluetoothLERawAdvertisement {\n"); diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 8a57d453fd..8c516adcbe 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -971,11 +971,13 @@ class RepeatedTypeInfo(TypeInfo): def build_type_usage_map( file_desc: descriptor.FileDescriptorProto, -) -> tuple[dict[str, str | None], dict[str, str | None], dict[str, int], set[str]]: +) -> tuple[ + dict[str, str | None], dict[str, str | None], dict[str, int], set[str], set[str] +]: """Build mappings for both enums and messages to their ifdefs based on usage. Returns: - tuple: (enum_ifdef_map, message_ifdef_map, message_source_map, used_enums) + tuple: (enum_ifdef_map, message_ifdef_map, message_source_map, used_enums, used_messages) """ enum_ifdef_map: dict[str, str | None] = {} message_ifdef_map: dict[str, str | None] = {} @@ -991,6 +993,7 @@ def build_type_usage_map( used_enums: set[str] = ( set() ) # Track which enums are actually used by non-deprecated fields + used_messages: set[str] = set() # Track which messages are actually used # Build message name to ifdef mapping for quick lookup message_to_ifdef: dict[str, str | None] = { @@ -1019,6 +1022,7 @@ def build_type_usage_map( # Track message usage elif field.type == 11: # TYPE_MESSAGE message_usage.setdefault(type_name, set()).add(message.name) + used_messages.add(type_name) # Helper to get unique ifdef from a set of messages def get_unique_ifdef(message_names: set[str]) -> str | None: @@ -1081,12 +1085,18 @@ def build_type_usage_map( # Build message source map # First pass: Get explicit sources for messages with source option or id for msg in file_desc.message_type: + # Skip deprecated messages + if msg.options.deprecated: + continue + if msg.options.HasExtension(pb.source): # Explicit source option takes precedence message_source_map[msg.name] = get_opt(msg, pb.source, SOURCE_BOTH) elif msg.options.HasExtension(pb.id): # Service messages (with id) default to SOURCE_BOTH message_source_map[msg.name] = SOURCE_BOTH + # Service messages are always used + used_messages.add(msg.name) # Second pass: Determine sources for embedded messages based on their usage for msg in file_desc.message_type: @@ -1115,7 +1125,13 @@ def build_type_usage_map( # Not used by any message and no explicit source - default to encode-only message_source_map[msg.name] = SOURCE_SERVER - return enum_ifdef_map, message_ifdef_map, message_source_map, used_enums + return ( + enum_ifdef_map, + message_ifdef_map, + message_source_map, + used_enums, + used_messages, + ) def build_enum_type(desc, enum_ifdef_map) -> tuple[str, str, str]: @@ -1702,7 +1718,7 @@ namespace api { content += "namespace enums {\n\n" # Build dynamic ifdef mappings for both enums and messages - enum_ifdef_map, message_ifdef_map, message_source_map, used_enums = ( + enum_ifdef_map, message_ifdef_map, message_source_map, used_enums, used_messages = ( build_type_usage_map(file) ) @@ -1772,6 +1788,10 @@ namespace api { if m.options.deprecated: continue + # Skip messages that aren't used (unless they have an ID/service message) + if m.name not in used_messages and not m.options.HasExtension(pb.id): + continue + s, c, dc = build_message_type(m, base_class_fields, message_source_map) msg_ifdef = message_ifdef_map.get(m.name) From d6422b6d253f82651d9868c02b549de03ef63229 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 09:07:29 -1000 Subject: [PATCH 37/38] missed some more --- esphome/components/api/api.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index b1ad674d39..a77309a2a8 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1376,7 +1376,9 @@ message SubscribeBluetoothLEAdvertisementsRequest { uint32 flags = 1; } +// Deprecated - only used by deprecated BluetoothLEAdvertisementResponse message BluetoothServiceData { + option deprecated = true; string uuid = 1; // Deprecated in API version 1.7 repeated uint32 legacy_data = 2 [deprecated=true]; // Removed in api version 1.7 From cc1abfcdb364e849c95e225ffdbf7a20b9a4718c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 18 Jul 2025 09:24:24 -1000 Subject: [PATCH 38/38] fixed unref enum tracking --- esphome/components/api/api.proto | 3 +++ esphome/components/api/api_pb2.h | 9 +++++++++ esphome/components/api/api_pb2_dump.cpp | 23 +++++++++++++++++++++++ script/api_protobuf/api_protobuf.py | 17 +++-------------- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index a77309a2a8..546c498ff3 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -422,7 +422,9 @@ message ListEntitiesFanResponse { repeated string supported_preset_modes = 12; uint32 device_id = 13 [(field_ifdef) = "USE_DEVICES"]; } +// Deprecated in API version 1.6 - only used in deprecated fields enum FanSpeed { + option deprecated = true; FAN_SPEED_LOW = 0; FAN_SPEED_MEDIUM = 1; FAN_SPEED_HIGH = 2; @@ -585,6 +587,7 @@ enum SensorStateClass { // Deprecated in API version 1.5 enum SensorLastResetType { + option deprecated = true; LAST_RESET_NONE = 0; LAST_RESET_NEVER = 1; LAST_RESET_AUTO = 2; diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 9788545e33..1c143818e4 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -185,6 +185,15 @@ enum BluetoothScannerMode : uint32_t { BLUETOOTH_SCANNER_MODE_ACTIVE = 1, }; #endif +enum VoiceAssistantSubscribeFlag : uint32_t { + VOICE_ASSISTANT_SUBSCRIBE_NONE = 0, + VOICE_ASSISTANT_SUBSCRIBE_API_AUDIO = 1, +}; +enum VoiceAssistantRequestFlag : uint32_t { + VOICE_ASSISTANT_REQUEST_NONE = 0, + VOICE_ASSISTANT_REQUEST_USE_VAD = 1, + VOICE_ASSISTANT_REQUEST_USE_WAKE_WORD = 2, +}; #ifdef USE_VOICE_ASSISTANT enum VoiceAssistantEvent : uint32_t { VOICE_ASSISTANT_ERROR = 0, diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 9e4a7e91fa..b4da15da0d 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -381,6 +381,29 @@ template<> const char *proto_enum_to_string(enums:: } } #endif +template<> +const char *proto_enum_to_string(enums::VoiceAssistantSubscribeFlag value) { + switch (value) { + case enums::VOICE_ASSISTANT_SUBSCRIBE_NONE: + return "VOICE_ASSISTANT_SUBSCRIBE_NONE"; + case enums::VOICE_ASSISTANT_SUBSCRIBE_API_AUDIO: + return "VOICE_ASSISTANT_SUBSCRIBE_API_AUDIO"; + default: + return "UNKNOWN"; + } +} +template<> const char *proto_enum_to_string(enums::VoiceAssistantRequestFlag value) { + switch (value) { + case enums::VOICE_ASSISTANT_REQUEST_NONE: + return "VOICE_ASSISTANT_REQUEST_NONE"; + case enums::VOICE_ASSISTANT_REQUEST_USE_VAD: + return "VOICE_ASSISTANT_REQUEST_USE_VAD"; + case enums::VOICE_ASSISTANT_REQUEST_USE_WAKE_WORD: + return "VOICE_ASSISTANT_REQUEST_USE_WAKE_WORD"; + default: + return "UNKNOWN"; + } +} #ifdef USE_VOICE_ASSISTANT template<> const char *proto_enum_to_string(enums::VoiceAssistantEvent value) { switch (value) { diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 8c516adcbe..50b264410f 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -971,13 +971,11 @@ class RepeatedTypeInfo(TypeInfo): def build_type_usage_map( file_desc: descriptor.FileDescriptorProto, -) -> tuple[ - dict[str, str | None], dict[str, str | None], dict[str, int], set[str], set[str] -]: +) -> tuple[dict[str, str | None], dict[str, str | None], dict[str, int], set[str]]: """Build mappings for both enums and messages to their ifdefs based on usage. Returns: - tuple: (enum_ifdef_map, message_ifdef_map, message_source_map, used_enums, used_messages) + tuple: (enum_ifdef_map, message_ifdef_map, message_source_map, used_messages) """ enum_ifdef_map: dict[str, str | None] = {} message_ifdef_map: dict[str, str | None] = {} @@ -990,9 +988,6 @@ def build_type_usage_map( message_usage: dict[ str, set[str] ] = {} # message_name -> set of message names that use it - used_enums: set[str] = ( - set() - ) # Track which enums are actually used by non-deprecated fields used_messages: set[str] = set() # Track which messages are actually used # Build message name to ifdef mapping for quick lookup @@ -1018,7 +1013,6 @@ def build_type_usage_map( # Track enum usage (only from non-deprecated fields) if field.type == 14: # TYPE_ENUM enum_usage.setdefault(type_name, set()).add(message.name) - used_enums.add(type_name) # Track message usage elif field.type == 11: # TYPE_MESSAGE message_usage.setdefault(type_name, set()).add(message.name) @@ -1129,7 +1123,6 @@ def build_type_usage_map( enum_ifdef_map, message_ifdef_map, message_source_map, - used_enums, used_messages, ) @@ -1718,7 +1711,7 @@ namespace api { content += "namespace enums {\n\n" # Build dynamic ifdef mappings for both enums and messages - enum_ifdef_map, message_ifdef_map, message_source_map, used_enums, used_messages = ( + enum_ifdef_map, message_ifdef_map, message_source_map, used_messages = ( build_type_usage_map(file) ) @@ -1730,10 +1723,6 @@ namespace api { if enum.options.deprecated: continue - # Skip enums that aren't used by any non-deprecated fields - if enum.name not in used_enums: - continue - s, c, dc = build_enum_type(enum, enum_ifdef_map) enum_ifdef = enum_ifdef_map.get(enum.name)