From 00d9baed11fa3c8a7330f88c963cd7d87a624817 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 1 Aug 2025 20:26:00 -1000 Subject: [PATCH] [bluetooth_proxy] Eliminate heap allocations in connection state reporting (#10010) --- esphome/components/api/api.proto | 5 ++- esphome/components/api/api_connection.cpp | 6 +-- esphome/components/api/api_options.proto | 1 + esphome/components/api/api_pb2.cpp | 10 +++-- esphome/components/api/api_pb2.h | 4 +- .../components/bluetooth_proxy/__init__.py | 4 ++ .../bluetooth_proxy/bluetooth_connection.cpp | 24 +++++++++++ .../bluetooth_proxy/bluetooth_connection.h | 3 ++ .../bluetooth_proxy/bluetooth_proxy.cpp | 33 ++++---------- .../bluetooth_proxy/bluetooth_proxy.h | 8 ++-- .../esp32_ble_client/ble_client_base.h | 2 +- esphome/core/defines.h | 1 + script/api_protobuf/api_protobuf.py | 43 ++++++++++++++++++- 13 files changed, 104 insertions(+), 40 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 4aa5cc4be0..e0b2c19a21 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1621,7 +1621,10 @@ message BluetoothConnectionsFreeResponse { uint32 free = 1; uint32 limit = 2; - repeated uint64 allocated = 3; + repeated uint64 allocated = 3 [ + (fixed_array_size_define) = "BLUETOOTH_PROXY_MAX_CONNECTIONS", + (fixed_array_skip_zero) = true + ]; } message BluetoothGATTErrorResponse { diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 8ac6c3b71e..5fff270c99 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1105,10 +1105,8 @@ void APIConnection::bluetooth_gatt_notify(const BluetoothGATTNotifyRequest &msg) bool APIConnection::send_subscribe_bluetooth_connections_free_response( const SubscribeBluetoothConnectionsFreeRequest &msg) { - BluetoothConnectionsFreeResponse resp; - resp.free = bluetooth_proxy::global_bluetooth_proxy->get_bluetooth_connections_free(); - resp.limit = bluetooth_proxy::global_bluetooth_proxy->get_bluetooth_connections_limit(); - return this->send_message(resp, BluetoothConnectionsFreeResponse::MESSAGE_TYPE); + bluetooth_proxy::global_bluetooth_proxy->send_connections_free(this); + return true; } void APIConnection::bluetooth_scanner_set_mode(const BluetoothScannerSetModeRequest &msg) { diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index d4b5700024..ed0e0d7455 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -29,6 +29,7 @@ extend google.protobuf.FieldOptions { optional uint32 fixed_array_size = 50007; optional bool no_zero_copy = 50008 [default=false]; optional bool fixed_array_skip_zero = 50009 [default=false]; + optional string fixed_array_size_define = 50010; // container_pointer: Zero-copy optimization for repeated fields. // diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 29d0f2842c..8c14153155 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -2073,15 +2073,17 @@ void BluetoothGATTNotifyDataResponse::calculate_size(ProtoSize &size) const { void BluetoothConnectionsFreeResponse::encode(ProtoWriteBuffer buffer) const { buffer.encode_uint32(1, this->free); buffer.encode_uint32(2, this->limit); - for (auto &it : this->allocated) { - buffer.encode_uint64(3, it, true); + for (const auto &it : this->allocated) { + if (it != 0) { + buffer.encode_uint64(3, it, true); + } } } void BluetoothConnectionsFreeResponse::calculate_size(ProtoSize &size) const { size.add_uint32(1, this->free); size.add_uint32(1, this->limit); - if (!this->allocated.empty()) { - for (const auto &it : this->allocated) { + for (const auto &it : this->allocated) { + if (it != 0) { size.add_uint64_force(1, it); } } diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 524674e6ef..0bc75ef00b 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -2076,13 +2076,13 @@ class SubscribeBluetoothConnectionsFreeRequest : public ProtoMessage { class BluetoothConnectionsFreeResponse : public ProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 81; - static constexpr uint8_t ESTIMATED_SIZE = 16; + static constexpr uint8_t ESTIMATED_SIZE = 20; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "bluetooth_connections_free_response"; } #endif uint32_t free{0}; uint32_t limit{0}; - std::vector allocated{}; + std::array allocated{}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(ProtoSize &size) const override; #ifdef HAS_PROTO_MESSAGE_DUMP diff --git a/esphome/components/bluetooth_proxy/__init__.py b/esphome/components/bluetooth_proxy/__init__.py index a1e9d464df..ec1df6a06c 100644 --- a/esphome/components/bluetooth_proxy/__init__.py +++ b/esphome/components/bluetooth_proxy/__init__.py @@ -87,6 +87,10 @@ async def to_code(config): cg.add(var.set_active(config[CONF_ACTIVE])) await esp32_ble_tracker.register_raw_ble_device(var, config) + # Define max connections for protobuf fixed array + connection_count = len(config.get(CONF_CONNECTIONS, [])) + cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", connection_count) + for connection_conf in config.get(CONF_CONNECTIONS, []): connection_var = cg.new_Pvariable(connection_conf[CONF_ID]) await cg.register_component(connection_var, connection_conf) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index fd1324dcdc..01c2aa3d22 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -78,6 +78,30 @@ void BluetoothConnection::dump_config() { BLEClientBase::dump_config(); } +void BluetoothConnection::update_allocated_slot_(uint64_t find_value, uint64_t set_value) { + auto &allocated = this->proxy_->connections_free_response_.allocated; + auto *it = std::find(allocated.begin(), allocated.end(), find_value); + if (it != allocated.end()) { + *it = set_value; + } +} + +void BluetoothConnection::set_address(uint64_t address) { + // If we're clearing an address (disconnecting), update the pre-allocated message + if (address == 0 && this->address_ != 0) { + this->proxy_->connections_free_response_.free++; + this->update_allocated_slot_(this->address_, 0); + } + // If we're setting a new address (connecting), update the pre-allocated message + else if (address != 0 && this->address_ == 0) { + this->proxy_->connections_free_response_.free--; + this->update_allocated_slot_(0, address); + } + + // Call parent implementation to actually set the address + BLEClientBase::set_address(address); +} + void BluetoothConnection::loop() { BLEClientBase::loop(); diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.h b/esphome/components/bluetooth_proxy/bluetooth_connection.h index 622d257bf8..042868e7a4 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.h +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.h @@ -24,12 +24,15 @@ class BluetoothConnection : public esp32_ble_client::BLEClientBase { esp_err_t notify_characteristic(uint16_t handle, bool enable); + void set_address(uint64_t address) override; + protected: friend class BluetoothProxy; bool supports_efficient_uuids_() const; void send_service_for_discovery_(); void reset_connection_(esp_err_t reason); + void update_allocated_slot_(uint64_t find_value, uint64_t set_value); // Memory optimized layout for 32-bit systems // Group 1: Pointers (4 bytes each, naturally aligned) diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index de5508c777..a59a33117a 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -35,6 +35,9 @@ void BluetoothProxy::setup() { // Don't pre-allocate pool - let it grow only if needed in busy environments // Many devices in quiet areas will never need the overflow pool + this->connections_free_response_.limit = this->connections_.size(); + this->connections_free_response_.free = this->connections_.size(); + this->parent_->add_scanner_state_callback([this](esp32_ble_tracker::ScannerState state) { if (this->api_connection_ != nullptr) { this->send_bluetooth_scanner_state_(state); @@ -134,20 +137,6 @@ void BluetoothProxy::dump_config() { YESNO(this->active_), this->connections_.size()); } -int BluetoothProxy::get_bluetooth_connections_free() { - int free = 0; - for (auto *connection : this->connections_) { - if (connection->address_ == 0) { - free++; - ESP_LOGV(TAG, "[%d] Free connection", connection->get_connection_index()); - } else { - ESP_LOGV(TAG, "[%d] Used connection by [%s]", connection->get_connection_index(), - connection->address_str().c_str()); - } - } - return free; -} - void BluetoothProxy::loop() { if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr) { for (auto *connection : this->connections_) { @@ -439,17 +428,13 @@ void BluetoothProxy::send_device_connection(uint64_t address, bool connected, ui this->api_connection_->send_message(call, api::BluetoothDeviceConnectionResponse::MESSAGE_TYPE); } void BluetoothProxy::send_connections_free() { - if (this->api_connection_ == nullptr) - return; - api::BluetoothConnectionsFreeResponse call; - call.free = this->get_bluetooth_connections_free(); - call.limit = this->get_bluetooth_connections_limit(); - for (auto *connection : this->connections_) { - if (connection->address_ != 0) { - call.allocated.push_back(connection->address_); - } + if (this->api_connection_ != nullptr) { + this->send_connections_free(this->api_connection_); } - this->api_connection_->send_message(call, api::BluetoothConnectionsFreeResponse::MESSAGE_TYPE); +} + +void BluetoothProxy::send_connections_free(api::APIConnection *api_connection) { + api_connection->send_message(this->connections_free_response_, api::BluetoothConnectionsFreeResponse::MESSAGE_TYPE); } void BluetoothProxy::send_gatt_services_done(uint64_t address) { diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.h b/esphome/components/bluetooth_proxy/bluetooth_proxy.h index d249515fdf..70deef1ebd 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.h +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.h @@ -49,6 +49,7 @@ enum BluetoothProxySubscriptionFlag : uint32_t { }; class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Component { + friend class BluetoothConnection; // Allow connection to update connections_free_response_ public: BluetoothProxy(); #ifdef USE_ESP32_BLE_DEVICE @@ -74,15 +75,13 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com void bluetooth_gatt_send_services(const api::BluetoothGATTGetServicesRequest &msg); void bluetooth_gatt_notify(const api::BluetoothGATTNotifyRequest &msg); - int get_bluetooth_connections_free(); - int get_bluetooth_connections_limit() { return this->connections_.size(); } - void subscribe_api_connection(api::APIConnection *api_connection, uint32_t flags); void unsubscribe_api_connection(api::APIConnection *api_connection); api::APIConnection *get_api_connection() { return this->api_connection_; } void send_device_connection(uint64_t address, bool connected, uint16_t mtu = 0, esp_err_t error = ESP_OK); void send_connections_free(); + void send_connections_free(api::APIConnection *api_connection); void send_gatt_services_done(uint64_t address); void send_gatt_error(uint64_t address, uint16_t handle, esp_err_t error); void send_device_pairing(uint64_t address, bool paired, esp_err_t error = ESP_OK); @@ -149,6 +148,9 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com // Group 3: 4-byte types uint32_t last_advertisement_flush_time_{0}; + // Pre-allocated response message - always ready to send + api::BluetoothConnectionsFreeResponse connections_free_response_; + // Group 4: 1-byte types grouped together bool active_; uint8_t advertisement_count_{0}; diff --git a/esphome/components/esp32_ble_client/ble_client_base.h b/esphome/components/esp32_ble_client/ble_client_base.h index 457a88ec1d..0a2fda4476 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.h +++ b/esphome/components/esp32_ble_client/ble_client_base.h @@ -48,7 +48,7 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void set_auto_connect(bool auto_connect) { this->auto_connect_ = auto_connect; } - void set_address(uint64_t address) { + virtual void set_address(uint64_t address) { this->address_ = address; this->remote_bda_[0] = (address >> 40) & 0xFF; this->remote_bda_[1] = (address >> 32) & 0xFF; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index e226f748a8..55652e443e 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -147,6 +147,7 @@ #define USE_ESPHOME_TASK_LOG_BUFFER #define USE_BLUETOOTH_PROXY +#define BLUETOOTH_PROXY_MAX_CONNECTIONS 3 #define USE_CAPTIVE_PORTAL #define USE_ESP32_BLE #define USE_ESP32_BLE_CLIENT diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 24e2b25e90..fa2f87d98d 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -342,6 +342,11 @@ def create_field_type_info( # Check if this repeated field has fixed_array_size option if (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None: return FixedArrayRepeatedType(field, fixed_size) + # Check if this repeated field has fixed_array_size_define option + if ( + size_define := get_field_opt(field, pb.fixed_array_size_define) + ) is not None: + return FixedArrayRepeatedType(field, size_define) return RepeatedTypeInfo(field) # Check for fixed_array_size option on bytes fields @@ -1066,9 +1071,10 @@ class FixedArrayRepeatedType(TypeInfo): control how many items we receive when decoding. """ - def __init__(self, field: descriptor.FieldDescriptorProto, size: int) -> None: + def __init__(self, field: descriptor.FieldDescriptorProto, size: int | str) -> None: super().__init__(field) self.array_size = size + self.is_define = isinstance(size, str) # Check if we should skip encoding when all elements are zero # Use getattr to handle older versions of api_options_pb2 self.skip_zero = get_field_opt( @@ -1113,6 +1119,14 @@ class FixedArrayRepeatedType(TypeInfo): # If skip_zero is enabled, wrap encoding in a zero check if self.skip_zero: + if self.is_define: + # When using a define, we need to use a loop-based approach + o = f"for (const auto &it : this->{self.field_name}) {{\n" + o += " if (it != 0) {\n" + o += f" {encode_element('it')}\n" + o += " }\n" + o += "}" + return o # Build the condition to check if at least one element is non-zero non_zero_checks = " || ".join( [f"this->{self.field_name}[{i}] != 0" for i in range(self.array_size)] @@ -1123,6 +1137,13 @@ class FixedArrayRepeatedType(TypeInfo): ] return f"if ({non_zero_checks}) {{\n" + "\n".join(encode_lines) + "\n}" + # When using a define, always use loop-based approach + if self.is_define: + o = f"for (const auto &it : this->{self.field_name}) {{\n" + o += f" {encode_element('it')}\n" + o += "}" + return o + # Unroll small arrays for efficiency if self.array_size == 1: return encode_element(f"this->{self.field_name}[0]") @@ -1153,6 +1174,14 @@ class FixedArrayRepeatedType(TypeInfo): def get_size_calculation(self, name: str, force: bool = False) -> str: # If skip_zero is enabled, wrap size calculation in a zero check if self.skip_zero: + if self.is_define: + # When using a define, we need to use a loop-based approach + o = f"for (const auto &it : {name}) {{\n" + o += " if (it != 0) {\n" + o += f" {self._ti.get_size_calculation('it', True)}\n" + o += " }\n" + o += "}" + return o # Build the condition to check if at least one element is non-zero non_zero_checks = " || ".join( [f"{name}[{i}] != 0" for i in range(self.array_size)] @@ -1163,6 +1192,13 @@ class FixedArrayRepeatedType(TypeInfo): ] return f"if ({non_zero_checks}) {{\n" + "\n".join(size_lines) + "\n}" + # When using a define, always use loop-based approach + if self.is_define: + o = f"for (const auto &it : {name}) {{\n" + o += f" {self._ti.get_size_calculation('it', True)}\n" + o += "}" + return o + # For fixed arrays, we always encode all elements # Special case for single-element arrays - no loop needed @@ -1186,6 +1222,11 @@ class FixedArrayRepeatedType(TypeInfo): def get_estimated_size(self) -> int: # For fixed arrays, estimate underlying type size * array size underlying_size = self._ti.get_estimated_size() + if self.is_define: + # When using a define, we don't know the actual size so just guess 3 + # This is only used for documentation and never actually used since + # fixed arrays are only for SOURCE_SERVER (encode-only) messages + return underlying_size * 3 return underlying_size * self.array_size