[bluetooth_proxy] Eliminate heap allocations in connection state reporting (#10010)

esphome/components/api/api.proto

@@ -1621,7 +1621,10 @@ message BluetoothConnectionsFreeResponse {
 
   uint32 free = 1;
   uint32 limit = 2;
-  repeated uint64 allocated = 3;
+  repeated uint64 allocated = 3 [
+    (fixed_array_size_define) = "BLUETOOTH_PROXY_MAX_CONNECTIONS",
+    (fixed_array_skip_zero) = true
+  ];
 }
 
 message BluetoothGATTErrorResponse {

esphome/components/api/api_connection.cpp

@@ -1105,10 +1105,8 @@ void APIConnection::bluetooth_gatt_notify(const BluetoothGATTNotifyRequest &msg)
 
 bool APIConnection::send_subscribe_bluetooth_connections_free_response(
     const SubscribeBluetoothConnectionsFreeRequest &msg) {
-  BluetoothConnectionsFreeResponse resp;
-  resp.free = bluetooth_proxy::global_bluetooth_proxy->get_bluetooth_connections_free();
-  resp.limit = bluetooth_proxy::global_bluetooth_proxy->get_bluetooth_connections_limit();
-  return this->send_message(resp, BluetoothConnectionsFreeResponse::MESSAGE_TYPE);
+  bluetooth_proxy::global_bluetooth_proxy->send_connections_free(this);
+  return true;
 }
 
 void APIConnection::bluetooth_scanner_set_mode(const BluetoothScannerSetModeRequest &msg) {

esphome/components/api/api_options.proto

@@ -29,6 +29,7 @@ extend google.protobuf.FieldOptions {
     optional uint32 fixed_array_size = 50007;
     optional bool no_zero_copy = 50008 [default=false];
     optional bool fixed_array_skip_zero = 50009 [default=false];
+    optional string fixed_array_size_define = 50010;
 
     // container_pointer: Zero-copy optimization for repeated fields.
     //

esphome/components/api/api_pb2.cpp

@@ -2073,15 +2073,17 @@ void BluetoothGATTNotifyDataResponse::calculate_size(ProtoSize &size) const {
 void BluetoothConnectionsFreeResponse::encode(ProtoWriteBuffer buffer) const {
   buffer.encode_uint32(1, this->free);
   buffer.encode_uint32(2, this->limit);
-  for (auto &it : this->allocated) {
-    buffer.encode_uint64(3, it, true);
+  for (const auto &it : this->allocated) {
+    if (it != 0) {
+      buffer.encode_uint64(3, it, true);
+    }
   }
 }
 void BluetoothConnectionsFreeResponse::calculate_size(ProtoSize &size) const {
   size.add_uint32(1, this->free);
   size.add_uint32(1, this->limit);
-  if (!this->allocated.empty()) {
-    for (const auto &it : this->allocated) {
+  for (const auto &it : this->allocated) {
+    if (it != 0) {
       size.add_uint64_force(1, it);
     }
   }

esphome/components/api/api_pb2.h

@@ -2076,13 +2076,13 @@ class SubscribeBluetoothConnectionsFreeRequest : public ProtoMessage {
 class BluetoothConnectionsFreeResponse : public ProtoMessage {
  public:
   static constexpr uint8_t MESSAGE_TYPE = 81;
-  static constexpr uint8_t ESTIMATED_SIZE = 16;
+  static constexpr uint8_t ESTIMATED_SIZE = 20;
 #ifdef HAS_PROTO_MESSAGE_DUMP
   const char *message_name() const override { return "bluetooth_connections_free_response"; }
 #endif
   uint32_t free{0};
   uint32_t limit{0};
-  std::vector<uint64_t> allocated{};
+  std::array<uint64_t, BLUETOOTH_PROXY_MAX_CONNECTIONS> allocated{};
   void encode(ProtoWriteBuffer buffer) const override;
   void calculate_size(ProtoSize &size) const override;
 #ifdef HAS_PROTO_MESSAGE_DUMP

esphome/components/bluetooth_proxy/__init__.py

@@ -87,6 +87,10 @@ async def to_code(config):
     cg.add(var.set_active(config[CONF_ACTIVE]))
     await esp32_ble_tracker.register_raw_ble_device(var, config)
 
+    # Define max connections for protobuf fixed array
+    connection_count = len(config.get(CONF_CONNECTIONS, []))
+    cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", connection_count)
+
     for connection_conf in config.get(CONF_CONNECTIONS, []):
         connection_var = cg.new_Pvariable(connection_conf[CONF_ID])
         await cg.register_component(connection_var, connection_conf)

esphome/components/bluetooth_proxy/bluetooth_connection.cpp

@@ -78,6 +78,30 @@ void BluetoothConnection::dump_config() {
   BLEClientBase::dump_config();
 }
 
+void BluetoothConnection::update_allocated_slot_(uint64_t find_value, uint64_t set_value) {
+  auto &allocated = this->proxy_->connections_free_response_.allocated;
+  auto *it = std::find(allocated.begin(), allocated.end(), find_value);
+  if (it != allocated.end()) {
+    *it = set_value;
+  }
+}
+
+void BluetoothConnection::set_address(uint64_t address) {
+  // If we're clearing an address (disconnecting), update the pre-allocated message
+  if (address == 0 && this->address_ != 0) {
+    this->proxy_->connections_free_response_.free++;
+    this->update_allocated_slot_(this->address_, 0);
+  }
+  // If we're setting a new address (connecting), update the pre-allocated message
+  else if (address != 0 && this->address_ == 0) {
+    this->proxy_->connections_free_response_.free--;
+    this->update_allocated_slot_(0, address);
+  }
+
+  // Call parent implementation to actually set the address
+  BLEClientBase::set_address(address);
+}
+
 void BluetoothConnection::loop() {
   BLEClientBase::loop();
 

esphome/components/bluetooth_proxy/bluetooth_connection.h

@@ -24,12 +24,15 @@ class BluetoothConnection : public esp32_ble_client::BLEClientBase {
 
   esp_err_t notify_characteristic(uint16_t handle, bool enable);
 
+  void set_address(uint64_t address) override;
+
  protected:
   friend class BluetoothProxy;
 
   bool supports_efficient_uuids_() const;
   void send_service_for_discovery_();
   void reset_connection_(esp_err_t reason);
+  void update_allocated_slot_(uint64_t find_value, uint64_t set_value);
 
   // Memory optimized layout for 32-bit systems
   // Group 1: Pointers (4 bytes each, naturally aligned)

esphome/components/bluetooth_proxy/bluetooth_proxy.cpp

@@ -35,6 +35,9 @@ void BluetoothProxy::setup() {
   // Don't pre-allocate pool - let it grow only if needed in busy environments
   // Many devices in quiet areas will never need the overflow pool
 
+  this->connections_free_response_.limit = this->connections_.size();
+  this->connections_free_response_.free = this->connections_.size();
+
   this->parent_->add_scanner_state_callback([this](esp32_ble_tracker::ScannerState state) {
     if (this->api_connection_ != nullptr) {
       this->send_bluetooth_scanner_state_(state);
@@ -134,20 +137,6 @@ void BluetoothProxy::dump_config() {
                 YESNO(this->active_), this->connections_.size());
 }
 
-int BluetoothProxy::get_bluetooth_connections_free() {
-  int free = 0;
-  for (auto *connection : this->connections_) {
-    if (connection->address_ == 0) {
-      free++;
-      ESP_LOGV(TAG, "[%d] Free connection", connection->get_connection_index());
-    } else {
-      ESP_LOGV(TAG, "[%d] Used connection by [%s]", connection->get_connection_index(),
-               connection->address_str().c_str());
-    }
-  }
-  return free;
-}
-
 void BluetoothProxy::loop() {
   if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr) {
     for (auto *connection : this->connections_) {
@@ -439,17 +428,13 @@ void BluetoothProxy::send_device_connection(uint64_t address, bool connected, ui
   this->api_connection_->send_message(call, api::BluetoothDeviceConnectionResponse::MESSAGE_TYPE);
 }
 void BluetoothProxy::send_connections_free() {
-  if (this->api_connection_ == nullptr)
-    return;
-  api::BluetoothConnectionsFreeResponse call;
-  call.free = this->get_bluetooth_connections_free();
-  call.limit = this->get_bluetooth_connections_limit();
-  for (auto *connection : this->connections_) {
-    if (connection->address_ != 0) {
-      call.allocated.push_back(connection->address_);
-    }
+  if (this->api_connection_ != nullptr) {
+    this->send_connections_free(this->api_connection_);
   }
-  this->api_connection_->send_message(call, api::BluetoothConnectionsFreeResponse::MESSAGE_TYPE);
+}
+
+void BluetoothProxy::send_connections_free(api::APIConnection *api_connection) {
+  api_connection->send_message(this->connections_free_response_, api::BluetoothConnectionsFreeResponse::MESSAGE_TYPE);
 }
 
 void BluetoothProxy::send_gatt_services_done(uint64_t address) {

esphome/components/bluetooth_proxy/bluetooth_proxy.h

@@ -49,6 +49,7 @@ enum BluetoothProxySubscriptionFlag : uint32_t {
 };
 
 class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Component {
+  friend class BluetoothConnection;  // Allow connection to update connections_free_response_
  public:
   BluetoothProxy();
 #ifdef USE_ESP32_BLE_DEVICE
@@ -74,15 +75,13 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com
   void bluetooth_gatt_send_services(const api::BluetoothGATTGetServicesRequest &msg);
   void bluetooth_gatt_notify(const api::BluetoothGATTNotifyRequest &msg);
 
-  int get_bluetooth_connections_free();
-  int get_bluetooth_connections_limit() { return this->connections_.size(); }
-
   void subscribe_api_connection(api::APIConnection *api_connection, uint32_t flags);
   void unsubscribe_api_connection(api::APIConnection *api_connection);
   api::APIConnection *get_api_connection() { return this->api_connection_; }
 
   void send_device_connection(uint64_t address, bool connected, uint16_t mtu = 0, esp_err_t error = ESP_OK);
   void send_connections_free();
+  void send_connections_free(api::APIConnection *api_connection);
   void send_gatt_services_done(uint64_t address);
   void send_gatt_error(uint64_t address, uint16_t handle, esp_err_t error);
   void send_device_pairing(uint64_t address, bool paired, esp_err_t error = ESP_OK);
@@ -149,6 +148,9 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com
   // Group 3: 4-byte types
   uint32_t last_advertisement_flush_time_{0};
 
+  // Pre-allocated response message - always ready to send
+  api::BluetoothConnectionsFreeResponse connections_free_response_;
+
   // Group 4: 1-byte types grouped together
   bool active_;
   uint8_t advertisement_count_{0};

esphome/components/esp32_ble_client/ble_client_base.h

@@ -48,7 +48,7 @@ class BLEClientBase : public espbt::ESPBTClient, public Component {
 
   void set_auto_connect(bool auto_connect) { this->auto_connect_ = auto_connect; }
 
-  void set_address(uint64_t address) {
+  virtual void set_address(uint64_t address) {
     this->address_ = address;
     this->remote_bda_[0] = (address >> 40) & 0xFF;
     this->remote_bda_[1] = (address >> 32) & 0xFF;

esphome/core/defines.h

@@ -147,6 +147,7 @@
 #define USE_ESPHOME_TASK_LOG_BUFFER
 
 #define USE_BLUETOOTH_PROXY
+#define BLUETOOTH_PROXY_MAX_CONNECTIONS 3
 #define USE_CAPTIVE_PORTAL
 #define USE_ESP32_BLE
 #define USE_ESP32_BLE_CLIENT

script/api_protobuf/api_protobuf.py

@@ -342,6 +342,11 @@ def create_field_type_info(
         # Check if this repeated field has fixed_array_size option
         if (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None:
             return FixedArrayRepeatedType(field, fixed_size)
+        # Check if this repeated field has fixed_array_size_define option
+        if (
+            size_define := get_field_opt(field, pb.fixed_array_size_define)
+        ) is not None:
+            return FixedArrayRepeatedType(field, size_define)
         return RepeatedTypeInfo(field)
 
     # Check for fixed_array_size option on bytes fields
@@ -1066,9 +1071,10 @@ class FixedArrayRepeatedType(TypeInfo):
     control how many items we receive when decoding.
     """
 
-    def __init__(self, field: descriptor.FieldDescriptorProto, size: int) -> None:
+    def __init__(self, field: descriptor.FieldDescriptorProto, size: int | str) -> None:
         super().__init__(field)
         self.array_size = size
+        self.is_define = isinstance(size, str)
         # Check if we should skip encoding when all elements are zero
         # Use getattr to handle older versions of api_options_pb2
         self.skip_zero = get_field_opt(
@@ -1113,6 +1119,14 @@ class FixedArrayRepeatedType(TypeInfo):
 
         # If skip_zero is enabled, wrap encoding in a zero check
         if self.skip_zero:
+            if self.is_define:
+                # When using a define, we need to use a loop-based approach
+                o = f"for (const auto &it : this->{self.field_name}) {{\n"
+                o += "  if (it != 0) {\n"
+                o += f"    {encode_element('it')}\n"
+                o += "  }\n"
+                o += "}"
+                return o
             # Build the condition to check if at least one element is non-zero
             non_zero_checks = " || ".join(
                 [f"this->{self.field_name}[{i}] != 0" for i in range(self.array_size)]
@@ -1123,6 +1137,13 @@ class FixedArrayRepeatedType(TypeInfo):
             ]
             return f"if ({non_zero_checks}) {{\n" + "\n".join(encode_lines) + "\n}"
 
+        # When using a define, always use loop-based approach
+        if self.is_define:
+            o = f"for (const auto &it : this->{self.field_name}) {{\n"
+            o += f"  {encode_element('it')}\n"
+            o += "}"
+            return o
+
         # Unroll small arrays for efficiency
         if self.array_size == 1:
             return encode_element(f"this->{self.field_name}[0]")
@@ -1153,6 +1174,14 @@ class FixedArrayRepeatedType(TypeInfo):
     def get_size_calculation(self, name: str, force: bool = False) -> str:
         # If skip_zero is enabled, wrap size calculation in a zero check
         if self.skip_zero:
+            if self.is_define:
+                # When using a define, we need to use a loop-based approach
+                o = f"for (const auto &it : {name}) {{\n"
+                o += "  if (it != 0) {\n"
+                o += f"    {self._ti.get_size_calculation('it', True)}\n"
+                o += "  }\n"
+                o += "}"
+                return o
             # Build the condition to check if at least one element is non-zero
             non_zero_checks = " || ".join(
                 [f"{name}[{i}] != 0" for i in range(self.array_size)]
@@ -1163,6 +1192,13 @@ class FixedArrayRepeatedType(TypeInfo):
             ]
             return f"if ({non_zero_checks}) {{\n" + "\n".join(size_lines) + "\n}"
 
+        # When using a define, always use loop-based approach
+        if self.is_define:
+            o = f"for (const auto &it : {name}) {{\n"
+            o += f"  {self._ti.get_size_calculation('it', True)}\n"
+            o += "}"
+            return o
+
         # For fixed arrays, we always encode all elements
 
         # Special case for single-element arrays - no loop needed
@@ -1186,6 +1222,11 @@ class FixedArrayRepeatedType(TypeInfo):
     def get_estimated_size(self) -> int:
         # For fixed arrays, estimate underlying type size * array size
         underlying_size = self._ti.get_estimated_size()
+        if self.is_define:
+            # When using a define, we don't know the actual size so just guess 3
+            # This is only used for documentation and never actually used since
+            # fixed arrays are only for SOURCE_SERVER (encode-only) messages
+            return underlying_size * 3
         return underlying_size * self.array_size