Fix bluetooth_proxy heap allocations during BLE scanning (#9633)

esphome/components/api/api.proto

@@ -1381,7 +1381,7 @@ message BluetoothLERawAdvertisement {
   sint32 rssi = 2;
   uint32 address_type = 3;
 
-  bytes data = 4;
+  bytes data = 4 [(fixed_array_size) = 62];
 }
 
 message BluetoothLERawAdvertisementsResponse {

esphome/components/api/api_options.proto

@@ -26,4 +26,5 @@ extend google.protobuf.MessageOptions {
 
 extend google.protobuf.FieldOptions {
     optional string field_ifdef = 1042;
+    optional uint32 fixed_array_size = 50007;
 }

esphome/components/api/api_pb2.cpp

@@ -3,6 +3,7 @@
 #include "api_pb2.h"
 #include "esphome/core/log.h"
 #include "esphome/core/helpers.h"
+#include <cstring>
 
 namespace esphome {
 namespace api {
@@ -1916,13 +1917,15 @@ void BluetoothLERawAdvertisement::encode(ProtoWriteBuffer buffer) const {
   buffer.encode_uint64(1, this->address);
   buffer.encode_sint32(2, this->rssi);
   buffer.encode_uint32(3, this->address_type);
-  buffer.encode_bytes(4, reinterpret_cast<const uint8_t *>(this->data.data()), this->data.size());
+  buffer.encode_bytes(4, this->data, this->data_len);
 }
 void BluetoothLERawAdvertisement::calculate_size(uint32_t &total_size) const {
   ProtoSize::add_uint64_field(total_size, 1, this->address);
   ProtoSize::add_sint32_field(total_size, 1, this->rssi);
   ProtoSize::add_uint32_field(total_size, 1, this->address_type);
-  ProtoSize::add_string_field(total_size, 1, this->data);
+  if (this->data_len != 0) {
+    total_size += 1 + ProtoSize::varint(static_cast<uint32_t>(this->data_len)) + this->data_len;
+  }
 }
 void BluetoothLERawAdvertisementsResponse::encode(ProtoWriteBuffer buffer) const {
   for (auto &it : this->advertisements) {

esphome/components/api/api_pb2.h

@@ -1768,7 +1768,8 @@ class BluetoothLERawAdvertisement : public ProtoMessage {
   uint64_t address{0};
   int32_t rssi{0};
   uint32_t address_type{0};
-  std::string data{};
+  uint8_t data[62]{};
+  uint8_t data_len{0};
   void encode(ProtoWriteBuffer buffer) const override;
   void calculate_size(uint32_t &total_size) const override;
 #ifdef HAS_PROTO_MESSAGE_DUMP

esphome/components/api/api_pb2_dump.cpp

@@ -3132,7 +3132,7 @@ void BluetoothLERawAdvertisement::dump_to(std::string &out) const {
   out.append("\n");
 
   out.append("  data: ");
-  out.append(format_hex_pretty(this->data));
+  out.append(format_hex_pretty(this->data, this->data_len));
   out.append("\n");
   out.append("}");
 }

esphome/components/bluetooth_proxy/bluetooth_proxy.cpp

@@ -3,6 +3,7 @@
 #include "esphome/core/log.h"
 #include "esphome/core/macros.h"
 #include "esphome/core/application.h"
+#include <cstring>
 
 #ifdef USE_ESP32
 
@@ -24,9 +25,30 @@ std::vector<uint64_t> get_128bit_uuid_vec(esp_bt_uuid_t uuid_source) {
                                    ((uint64_t) uuid.uuid.uuid128[1] << 8) | ((uint64_t) uuid.uuid.uuid128[0])};
 }
 
+// Batch size for BLE advertisements to maximize WiFi efficiency
+// Each advertisement is up to 80 bytes when packaged (including protocol overhead)
+// Most advertisements are 20-30 bytes, allowing even more to fit per packet
+// 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload
+// This achieves ~97% WiFi MTU utilization while staying under the limit
+static constexpr size_t FLUSH_BATCH_SIZE = 16;
+
+// Verify BLE advertisement data array size matches the BLE specification (31 bytes adv + 31 bytes scan response)
+static_assert(sizeof(((api::BluetoothLERawAdvertisement *) nullptr)->data) == 62,
+              "BLE advertisement data array size mismatch");
+
 BluetoothProxy::BluetoothProxy() { global_bluetooth_proxy = this; }
 
 void BluetoothProxy::setup() {
+  // Pre-allocate response object
+  this->response_ = std::make_unique<api::BluetoothLERawAdvertisementsResponse>();
+
+  // Reserve capacity but start with size 0
+  // Reserve 50% since we'll grow naturally and flush at FLUSH_BATCH_SIZE
+  this->response_->advertisements.reserve(FLUSH_BATCH_SIZE / 2);
+
+  // Don't pre-allocate pool - let it grow only if needed in busy environments
+  // Many devices in quiet areas will never need the overflow pool
+
   this->parent_->add_scanner_state_callback([this](esp32_ble_tracker::ScannerState state) {
     if (this->api_connection_ != nullptr) {
       this->send_bluetooth_scanner_state_(state);
@@ -50,68 +72,72 @@ bool BluetoothProxy::parse_device(const esp32_ble_tracker::ESPBTDevice &device)
 }
 #endif
 
-// Batch size for BLE advertisements to maximize WiFi efficiency
-// Each advertisement is up to 80 bytes when packaged (including protocol overhead)
-// Most advertisements are 20-30 bytes, allowing even more to fit per packet
-// 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload
-// This achieves ~97% WiFi MTU utilization while staying under the limit
-static constexpr size_t FLUSH_BATCH_SIZE = 16;
-
-namespace {
-// Batch buffer in anonymous namespace to avoid guard variable (saves 8 bytes)
-// This is initialized at program startup before any threads
-// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-std::vector<api::BluetoothLERawAdvertisement> batch_buffer;
-}  // namespace
-
-static std::vector<api::BluetoothLERawAdvertisement> &get_batch_buffer() { return batch_buffer; }
-
 bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, size_t count) {
   if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr)
     return false;
 
-  // Get the batch buffer reference
-  auto &batch_buffer = get_batch_buffer();
+  auto &advertisements = this->response_->advertisements;
 
-  // Reserve additional capacity if needed
-  size_t new_size = batch_buffer.size() + count;
-  if (batch_buffer.capacity() < new_size) {
-    batch_buffer.reserve(new_size);
-  }
-
-  // Add new advertisements to the batch buffer
   for (size_t i = 0; i < count; i++) {
     auto &result = scan_results[i];
     uint8_t length = result.adv_data_len + result.scan_rsp_len;
 
-    batch_buffer.emplace_back();
-    auto &adv = batch_buffer.back();
+    // Check if we need to expand the vector
+    if (this->advertisement_count_ >= advertisements.size()) {
+      if (this->advertisement_pool_.empty()) {
+        // No room in pool, need to allocate
+        advertisements.emplace_back();
+      } else {
+        // Pull from pool
+        advertisements.push_back(std::move(this->advertisement_pool_.back()));
+        this->advertisement_pool_.pop_back();
+      }
+    }
+
+    // Fill in the data directly at current position
+    auto &adv = advertisements[this->advertisement_count_];
     adv.address = esp32_ble::ble_addr_to_uint64(result.bda);
     adv.rssi = result.rssi;
     adv.address_type = result.ble_addr_type;
-    adv.data.assign(&result.ble_adv[0], &result.ble_adv[length]);
+    adv.data_len = length;
+    std::memcpy(adv.data, result.ble_adv, length);
+
+    this->advertisement_count_++;
 
     ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d. RSSI: %d dB", result.bda[0],
              result.bda[1], result.bda[2], result.bda[3], result.bda[4], result.bda[5], length, result.rssi);
-  }
 
-  // Only send if we've accumulated a good batch size to maximize batching efficiency
-  // https://github.com/esphome/backlog/issues/21
-  if (batch_buffer.size() >= FLUSH_BATCH_SIZE) {
-    this->flush_pending_advertisements();
+    // Flush if we have reached FLUSH_BATCH_SIZE
+    if (this->advertisement_count_ >= FLUSH_BATCH_SIZE) {
+      this->flush_pending_advertisements();
+    }
   }
 
   return true;
 }
 
 void BluetoothProxy::flush_pending_advertisements() {
-  auto &batch_buffer = get_batch_buffer();
-  if (batch_buffer.empty() || !api::global_api_server->is_connected() || this->api_connection_ == nullptr)
+  if (this->advertisement_count_ == 0 || !api::global_api_server->is_connected() || this->api_connection_ == nullptr)
     return;
 
-  api::BluetoothLERawAdvertisementsResponse resp;
-  resp.advertisements.swap(batch_buffer);
-  this->api_connection_->send_message(resp, api::BluetoothLERawAdvertisementsResponse::MESSAGE_TYPE);
+  auto &advertisements = this->response_->advertisements;
+
+  // Return any items beyond advertisement_count_ to the pool
+  if (advertisements.size() > this->advertisement_count_) {
+    // Move unused items back to pool
+    this->advertisement_pool_.insert(this->advertisement_pool_.end(),
+                                     std::make_move_iterator(advertisements.begin() + this->advertisement_count_),
+                                     std::make_move_iterator(advertisements.end()));
+
+    // Resize to actual count
+    advertisements.resize(this->advertisement_count_);
+  }
+
+  // Send the message
+  this->api_connection_->send_message(*this->response_, api::BluetoothLERawAdvertisementsResponse::MESSAGE_TYPE);
+
+  // Reset count - existing items will be overwritten in next batch
+  this->advertisement_count_ = 0;
 }
 
 #ifdef USE_ESP32_BLE_DEVICE

esphome/components/bluetooth_proxy/bluetooth_proxy.h

@@ -145,9 +145,14 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com
   // Group 2: Container types (typically 12 bytes on 32-bit)
   std::vector<BluetoothConnection *> connections_{};
 
+  // BLE advertisement batching
+  std::vector<api::BluetoothLERawAdvertisement> advertisement_pool_;
+  std::unique_ptr<api::BluetoothLERawAdvertisementsResponse> response_;
+
   // Group 3: 1-byte types grouped together
   bool active_;
-  // 1 byte used, 3 bytes padding
+  uint8_t advertisement_count_{0};
+  // 2 bytes used, 2 bytes padding
 };
 
 extern BluetoothProxy *global_bluetooth_proxy;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)

script/api_protobuf/api_protobuf.py

@@ -313,13 +313,18 @@ def validate_field_type(field_type: int, field_name: str = "") -> None:
         )
 
 
-def get_type_info_for_field(field: descriptor.FieldDescriptorProto) -> TypeInfo:
-    """Get the appropriate TypeInfo for a field, handling repeated fields.
-
-    Also validates that the field type is supported.
-    """
+def create_field_type_info(field: descriptor.FieldDescriptorProto) -> TypeInfo:
+    """Create the appropriate TypeInfo instance for a field, handling repeated fields and custom options."""
     if field.label == 3:  # repeated
         return RepeatedTypeInfo(field)
+
+    # Check for fixed_array_size option on bytes fields
+    if (
+        field.type == 12
+        and (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None
+    ):
+        return FixedArrayBytesType(field, fixed_size)
+
     validate_field_type(field.type, field.name)
     return TYPE_INFO[field.type](field)
 
@@ -603,6 +608,85 @@ class BytesType(TypeInfo):
         return self.calculate_field_id_size() + 8  # field ID + 8 bytes typical bytes
 
 
+class FixedArrayBytesType(TypeInfo):
+    """Special type for fixed-size byte arrays."""
+
+    def __init__(self, field: descriptor.FieldDescriptorProto, size: int) -> None:
+        super().__init__(field)
+        self.array_size = size
+
+    @property
+    def cpp_type(self) -> str:
+        return "uint8_t"
+
+    @property
+    def default_value(self) -> str:
+        return "{}"
+
+    @property
+    def reference_type(self) -> str:
+        return f"uint8_t (&)[{self.array_size}]"
+
+    @property
+    def const_reference_type(self) -> str:
+        return f"const uint8_t (&)[{self.array_size}]"
+
+    @property
+    def public_content(self) -> list[str]:
+        # Add both the array and length fields
+        return [
+            f"uint8_t {self.field_name}[{self.array_size}]{{}};",
+            f"uint8_t {self.field_name}_len{{0}};",
+        ]
+
+    @property
+    def decode_length_content(self) -> str:
+        o = f"case {self.number}: {{\n"
+        o += "  const std::string &data_str = value.as_string();\n"
+        o += f"  this->{self.field_name}_len = data_str.size();\n"
+        o += f"  if (this->{self.field_name}_len > {self.array_size}) {{\n"
+        o += f"    this->{self.field_name}_len = {self.array_size};\n"
+        o += "  }\n"
+        o += f"  memcpy(this->{self.field_name}, data_str.data(), this->{self.field_name}_len);\n"
+        o += "  break;\n"
+        o += "}"
+        return o
+
+    @property
+    def encode_content(self) -> str:
+        return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);"
+
+    def dump(self, name: str) -> str:
+        o = f"out.append(format_hex_pretty({name}, {name}_len));"
+        return o
+
+    def get_size_calculation(self, name: str, force: bool = False) -> str:
+        # Use the actual length stored in the _len field
+        length_field = f"this->{self.field_name}_len"
+        field_id_size = self.calculate_field_id_size()
+
+        if force:
+            # For repeated fields, always calculate size
+            return f"total_size += {field_id_size} + ProtoSize::varint(static_cast<uint32_t>({length_field})) + {length_field};"
+        else:
+            # For non-repeated fields, skip if length is 0 (matching encode_string behavior)
+            return (
+                f"if ({length_field} != 0) {{\n"
+                f"  total_size += {field_id_size} + ProtoSize::varint(static_cast<uint32_t>({length_field})) + {length_field};\n"
+                f"}}"
+            )
+
+    def get_estimated_size(self) -> int:
+        # Estimate based on typical BLE advertisement size
+        return (
+            self.calculate_field_id_size() + 1 + 31
+        )  # field ID + length byte + typical 31 bytes
+
+    @property
+    def wire_type(self) -> WireType:
+        return WireType.LENGTH_DELIMITED
+
+
 @register_type(13)
 class UInt32Type(TypeInfo):
     cpp_type = "uint32_t"
@@ -748,6 +832,16 @@ class SInt64Type(TypeInfo):
 class RepeatedTypeInfo(TypeInfo):
     def __init__(self, field: descriptor.FieldDescriptorProto) -> None:
         super().__init__(field)
+        # For repeated fields, we need to get the base type info
+        # but we can't call create_field_type_info as it would cause recursion
+        # So we extract just the type creation logic
+        if (
+            field.type == 12
+            and (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None
+        ):
+            self._ti: TypeInfo = FixedArrayBytesType(field, fixed_size)
+            return
+
         validate_field_type(field.type, field.name)
         self._ti: TypeInfo = TYPE_INFO[field.type](field)
 
@@ -1051,7 +1145,7 @@ def calculate_message_estimated_size(desc: descriptor.DescriptorProto) -> int:
     total_size = 0
 
     for field in desc.field:
-        ti = get_type_info_for_field(field)
+        ti = create_field_type_info(field)
 
         # Add estimated size for this field
         total_size += ti.get_estimated_size()
@@ -1119,10 +1213,7 @@ def build_message_type(
         public_content.append("#endif")
 
     for field in desc.field:
-        if field.label == 3:
-            ti = RepeatedTypeInfo(field)
-        else:
-            ti = TYPE_INFO[field.type](field)
+        ti = create_field_type_info(field)
 
         # Skip field declarations for fields that are in the base class
         # but include their encode/decode logic
@@ -1327,6 +1418,17 @@ def get_opt(
     return desc.options.Extensions[opt]
 
 
+def get_field_opt(
+    field: descriptor.FieldDescriptorProto,
+    opt: descriptor.FieldOptions,
+    default: Any = None,
+) -> Any:
+    """Get the option from a field descriptor."""
+    if not field.options.HasExtension(opt):
+        return default
+    return field.options.Extensions[opt]
+
+
 def get_base_class(desc: descriptor.DescriptorProto) -> str | None:
     """Get the base_class option from a message descriptor."""
     if not desc.options.HasExtension(pb.base_class):
@@ -1401,7 +1503,7 @@ def build_base_class(
     # For base classes, we only declare the fields but don't handle encode/decode
     # The derived classes will handle encoding/decoding with their specific field numbers
     for field in common_fields:
-        ti = get_type_info_for_field(field)
+        ti = create_field_type_info(field)
 
         # Only add field declarations, not encode/decode logic
         protected_content.extend(ti.protected_content)
@@ -1543,6 +1645,7 @@ namespace api {
     #include "api_pb2.h"
     #include "esphome/core/log.h"
     #include "esphome/core/helpers.h"
+    #include <cstring>
 
 namespace esphome {
 namespace api {