Eliminate memory allocations in Bluetooth proxy BLE advertisement batching

esphome/components/api/api.proto

@@ -1381,7 +1381,7 @@ message BluetoothLERawAdvertisement {
   sint32 rssi = 2;
   uint32 address_type = 3;
 
-  bytes data = 4;
+  bytes data = 4 [(fixed_array_size) = 62];
 }
 
 message BluetoothLERawAdvertisementsResponse {

esphome/components/api/api_options.proto

@@ -23,3 +23,7 @@ extend google.protobuf.MessageOptions {
     optional bool no_delay = 1040 [default=false];
     optional string base_class = 1041;
 }
+
+extend google.protobuf.FieldOptions {
+    optional uint32 fixed_array_size = 50007;
+}

esphome/components/api/api_pb2.cpp

@@ -3,6 +3,7 @@
 #include "api_pb2.h"
 #include "esphome/core/log.h"
 #include "esphome/core/helpers.h"
+#include <cstring>
 
 namespace esphome {
 namespace api {
@@ -3825,9 +3826,14 @@ bool BluetoothLERawAdvertisement::decode_varint(uint32_t field_id, ProtoVarInt v
 }
 bool BluetoothLERawAdvertisement::decode_length(uint32_t field_id, ProtoLengthDelimited value) {
   switch (field_id) {
-    case 4:
+    case 4: {
-      this->data = value.as_string();
+      this->data_len = value.as_string().size();
+      if (this->data_len > 62) {
+        this->data_len = 62;
+      }
+      memcpy(this->data, value.as_string().data(), this->data_len);
       break;
+    }
     default:
       return false;
   }
@@ -3837,13 +3843,13 @@ void BluetoothLERawAdvertisement::encode(ProtoWriteBuffer buffer) const {
   buffer.encode_uint64(1, this->address);
   buffer.encode_sint32(2, this->rssi);
   buffer.encode_uint32(3, this->address_type);
-  buffer.encode_bytes(4, reinterpret_cast<const uint8_t *>(this->data.data()), this->data.size());
+  buffer.encode_bytes(4, this->data, this->data_len);
 }
 void BluetoothLERawAdvertisement::calculate_size(uint32_t &total_size) const {
   ProtoSize::add_uint64_field(total_size, 1, this->address);
   ProtoSize::add_sint32_field(total_size, 1, this->rssi);
   ProtoSize::add_uint32_field(total_size, 1, this->address_type);
-  ProtoSize::add_string_field(total_size, 1, this->data);
+  total_size += 1 + ProtoSize::varint(static_cast<uint32_t>(this->data_len)) + this->data_len;
 }
 bool BluetoothLERawAdvertisementsResponse::decode_length(uint32_t field_id, ProtoLengthDelimited value) {
   switch (field_id) {

esphome/components/api/api_pb2.h

@@ -1887,7 +1887,8 @@ class BluetoothLERawAdvertisement : public ProtoMessage {
   uint64_t address{0};
   int32_t rssi{0};
   uint32_t address_type{0};
-  std::string data{};
+  uint8_t data[62]{};
+  uint8_t data_len{0};
   void encode(ProtoWriteBuffer buffer) const override;
   void calculate_size(uint32_t &total_size) const override;
 #ifdef HAS_PROTO_MESSAGE_DUMP

esphome/components/api/api_pb2_dump.cpp

@@ -3013,7 +3013,7 @@ void BluetoothLERawAdvertisement::dump_to(std::string &out) const {
   out.append("\n");
 
   out.append("  data: ");
-  out.append(format_hex_pretty(this->data));
+  out.append(format_hex_pretty(reinterpret_cast<const char *>(this->this->data), this->this->data_len));
   out.append("\n");
   out.append("}");
 }

esphome/components/bluetooth_proxy/bluetooth_proxy.cpp

@@ -3,6 +3,8 @@
 #include "esphome/core/log.h"
 #include "esphome/core/macros.h"
 #include "esphome/core/application.h"
+#include <cstring>
+#include <memory>
 
 #ifdef USE_ESP32
 
@@ -60,13 +62,39 @@ bool BluetoothProxy::parse_device(const esp32_ble_tracker::ESPBTDevice &device)
 static constexpr size_t FLUSH_BATCH_SIZE = 16;
 
 namespace {
-// Batch buffer in anonymous namespace to avoid guard variable (saves 8 bytes)
+// Memory pool for BluetoothLERawAdvertisement objects
-// This is initialized at program startup before any threads
 // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-std::vector<api::BluetoothLERawAdvertisement> batch_buffer;
+std::vector<api::BluetoothLERawAdvertisement> advertisement_pool;
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+std::vector<api::BluetoothLERawAdvertisement *> free_advertisements;
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+std::vector<api::BluetoothLERawAdvertisement *> batch_buffer;
+
+// Pre-allocated response object to avoid heap allocation during runtime
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+std::unique_ptr<api::BluetoothLERawAdvertisementsResponse> response =
+    std::make_unique<api::BluetoothLERawAdvertisementsResponse>();
+
+// Initialize the pool
+struct PoolInitializer {
+  PoolInitializer() {
+    // Pre-allocate all vectors
+    advertisement_pool.resize(FLUSH_BATCH_SIZE);
+    free_advertisements.resize(FLUSH_BATCH_SIZE);
+    batch_buffer.reserve(FLUSH_BATCH_SIZE);
+    response->advertisements.reserve(FLUSH_BATCH_SIZE);
+
+    // Populate free pool with pointers to all advertisements
+    for (size_t i = 0; i < FLUSH_BATCH_SIZE; i++) {
+      free_advertisements[i] = &advertisement_pool[i];
+    }
+  }
+} pool_initializer;
 }  // namespace
 
-static std::vector<api::BluetoothLERawAdvertisement> &get_batch_buffer() { return batch_buffer; }
+static std::vector<api::BluetoothLERawAdvertisement *> &get_batch_buffer() { return batch_buffer; }
 
 bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, size_t count) {
   if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr || !this->raw_advertisements_)
@@ -75,26 +103,35 @@ bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results,
   // Get the batch buffer reference
   auto &batch_buffer = get_batch_buffer();
 
-  // Reserve additional capacity if needed
-  size_t new_size = batch_buffer.size() + count;
-  if (batch_buffer.capacity() < new_size) {
-    batch_buffer.reserve(new_size);
-  }
-
   // Add new advertisements to the batch buffer
   for (size_t i = 0; i < count; i++) {
+    // Check if we have free advertisements available
+    if (free_advertisements.empty()) {
+      // No free advertisements, flush current batch now
+      ESP_LOGV(TAG, "Advertisement pool exhausted, flushing batch");
+      this->flush_pending_advertisements();
+    }
+
     auto &result = scan_results[i];
     uint8_t length = result.adv_data_len + result.scan_rsp_len;
 
-    batch_buffer.emplace_back();
+    // Get an advertisement from the free pool
-    auto &adv = batch_buffer.back();
+    auto *adv = free_advertisements.back();
-    adv.address = esp32_ble::ble_addr_to_uint64(result.bda);
+    free_advertisements.pop_back();
-    adv.rssi = result.rssi;
-    adv.address_type = result.ble_addr_type;
-    adv.data.assign(&result.ble_adv[0], &result.ble_adv[length]);
 
-    ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d. RSSI: %d dB", result.bda[0],
+    // Fill in the advertisement data
-             result.bda[1], result.bda[2], result.bda[3], result.bda[4], result.bda[5], length, result.rssi);
+    adv->address = esp32_ble::ble_addr_to_uint64(result.bda);
+    adv->rssi = result.rssi;
+    adv->address_type = result.ble_addr_type;
+    adv->data_len = length;
+    std::memcpy(adv->data, result.ble_adv, length);
+
+    // Add to batch buffer
+    batch_buffer.push_back(adv);
+
+    ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d (adv:%d, rsp:%d). RSSI: %d dB",
+             result.bda[0], result.bda[1], result.bda[2], result.bda[3], result.bda[4], result.bda[5], length,
+             result.adv_data_len, result.scan_rsp_len, result.rssi);
   }
 
   // Only send if we've accumulated a good batch size to maximize batching efficiency
@@ -111,9 +148,30 @@ void BluetoothProxy::flush_pending_advertisements() {
   if (batch_buffer.empty() || !api::global_api_server->is_connected() || this->api_connection_ == nullptr)
     return;
 
-  api::BluetoothLERawAdvertisementsResponse resp;
+  // Defensive check
-  resp.advertisements.swap(batch_buffer);
+  if (batch_buffer.size() > FLUSH_BATCH_SIZE) {
-  this->api_connection_->send_message(resp);
+    ESP_LOGW(TAG, "Batch buffer size %d exceeds maximum %d", batch_buffer.size(), FLUSH_BATCH_SIZE);
+    return;
+  }
+
+  // Clear any previous data in response
+  response->advertisements.clear();
+
+  // Copy data from pool objects to response
+  // Use resize + index assignment instead of push_back to avoid temporaries
+  response->advertisements.resize(batch_buffer.size());
+  for (size_t i = 0; i < batch_buffer.size(); i++) {
+    response->advertisements[i] = *batch_buffer[i];
+  }
+
+  // Send the message
+  this->api_connection_->send_message(*response);
+
+  // Return all advertisements to the free pool
+  free_advertisements.insert(free_advertisements.end(), batch_buffer.begin(), batch_buffer.end());
+
+  // Clear the batch buffer
+  batch_buffer.clear();
 }
 
 void BluetoothProxy::send_api_packet_(const esp32_ble_tracker::ESPBTDevice &device) {

script/api_protobuf/api_protobuf.py

@@ -564,6 +564,75 @@ class BytesType(TypeInfo):
         return self.calculate_field_id_size() + 8  # field ID + 8 bytes typical bytes
 
 
+class FixedArrayBytesType(TypeInfo):
+    """Special type for fixed-size byte arrays."""
+
+    def __init__(self, field: descriptor.FieldDescriptorProto, size: int) -> None:
+        super().__init__(field)
+        self.array_size = size
+
+    @property
+    def cpp_type(self) -> str:
+        return "uint8_t"
+
+    @property
+    def default_value(self) -> str:
+        return "{}"
+
+    @property
+    def reference_type(self) -> str:
+        return f"uint8_t (&)[{self.array_size}]"
+
+    @property
+    def const_reference_type(self) -> str:
+        return f"const uint8_t (&)[{self.array_size}]"
+
+    @property
+    def public_content(self) -> list[str]:
+        # Add both the array and length fields
+        return [
+            f"uint8_t {self.field_name}[{self.array_size}]{{}};",
+            f"uint8_t {self.field_name}_len{{0}};",
+        ]
+
+    @property
+    def decode_length_content(self) -> str:
+        o = f"case {self.number}: {{\n"
+        o += f"  this->{self.field_name}_len = value.as_string().size();\n"
+        o += f"  if (this->{self.field_name}_len > {self.array_size}) {{\n"
+        o += f"    this->{self.field_name}_len = {self.array_size};\n"
+        o += "  }\n"
+        o += f"  memcpy(this->{self.field_name}, value.as_string().data(), this->{self.field_name}_len);\n"
+        o += "  break;\n"
+        o += "}"
+        return o
+
+    @property
+    def encode_content(self) -> str:
+        return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);"
+
+    def dump(self, name: str) -> str:
+        o = f"out.append(format_hex_pretty(reinterpret_cast<const char*>(this->{name}), this->{name}_len));"
+        return o
+
+    def get_size_calculation(self, name: str, force: bool = False) -> str:
+        # Use the actual length stored in the _len field
+        length_field = f"this->{self.field_name}_len"
+        # Size = field_id_size + varint(length) + actual_data_bytes
+        field_id_size = self.calculate_field_id_size()
+        return f"  total_size += {field_id_size} + ProtoSize::varint(static_cast<uint32_t>({length_field})) + {length_field};\n"
+
+    def get_estimated_size(self) -> int:
+        # Estimate based on typical BLE advertisement size
+        return (
+            self.calculate_field_id_size() + 1 + 31
+        )  # field ID + length byte + typical 31 bytes
+
+    @property
+    def wire_type(self) -> WireType:
+        return WireType.LENGTH_DELIMITED
+
+
 @register_type(13)
 class UInt32Type(TypeInfo):
     cpp_type = "uint32_t"
@@ -971,10 +1040,7 @@ def calculate_message_estimated_size(desc: descriptor.DescriptorProto) -> int:
     total_size = 0
 
     for field in desc.field:
-        if field.label == 3:  # repeated
+        ti = create_field_type_info(field)
-            ti = RepeatedTypeInfo(field)
-        else:
-            ti = TYPE_INFO[field.type](field)
 
         # Add estimated size for this field
         total_size += ti.get_estimated_size()
@@ -1036,10 +1102,7 @@ def build_message_type(
         public_content.append("#endif")
 
     for field in desc.field:
-        if field.label == 3:
+        ti = create_field_type_info(field)
-            ti = RepeatedTypeInfo(field)
-        else:
-            ti = TYPE_INFO[field.type](field)
 
         # Skip field declarations for fields that are in the base class
         # but include their encode/decode logic
@@ -1203,6 +1266,31 @@ def get_opt(
     return desc.options.Extensions[opt]
 
 
+def get_field_opt(
+    field: descriptor.FieldDescriptorProto,
+    opt: descriptor.FieldOptions,
+    default: Any = None,
+) -> Any:
+    """Get the option from a field descriptor."""
+    if not field.options.HasExtension(opt):
+        return default
+    return field.options.Extensions[opt]
+
+
+def create_field_type_info(field: descriptor.FieldDescriptorProto) -> TypeInfo:
+    """Create the appropriate TypeInfo instance for a field, handling repeated fields and custom options."""
+    if field.label == 3:  # repeated
+        return RepeatedTypeInfo(field)
+
+    # Check for fixed_array_size option on bytes fields
+    if field.type == 12:  # TYPE_BYTES
+        fixed_size = get_field_opt(field, pb.fixed_array_size)
+        if fixed_size is not None:
+            return FixedArrayBytesType(field, fixed_size)
+
+    return TYPE_INFO[field.type](field)
+
+
 def get_base_class(desc: descriptor.DescriptorProto) -> str | None:
     """Get the base_class option from a message descriptor."""
     if not desc.options.HasExtension(pb.base_class):
@@ -1276,10 +1364,7 @@ def build_base_class(
     # For base classes, we only declare the fields but don't handle encode/decode
     # The derived classes will handle encoding/decoding with their specific field numbers
     for field in common_fields:
-        if field.label == 3:  # repeated
+        ti = create_field_type_info(field)
-            ti = RepeatedTypeInfo(field)
-        else:
-            ti = TYPE_INFO[field.type](field)
 
         # Only add field declarations, not encode/decode logic
         protected_content.extend(ti.protected_content)
@@ -1412,6 +1497,7 @@ namespace api {
     #include "api_pb2.h"
     #include "esphome/core/log.h"
     #include "esphome/core/helpers.h"
+    #include <cstring>
 
 namespace esphome {
 namespace api {