diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 861b3471d7..9f1c34e759 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1381,7 +1381,7 @@ message BluetoothLERawAdvertisement { sint32 rssi = 2; uint32 address_type = 3; - bytes data = 4; + bytes data = 4 [(fixed_array_size) = 62]; } message BluetoothLERawAdvertisementsResponse { diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index 3a547b8688..6903e6258a 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -23,3 +23,7 @@ extend google.protobuf.MessageOptions { optional bool no_delay = 1040 [default=false]; optional string base_class = 1041; } + +extend google.protobuf.FieldOptions { + optional uint32 fixed_array_size = 50007; +} diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 1ad8ba62af..9108457d72 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -3,6 +3,7 @@ #include "api_pb2.h" #include "esphome/core/log.h" #include "esphome/core/helpers.h" +#include namespace esphome { namespace api { @@ -3825,9 +3826,14 @@ bool BluetoothLERawAdvertisement::decode_varint(uint32_t field_id, ProtoVarInt v } bool BluetoothLERawAdvertisement::decode_length(uint32_t field_id, ProtoLengthDelimited value) { switch (field_id) { - case 4: - this->data = value.as_string(); + case 4: { + this->data_len = value.as_string().size(); + if (this->data_len > 62) { + this->data_len = 62; + } + memcpy(this->data, value.as_string().data(), this->data_len); break; + } default: return false; } @@ -3837,13 +3843,13 @@ void BluetoothLERawAdvertisement::encode(ProtoWriteBuffer buffer) const { buffer.encode_uint64(1, this->address); buffer.encode_sint32(2, this->rssi); buffer.encode_uint32(3, this->address_type); - buffer.encode_bytes(4, reinterpret_cast(this->data.data()), this->data.size()); + buffer.encode_bytes(4, this->data, this->data_len); } void BluetoothLERawAdvertisement::calculate_size(uint32_t &total_size) const { ProtoSize::add_uint64_field(total_size, 1, this->address); ProtoSize::add_sint32_field(total_size, 1, this->rssi); ProtoSize::add_uint32_field(total_size, 1, this->address_type); - ProtoSize::add_string_field(total_size, 1, this->data); + total_size += 1 + ProtoSize::varint(static_cast(this->data_len)) + this->data_len; } bool BluetoothLERawAdvertisementsResponse::decode_length(uint32_t field_id, ProtoLengthDelimited value) { switch (field_id) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 6a95055c2b..313e465f6b 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1887,7 +1887,8 @@ class BluetoothLERawAdvertisement : public ProtoMessage { uint64_t address{0}; int32_t rssi{0}; uint32_t address_type{0}; - std::string data{}; + uint8_t data[62]{}; + uint8_t data_len{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(uint32_t &total_size) const override; #ifdef HAS_PROTO_MESSAGE_DUMP diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index f6509f47cc..4b0ba9b5c5 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -3013,7 +3013,7 @@ void BluetoothLERawAdvertisement::dump_to(std::string &out) const { out.append("\n"); out.append(" data: "); - out.append(format_hex_pretty(this->data)); + out.append(format_hex_pretty(reinterpret_cast(this->this->data), this->this->data_len)); out.append("\n"); out.append("}"); } diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index a5e8ec0860..4ccfb79fb9 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -3,6 +3,8 @@ #include "esphome/core/log.h" #include "esphome/core/macros.h" #include "esphome/core/application.h" +#include +#include #ifdef USE_ESP32 @@ -60,13 +62,39 @@ bool BluetoothProxy::parse_device(const esp32_ble_tracker::ESPBTDevice &device) static constexpr size_t FLUSH_BATCH_SIZE = 16; namespace { -// Batch buffer in anonymous namespace to avoid guard variable (saves 8 bytes) -// This is initialized at program startup before any threads +// Memory pool for BluetoothLERawAdvertisement objects // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -std::vector batch_buffer; +std::vector advertisement_pool; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +std::vector free_advertisements; + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +std::vector batch_buffer; + +// Pre-allocated response object to avoid heap allocation during runtime +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +std::unique_ptr response = + std::make_unique(); + +// Initialize the pool +struct PoolInitializer { + PoolInitializer() { + // Pre-allocate all vectors + advertisement_pool.resize(FLUSH_BATCH_SIZE); + free_advertisements.resize(FLUSH_BATCH_SIZE); + batch_buffer.reserve(FLUSH_BATCH_SIZE); + response->advertisements.reserve(FLUSH_BATCH_SIZE); + + // Populate free pool with pointers to all advertisements + for (size_t i = 0; i < FLUSH_BATCH_SIZE; i++) { + free_advertisements[i] = &advertisement_pool[i]; + } + } +} pool_initializer; } // namespace -static std::vector &get_batch_buffer() { return batch_buffer; } +static std::vector &get_batch_buffer() { return batch_buffer; } bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, size_t count) { if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr || !this->raw_advertisements_) @@ -75,26 +103,35 @@ bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, // Get the batch buffer reference auto &batch_buffer = get_batch_buffer(); - // Reserve additional capacity if needed - size_t new_size = batch_buffer.size() + count; - if (batch_buffer.capacity() < new_size) { - batch_buffer.reserve(new_size); - } - // Add new advertisements to the batch buffer for (size_t i = 0; i < count; i++) { + // Check if we have free advertisements available + if (free_advertisements.empty()) { + // No free advertisements, flush current batch now + ESP_LOGV(TAG, "Advertisement pool exhausted, flushing batch"); + this->flush_pending_advertisements(); + } + auto &result = scan_results[i]; uint8_t length = result.adv_data_len + result.scan_rsp_len; - batch_buffer.emplace_back(); - auto &adv = batch_buffer.back(); - adv.address = esp32_ble::ble_addr_to_uint64(result.bda); - adv.rssi = result.rssi; - adv.address_type = result.ble_addr_type; - adv.data.assign(&result.ble_adv[0], &result.ble_adv[length]); + // Get an advertisement from the free pool + auto *adv = free_advertisements.back(); + free_advertisements.pop_back(); - ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d. RSSI: %d dB", result.bda[0], - result.bda[1], result.bda[2], result.bda[3], result.bda[4], result.bda[5], length, result.rssi); + // Fill in the advertisement data + adv->address = esp32_ble::ble_addr_to_uint64(result.bda); + adv->rssi = result.rssi; + adv->address_type = result.ble_addr_type; + adv->data_len = length; + std::memcpy(adv->data, result.ble_adv, length); + + // Add to batch buffer + batch_buffer.push_back(adv); + + ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d (adv:%d, rsp:%d). RSSI: %d dB", + result.bda[0], result.bda[1], result.bda[2], result.bda[3], result.bda[4], result.bda[5], length, + result.adv_data_len, result.scan_rsp_len, result.rssi); } // Only send if we've accumulated a good batch size to maximize batching efficiency @@ -111,9 +148,30 @@ void BluetoothProxy::flush_pending_advertisements() { if (batch_buffer.empty() || !api::global_api_server->is_connected() || this->api_connection_ == nullptr) return; - api::BluetoothLERawAdvertisementsResponse resp; - resp.advertisements.swap(batch_buffer); - this->api_connection_->send_message(resp); + // Defensive check + if (batch_buffer.size() > FLUSH_BATCH_SIZE) { + ESP_LOGW(TAG, "Batch buffer size %d exceeds maximum %d", batch_buffer.size(), FLUSH_BATCH_SIZE); + return; + } + + // Clear any previous data in response + response->advertisements.clear(); + + // Copy data from pool objects to response + // Use resize + index assignment instead of push_back to avoid temporaries + response->advertisements.resize(batch_buffer.size()); + for (size_t i = 0; i < batch_buffer.size(); i++) { + response->advertisements[i] = *batch_buffer[i]; + } + + // Send the message + this->api_connection_->send_message(*response); + + // Return all advertisements to the free pool + free_advertisements.insert(free_advertisements.end(), batch_buffer.begin(), batch_buffer.end()); + + // Clear the batch buffer + batch_buffer.clear(); } void BluetoothProxy::send_api_packet_(const esp32_ble_tracker::ESPBTDevice &device) { diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 938c613e53..f3c7c56d1a 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -564,6 +564,75 @@ class BytesType(TypeInfo): return self.calculate_field_id_size() + 8 # field ID + 8 bytes typical bytes +class FixedArrayBytesType(TypeInfo): + """Special type for fixed-size byte arrays.""" + + def __init__(self, field: descriptor.FieldDescriptorProto, size: int) -> None: + super().__init__(field) + self.array_size = size + + @property + def cpp_type(self) -> str: + return "uint8_t" + + @property + def default_value(self) -> str: + return "{}" + + @property + def reference_type(self) -> str: + return f"uint8_t (&)[{self.array_size}]" + + @property + def const_reference_type(self) -> str: + return f"const uint8_t (&)[{self.array_size}]" + + @property + def public_content(self) -> list[str]: + # Add both the array and length fields + return [ + f"uint8_t {self.field_name}[{self.array_size}]{{}};", + f"uint8_t {self.field_name}_len{{0}};", + ] + + @property + def decode_length_content(self) -> str: + o = f"case {self.number}: {{\n" + o += f" this->{self.field_name}_len = value.as_string().size();\n" + o += f" if (this->{self.field_name}_len > {self.array_size}) {{\n" + o += f" this->{self.field_name}_len = {self.array_size};\n" + o += " }\n" + o += f" memcpy(this->{self.field_name}, value.as_string().data(), this->{self.field_name}_len);\n" + o += " break;\n" + o += "}" + return o + + @property + def encode_content(self) -> str: + return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);" + + def dump(self, name: str) -> str: + o = f"out.append(format_hex_pretty(reinterpret_cast(this->{name}), this->{name}_len));" + return o + + def get_size_calculation(self, name: str, force: bool = False) -> str: + # Use the actual length stored in the _len field + length_field = f"this->{self.field_name}_len" + # Size = field_id_size + varint(length) + actual_data_bytes + field_id_size = self.calculate_field_id_size() + return f" total_size += {field_id_size} + ProtoSize::varint(static_cast({length_field})) + {length_field};\n" + + def get_estimated_size(self) -> int: + # Estimate based on typical BLE advertisement size + return ( + self.calculate_field_id_size() + 1 + 31 + ) # field ID + length byte + typical 31 bytes + + @property + def wire_type(self) -> WireType: + return WireType.LENGTH_DELIMITED + + @register_type(13) class UInt32Type(TypeInfo): cpp_type = "uint32_t" @@ -971,10 +1040,7 @@ def calculate_message_estimated_size(desc: descriptor.DescriptorProto) -> int: total_size = 0 for field in desc.field: - if field.label == 3: # repeated - ti = RepeatedTypeInfo(field) - else: - ti = TYPE_INFO[field.type](field) + ti = create_field_type_info(field) # Add estimated size for this field total_size += ti.get_estimated_size() @@ -1036,10 +1102,7 @@ def build_message_type( public_content.append("#endif") for field in desc.field: - if field.label == 3: - ti = RepeatedTypeInfo(field) - else: - ti = TYPE_INFO[field.type](field) + ti = create_field_type_info(field) # Skip field declarations for fields that are in the base class # but include their encode/decode logic @@ -1203,6 +1266,31 @@ def get_opt( return desc.options.Extensions[opt] +def get_field_opt( + field: descriptor.FieldDescriptorProto, + opt: descriptor.FieldOptions, + default: Any = None, +) -> Any: + """Get the option from a field descriptor.""" + if not field.options.HasExtension(opt): + return default + return field.options.Extensions[opt] + + +def create_field_type_info(field: descriptor.FieldDescriptorProto) -> TypeInfo: + """Create the appropriate TypeInfo instance for a field, handling repeated fields and custom options.""" + if field.label == 3: # repeated + return RepeatedTypeInfo(field) + + # Check for fixed_array_size option on bytes fields + if field.type == 12: # TYPE_BYTES + fixed_size = get_field_opt(field, pb.fixed_array_size) + if fixed_size is not None: + return FixedArrayBytesType(field, fixed_size) + + return TYPE_INFO[field.type](field) + + def get_base_class(desc: descriptor.DescriptorProto) -> str | None: """Get the base_class option from a message descriptor.""" if not desc.options.HasExtension(pb.base_class): @@ -1276,10 +1364,7 @@ def build_base_class( # For base classes, we only declare the fields but don't handle encode/decode # The derived classes will handle encoding/decoding with their specific field numbers for field in common_fields: - if field.label == 3: # repeated - ti = RepeatedTypeInfo(field) - else: - ti = TYPE_INFO[field.type](field) + ti = create_field_type_info(field) # Only add field declarations, not encode/decode logic protected_content.extend(ti.protected_content) @@ -1412,6 +1497,7 @@ namespace api { #include "api_pb2.h" #include "esphome/core/log.h" #include "esphome/core/helpers.h" + #include namespace esphome { namespace api {