diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index c8b046c1e2..b0ce21b1ce 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1381,7 +1381,7 @@ message BluetoothLERawAdvertisement { sint32 rssi = 2; uint32 address_type = 3; - bytes data = 4; + bytes data = 4 [(fixed_array_size) = 62]; } message BluetoothLERawAdvertisementsResponse { diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index 022cd8b3d2..bb3947e8a3 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -26,4 +26,5 @@ extend google.protobuf.MessageOptions { extend google.protobuf.FieldOptions { optional string field_ifdef = 1042; + optional uint32 fixed_array_size = 50007; } diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index b7a69a5d95..437c9ece1d 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -3,6 +3,7 @@ #include "api_pb2.h" #include "esphome/core/log.h" #include "esphome/core/helpers.h" +#include namespace esphome { namespace api { @@ -1916,13 +1917,15 @@ void BluetoothLERawAdvertisement::encode(ProtoWriteBuffer buffer) const { buffer.encode_uint64(1, this->address); buffer.encode_sint32(2, this->rssi); buffer.encode_uint32(3, this->address_type); - buffer.encode_bytes(4, reinterpret_cast(this->data.data()), this->data.size()); + buffer.encode_bytes(4, this->data, this->data_len); } void BluetoothLERawAdvertisement::calculate_size(uint32_t &total_size) const { ProtoSize::add_uint64_field(total_size, 1, this->address); ProtoSize::add_sint32_field(total_size, 1, this->rssi); ProtoSize::add_uint32_field(total_size, 1, this->address_type); - ProtoSize::add_string_field(total_size, 1, this->data); + if (this->data_len != 0) { + total_size += 1 + ProtoSize::varint(static_cast(this->data_len)) + this->data_len; + } } void BluetoothLERawAdvertisementsResponse::encode(ProtoWriteBuffer buffer) const { for (auto &it : this->advertisements) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 99486f57d7..39f00b4adc 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1768,7 +1768,8 @@ class BluetoothLERawAdvertisement : public ProtoMessage { uint64_t address{0}; int32_t rssi{0}; uint32_t address_type{0}; - std::string data{}; + uint8_t data[62]{}; + uint8_t data_len{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(uint32_t &total_size) const override; #ifdef HAS_PROTO_MESSAGE_DUMP diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 7d4150a857..ad5a5fdcaa 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -3132,7 +3132,7 @@ void BluetoothLERawAdvertisement::dump_to(std::string &out) const { out.append("\n"); out.append(" data: "); - out.append(format_hex_pretty(this->data)); + out.append(format_hex_pretty(this->data, this->data_len)); out.append("\n"); out.append("}"); } diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index fea8975060..7d12842a24 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -3,6 +3,7 @@ #include "esphome/core/log.h" #include "esphome/core/macros.h" #include "esphome/core/application.h" +#include #ifdef USE_ESP32 @@ -24,9 +25,30 @@ std::vector get_128bit_uuid_vec(esp_bt_uuid_t uuid_source) { ((uint64_t) uuid.uuid.uuid128[1] << 8) | ((uint64_t) uuid.uuid.uuid128[0])}; } +// Batch size for BLE advertisements to maximize WiFi efficiency +// Each advertisement is up to 80 bytes when packaged (including protocol overhead) +// Most advertisements are 20-30 bytes, allowing even more to fit per packet +// 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload +// This achieves ~97% WiFi MTU utilization while staying under the limit +static constexpr size_t FLUSH_BATCH_SIZE = 16; + +// Verify BLE advertisement data array size matches the BLE specification (31 bytes adv + 31 bytes scan response) +static_assert(sizeof(((api::BluetoothLERawAdvertisement *) nullptr)->data) == 62, + "BLE advertisement data array size mismatch"); + BluetoothProxy::BluetoothProxy() { global_bluetooth_proxy = this; } void BluetoothProxy::setup() { + // Pre-allocate response object + this->response_ = std::make_unique(); + + // Reserve capacity but start with size 0 + // Reserve 50% since we'll grow naturally and flush at FLUSH_BATCH_SIZE + this->response_->advertisements.reserve(FLUSH_BATCH_SIZE / 2); + + // Don't pre-allocate pool - let it grow only if needed in busy environments + // Many devices in quiet areas will never need the overflow pool + this->parent_->add_scanner_state_callback([this](esp32_ble_tracker::ScannerState state) { if (this->api_connection_ != nullptr) { this->send_bluetooth_scanner_state_(state); @@ -50,68 +72,72 @@ bool BluetoothProxy::parse_device(const esp32_ble_tracker::ESPBTDevice &device) } #endif -// Batch size for BLE advertisements to maximize WiFi efficiency -// Each advertisement is up to 80 bytes when packaged (including protocol overhead) -// Most advertisements are 20-30 bytes, allowing even more to fit per packet -// 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload -// This achieves ~97% WiFi MTU utilization while staying under the limit -static constexpr size_t FLUSH_BATCH_SIZE = 16; - -namespace { -// Batch buffer in anonymous namespace to avoid guard variable (saves 8 bytes) -// This is initialized at program startup before any threads -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -std::vector batch_buffer; -} // namespace - -static std::vector &get_batch_buffer() { return batch_buffer; } - bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, size_t count) { if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr) return false; - // Get the batch buffer reference - auto &batch_buffer = get_batch_buffer(); + auto &advertisements = this->response_->advertisements; - // Reserve additional capacity if needed - size_t new_size = batch_buffer.size() + count; - if (batch_buffer.capacity() < new_size) { - batch_buffer.reserve(new_size); - } - - // Add new advertisements to the batch buffer for (size_t i = 0; i < count; i++) { auto &result = scan_results[i]; uint8_t length = result.adv_data_len + result.scan_rsp_len; - batch_buffer.emplace_back(); - auto &adv = batch_buffer.back(); + // Check if we need to expand the vector + if (this->advertisement_count_ >= advertisements.size()) { + if (this->advertisement_pool_.empty()) { + // No room in pool, need to allocate + advertisements.emplace_back(); + } else { + // Pull from pool + advertisements.push_back(std::move(this->advertisement_pool_.back())); + this->advertisement_pool_.pop_back(); + } + } + + // Fill in the data directly at current position + auto &adv = advertisements[this->advertisement_count_]; adv.address = esp32_ble::ble_addr_to_uint64(result.bda); adv.rssi = result.rssi; adv.address_type = result.ble_addr_type; - adv.data.assign(&result.ble_adv[0], &result.ble_adv[length]); + adv.data_len = length; + std::memcpy(adv.data, result.ble_adv, length); + + this->advertisement_count_++; ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d. RSSI: %d dB", result.bda[0], result.bda[1], result.bda[2], result.bda[3], result.bda[4], result.bda[5], length, result.rssi); - } - // Only send if we've accumulated a good batch size to maximize batching efficiency - // https://github.com/esphome/backlog/issues/21 - if (batch_buffer.size() >= FLUSH_BATCH_SIZE) { - this->flush_pending_advertisements(); + // Flush if we have reached FLUSH_BATCH_SIZE + if (this->advertisement_count_ >= FLUSH_BATCH_SIZE) { + this->flush_pending_advertisements(); + } } return true; } void BluetoothProxy::flush_pending_advertisements() { - auto &batch_buffer = get_batch_buffer(); - if (batch_buffer.empty() || !api::global_api_server->is_connected() || this->api_connection_ == nullptr) + if (this->advertisement_count_ == 0 || !api::global_api_server->is_connected() || this->api_connection_ == nullptr) return; - api::BluetoothLERawAdvertisementsResponse resp; - resp.advertisements.swap(batch_buffer); - this->api_connection_->send_message(resp, api::BluetoothLERawAdvertisementsResponse::MESSAGE_TYPE); + auto &advertisements = this->response_->advertisements; + + // Return any items beyond advertisement_count_ to the pool + if (advertisements.size() > this->advertisement_count_) { + // Move unused items back to pool + this->advertisement_pool_.insert(this->advertisement_pool_.end(), + std::make_move_iterator(advertisements.begin() + this->advertisement_count_), + std::make_move_iterator(advertisements.end())); + + // Resize to actual count + advertisements.resize(this->advertisement_count_); + } + + // Send the message + this->api_connection_->send_message(*this->response_, api::BluetoothLERawAdvertisementsResponse::MESSAGE_TYPE); + + // Reset count - existing items will be overwritten in next batch + this->advertisement_count_ = 0; } #ifdef USE_ESP32_BLE_DEVICE diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.h b/esphome/components/bluetooth_proxy/bluetooth_proxy.h index 3ccf0706a7..52f1d0f88a 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.h +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.h @@ -145,9 +145,14 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com // Group 2: Container types (typically 12 bytes on 32-bit) std::vector connections_{}; + // BLE advertisement batching + std::vector advertisement_pool_; + std::unique_ptr response_; + // Group 3: 1-byte types grouped together bool active_; - // 1 byte used, 3 bytes padding + uint8_t advertisement_count_{0}; + // 2 bytes used, 2 bytes padding }; extern BluetoothProxy *global_bluetooth_proxy; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 46976918f9..23d8a53b70 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -313,13 +313,18 @@ def validate_field_type(field_type: int, field_name: str = "") -> None: ) -def get_type_info_for_field(field: descriptor.FieldDescriptorProto) -> TypeInfo: - """Get the appropriate TypeInfo for a field, handling repeated fields. - - Also validates that the field type is supported. - """ +def create_field_type_info(field: descriptor.FieldDescriptorProto) -> TypeInfo: + """Create the appropriate TypeInfo instance for a field, handling repeated fields and custom options.""" if field.label == 3: # repeated return RepeatedTypeInfo(field) + + # Check for fixed_array_size option on bytes fields + if ( + field.type == 12 + and (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None + ): + return FixedArrayBytesType(field, fixed_size) + validate_field_type(field.type, field.name) return TYPE_INFO[field.type](field) @@ -603,6 +608,85 @@ class BytesType(TypeInfo): return self.calculate_field_id_size() + 8 # field ID + 8 bytes typical bytes +class FixedArrayBytesType(TypeInfo): + """Special type for fixed-size byte arrays.""" + + def __init__(self, field: descriptor.FieldDescriptorProto, size: int) -> None: + super().__init__(field) + self.array_size = size + + @property + def cpp_type(self) -> str: + return "uint8_t" + + @property + def default_value(self) -> str: + return "{}" + + @property + def reference_type(self) -> str: + return f"uint8_t (&)[{self.array_size}]" + + @property + def const_reference_type(self) -> str: + return f"const uint8_t (&)[{self.array_size}]" + + @property + def public_content(self) -> list[str]: + # Add both the array and length fields + return [ + f"uint8_t {self.field_name}[{self.array_size}]{{}};", + f"uint8_t {self.field_name}_len{{0}};", + ] + + @property + def decode_length_content(self) -> str: + o = f"case {self.number}: {{\n" + o += " const std::string &data_str = value.as_string();\n" + o += f" this->{self.field_name}_len = data_str.size();\n" + o += f" if (this->{self.field_name}_len > {self.array_size}) {{\n" + o += f" this->{self.field_name}_len = {self.array_size};\n" + o += " }\n" + o += f" memcpy(this->{self.field_name}, data_str.data(), this->{self.field_name}_len);\n" + o += " break;\n" + o += "}" + return o + + @property + def encode_content(self) -> str: + return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);" + + def dump(self, name: str) -> str: + o = f"out.append(format_hex_pretty({name}, {name}_len));" + return o + + def get_size_calculation(self, name: str, force: bool = False) -> str: + # Use the actual length stored in the _len field + length_field = f"this->{self.field_name}_len" + field_id_size = self.calculate_field_id_size() + + if force: + # For repeated fields, always calculate size + return f"total_size += {field_id_size} + ProtoSize::varint(static_cast({length_field})) + {length_field};" + else: + # For non-repeated fields, skip if length is 0 (matching encode_string behavior) + return ( + f"if ({length_field} != 0) {{\n" + f" total_size += {field_id_size} + ProtoSize::varint(static_cast({length_field})) + {length_field};\n" + f"}}" + ) + + def get_estimated_size(self) -> int: + # Estimate based on typical BLE advertisement size + return ( + self.calculate_field_id_size() + 1 + 31 + ) # field ID + length byte + typical 31 bytes + + @property + def wire_type(self) -> WireType: + return WireType.LENGTH_DELIMITED + + @register_type(13) class UInt32Type(TypeInfo): cpp_type = "uint32_t" @@ -748,6 +832,16 @@ class SInt64Type(TypeInfo): class RepeatedTypeInfo(TypeInfo): def __init__(self, field: descriptor.FieldDescriptorProto) -> None: super().__init__(field) + # For repeated fields, we need to get the base type info + # but we can't call create_field_type_info as it would cause recursion + # So we extract just the type creation logic + if ( + field.type == 12 + and (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None + ): + self._ti: TypeInfo = FixedArrayBytesType(field, fixed_size) + return + validate_field_type(field.type, field.name) self._ti: TypeInfo = TYPE_INFO[field.type](field) @@ -1051,7 +1145,7 @@ def calculate_message_estimated_size(desc: descriptor.DescriptorProto) -> int: total_size = 0 for field in desc.field: - ti = get_type_info_for_field(field) + ti = create_field_type_info(field) # Add estimated size for this field total_size += ti.get_estimated_size() @@ -1119,10 +1213,7 @@ def build_message_type( public_content.append("#endif") for field in desc.field: - if field.label == 3: - ti = RepeatedTypeInfo(field) - else: - ti = TYPE_INFO[field.type](field) + ti = create_field_type_info(field) # Skip field declarations for fields that are in the base class # but include their encode/decode logic @@ -1327,6 +1418,17 @@ def get_opt( return desc.options.Extensions[opt] +def get_field_opt( + field: descriptor.FieldDescriptorProto, + opt: descriptor.FieldOptions, + default: Any = None, +) -> Any: + """Get the option from a field descriptor.""" + if not field.options.HasExtension(opt): + return default + return field.options.Extensions[opt] + + def get_base_class(desc: descriptor.DescriptorProto) -> str | None: """Get the base_class option from a message descriptor.""" if not desc.options.HasExtension(pb.base_class): @@ -1401,7 +1503,7 @@ def build_base_class( # For base classes, we only declare the fields but don't handle encode/decode # The derived classes will handle encoding/decoding with their specific field numbers for field in common_fields: - ti = get_type_info_for_field(field) + ti = create_field_type_info(field) # Only add field declarations, not encode/decode logic protected_content.extend(ti.protected_content) @@ -1543,6 +1645,7 @@ namespace api { #include "api_pb2.h" #include "esphome/core/log.h" #include "esphome/core/helpers.h" + #include namespace esphome { namespace api {