From ac08fb314f21ad05f8e7498872b9e0df845de057 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 21 Jul 2025 23:50:49 -1000 Subject: [PATCH] [api] Optimize protobuf memory usage with fixed-size arrays for Bluetooth UUIDs (#9782) --- esphome/components/api/api.proto | 8 +- esphome/components/api/api_pb2.cpp | 42 ++---- esphome/components/api/api_pb2.h | 10 +- .../bluetooth_proxy/bluetooth_connection.cpp | 27 ++-- script/api_protobuf/api_protobuf.py | 129 ++++++++++++++++++ 5 files changed, 165 insertions(+), 51 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index e7c2fcaf8a..93e84702e2 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1463,19 +1463,19 @@ message BluetoothGATTGetServicesRequest { } message BluetoothGATTDescriptor { - repeated uint64 uuid = 1; + repeated uint64 uuid = 1 [(fixed_array_size) = 2]; uint32 handle = 2; } message BluetoothGATTCharacteristic { - repeated uint64 uuid = 1; + repeated uint64 uuid = 1 [(fixed_array_size) = 2]; uint32 handle = 2; uint32 properties = 3; repeated BluetoothGATTDescriptor descriptors = 4; } message BluetoothGATTService { - repeated uint64 uuid = 1; + repeated uint64 uuid = 1 [(fixed_array_size) = 2]; uint32 handle = 2; repeated BluetoothGATTCharacteristic characteristics = 3; } @@ -1486,7 +1486,7 @@ message BluetoothGATTGetServicesResponse { option (ifdef) = "USE_BLUETOOTH_PROXY"; uint64 address = 1; - repeated BluetoothGATTService services = 2; + repeated BluetoothGATTService services = 2 [(fixed_array_size) = 1]; } message BluetoothGATTGetServicesDoneResponse { diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 28d135ed6d..09a1522fb4 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -1891,23 +1891,18 @@ bool BluetoothGATTGetServicesRequest::decode_varint(uint32_t field_id, ProtoVarI return true; } void BluetoothGATTDescriptor::encode(ProtoWriteBuffer buffer) const { - for (auto &it : this->uuid) { - buffer.encode_uint64(1, it, true); - } + buffer.encode_uint64(1, this->uuid[0], true); + buffer.encode_uint64(1, this->uuid[1], true); buffer.encode_uint32(2, this->handle); } void BluetoothGATTDescriptor::calculate_size(uint32_t &total_size) const { - if (!this->uuid.empty()) { - for (const auto &it : this->uuid) { - ProtoSize::add_uint64_field_repeated(total_size, 1, it); - } - } + ProtoSize::add_uint64_field_repeated(total_size, 1, this->uuid[0]); + ProtoSize::add_uint64_field_repeated(total_size, 1, this->uuid[1]); ProtoSize::add_uint32_field(total_size, 1, this->handle); } void BluetoothGATTCharacteristic::encode(ProtoWriteBuffer buffer) const { - for (auto &it : this->uuid) { - buffer.encode_uint64(1, it, true); - } + buffer.encode_uint64(1, this->uuid[0], true); + buffer.encode_uint64(1, this->uuid[1], true); buffer.encode_uint32(2, this->handle); buffer.encode_uint32(3, this->properties); for (auto &it : this->descriptors) { @@ -1915,42 +1910,33 @@ void BluetoothGATTCharacteristic::encode(ProtoWriteBuffer buffer) const { } } void BluetoothGATTCharacteristic::calculate_size(uint32_t &total_size) const { - if (!this->uuid.empty()) { - for (const auto &it : this->uuid) { - ProtoSize::add_uint64_field_repeated(total_size, 1, it); - } - } + ProtoSize::add_uint64_field_repeated(total_size, 1, this->uuid[0]); + ProtoSize::add_uint64_field_repeated(total_size, 1, this->uuid[1]); ProtoSize::add_uint32_field(total_size, 1, this->handle); ProtoSize::add_uint32_field(total_size, 1, this->properties); ProtoSize::add_repeated_message(total_size, 1, this->descriptors); } void BluetoothGATTService::encode(ProtoWriteBuffer buffer) const { - for (auto &it : this->uuid) { - buffer.encode_uint64(1, it, true); - } + buffer.encode_uint64(1, this->uuid[0], true); + buffer.encode_uint64(1, this->uuid[1], true); buffer.encode_uint32(2, this->handle); for (auto &it : this->characteristics) { buffer.encode_message(3, it, true); } } void BluetoothGATTService::calculate_size(uint32_t &total_size) const { - if (!this->uuid.empty()) { - for (const auto &it : this->uuid) { - ProtoSize::add_uint64_field_repeated(total_size, 1, it); - } - } + ProtoSize::add_uint64_field_repeated(total_size, 1, this->uuid[0]); + ProtoSize::add_uint64_field_repeated(total_size, 1, this->uuid[1]); ProtoSize::add_uint32_field(total_size, 1, this->handle); ProtoSize::add_repeated_message(total_size, 1, this->characteristics); } void BluetoothGATTGetServicesResponse::encode(ProtoWriteBuffer buffer) const { buffer.encode_uint64(1, this->address); - for (auto &it : this->services) { - buffer.encode_message(2, it, true); - } + buffer.encode_message(2, this->services[0], true); } void BluetoothGATTGetServicesResponse::calculate_size(uint32_t &total_size) const { ProtoSize::add_uint64_field(total_size, 1, this->address); - ProtoSize::add_repeated_message(total_size, 1, this->services); + ProtoSize::add_message_object_repeated(total_size, 1, this->services[0]); } void BluetoothGATTGetServicesDoneResponse::encode(ProtoWriteBuffer buffer) const { buffer.encode_uint64(1, this->address); diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 7255aa7903..1d052f6114 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1796,7 +1796,7 @@ class BluetoothGATTGetServicesRequest : public ProtoDecodableMessage { }; class BluetoothGATTDescriptor : public ProtoMessage { public: - std::vector uuid{}; + std::array uuid{}; uint32_t handle{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(uint32_t &total_size) const override; @@ -1808,7 +1808,7 @@ class BluetoothGATTDescriptor : public ProtoMessage { }; class BluetoothGATTCharacteristic : public ProtoMessage { public: - std::vector uuid{}; + std::array uuid{}; uint32_t handle{0}; uint32_t properties{0}; std::vector descriptors{}; @@ -1822,7 +1822,7 @@ class BluetoothGATTCharacteristic : public ProtoMessage { }; class BluetoothGATTService : public ProtoMessage { public: - std::vector uuid{}; + std::array uuid{}; uint32_t handle{0}; std::vector characteristics{}; void encode(ProtoWriteBuffer buffer) const override; @@ -1836,12 +1836,12 @@ class BluetoothGATTService : public ProtoMessage { class BluetoothGATTGetServicesResponse : public ProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 71; - static constexpr uint8_t ESTIMATED_SIZE = 38; + static constexpr uint8_t ESTIMATED_SIZE = 21; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "bluetooth_gatt_get_services_response"; } #endif uint64_t address{0}; - std::vector services{}; + std::array services{}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(uint32_t &total_size) const override; #ifdef HAS_PROTO_MESSAGE_DUMP diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 7c883b74a2..616dba891a 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -13,16 +13,16 @@ namespace bluetooth_proxy { static const char *const TAG = "bluetooth_proxy.connection"; -static std::vector get_128bit_uuid_vec(esp_bt_uuid_t uuid_source) { +static void fill_128bit_uuid_array(std::array &out, esp_bt_uuid_t uuid_source) { esp_bt_uuid_t uuid = espbt::ESPBTUUID::from_uuid(uuid_source).as_128bit().get_uuid(); - return std::vector{((uint64_t) uuid.uuid.uuid128[15] << 56) | ((uint64_t) uuid.uuid.uuid128[14] << 48) | - ((uint64_t) uuid.uuid.uuid128[13] << 40) | ((uint64_t) uuid.uuid.uuid128[12] << 32) | - ((uint64_t) uuid.uuid.uuid128[11] << 24) | ((uint64_t) uuid.uuid.uuid128[10] << 16) | - ((uint64_t) uuid.uuid.uuid128[9] << 8) | ((uint64_t) uuid.uuid.uuid128[8]), - ((uint64_t) uuid.uuid.uuid128[7] << 56) | ((uint64_t) uuid.uuid.uuid128[6] << 48) | - ((uint64_t) uuid.uuid.uuid128[5] << 40) | ((uint64_t) uuid.uuid.uuid128[4] << 32) | - ((uint64_t) uuid.uuid.uuid128[3] << 24) | ((uint64_t) uuid.uuid.uuid128[2] << 16) | - ((uint64_t) uuid.uuid.uuid128[1] << 8) | ((uint64_t) uuid.uuid.uuid128[0])}; + out[0] = ((uint64_t) uuid.uuid.uuid128[15] << 56) | ((uint64_t) uuid.uuid.uuid128[14] << 48) | + ((uint64_t) uuid.uuid.uuid128[13] << 40) | ((uint64_t) uuid.uuid.uuid128[12] << 32) | + ((uint64_t) uuid.uuid.uuid128[11] << 24) | ((uint64_t) uuid.uuid.uuid128[10] << 16) | + ((uint64_t) uuid.uuid.uuid128[9] << 8) | ((uint64_t) uuid.uuid.uuid128[8]); + out[1] = ((uint64_t) uuid.uuid.uuid128[7] << 56) | ((uint64_t) uuid.uuid.uuid128[6] << 48) | + ((uint64_t) uuid.uuid.uuid128[5] << 40) | ((uint64_t) uuid.uuid.uuid128[4] << 32) | + ((uint64_t) uuid.uuid.uuid128[3] << 24) | ((uint64_t) uuid.uuid.uuid128[2] << 16) | + ((uint64_t) uuid.uuid.uuid128[1] << 8) | ((uint64_t) uuid.uuid.uuid128[0]); } void BluetoothConnection::dump_config() { @@ -95,9 +95,8 @@ void BluetoothConnection::send_service_for_discovery_() { api::BluetoothGATTGetServicesResponse resp; resp.address = this->address_; - resp.services.emplace_back(); - auto &service_resp = resp.services.back(); - service_resp.uuid = get_128bit_uuid_vec(service_result.uuid); + auto &service_resp = resp.services[0]; + fill_128bit_uuid_array(service_resp.uuid, service_result.uuid); service_resp.handle = service_result.start_handle; // Get the number of characteristics directly with one call @@ -136,7 +135,7 @@ void BluetoothConnection::send_service_for_discovery_() { service_resp.characteristics.emplace_back(); auto &characteristic_resp = service_resp.characteristics.back(); - characteristic_resp.uuid = get_128bit_uuid_vec(char_result.uuid); + fill_128bit_uuid_array(characteristic_resp.uuid, char_result.uuid); characteristic_resp.handle = char_result.char_handle; characteristic_resp.properties = char_result.properties; char_offset++; @@ -176,7 +175,7 @@ void BluetoothConnection::send_service_for_discovery_() { characteristic_resp.descriptors.emplace_back(); auto &descriptor_resp = characteristic_resp.descriptors.back(); - descriptor_resp.uuid = get_128bit_uuid_vec(desc_result.uuid); + fill_128bit_uuid_array(descriptor_resp.uuid, desc_result.uuid); descriptor_resp.handle = desc_result.handle; desc_offset++; } diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 2678b7009a..2d49ac5ece 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -327,6 +327,9 @@ def create_field_type_info( ) -> TypeInfo: """Create the appropriate TypeInfo instance for a field, handling repeated fields and custom options.""" if field.label == 3: # repeated + # Check if this repeated field has fixed_array_size option + if (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None: + return FixedArrayRepeatedType(field, fixed_size) return RepeatedTypeInfo(field) # Check for fixed_array_size option on bytes fields @@ -593,6 +596,8 @@ class MessageType(TypeInfo): return self._get_simple_size_calculation(name, force, "add_message_object") def get_estimated_size(self) -> int: + # For message types, we can't easily estimate the submessage size without + # access to the actual message definition. This is just a rough estimate. return ( self.calculate_field_id_size() + 16 ) # field ID + 16 bytes estimated submessage @@ -883,6 +888,111 @@ class SInt64Type(TypeInfo): return self.calculate_field_id_size() + 3 # field ID + 3 bytes typical varint +class FixedArrayRepeatedType(TypeInfo): + """Special type for fixed-size repeated fields using std::array. + + Fixed arrays are only supported for encoding (SOURCE_SERVER) since we cannot + control how many items we receive when decoding. + """ + + def __init__(self, field: descriptor.FieldDescriptorProto, size: int) -> None: + super().__init__(field) + self.array_size = size + # Create the element type info + validate_field_type(field.type, field.name) + self._ti: TypeInfo = TYPE_INFO[field.type](field) + + @property + def cpp_type(self) -> str: + return f"std::array<{self._ti.cpp_type}, {self.array_size}>" + + @property + def reference_type(self) -> str: + return f"{self.cpp_type} &" + + @property + def const_reference_type(self) -> str: + return f"const {self.cpp_type} &" + + @property + def wire_type(self) -> WireType: + """Get the wire type for this fixed array field.""" + return self._ti.wire_type + + @property + def public_content(self) -> list[str]: + # Just the array member, no index needed since we don't decode + return [f"{self.cpp_type} {self.field_name}{{}};"] + + # No decode methods needed - fixed arrays don't support decoding + # The base class TypeInfo already returns None for all decode properties + + @property + def encode_content(self) -> str: + # Helper to generate encode statement for a single element + def encode_element(element: str) -> str: + if isinstance(self._ti, EnumType): + return f"buffer.{self._ti.encode_func}({self.number}, static_cast({element}), true);" + else: + return f"buffer.{self._ti.encode_func}({self.number}, {element}, true);" + + # Unroll small arrays for efficiency + if self.array_size == 1: + return encode_element(f"this->{self.field_name}[0]") + elif self.array_size == 2: + return ( + encode_element(f"this->{self.field_name}[0]") + + "\n " + + encode_element(f"this->{self.field_name}[1]") + ) + + # Use loops for larger arrays + o = f"for (const auto &it : this->{self.field_name}) {{\n" + o += f" {encode_element('it')}\n" + o += "}" + return o + + @property + def dump_content(self) -> str: + o = f"for (const auto &it : this->{self.field_name}) {{\n" + o += f' out.append(" {self.name}: ");\n' + o += indent(self._ti.dump("it")) + "\n" + o += ' out.append("\\n");\n' + o += "}\n" + return o + + def dump(self, name: str) -> str: + # This is used when dumping the array itself (not its elements) + # Since dump_content handles the iteration, this is not used directly + return "" + + def get_size_calculation(self, name: str, force: bool = False) -> str: + # For fixed arrays, we always encode all elements + + # Special case for single-element arrays - no loop needed + if self.array_size == 1: + return self._ti.get_size_calculation(f"{name}[0]", True) + + # Special case for 2-element arrays - unroll the calculation + if self.array_size == 2: + return ( + self._ti.get_size_calculation(f"{name}[0]", True) + + "\n " + + self._ti.get_size_calculation(f"{name}[1]", True) + ) + + # Use loops for larger arrays + o = f"for (const auto &it : {name}) {{\n" + o += f" {self._ti.get_size_calculation('it', True)}\n" + o += "}" + return o + + def get_estimated_size(self) -> int: + # For fixed arrays, estimate underlying type size * array size + underlying_size = self._ti.get_estimated_size() + return underlying_size * self.array_size + + class RepeatedTypeInfo(TypeInfo): def __init__(self, field: descriptor.FieldDescriptorProto) -> None: super().__init__(field) @@ -1311,6 +1421,19 @@ def build_message_type( if field.options.deprecated: continue + # Validate that fixed_array_size is only used in encode-only messages + if ( + needs_decode + and field.label == 3 + and get_field_opt(field, pb.fixed_array_size) is not None + ): + raise ValueError( + f"Message '{desc.name}' uses fixed_array_size on field '{field.name}' " + f"but has source={SOURCE_NAMES[source]}. " + f"Fixed arrays are only supported for SOURCE_SERVER (encode-only) messages " + f"since we cannot trust or control the number of items received from clients." + ) + ti = create_field_type_info(field, needs_decode, needs_encode) # Skip field declarations for fields that are in the base class @@ -1500,6 +1623,12 @@ SOURCE_BOTH = 0 SOURCE_SERVER = 1 SOURCE_CLIENT = 2 +SOURCE_NAMES = { + SOURCE_BOTH: "SOURCE_BOTH", + SOURCE_SERVER: "SOURCE_SERVER", + SOURCE_CLIENT: "SOURCE_CLIENT", +} + RECEIVE_CASES: dict[int, tuple[str, str | None]] = {} ifdefs: dict[str, str] = {}