Reduce API flash usage by eliminating unnecessary template instantiations (#9452)

esphome/components/api/proto.h

@@ -59,7 +59,6 @@ class ProtoVarInt {
   uint32_t as_uint32() const { return this->value_; }
   uint64_t as_uint64() const { return this->value_; }
   bool as_bool() const { return this->value_; }
-  template<typename T> T as_enum() const { return static_cast<T>(this->as_uint32()); }
   int32_t as_int32() const {
     // Not ZigZag encoded
     return static_cast<int32_t>(this->as_int64());
@@ -133,15 +132,24 @@ class ProtoVarInt {
   uint64_t value_;
 };
 
+// Forward declaration for decode_to_message and encode_to_writer
+class ProtoMessage;
+
 class ProtoLengthDelimited {
  public:
   explicit ProtoLengthDelimited(const uint8_t *value, size_t length) : value_(value), length_(length) {}
   std::string as_string() const { return std::string(reinterpret_cast<const char *>(this->value_), this->length_); }
-  template<class C> C as_message() const {
-    auto msg = C();
-    msg.decode(this->value_, this->length_);
-    return msg;
-  }
+
+  /**
+   * Decode the length-delimited data into an existing ProtoMessage instance.
+   *
+   * This method allows decoding without templates, enabling use in contexts
+   * where the message type is not known at compile time. The ProtoMessage's
+   * decode() method will be called with the raw data and length.
+   *
+   * @param msg The ProtoMessage instance to decode into
+   */
+  void decode_to_message(ProtoMessage &msg) const;
 
  protected:
   const uint8_t *const value_;
@@ -263,9 +271,6 @@ class ProtoWriteBuffer {
     this->write((value >> 48) & 0xFF);
     this->write((value >> 56) & 0xFF);
   }
-  template<typename T> void encode_enum(uint32_t field_id, T value, bool force = false) {
-    this->encode_uint32(field_id, static_cast<uint32_t>(value), force);
-  }
   void encode_float(uint32_t field_id, float value, bool force = false) {
     if (value == 0.0f && !force)
       return;
@@ -306,18 +311,7 @@ class ProtoWriteBuffer {
     }
     this->encode_uint64(field_id, uvalue, force);
   }
-  template<class C> void encode_message(uint32_t field_id, const C &value, bool force = false) {
-    this->encode_field_raw(field_id, 2);  // type 2: Length-delimited message
-    size_t begin = this->buffer_->size();
-
-    value.encode(*this);
-
-    const uint32_t nested_length = this->buffer_->size() - begin;
-    // add size varint
-    std::vector<uint8_t> var;
-    ProtoVarInt(nested_length).encode(var);
-    this->buffer_->insert(this->buffer_->begin() + begin, var.begin(), var.end());
-  }
+  void encode_message(uint32_t field_id, const ProtoMessage &value, bool force = false);
   std::vector<uint8_t> *get_buffer() const { return buffer_; }
 
  protected:
@@ -345,6 +339,25 @@ class ProtoMessage {
   virtual bool decode_64bit(uint32_t field_id, Proto64Bit value) { return false; }
 };
 
+// Implementation of encode_message - must be after ProtoMessage is defined
+inline void ProtoWriteBuffer::encode_message(uint32_t field_id, const ProtoMessage &value, bool force) {
+  this->encode_field_raw(field_id, 2);  // type 2: Length-delimited message
+  size_t begin = this->buffer_->size();
+
+  value.encode(*this);
+
+  const uint32_t nested_length = this->buffer_->size() - begin;
+  // add size varint
+  std::vector<uint8_t> var;
+  ProtoVarInt(nested_length).encode(var);
+  this->buffer_->insert(this->buffer_->begin() + begin, var.begin(), var.end());
+}
+
+// Implementation of decode_to_message - must be after ProtoMessage is defined
+inline void ProtoLengthDelimited::decode_to_message(ProtoMessage &msg) const {
+  msg.decode(this->value_, this->length_);
+}
+
 template<typename T> const char *proto_enum_to_string(T value);
 
 class ProtoService {

script/api_protobuf/api_protobuf.py

@@ -536,11 +536,26 @@ class MessageType(TypeInfo):
 
     @property
     def encode_func(self) -> str:
-        return f"encode_message<{self.cpp_type}>"
+        return "encode_message"
 
     @property
     def decode_length(self) -> str:
-        return f"value.as_message<{self.cpp_type}>()"
+        # Override to return None for message types because we can't use template-based
+        # decoding when the specific message type isn't known at compile time.
+        # Instead, we use the non-template decode_to_message() method which allows
+        # runtime polymorphism through virtual function calls.
+        return None
+
+    @property
+    def decode_length_content(self) -> str:
+        # Custom decode that doesn't use templates
+        return dedent(
+            f"""\
+        case {self.number}: {{
+          value.decode_to_message(this->{self.field_name});
+          return true;
+        }}"""
+        )
 
     def dump(self, name: str) -> str:
         o = f"{name}.dump_to(out);"
@@ -608,14 +623,18 @@ class EnumType(TypeInfo):
 
     @property
     def decode_varint(self) -> str:
-        return f"value.as_enum<{self.cpp_type}>()"
+        return f"static_cast<{self.cpp_type}>(value.as_uint32())"
 
     default_value = ""
     wire_type = WireType.VARINT  # Uses wire type 0
 
     @property
     def encode_func(self) -> str:
-        return f"encode_enum<{self.cpp_type}>"
+        return "encode_uint32"
+
+    @property
+    def encode_content(self) -> str:
+        return f"buffer.{self.encode_func}({self.number}, static_cast<uint32_t>(this->{self.field_name}));"
 
     def dump(self, name: str) -> str:
         o = f"out.append(proto_enum_to_string<{self.cpp_type}>({name}));"
@@ -757,6 +776,16 @@ class RepeatedTypeInfo(TypeInfo):
     @property
     def decode_length_content(self) -> str:
         content = self._ti.decode_length
+        if content is None and isinstance(self._ti, MessageType):
+            # Special handling for non-template message decoding
+            return dedent(
+                f"""\
+        case {self.number}: {{
+          this->{self.field_name}.emplace_back();
+          value.decode_to_message(this->{self.field_name}.back());
+          return true;
+        }}"""
+            )
         if content is None:
             return None
         return dedent(
@@ -801,6 +830,9 @@ class RepeatedTypeInfo(TypeInfo):
     @property
     def encode_content(self) -> str:
         o = f"for (auto {'' if self._ti_is_bool else '&'}it : this->{self.field_name}) {{\n"
+        if isinstance(self._ti, EnumType):
+            o += f"  buffer.{self._ti.encode_func}({self.number}, static_cast<uint32_t>(it), true);\n"
+        else:
             o += f"  buffer.{self._ti.encode_func}({self.number}, it, true);\n"
         o += "}"
         return o