Optimize API proto size calculations by removing redundant force parameter (#9449)

This commit is contained in:
J. Nick Koston 2025-07-11 16:08:52 -10:00 committed by Jesse Hills
parent 1b222ceca3
commit 7a6894e087
No known key found for this signature in database
GPG Key ID: BEAAE804EFD8E83A
3 changed files with 953 additions and 804 deletions

File diff suppressed because it is too large Load Diff

View File

@ -141,9 +141,9 @@ class ProtoSize {
/**
* @brief Calculates and adds the size of an int32 field to the total message size
*/
static inline void add_int32_field(uint32_t &total_size, uint32_t field_id_size, int32_t value, bool force = false) {
// Skip calculation if value is zero and not forced
if (value == 0 && !force) {
static inline void add_int32_field(uint32_t &total_size, uint32_t field_id_size, int32_t value) {
// Skip calculation if value is zero
if (value == 0) {
return; // No need to update total_size
}
@ -157,13 +157,26 @@ class ProtoSize {
}
}
/**
* @brief Calculates and adds the size of an int32 field to the total message size (repeated field version)
*/
static inline void add_int32_field_repeated(uint32_t &total_size, uint32_t field_id_size, int32_t value) {
// Always calculate size for repeated fields
if (value < 0) {
// Negative values are encoded as 10-byte varints in protobuf
total_size += field_id_size + 10;
} else {
// For non-negative values, use the standard varint size
total_size += field_id_size + varint(static_cast<uint32_t>(value));
}
}
/**
* @brief Calculates and adds the size of a uint32 field to the total message size
*/
static inline void add_uint32_field(uint32_t &total_size, uint32_t field_id_size, uint32_t value,
bool force = false) {
// Skip calculation if value is zero and not forced
if (value == 0 && !force) {
static inline void add_uint32_field(uint32_t &total_size, uint32_t field_id_size, uint32_t value) {
// Skip calculation if value is zero
if (value == 0) {
return; // No need to update total_size
}
@ -171,12 +184,20 @@ class ProtoSize {
total_size += field_id_size + varint(value);
}
/**
* @brief Calculates and adds the size of a uint32 field to the total message size (repeated field version)
*/
static inline void add_uint32_field_repeated(uint32_t &total_size, uint32_t field_id_size, uint32_t value) {
// Always calculate size for repeated fields
total_size += field_id_size + varint(value);
}
/**
* @brief Calculates and adds the size of a boolean field to the total message size
*/
static inline void add_bool_field(uint32_t &total_size, uint32_t field_id_size, bool value, bool force = false) {
// Skip calculation if value is false and not forced
if (!value && !force) {
static inline void add_bool_field(uint32_t &total_size, uint32_t field_id_size, bool value) {
// Skip calculation if value is false
if (!value) {
return; // No need to update total_size
}
@ -184,6 +205,15 @@ class ProtoSize {
total_size += field_id_size + 1;
}
/**
* @brief Calculates and adds the size of a boolean field to the total message size (repeated field version)
*/
static inline void add_bool_field_repeated(uint32_t &total_size, uint32_t field_id_size, bool value) {
// Always calculate size for repeated fields
// Boolean fields always use 1 byte
total_size += field_id_size + 1;
}
/**
* @brief Calculates and adds the size of a fixed field to the total message size
*
@ -193,10 +223,9 @@ class ProtoSize {
* @param is_nonzero Whether the value is non-zero
*/
template<uint32_t NumBytes>
static inline void add_fixed_field(uint32_t &total_size, uint32_t field_id_size, bool is_nonzero,
bool force = false) {
// Skip calculation if value is zero and not forced
if (!is_nonzero && !force) {
static inline void add_fixed_field(uint32_t &total_size, uint32_t field_id_size, bool is_nonzero) {
// Skip calculation if value is zero
if (!is_nonzero) {
return; // No need to update total_size
}
@ -209,9 +238,9 @@ class ProtoSize {
*
* Enum fields are encoded as uint32 varints.
*/
static inline void add_enum_field(uint32_t &total_size, uint32_t field_id_size, uint32_t value, bool force = false) {
// Skip calculation if value is zero and not forced
if (value == 0 && !force) {
static inline void add_enum_field(uint32_t &total_size, uint32_t field_id_size, uint32_t value) {
// Skip calculation if value is zero
if (value == 0) {
return; // No need to update total_size
}
@ -219,14 +248,25 @@ class ProtoSize {
total_size += field_id_size + varint(value);
}
/**
* @brief Calculates and adds the size of an enum field to the total message size (repeated field version)
*
* Enum fields are encoded as uint32 varints.
*/
static inline void add_enum_field_repeated(uint32_t &total_size, uint32_t field_id_size, uint32_t value) {
// Always calculate size for repeated fields
// Enums are encoded as uint32
total_size += field_id_size + varint(value);
}
/**
* @brief Calculates and adds the size of a sint32 field to the total message size
*
* Sint32 fields use ZigZag encoding, which is more efficient for negative values.
*/
static inline void add_sint32_field(uint32_t &total_size, uint32_t field_id_size, int32_t value, bool force = false) {
// Skip calculation if value is zero and not forced
if (value == 0 && !force) {
static inline void add_sint32_field(uint32_t &total_size, uint32_t field_id_size, int32_t value) {
// Skip calculation if value is zero
if (value == 0) {
return; // No need to update total_size
}
@ -235,12 +275,24 @@ class ProtoSize {
total_size += field_id_size + varint(zigzag);
}
/**
* @brief Calculates and adds the size of a sint32 field to the total message size (repeated field version)
*
* Sint32 fields use ZigZag encoding, which is more efficient for negative values.
*/
static inline void add_sint32_field_repeated(uint32_t &total_size, uint32_t field_id_size, int32_t value) {
// Always calculate size for repeated fields
// ZigZag encoding for sint32: (n << 1) ^ (n >> 31)
uint32_t zigzag = (static_cast<uint32_t>(value) << 1) ^ (static_cast<uint32_t>(value >> 31));
total_size += field_id_size + varint(zigzag);
}
/**
* @brief Calculates and adds the size of an int64 field to the total message size
*/
static inline void add_int64_field(uint32_t &total_size, uint32_t field_id_size, int64_t value, bool force = false) {
// Skip calculation if value is zero and not forced
if (value == 0 && !force) {
static inline void add_int64_field(uint32_t &total_size, uint32_t field_id_size, int64_t value) {
// Skip calculation if value is zero
if (value == 0) {
return; // No need to update total_size
}
@ -248,13 +300,20 @@ class ProtoSize {
total_size += field_id_size + varint(value);
}
/**
* @brief Calculates and adds the size of an int64 field to the total message size (repeated field version)
*/
static inline void add_int64_field_repeated(uint32_t &total_size, uint32_t field_id_size, int64_t value) {
// Always calculate size for repeated fields
total_size += field_id_size + varint(value);
}
/**
* @brief Calculates and adds the size of a uint64 field to the total message size
*/
static inline void add_uint64_field(uint32_t &total_size, uint32_t field_id_size, uint64_t value,
bool force = false) {
// Skip calculation if value is zero and not forced
if (value == 0 && !force) {
static inline void add_uint64_field(uint32_t &total_size, uint32_t field_id_size, uint64_t value) {
// Skip calculation if value is zero
if (value == 0) {
return; // No need to update total_size
}
@ -262,14 +321,22 @@ class ProtoSize {
total_size += field_id_size + varint(value);
}
/**
* @brief Calculates and adds the size of a uint64 field to the total message size (repeated field version)
*/
static inline void add_uint64_field_repeated(uint32_t &total_size, uint32_t field_id_size, uint64_t value) {
// Always calculate size for repeated fields
total_size += field_id_size + varint(value);
}
/**
* @brief Calculates and adds the size of a sint64 field to the total message size
*
* Sint64 fields use ZigZag encoding, which is more efficient for negative values.
*/
static inline void add_sint64_field(uint32_t &total_size, uint32_t field_id_size, int64_t value, bool force = false) {
// Skip calculation if value is zero and not forced
if (value == 0 && !force) {
static inline void add_sint64_field(uint32_t &total_size, uint32_t field_id_size, int64_t value) {
// Skip calculation if value is zero
if (value == 0) {
return; // No need to update total_size
}
@ -278,13 +345,24 @@ class ProtoSize {
total_size += field_id_size + varint(zigzag);
}
/**
* @brief Calculates and adds the size of a sint64 field to the total message size (repeated field version)
*
* Sint64 fields use ZigZag encoding, which is more efficient for negative values.
*/
static inline void add_sint64_field_repeated(uint32_t &total_size, uint32_t field_id_size, int64_t value) {
// Always calculate size for repeated fields
// ZigZag encoding for sint64: (n << 1) ^ (n >> 63)
uint64_t zigzag = (static_cast<uint64_t>(value) << 1) ^ (static_cast<uint64_t>(value >> 63));
total_size += field_id_size + varint(zigzag);
}
/**
* @brief Calculates and adds the size of a string/bytes field to the total message size
*/
static inline void add_string_field(uint32_t &total_size, uint32_t field_id_size, const std::string &str,
bool force = false) {
// Skip calculation if string is empty and not forced
if (str.empty() && !force) {
static inline void add_string_field(uint32_t &total_size, uint32_t field_id_size, const std::string &str) {
// Skip calculation if string is empty
if (str.empty()) {
return; // No need to update total_size
}
@ -293,18 +371,26 @@ class ProtoSize {
total_size += field_id_size + varint(str_size) + str_size;
}
/**
* @brief Calculates and adds the size of a string/bytes field to the total message size (repeated field version)
*/
static inline void add_string_field_repeated(uint32_t &total_size, uint32_t field_id_size, const std::string &str) {
// Always calculate size for repeated fields
const uint32_t str_size = static_cast<uint32_t>(str.size());
total_size += field_id_size + varint(str_size) + str_size;
}
/**
* @brief Calculates and adds the size of a nested message field to the total message size
*
* This helper function directly updates the total_size reference if the nested size
* is greater than zero or force is true.
* is greater than zero.
*
* @param nested_size The pre-calculated size of the nested message
*/
static inline void add_message_field(uint32_t &total_size, uint32_t field_id_size, uint32_t nested_size,
bool force = false) {
// Skip calculation if nested message is empty and not forced
if (nested_size == 0 && !force) {
static inline void add_message_field(uint32_t &total_size, uint32_t field_id_size, uint32_t nested_size) {
// Skip calculation if nested message is empty
if (nested_size == 0) {
return; // No need to update total_size
}
@ -313,6 +399,17 @@ class ProtoSize {
total_size += field_id_size + varint(nested_size) + nested_size;
}
/**
* @brief Calculates and adds the size of a nested message field to the total message size (repeated field version)
*
* @param nested_size The pre-calculated size of the nested message
*/
static inline void add_message_field_repeated(uint32_t &total_size, uint32_t field_id_size, uint32_t nested_size) {
// Always calculate size for repeated fields
// Field ID + length varint + nested message content
total_size += field_id_size + varint(nested_size) + nested_size;
}
/**
* @brief Calculates and adds the size of a nested message field to the total message size
*
@ -322,13 +419,26 @@ class ProtoSize {
*
* @param message The nested message object
*/
static inline void add_message_object(uint32_t &total_size, uint32_t field_id_size, const ProtoMessage &message,
bool force = false) {
static inline void add_message_object(uint32_t &total_size, uint32_t field_id_size, const ProtoMessage &message) {
uint32_t nested_size = 0;
message.calculate_size(nested_size);
// Use the base implementation with the calculated nested_size
add_message_field(total_size, field_id_size, nested_size, force);
add_message_field(total_size, field_id_size, nested_size);
}
/**
* @brief Calculates and adds the size of a nested message field to the total message size (repeated field version)
*
* @param message The nested message object
*/
static inline void add_message_object_repeated(uint32_t &total_size, uint32_t field_id_size,
const ProtoMessage &message) {
uint32_t nested_size = 0;
message.calculate_size(nested_size);
// Use the base implementation with the calculated nested_size
add_message_field_repeated(total_size, field_id_size, nested_size);
}
/**
@ -348,9 +458,9 @@ class ProtoSize {
return;
}
// For repeated fields, always use force=true
// Use the repeated field version for all messages
for (const auto &message : messages) {
add_message_object(total_size, field_id_size, message, true);
add_message_object_repeated(total_size, field_id_size, message);
}
}
};

View File

@ -249,6 +249,42 @@ class TypeInfo(ABC):
return 4 # 28 bits
return 5 # 32 bits (maximum for uint32_t)
def _get_simple_size_calculation(
self, name: str, force: bool, base_method: str, value_expr: str = None
) -> str:
"""Helper for simple size calculations.
Args:
name: Field name
force: Whether this is for a repeated field
base_method: Base method name (e.g., "add_int32_field")
value_expr: Optional value expression (defaults to name)
"""
field_id_size = self.calculate_field_id_size()
method = f"{base_method}_repeated" if force else base_method
value = value_expr if value_expr else name
return f"ProtoSize::{method}(total_size, {field_id_size}, {value});"
def _get_fixed_size_calculation(
self, name: str, force: bool, num_bytes: int, zero_check: str
) -> str:
"""Helper for fixed-size field calculations.
Args:
name: Field name
force: Whether this is for a repeated field
num_bytes: Number of bytes (4 or 8)
zero_check: Expression to check for zero value (e.g., "!= 0.0f")
"""
field_id_size = self.calculate_field_id_size()
# Fixed-size repeated fields are handled differently in RepeatedTypeInfo
# so we should never get force=True here
assert not force, (
"Fixed-size repeated fields should be handled by RepeatedTypeInfo"
)
method = f"add_fixed_field<{num_bytes}>"
return f"ProtoSize::{method}(total_size, {field_id_size}, {name} {zero_check});"
@abstractmethod
def get_size_calculation(self, name: str, force: bool = False) -> str:
"""Calculate the size needed for encoding this field.
@ -258,6 +294,14 @@ class TypeInfo(ABC):
force: Whether to force encoding the field even if it has a default value
"""
def get_fixed_size_bytes(self) -> int | None:
"""Get the number of bytes for fixed-size fields (float, double, fixed32, etc).
Returns:
The number of bytes (4 or 8) for fixed-size fields, None for variable-size fields.
"""
return None
@abstractmethod
def get_estimated_size(self) -> int:
"""Get estimated size in bytes for this field with typical values.
@ -295,9 +339,10 @@ class DoubleType(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_fixed_field<8>(total_size, {field_id_size}, {name} != 0.0, {force_str(force)});"
return o
return self._get_fixed_size_calculation(name, force, 8, "!= 0.0")
def get_fixed_size_bytes(self) -> int:
return 8
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 8 # field ID + 8 bytes for double
@ -317,9 +362,10 @@ class FloatType(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_fixed_field<4>(total_size, {field_id_size}, {name} != 0.0f, {force_str(force)});"
return o
return self._get_fixed_size_calculation(name, force, 4, "!= 0.0f")
def get_fixed_size_bytes(self) -> int:
return 4
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 4 # field ID + 4 bytes for float
@ -339,9 +385,7 @@ class Int64Type(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_int64_field(total_size, {field_id_size}, {name}, {force_str(force)});"
return o
return self._get_simple_size_calculation(name, force, "add_int64_field")
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 3 # field ID + 3 bytes typical varint
@ -361,9 +405,7 @@ class UInt64Type(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_uint64_field(total_size, {field_id_size}, {name}, {force_str(force)});"
return o
return self._get_simple_size_calculation(name, force, "add_uint64_field")
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 3 # field ID + 3 bytes typical varint
@ -383,9 +425,7 @@ class Int32Type(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_int32_field(total_size, {field_id_size}, {name}, {force_str(force)});"
return o
return self._get_simple_size_calculation(name, force, "add_int32_field")
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 3 # field ID + 3 bytes typical varint
@ -405,9 +445,10 @@ class Fixed64Type(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_fixed_field<8>(total_size, {field_id_size}, {name} != 0, {force_str(force)});"
return o
return self._get_fixed_size_calculation(name, force, 8, "!= 0")
def get_fixed_size_bytes(self) -> int:
return 8
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 8 # field ID + 8 bytes fixed
@ -427,9 +468,10 @@ class Fixed32Type(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_fixed_field<4>(total_size, {field_id_size}, {name} != 0, {force_str(force)});"
return o
return self._get_fixed_size_calculation(name, force, 4, "!= 0")
def get_fixed_size_bytes(self) -> int:
return 4
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 4 # field ID + 4 bytes fixed
@ -448,9 +490,7 @@ class BoolType(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_bool_field(total_size, {field_id_size}, {name}, {force_str(force)});"
return o
return self._get_simple_size_calculation(name, force, "add_bool_field")
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 1 # field ID + 1 byte
@ -471,9 +511,7 @@ class StringType(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_string_field(total_size, {field_id_size}, {name}, {force_str(force)});"
return o
return self._get_simple_size_calculation(name, force, "add_string_field")
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 8 # field ID + 8 bytes typical string
@ -509,9 +547,7 @@ class MessageType(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_message_object(total_size, {field_id_size}, {name}, {force_str(force)});"
return o
return self._get_simple_size_calculation(name, force, "add_message_object")
def get_estimated_size(self) -> int:
return (
@ -538,9 +574,7 @@ class BytesType(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_string_field(total_size, {field_id_size}, {name}, {force_str(force)});"
return o
return self._get_simple_size_calculation(name, force, "add_string_field")
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 8 # field ID + 8 bytes typical bytes
@ -560,9 +594,7 @@ class UInt32Type(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_uint32_field(total_size, {field_id_size}, {name}, {force_str(force)});"
return o
return self._get_simple_size_calculation(name, force, "add_uint32_field")
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 3 # field ID + 3 bytes typical varint
@ -590,9 +622,9 @@ class EnumType(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_enum_field(total_size, {field_id_size}, static_cast<uint32_t>({name}), {force_str(force)});"
return o
return self._get_simple_size_calculation(
name, force, "add_enum_field", f"static_cast<uint32_t>({name})"
)
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 1 # field ID + 1 byte typical enum
@ -612,9 +644,10 @@ class SFixed32Type(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_fixed_field<4>(total_size, {field_id_size}, {name} != 0, {force_str(force)});"
return o
return self._get_fixed_size_calculation(name, force, 4, "!= 0")
def get_fixed_size_bytes(self) -> int:
return 4
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 4 # field ID + 4 bytes fixed
@ -634,9 +667,10 @@ class SFixed64Type(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_fixed_field<8>(total_size, {field_id_size}, {name} != 0, {force_str(force)});"
return o
return self._get_fixed_size_calculation(name, force, 8, "!= 0")
def get_fixed_size_bytes(self) -> int:
return 8
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 8 # field ID + 8 bytes fixed
@ -656,9 +690,7 @@ class SInt32Type(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_sint32_field(total_size, {field_id_size}, {name}, {force_str(force)});"
return o
return self._get_simple_size_calculation(name, force, "add_sint32_field")
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 3 # field ID + 3 bytes typical varint
@ -678,9 +710,7 @@ class SInt64Type(TypeInfo):
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
field_id_size = self.calculate_field_id_size()
o = f"ProtoSize::add_sint64_field(total_size, {field_id_size}, {name}, {force_str(force)});"
return o
return self._get_simple_size_calculation(name, force, "add_sint64_field")
def get_estimated_size(self) -> int:
return self.calculate_field_id_size() + 3 # field ID + 3 bytes typical varint
@ -795,11 +825,23 @@ class RepeatedTypeInfo(TypeInfo):
field_id_size = self._ti.calculate_field_id_size()
o = f"ProtoSize::add_repeated_message(total_size, {field_id_size}, {name});"
return o
# For other repeated types, use the underlying type's size calculation with force=True
o = f"if (!{name}.empty()) {{\n"
o += f" for (const auto {'' if self._ti_is_bool else '&'}it : {name}) {{\n"
o += f" {self._ti.get_size_calculation('it', True)}\n"
o += " }\n"
# Check if this is a fixed-size type by seeing if it has a fixed byte count
num_bytes = self._ti.get_fixed_size_bytes()
if num_bytes is not None:
# Fixed types have constant size per element, so we can multiply
field_id_size = self._ti.calculate_field_id_size()
# Pre-calculate the total bytes per element
bytes_per_element = field_id_size + num_bytes
o += f" total_size += {name}.size() * {bytes_per_element};\n"
else:
# Other types need the actual value
o += f" for (const auto {'' if self._ti_is_bool else '&'}it : {name}) {{\n"
o += f" {self._ti.get_size_calculation('it', True)}\n"
o += " }\n"
o += "}"
return o
@ -1712,7 +1754,6 @@ static const char *const TAG = "api.service";
exec_clang_format(root / "api_pb2_service.cpp")
exec_clang_format(root / "api_pb2.h")
exec_clang_format(root / "api_pb2.cpp")
exec_clang_format(root / "api_pb2_dump.h")
exec_clang_format(root / "api_pb2_dump.cpp")
except ImportError:
pass