jeans/esphome
jeans/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-08-09 11:57:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'scheduler_copy' into integration

Browse Source
This commit is contained in:
J. Nick Koston 2025-06-28 15:52:09 -05:00
commit 3d08eae8e4
No known key found for this signature in database
9 changed files with 249 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
esphome/components/api/api_connection.cpp
View File

@ -94,6 +94,19 @@ APIConnection::~APIConnection() {
#endif
}
#ifdef HAS_PROTO_MESSAGE_DUMP
void APIConnection::log_batch_item_(const DeferredBatch::BatchItem &item) {
// Set log-only mode
this->log_only_mode_ = true;
// Call the creator - it will create the message and log it via encode_message_to_buffer
item.creator(item.entity, this, std::numeric_limits<uint16_t>::max(), true, item.message_type);
// Clear log-only mode
this->log_only_mode_ = false;
}
#endif
void APIConnection::loop() {
if (this->next_close_) {
// requested a disconnect
@ -249,6 +262,14 @@ void APIConnection::on_disconnect_response(const DisconnectResponse &value) {
// including header and footer overhead. Returns 0 if the message doesn't fit.
uint16_t APIConnection::encode_message_to_buffer(ProtoMessage &msg, uint16_t message_type, APIConnection *conn,
uint32_t remaining_size, bool is_single) {
#ifdef HAS_PROTO_MESSAGE_DUMP
// If in log-only mode, just log and return
if (conn->log_only_mode_) {
conn->log_send_message_(msg.message_name(), msg.dump());
return 1; // Return non-zero to indicate "success" for logging
}
#endif
// Calculate size
uint32_t calculated_size = 0;
msg.calculate_size(calculated_size);
@ -276,11 +297,6 @@ uint16_t APIConnection::encode_message_to_buffer(ProtoMessage &msg, uint16_t mes
// Encode directly into buffer
msg.encode(buffer);
#ifdef HAS_PROTO_MESSAGE_DUMP
// Log the message for VV debugging
conn->log_send_message_(msg.message_name(), msg.dump());
#endif
// Calculate actual encoded size (not including header that was already added)
size_t actual_payload_size = shared_buf.size() - size_before_encode;
@ -1891,6 +1907,15 @@ void APIConnection::process_batch_() {
}
}
#ifdef HAS_PROTO_MESSAGE_DUMP
// Log messages after send attempt for VV debugging
// It's safe to use the buffer for logging at this point regardless of send result
for (size_t i = 0; i < items_processed; i++) {
const auto &item = this->deferred_batch_.items[i];
this->log_batch_item_(item);
}
#endif
// Handle remaining items more efficiently
if (items_processed < this->deferred_batch_.items.size()) {
// Remove processed items from the beginning

8
esphome/components/api/api_connection.h
View File

@ -470,6 +470,10 @@ class APIConnection : public APIServerConnection {
bool sent_ping_{false};
bool service_call_subscription_{false};
bool next_close_ = false;
#ifdef HAS_PROTO_MESSAGE_DUMP
// When true, encode_message_to_buffer will only log, not encode
bool log_only_mode_{false};
#endif
uint8_t ping_retries_{0};
// 8 bytes used, no padding needed
@ -627,6 +631,10 @@ class APIConnection : public APIServerConnection {
// State for batch buffer allocation
bool batch_first_message_{false};
#ifdef HAS_PROTO_MESSAGE_DUMP
void log_batch_item_(const DeferredBatch::BatchItem &item);
#endif
// Helper function to schedule a deferred message with known message type
bool schedule_message_(EntityBase *entity, MessageCreator creator, uint16_t message_type) {
this->deferred_batch_.add_item(entity, std::move(creator), message_type);

4
esphome/core/datatypes.h
View File

@ -11,7 +11,7 @@ namespace internal {
/// Wrapper class for memory using big endian data layout, transparently converting it to native order.
template<typename T> class BigEndianLayout {
public:
constexpr14 operator T() { return convert_big_endian(val_); }
constexpr operator T() { return convert_big_endian(val_); }
private:
T val_;
@ -20,7 +20,7 @@ template<typename T> class BigEndianLayout {
/// Wrapper class for memory using big endian data layout, transparently converting it to native order.
template<typename T> class LittleEndianLayout {
public:
constexpr14 operator T() { return convert_little_endian(val_); }
constexpr operator T() { return convert_little_endian(val_); }
private:
T val_;

15
esphome/core/helpers.cpp
View File

@ -76,23 +76,8 @@ static const uint16_t CRC16_1021_BE_LUT_H[] = {0x0000, 0x1231, 0x2462, 0x3653, 0
0x9188, 0x83b9, 0xb5ea, 0xa7db, 0xd94c, 0xcb7d, 0xfd2e, 0xef1f};
#endif
// STL backports
#if _GLIBCXX_RELEASE < 8
std::string to_string(int value) { return str_snprintf("%d", 32, value); } // NOLINT
std::string to_string(long value) { return str_snprintf("%ld", 32, value); } // NOLINT
std::string to_string(long long value) { return str_snprintf("%lld", 32, value); } // NOLINT
std::string to_string(unsigned value) { return str_snprintf("%u", 32, value); } // NOLINT
std::string to_string(unsigned long value) { return str_snprintf("%lu", 32, value); } // NOLINT
std::string to_string(unsigned long long value) { return str_snprintf("%llu", 32, value); } // NOLINT
std::string to_string(float value) { return str_snprintf("%f", 32, value); }
std::string to_string(double value) { return str_snprintf("%f", 32, value); }
std::string to_string(long double value) { return str_snprintf("%Lf", 32, value); }
#endif
// Mathematics
float lerp(float completion, float start, float end) { return start + (end - start) * completion; }
uint8_t crc8(const uint8_t *data, uint8_t len) {
uint8_t crc = 0;

109
esphome/core/helpers.h
View File

@ -37,89 +37,18 @@
#define ESPHOME_ALWAYS_INLINE __attribute__((always_inline))
#define PACKED __attribute__((packed))
// Various functions can be constexpr in C++14, but not in C++11 (because their body isn't just a return statement).
// Define a substitute constexpr keyword for those functions, until we can drop C++11 support.
#if __cplusplus >= 201402L
#define constexpr14 constexpr
#else
#define constexpr14 inline // constexpr implies inline
#endif
namespace esphome {
/// @name STL backports
///@{
// Backports for various STL features we like to use. Pull in the STL implementation wherever available, to avoid
// ambiguity and to provide a uniform API.
// std::to_string() from C++11, available from libstdc++/g++ 8
// See https://github.com/espressif/esp-idf/issues/1445
#if _GLIBCXX_RELEASE >= 8
// Keep "using" even after the removal of our backports, to avoid breaking existing code.
using std::to_string;
#else
std::string to_string(int value); // NOLINT
std::string to_string(long value); // NOLINT
std::string to_string(long long value); // NOLINT
std::string to_string(unsigned value); // NOLINT
std::string to_string(unsigned long value); // NOLINT
std::string to_string(unsigned long long value); // NOLINT
std::string to_string(float value);
std::string to_string(double value);
std::string to_string(long double value);
#endif
// std::is_trivially_copyable from C++11, implemented in libstdc++/g++ 5.1 (but minor releases can't be detected)
#if _GLIBCXX_RELEASE >= 6
using std::is_trivially_copyable;
#else
// Implementing this is impossible without compiler intrinsics, so don't bother. Invalid usage will be detected on
// other variants that use a newer compiler anyway.
// NOLINTNEXTLINE(readability-identifier-naming)
template<typename T> struct is_trivially_copyable : public std::integral_constant<bool, true> {};
#endif
// std::make_unique() from C++14
#if __cpp_lib_make_unique >= 201304
using std::make_unique;
#else
template<typename T, typename... Args> std::unique_ptr<T> make_unique(Args &&...args) {
return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
}
#endif
// std::enable_if_t from C++14
#if __cplusplus >= 201402L
using std::enable_if_t;
#else
template<bool B, class T = void> using enable_if_t = typename std::enable_if<B, T>::type;
#endif
// std::clamp from C++17
#if __cpp_lib_clamp >= 201603
using std::clamp;
#else
template<typename T, typename Compare> constexpr const T &clamp(const T &v, const T &lo, const T &hi, Compare comp) {
return comp(v, lo) ? lo : comp(hi, v) ? hi : v;
}
template<typename T> constexpr const T &clamp(const T &v, const T &lo, const T &hi) {
return clamp(v, lo, hi, std::less<T>{});
}
#endif
// std::is_invocable from C++17
#if __cpp_lib_is_invocable >= 201703
using std::is_invocable;
#else
// https://stackoverflow.com/a/37161919/8924614
template<class T, class... Args> struct is_invocable { // NOLINT(readability-identifier-naming)
template<class U> static auto test(U *p) -> decltype((*p)(std::declval<Args>()...), void(), std::true_type());
template<class U> static auto test(...) -> decltype(std::false_type());
static constexpr auto value = decltype(test<T>(nullptr))::value; // NOLINT
};
#endif
// std::bit_cast from C++20
#if __cpp_lib_bit_cast >= 201806
using std::bit_cast;
#else
@ -134,31 +63,29 @@ To bit_cast(const From &src) {
return dst;
}
#endif
using std::lerp;
// std::byteswap from C++23
template<typename T> constexpr14 T byteswap(T n) {
template<typename T> constexpr T byteswap(T n) {
T m;
for (size_t i = 0; i < sizeof(T); i++)
reinterpret_cast<uint8_t *>(&m)[i] = reinterpret_cast<uint8_t *>(&n)[sizeof(T) - 1 - i];
return m;
}
template<> constexpr14 uint8_t byteswap(uint8_t n) { return n; }
template<> constexpr14 uint16_t byteswap(uint16_t n) { return __builtin_bswap16(n); }
template<> constexpr14 uint32_t byteswap(uint32_t n) { return __builtin_bswap32(n); }
template<> constexpr14 uint64_t byteswap(uint64_t n) { return __builtin_bswap64(n); }
template<> constexpr14 int8_t byteswap(int8_t n) { return n; }
template<> constexpr14 int16_t byteswap(int16_t n) { return __builtin_bswap16(n); }
template<> constexpr14 int32_t byteswap(int32_t n) { return __builtin_bswap32(n); }
template<> constexpr14 int64_t byteswap(int64_t n) { return __builtin_bswap64(n); }
template<> constexpr uint8_t byteswap(uint8_t n) { return n; }
template<> constexpr uint16_t byteswap(uint16_t n) { return __builtin_bswap16(n); }
template<> constexpr uint32_t byteswap(uint32_t n) { return __builtin_bswap32(n); }
template<> constexpr uint64_t byteswap(uint64_t n) { return __builtin_bswap64(n); }
template<> constexpr int8_t byteswap(int8_t n) { return n; }
template<> constexpr int16_t byteswap(int16_t n) { return __builtin_bswap16(n); }
template<> constexpr int32_t byteswap(int32_t n) { return __builtin_bswap32(n); }
template<> constexpr int64_t byteswap(int64_t n) { return __builtin_bswap64(n); }
///@}
/// @name Mathematics
///@{
/// Linearly interpolate between \p start and \p end by \p completion (between 0 and 1).
float lerp(float completion, float start, float end);
/// Remap \p value from the range (\p min, \p max) to (\p min_out, \p max_out).
template<typename T, typename U> T remap(U value, U min, U max, T min_out, T max_out) {
return (value - min) * (max_out - min_out) / (max - min) + min_out;
@ -203,8 +130,7 @@ constexpr uint32_t encode_uint32(uint8_t byte1, uint8_t byte2, uint8_t byte3, ui
}
/// Encode a value from its constituent bytes (from most to least significant) in an array with length sizeof(T).
template<typename T, enable_if_t<std::is_unsigned<T>::value, int> = 0>
constexpr14 T encode_value(const uint8_t *bytes) {
template<typename T, enable_if_t<std::is_unsigned<T>::value, int> = 0> constexpr T encode_value(const uint8_t *bytes) {
T val = 0;
for (size_t i = 0; i < sizeof(T); i++) {
val <<= 8;
@ -214,12 +140,12 @@ constexpr14 T encode_value(const uint8_t *bytes) {
}
/// Encode a value from its constituent bytes (from most to least significant) in an std::array with length sizeof(T).
template<typename T, enable_if_t<std::is_unsigned<T>::value, int> = 0>
constexpr14 T encode_value(const std::array<uint8_t, sizeof(T)> bytes) {
constexpr T encode_value(const std::array<uint8_t, sizeof(T)> bytes) {
return encode_value<T>(bytes.data());
}
/// Decode a value into its constituent bytes (from most to least significant).
template<typename T, enable_if_t<std::is_unsigned<T>::value, int> = 0>
constexpr14 std::array<uint8_t, sizeof(T)> decode_value(T val) {
constexpr std::array<uint8_t, sizeof(T)> decode_value(T val) {
std::array<uint8_t, sizeof(T)> ret{};
for (size_t i = sizeof(T); i > 0; i--) {
ret[i - 1] = val & 0xFF;
@ -246,7 +172,7 @@ inline uint32_t reverse_bits(uint32_t x) {
}
/// Convert a value between host byte order and big endian (most significant byte first) order.
template<typename T> constexpr14 T convert_big_endian(T val) {
template<typename T> constexpr T convert_big_endian(T val) {
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
return byteswap(val);
#else
@ -255,7 +181,7 @@ template<typename T> constexpr14 T convert_big_endian(T val) {
}
/// Convert a value between host byte order and little endian (least significant byte first) order.
template<typename T> constexpr14 T convert_little_endian(T val) {
template<typename T> constexpr T convert_little_endian(T val) {
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
return val;
#else
@ -276,9 +202,6 @@ bool str_startswith(const std::string &str, const std::string &start);
/// Check whether a string ends with a value.
bool str_endswith(const std::string &str, const std::string &end);
/// Convert the value to a string (added as extra overload so that to_string() can be used on all stringifiable types).
inline std::string to_string(const std::string &val) { return val; }
/// Truncate a string to a specific length.
std::string str_truncate(const std::string &str, size_t length);

89
tests/integration/fixtures/api_vv_logging.yaml Normal file
View File

@ -0,0 +1,89 @@
esphome:
name: vv-logging-test
host:
api:
logger:
level: VERY_VERBOSE
# Enable VV logging for API components where the issue occurs
logs:
api.connection: VERY_VERBOSE
api.service: VERY_VERBOSE
api.proto: VERY_VERBOSE
sensor: VERY_VERBOSE
# Create many sensors that update frequently to generate API traffic
# This will cause many messages to be batched and sent, triggering the
# code path where VV logging could cause buffer corruption
sensor:
- platform: template
name: "Test Sensor 1"
lambda: 'return millis() / 1000.0;'
update_interval: 50ms
unit_of_measurement: "s"
- platform: template
name: "Test Sensor 2"
lambda: 'return (millis() / 1000.0) + 10;'
update_interval: 50ms
unit_of_measurement: "s"
- platform: template
name: "Test Sensor 3"
lambda: 'return (millis() / 1000.0) + 20;'
update_interval: 50ms
unit_of_measurement: "s"
- platform: template
name: "Test Sensor 4"
lambda: 'return (millis() / 1000.0) + 30;'
update_interval: 50ms
unit_of_measurement: "s"
- platform: template
name: "Test Sensor 5"
lambda: 'return (millis() / 1000.0) + 40;'
update_interval: 50ms
unit_of_measurement: "s"
- platform: template
name: "Test Sensor 6"
lambda: 'return (millis() / 1000.0) + 50;'
update_interval: 50ms
unit_of_measurement: "s"
- platform: template
name: "Test Sensor 7"
lambda: 'return (millis() / 1000.0) + 60;'
update_interval: 50ms
unit_of_measurement: "s"
- platform: template
name: "Test Sensor 8"
lambda: 'return (millis() / 1000.0) + 70;'
update_interval: 50ms
unit_of_measurement: "s"
- platform: template
name: "Test Sensor 9"
lambda: 'return (millis() / 1000.0) + 80;'
update_interval: 50ms
unit_of_measurement: "s"
- platform: template
name: "Test Sensor 10"
lambda: 'return (millis() / 1000.0) + 90;'
update_interval: 50ms
unit_of_measurement: "s"
# Add some binary sensors too for variety
binary_sensor:
- platform: template
name: "Test Binary 1"
lambda: 'return (millis() / 1000) % 2 == 0;'
- platform: template
name: "Test Binary 2"
lambda: 'return (millis() / 1000) % 3 == 0;'

14
tests/integration/fixtures/scheduler_string_test.yaml
View File

@ -67,20 +67,28 @@ script:
ESP_LOGI("test", "Empty string timeout fired");
});
// Test 5: Cancel timeout with const char* literal
App.scheduler.set_timeout(component1, "cancel_static_timeout", 5000, []() {
ESP_LOGI("test", "This static timeout should be cancelled");
});
// Cancel using const char* directly
App.scheduler.cancel_timeout(component1, "cancel_static_timeout");
ESP_LOGI("test", "Cancelled static timeout using const char*");
- id: test_dynamic_strings
then:
- logger.log: "Testing dynamic string timeouts and intervals"
- lambda: |-
auto *component2 = id(test_sensor2);
// Test 5: Dynamic string with set_timeout (std::string)
// Test 6: Dynamic string with set_timeout (std::string)
std::string dynamic_name = "dynamic_timeout_" + std::to_string(id(dynamic_counter)++);
App.scheduler.set_timeout(component2, dynamic_name, 100, []() {
ESP_LOGI("test", "Dynamic timeout fired");
id(timeout_counter) += 1;
});
// Test 6: Dynamic string with set_interval
// Test 7: Dynamic string with set_interval
std::string interval_name = "dynamic_interval_" + std::to_string(id(dynamic_counter)++);
App.scheduler.set_interval(component2, interval_name, 250, [interval_name]() {
ESP_LOGI("test", "Dynamic interval fired: %s", interval_name.c_str());
@ -91,7 +99,7 @@ script:
}
});
// Test 7: Cancel with different string object but same content
// Test 8: Cancel with different string object but same content
std::string cancel_name = "cancel_test";
App.scheduler.set_timeout(component2, cancel_name, 2000, []() {
ESP_LOGI("test", "This should be cancelled");

83
tests/integration/test_api_vv_logging.py Normal file
View File

@ -0,0 +1,83 @@
"""Integration test for API with VERY_VERBOSE logging to verify no buffer corruption."""
from __future__ import annotations
import asyncio
from typing import Any
from aioesphomeapi import LogLevel
import pytest
from .types import APIClientConnectedFactory, RunCompiledFunction
@pytest.mark.asyncio
async def test_api_vv_logging(
yaml_config: str,
run_compiled: RunCompiledFunction,
api_client_connected: APIClientConnectedFactory,
) -> None:
"""Test that VERY_VERBOSE logging doesn't cause buffer corruption with API messages."""
# Track that we're receiving VV log messages and sensor updates
vv_logs_received = 0
sensor_updates_received = 0
errors_detected = []
def on_log(msg: Any) -> None:
"""Capture log messages."""
nonlocal vv_logs_received
# msg is a SubscribeLogsResponse object with 'message' attribute
# The message field is always bytes
message_text = msg.message.decode("utf-8", errors="replace")
# Only count VV logs specifically
if "[VV]" in message_text:
vv_logs_received += 1
# Check for assertion or error messages
if "assert" in message_text.lower() or "error" in message_text.lower():
errors_detected.append(message_text)
# Write, compile and run the ESPHome device
async with run_compiled(yaml_config), api_client_connected() as client:
# Subscribe to VERY_VERBOSE logs - this enables the code path that could cause corruption
client.subscribe_logs(on_log, log_level=LogLevel.LOG_LEVEL_VERY_VERBOSE)
# Wait for device to be ready
device_info = await client.device_info()
assert device_info is not None
assert device_info.name == "vv-logging-test"
# Subscribe to sensor states
states = {}
def on_state(state):
nonlocal sensor_updates_received
sensor_updates_received += 1
states[state.key] = state
client.subscribe_states(on_state)
# List entities to find our test sensors
entity_info, _ = await client.list_entities_services()
# Count sensors
sensor_count = sum(1 for e in entity_info if hasattr(e, "unit_of_measurement"))
assert sensor_count >= 10, f"Expected at least 10 sensors, got {sensor_count}"
# Wait for sensor updates to flow with VV logging active
# The sensors update every 50ms, so we should get many updates
await asyncio.sleep(0.25)
# Verify we received both VV logs and sensor updates
assert vv_logs_received > 0, "Expected to receive VERY_VERBOSE log messages"
assert sensor_updates_received > 10, (
f"Expected many sensor updates, got {sensor_updates_received}"
)
# Check for any errors
if errors_detected:
pytest.fail(f"Errors detected during test: {errors_detected}")
# The test passes if we didn't hit any assertions or buffer corruption

10
tests/integration/test_scheduler_string_test.py
View File

@ -25,6 +25,7 @@ async def test_scheduler_string_test(
static_interval_fired = asyncio.Event()
static_interval_cancelled = asyncio.Event()
empty_string_timeout_fired = asyncio.Event()
static_timeout_cancelled = asyncio.Event()
dynamic_timeout_fired = asyncio.Event()
dynamic_interval_fired = asyncio.Event()
cancel_test_done = asyncio.Event()
@ -67,6 +68,10 @@ async def test_scheduler_string_test(
elif "Empty string timeout fired" in clean_line:
empty_string_timeout_fired.set()
# Check for static timeout cancellation
elif "Cancelled static timeout using const char*" in clean_line:
static_timeout_cancelled.set()
# Check for dynamic string tests
elif "Dynamic timeout fired" in clean_line:
dynamic_timeout_fired.set()
@ -123,6 +128,11 @@ async def test_scheduler_string_test(
f"Expected static interval to run at least 3 times, got {static_interval_count + 1}"
)
# Verify static timeout was cancelled
assert static_timeout_cancelled.is_set(), (
"Static timeout should have been cancelled"
)
# Wait for dynamic string tests
try:
await asyncio.wait_for(dynamic_timeout_fired.wait(), timeout=1.0)