From 0046e677273ec264201628f3c92a2406dd759f28 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 19 Jul 2025 15:06:42 -1000
Subject: [PATCH] wip

---
 esphome/components/api/api_frame_helper.cpp | 35 ++++++++++-----------
 esphome/components/api/api_frame_helper.h   |  1 +
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/esphome/components/api/api_frame_helper.cpp b/esphome/components/api/api_frame_helper.cpp
index b7c34bc44f..b99c3e2614 100644
--- a/esphome/components/api/api_frame_helper.cpp
+++ b/esphome/components/api/api_frame_helper.cpp
@@ -299,6 +299,19 @@ APIError APINoiseFrameHelper::init() {
   state_ = State::CLIENT_HELLO;
   return APIError::OK;
 }
+// Helper for handling handshake frame errors
+APIError APINoiseFrameHelper::handle_handshake_frame_error_(APIError aerr) {
+  if (aerr == APIError::BAD_INDICATOR) {
+    send_explicit_handshake_reject_("Bad indicator byte");
+    return aerr;
+  }
+  if (aerr == APIError::BAD_HANDSHAKE_PACKET_LEN) {
+    send_explicit_handshake_reject_("Bad handshake packet len");
+    return aerr;
+  }
+  return aerr;
+}
+
 /// Run through handshake messages (if in that phase)
 APIError APINoiseFrameHelper::loop() {
   // During handshake phase, process as many actions as possible until we can't progress
@@ -423,16 +436,9 @@ APIError APINoiseFrameHelper::state_action_() {
     // waiting for client hello
     ParsedFrame frame;
     aerr = try_read_frame_(&frame);
-    if (aerr == APIError::BAD_INDICATOR) {
-      send_explicit_handshake_reject_("Bad indicator byte");
-      return aerr;
+    if (aerr != APIError::OK) {
+      return handle_handshake_frame_error_(aerr);
     }
-    if (aerr == APIError::BAD_HANDSHAKE_PACKET_LEN) {
-      send_explicit_handshake_reject_("Bad handshake packet len");
-      return aerr;
-    }
-    if (aerr != APIError::OK)
-      return aerr;
     // ignore contents, may be used in future for flags
     // Reserve space for: existing prologue + 2 size bytes + frame data
     prologue_.reserve(prologue_.size() + 2 + frame.msg.size());
@@ -478,16 +484,9 @@ APIError APINoiseFrameHelper::state_action_() {
       // waiting for handshake msg
       ParsedFrame frame;
       aerr = try_read_frame_(&frame);
-      if (aerr == APIError::BAD_INDICATOR) {
-        send_explicit_handshake_reject_("Bad indicator byte");
-        return aerr;
+      if (aerr != APIError::OK) {
+        return handle_handshake_frame_error_(aerr);
       }
-      if (aerr == APIError::BAD_HANDSHAKE_PACKET_LEN) {
-        send_explicit_handshake_reject_("Bad handshake packet len");
-        return aerr;
-      }
-      if (aerr != APIError::OK)
-        return aerr;
 
       if (frame.msg.empty()) {
         send_explicit_handshake_reject_("Empty handshake message");
diff --git a/esphome/components/api/api_frame_helper.h b/esphome/components/api/api_frame_helper.h
index 9488468ba0..c2b5dcf9d5 100644
--- a/esphome/components/api/api_frame_helper.h
+++ b/esphome/components/api/api_frame_helper.h
@@ -212,6 +212,7 @@ class APINoiseFrameHelper : public APIFrameHelper {
   APIError init_handshake_();
   APIError check_handshake_finished_();
   void send_explicit_handshake_reject_(const std::string &reason);
+  APIError handle_handshake_frame_error_(APIError aerr);
 
   // Pointers first (4 bytes each)
   NoiseHandshakeState *handshake_{nullptr};