From 991f3d3a10e8f70453548224c20e0159305efff9 Mon Sep 17 00:00:00 2001 From: Craig Andrews Date: Wed, 23 Apr 2025 00:30:50 -0400 Subject: [PATCH] [http_request] Ability to get response headers (#8224) Co-authored-by: guillempages Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> --- esphome/components/http_request/__init__.py | 19 +++-- .../components/http_request/http_request.cpp | 20 ++++++ .../components/http_request/http_request.h | 72 +++++++++++++++---- .../http_request/http_request_arduino.cpp | 28 ++++++-- .../http_request/http_request_arduino.h | 7 +- .../http_request/http_request_idf.cpp | 38 +++++++++- .../http_request/http_request_idf.h | 13 +++- tests/components/http_request/common.yaml | 11 +-- 8 files changed, 168 insertions(+), 40 deletions(-) diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index 78064fb4b4..2a999532f8 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -47,6 +47,8 @@ CONF_BUFFER_SIZE_TX = "buffer_size_tx" CONF_MAX_RESPONSE_BUFFER_SIZE = "max_response_buffer_size" CONF_ON_RESPONSE = "on_response" CONF_HEADERS = "headers" +CONF_REQUEST_HEADERS = "request_headers" +CONF_COLLECT_HEADERS = "collect_headers" CONF_BODY = "body" CONF_JSON = "json" CONF_CAPTURE_RESPONSE = "capture_response" @@ -176,9 +178,13 @@ HTTP_REQUEST_ACTION_SCHEMA = cv.Schema( { cv.GenerateID(): cv.use_id(HttpRequestComponent), cv.Required(CONF_URL): cv.templatable(validate_url), - cv.Optional(CONF_HEADERS): cv.All( + cv.Optional(CONF_HEADERS): cv.invalid( + "The 'headers' options has been renamed to 'request_headers'" + ), + cv.Optional(CONF_REQUEST_HEADERS): cv.All( cv.Schema({cv.string: cv.templatable(cv.string)}) ), + cv.Optional(CONF_COLLECT_HEADERS): cv.ensure_list(cv.string), cv.Optional(CONF_VERIFY_SSL): cv.invalid( f"{CONF_VERIFY_SSL} has moved to the base component configuration." ), @@ -263,11 +269,12 @@ async def http_request_action_to_code(config, action_id, template_arg, args): for key in json_: template_ = await cg.templatable(json_[key], args, cg.std_string) cg.add(var.add_json(key, template_)) - for key in config.get(CONF_HEADERS, []): - template_ = await cg.templatable( - config[CONF_HEADERS][key], args, cg.const_char_ptr - ) - cg.add(var.add_header(key, template_)) + for key in config.get(CONF_REQUEST_HEADERS, []): + template_ = await cg.templatable(key, args, cg.std_string) + cg.add(var.add_request_header(key, template_)) + + for value in config.get(CONF_COLLECT_HEADERS, []): + cg.add(var.add_collect_header(value)) for conf in config.get(CONF_ON_RESPONSE, []): trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID]) diff --git a/esphome/components/http_request/http_request.cpp b/esphome/components/http_request/http_request.cpp index be8bef006e..ca9fd2c2dc 100644 --- a/esphome/components/http_request/http_request.cpp +++ b/esphome/components/http_request/http_request.cpp @@ -20,5 +20,25 @@ void HttpRequestComponent::dump_config() { } } +std::string HttpContainer::get_response_header(const std::string &header_name) { + auto response_headers = this->get_response_headers(); + auto header_name_lower_case = str_lower_case(header_name); + if (response_headers.count(header_name_lower_case) == 0) { + ESP_LOGW(TAG, "No header with name %s found", header_name_lower_case.c_str()); + return ""; + } else { + auto values = response_headers[header_name_lower_case]; + if (values.empty()) { + ESP_LOGE(TAG, "header with name %s returned an empty list, this shouldn't happen", + header_name_lower_case.c_str()); + return ""; + } else { + auto header_value = values.front(); + ESP_LOGD(TAG, "Header with name %s found with value %s", header_name_lower_case.c_str(), header_value.c_str()); + return header_value; + } + } +} + } // namespace http_request } // namespace esphome diff --git a/esphome/components/http_request/http_request.h b/esphome/components/http_request/http_request.h index e98fd1a475..a67b04eadc 100644 --- a/esphome/components/http_request/http_request.h +++ b/esphome/components/http_request/http_request.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -95,9 +96,19 @@ class HttpContainer : public Parented { size_t get_bytes_read() const { return this->bytes_read_; } + /** + * @brief Get response headers. + * + * @return The key is the lower case response header name, the value is the header value. + */ + std::map> get_response_headers() { return this->response_headers_; } + + std::string get_response_header(const std::string &header_name); + protected: size_t bytes_read_{0}; bool secure_{false}; + std::map> response_headers_{}; }; class HttpRequestResponseTrigger : public Trigger, std::string &> { @@ -119,21 +130,46 @@ class HttpRequestComponent : public Component { void set_follow_redirects(bool follow_redirects) { this->follow_redirects_ = follow_redirects; } void set_redirect_limit(uint16_t limit) { this->redirect_limit_ = limit; } - std::shared_ptr get(std::string url) { return this->start(std::move(url), "GET", "", {}); } - std::shared_ptr get(std::string url, std::list
headers) { - return this->start(std::move(url), "GET", "", std::move(headers)); + std::shared_ptr get(const std::string &url) { return this->start(url, "GET", "", {}); } + std::shared_ptr get(const std::string &url, const std::list
&request_headers) { + return this->start(url, "GET", "", request_headers); } - std::shared_ptr post(std::string url, std::string body) { - return this->start(std::move(url), "POST", std::move(body), {}); + std::shared_ptr get(const std::string &url, const std::list
&request_headers, + const std::set &collect_headers) { + return this->start(url, "GET", "", request_headers, collect_headers); } - std::shared_ptr post(std::string url, std::string body, std::list
headers) { - return this->start(std::move(url), "POST", std::move(body), std::move(headers)); + std::shared_ptr post(const std::string &url, const std::string &body) { + return this->start(url, "POST", body, {}); + } + std::shared_ptr post(const std::string &url, const std::string &body, + const std::list
&request_headers) { + return this->start(url, "POST", body, request_headers); + } + std::shared_ptr post(const std::string &url, const std::string &body, + const std::list
&request_headers, + const std::set &collect_headers) { + return this->start(url, "POST", body, request_headers, collect_headers); } - virtual std::shared_ptr start(std::string url, std::string method, std::string body, - std::list
headers) = 0; + std::shared_ptr start(const std::string &url, const std::string &method, const std::string &body, + const std::list
&request_headers) { + return this->start(url, method, body, request_headers, {}); + } + + std::shared_ptr start(const std::string &url, const std::string &method, const std::string &body, + const std::list
&request_headers, + const std::set &collect_headers) { + std::set lower_case_collect_headers; + for (const std::string &collect_header : collect_headers) { + lower_case_collect_headers.insert(str_lower_case(collect_header)); + } + return this->perform(url, method, body, request_headers, lower_case_collect_headers); + } protected: + virtual std::shared_ptr perform(std::string url, std::string method, std::string body, + std::list
request_headers, + std::set collect_headers) = 0; const char *useragent_{nullptr}; bool follow_redirects_{}; uint16_t redirect_limit_{}; @@ -149,7 +185,11 @@ template class HttpRequestSendAction : public Action { TEMPLATABLE_VALUE(std::string, body) TEMPLATABLE_VALUE(bool, capture_response) - void add_header(const char *key, TemplatableValue value) { this->headers_.insert({key, value}); } + void add_request_header(const char *key, TemplatableValue value) { + this->request_headers_.insert({key, value}); + } + + void add_collect_header(const char *value) { this->collect_headers_.insert(value); } void add_json(const char *key, TemplatableValue value) { this->json_.insert({key, value}); } @@ -176,16 +216,17 @@ template class HttpRequestSendAction : public Action { auto f = std::bind(&HttpRequestSendAction::encode_json_func_, this, x..., std::placeholders::_1); body = json::build_json(f); } - std::list
headers; - for (const auto &item : this->headers_) { + std::list
request_headers; + for (const auto &item : this->request_headers_) { auto val = item.second; Header header; header.name = item.first; header.value = val.value(x...); - headers.push_back(header); + request_headers.push_back(header); } - auto container = this->parent_->start(this->url_.value(x...), this->method_.value(x...), body, headers); + auto container = this->parent_->start(this->url_.value(x...), this->method_.value(x...), body, request_headers, + this->collect_headers_); if (container == nullptr) { for (auto *trigger : this->error_triggers_) @@ -238,7 +279,8 @@ template class HttpRequestSendAction : public Action { } void encode_json_func_(Ts... x, JsonObject root) { this->json_func_(x..., root); } HttpRequestComponent *parent_; - std::map> headers_{}; + std::map> request_headers_{}; + std::set collect_headers_{"content-type", "content-length"}; std::map> json_{}; std::function json_func_{nullptr}; std::vector response_triggers_{}; diff --git a/esphome/components/http_request/http_request_arduino.cpp b/esphome/components/http_request/http_request_arduino.cpp index b0067e7839..b4378cdce6 100644 --- a/esphome/components/http_request/http_request_arduino.cpp +++ b/esphome/components/http_request/http_request_arduino.cpp @@ -14,8 +14,9 @@ namespace http_request { static const char *const TAG = "http_request.arduino"; -std::shared_ptr HttpRequestArduino::start(std::string url, std::string method, std::string body, - std::list
headers) { +std::shared_ptr HttpRequestArduino::perform(std::string url, std::string method, std::string body, + std::list
request_headers, + std::set collect_headers) { if (!network::is_connected()) { this->status_momentary_error("failed", 1000); ESP_LOGW(TAG, "HTTP Request failed; Not connected to network"); @@ -95,14 +96,17 @@ std::shared_ptr HttpRequestArduino::start(std::string url, std::s if (this->useragent_ != nullptr) { container->client_.setUserAgent(this->useragent_); } - for (const auto &header : headers) { + for (const auto &header : request_headers) { container->client_.addHeader(header.name.c_str(), header.value.c_str(), false, true); } // returned needed headers must be collected before the requests - static const char *header_keys[] = {"Content-Length", "Content-Type"}; - static const size_t HEADER_COUNT = sizeof(header_keys) / sizeof(header_keys[0]); - container->client_.collectHeaders(header_keys, HEADER_COUNT); + const char *header_keys[collect_headers.size()]; + int index = 0; + for (auto const &header_name : collect_headers) { + header_keys[index++] = header_name.c_str(); + } + container->client_.collectHeaders(header_keys, index); App.feed_wdt(); container->status_code = container->client_.sendRequest(method.c_str(), body.c_str()); @@ -121,6 +125,18 @@ std::shared_ptr HttpRequestArduino::start(std::string url, std::s // Still return the container, so it can be used to get the status code and error message } + container->response_headers_ = {}; + auto header_count = container->client_.headers(); + for (int i = 0; i < header_count; i++) { + const std::string header_name = str_lower_case(container->client_.headerName(i).c_str()); + if (collect_headers.count(header_name) > 0) { + std::string header_value = container->client_.header(i).c_str(); + ESP_LOGD(TAG, "Received response header, name: %s, value: %s", header_name.c_str(), header_value.c_str()); + container->response_headers_[header_name].push_back(header_value); + break; + } + } + int content_length = container->client_.getSize(); ESP_LOGD(TAG, "Content-Length: %d", content_length); container->content_length = (size_t) content_length; diff --git a/esphome/components/http_request/http_request_arduino.h b/esphome/components/http_request/http_request_arduino.h index dfdf4a35e2..ac9ddffbb0 100644 --- a/esphome/components/http_request/http_request_arduino.h +++ b/esphome/components/http_request/http_request_arduino.h @@ -29,9 +29,10 @@ class HttpContainerArduino : public HttpContainer { }; class HttpRequestArduino : public HttpRequestComponent { - public: - std::shared_ptr start(std::string url, std::string method, std::string body, - std::list
headers) override; + protected: + std::shared_ptr perform(std::string url, std::string method, std::string body, + std::list
request_headers, + std::set collect_headers) override; }; } // namespace http_request diff --git a/esphome/components/http_request/http_request_idf.cpp b/esphome/components/http_request/http_request_idf.cpp index 78c37403f5..0923062822 100644 --- a/esphome/components/http_request/http_request_idf.cpp +++ b/esphome/components/http_request/http_request_idf.cpp @@ -19,14 +19,41 @@ namespace http_request { static const char *const TAG = "http_request.idf"; +struct UserData { + const std::set &collect_headers; + std::map> response_headers; +}; + void HttpRequestIDF::dump_config() { HttpRequestComponent::dump_config(); ESP_LOGCONFIG(TAG, " Buffer Size RX: %u", this->buffer_size_rx_); ESP_LOGCONFIG(TAG, " Buffer Size TX: %u", this->buffer_size_tx_); } -std::shared_ptr HttpRequestIDF::start(std::string url, std::string method, std::string body, - std::list
headers) { +esp_err_t HttpRequestIDF::http_event_handler(esp_http_client_event_t *evt) { + UserData *user_data = (UserData *) evt->user_data; + + switch (evt->event_id) { + case HTTP_EVENT_ON_HEADER: { + const std::string header_name = str_lower_case(evt->header_key); + if (user_data->collect_headers.count(header_name)) { + const std::string header_value = evt->header_value; + ESP_LOGD(TAG, "Received response header, name: %s, value: %s", header_name.c_str(), header_value.c_str()); + user_data->response_headers[header_name].push_back(header_value); + break; + } + break; + } + default: { + break; + } + } + return ESP_OK; +} + +std::shared_ptr HttpRequestIDF::perform(std::string url, std::string method, std::string body, + std::list
request_headers, + std::set collect_headers) { if (!network::is_connected()) { this->status_momentary_error("failed", 1000); ESP_LOGE(TAG, "HTTP Request failed; Not connected to network"); @@ -76,6 +103,10 @@ std::shared_ptr HttpRequestIDF::start(std::string url, std::strin const uint32_t start = millis(); watchdog::WatchdogManager wdm(this->get_watchdog_timeout()); + config.event_handler = http_event_handler; + auto user_data = UserData{collect_headers, {}}; + config.user_data = static_cast(&user_data); + esp_http_client_handle_t client = esp_http_client_init(&config); std::shared_ptr container = std::make_shared(client); @@ -83,7 +114,7 @@ std::shared_ptr HttpRequestIDF::start(std::string url, std::strin container->set_secure(secure); - for (const auto &header : headers) { + for (const auto &header : request_headers) { esp_http_client_set_header(client, header.name.c_str(), header.value.c_str()); } @@ -124,6 +155,7 @@ std::shared_ptr HttpRequestIDF::start(std::string url, std::strin container->feed_wdt(); container->status_code = esp_http_client_get_status_code(client); container->feed_wdt(); + container->set_response_headers(user_data.response_headers); if (is_success(container->status_code)) { container->duration_ms = millis() - start; return container; diff --git a/esphome/components/http_request/http_request_idf.h b/esphome/components/http_request/http_request_idf.h index 2ed50698b9..5c5b784853 100644 --- a/esphome/components/http_request/http_request_idf.h +++ b/esphome/components/http_request/http_request_idf.h @@ -21,6 +21,10 @@ class HttpContainerIDF : public HttpContainer { /// @brief Feeds the watchdog timer if the executing task has one attached void feed_wdt(); + void set_response_headers(std::map> &response_headers) { + this->response_headers_ = std::move(response_headers); + } + protected: esp_http_client_handle_t client_; }; @@ -29,16 +33,19 @@ class HttpRequestIDF : public HttpRequestComponent { public: void dump_config() override; - std::shared_ptr start(std::string url, std::string method, std::string body, - std::list
headers) override; - void set_buffer_size_rx(uint16_t buffer_size_rx) { this->buffer_size_rx_ = buffer_size_rx; } void set_buffer_size_tx(uint16_t buffer_size_tx) { this->buffer_size_tx_ = buffer_size_tx; } protected: + std::shared_ptr perform(std::string url, std::string method, std::string body, + std::list
request_headers, + std::set collect_headers) override; // if zero ESP-IDF will use DEFAULT_HTTP_BUF_SIZE uint16_t buffer_size_rx_{}; uint16_t buffer_size_tx_{}; + + /// @brief Monitors the http client events to gather response headers + static esp_err_t http_event_handler(esp_http_client_event_t *evt); }; } // namespace http_request diff --git a/tests/components/http_request/common.yaml b/tests/components/http_request/common.yaml index 8408f27a05..4a9b8a0e62 100644 --- a/tests/components/http_request/common.yaml +++ b/tests/components/http_request/common.yaml @@ -10,27 +10,30 @@ esphome: then: - http_request.get: url: https://esphome.io - headers: + request_headers: Content-Type: application/json + collect_headers: + - age on_error: logger.log: "Request failed" on_response: then: - logger.log: - format: "Response status: %d, Duration: %lu ms" + format: "Response status: %d, Duration: %lu ms, age: %s" args: - response->status_code - (long) response->duration_ms + - response->get_response_header("age").c_str() - http_request.post: url: https://esphome.io - headers: + request_headers: Content-Type: application/json json: key: value - http_request.send: method: PUT url: https://esphome.io - headers: + request_headers: Content-Type: application/json body: "Some data"