[http_request] Ability to get response headers (#8224)

Co-authored-by: guillempages <guillempages@users.noreply.github.com>
Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com>

esphome/components/http_request/__init__.py

@@ -47,6 +47,8 @@ CONF_BUFFER_SIZE_TX = "buffer_size_tx"
 CONF_MAX_RESPONSE_BUFFER_SIZE = "max_response_buffer_size"
 CONF_ON_RESPONSE = "on_response"
 CONF_HEADERS = "headers"
+CONF_REQUEST_HEADERS = "request_headers"
+CONF_COLLECT_HEADERS = "collect_headers"
 CONF_BODY = "body"
 CONF_JSON = "json"
 CONF_CAPTURE_RESPONSE = "capture_response"
@@ -176,9 +178,13 @@ HTTP_REQUEST_ACTION_SCHEMA = cv.Schema(
     {
         cv.GenerateID(): cv.use_id(HttpRequestComponent),
         cv.Required(CONF_URL): cv.templatable(validate_url),
-        cv.Optional(CONF_HEADERS): cv.All(
+        cv.Optional(CONF_HEADERS): cv.invalid(
+            "The 'headers' options has been renamed to 'request_headers'"
+        ),
+        cv.Optional(CONF_REQUEST_HEADERS): cv.All(
             cv.Schema({cv.string: cv.templatable(cv.string)})
         ),
+        cv.Optional(CONF_COLLECT_HEADERS): cv.ensure_list(cv.string),
         cv.Optional(CONF_VERIFY_SSL): cv.invalid(
             f"{CONF_VERIFY_SSL} has moved to the base component configuration."
         ),
@@ -263,11 +269,12 @@ async def http_request_action_to_code(config, action_id, template_arg, args):
             for key in json_:
                 template_ = await cg.templatable(json_[key], args, cg.std_string)
                 cg.add(var.add_json(key, template_))
-    for key in config.get(CONF_HEADERS, []):
+    for key in config.get(CONF_REQUEST_HEADERS, []):
-        template_ = await cg.templatable(
+        template_ = await cg.templatable(key, args, cg.std_string)
-            config[CONF_HEADERS][key], args, cg.const_char_ptr
+        cg.add(var.add_request_header(key, template_))
-        )
+
-        cg.add(var.add_header(key, template_))
+    for value in config.get(CONF_COLLECT_HEADERS, []):
+        cg.add(var.add_collect_header(value))
 
     for conf in config.get(CONF_ON_RESPONSE, []):
         trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID])

esphome/components/http_request/http_request.cpp

@@ -20,5 +20,25 @@ void HttpRequestComponent::dump_config() {
   }
 }
 
+std::string HttpContainer::get_response_header(const std::string &header_name) {
+  auto response_headers = this->get_response_headers();
+  auto header_name_lower_case = str_lower_case(header_name);
+  if (response_headers.count(header_name_lower_case) == 0) {
+    ESP_LOGW(TAG, "No header with name %s found", header_name_lower_case.c_str());
+    return "";
+  } else {
+    auto values = response_headers[header_name_lower_case];
+    if (values.empty()) {
+      ESP_LOGE(TAG, "header with name %s returned an empty list, this shouldn't happen",
+               header_name_lower_case.c_str());
+      return "";
+    } else {
+      auto header_value = values.front();
+      ESP_LOGD(TAG, "Header with name %s found with value %s", header_name_lower_case.c_str(), header_value.c_str());
+      return header_value;
+    }
+  }
+}
+
 }  // namespace http_request
 }  // namespace esphome

esphome/components/http_request/http_request.h

@@ -3,6 +3,7 @@
 #include <list>
 #include <map>
 #include <memory>
+#include <set>
 #include <utility>
 #include <vector>
 
@@ -95,9 +96,19 @@ class HttpContainer : public Parented<HttpRequestComponent> {
 
   size_t get_bytes_read() const { return this->bytes_read_; }
 
+  /**
+   * @brief Get response headers.
+   *
+   * @return The key is the lower case response header name, the value is the header value.
+   */
+  std::map<std::string, std::list<std::string>> get_response_headers() { return this->response_headers_; }
+
+  std::string get_response_header(const std::string &header_name);
+
  protected:
   size_t bytes_read_{0};
   bool secure_{false};
+  std::map<std::string, std::list<std::string>> response_headers_{};
 };
 
 class HttpRequestResponseTrigger : public Trigger<std::shared_ptr<HttpContainer>, std::string &> {
@@ -119,21 +130,46 @@ class HttpRequestComponent : public Component {
   void set_follow_redirects(bool follow_redirects) { this->follow_redirects_ = follow_redirects; }
   void set_redirect_limit(uint16_t limit) { this->redirect_limit_ = limit; }
 
-  std::shared_ptr<HttpContainer> get(std::string url) { return this->start(std::move(url), "GET", "", {}); }
+  std::shared_ptr<HttpContainer> get(const std::string &url) { return this->start(url, "GET", "", {}); }
-  std::shared_ptr<HttpContainer> get(std::string url, std::list<Header> headers) {
+  std::shared_ptr<HttpContainer> get(const std::string &url, const std::list<Header> &request_headers) {
-    return this->start(std::move(url), "GET", "", std::move(headers));
+    return this->start(url, "GET", "", request_headers);
   }
-  std::shared_ptr<HttpContainer> post(std::string url, std::string body) {
+  std::shared_ptr<HttpContainer> get(const std::string &url, const std::list<Header> &request_headers,
-    return this->start(std::move(url), "POST", std::move(body), {});
+                                     const std::set<std::string> &collect_headers) {
+    return this->start(url, "GET", "", request_headers, collect_headers);
   }
-  std::shared_ptr<HttpContainer> post(std::string url, std::string body, std::list<Header> headers) {
+  std::shared_ptr<HttpContainer> post(const std::string &url, const std::string &body) {
-    return this->start(std::move(url), "POST", std::move(body), std::move(headers));
+    return this->start(url, "POST", body, {});
+  }
+  std::shared_ptr<HttpContainer> post(const std::string &url, const std::string &body,
+                                      const std::list<Header> &request_headers) {
+    return this->start(url, "POST", body, request_headers);
+  }
+  std::shared_ptr<HttpContainer> post(const std::string &url, const std::string &body,
+                                      const std::list<Header> &request_headers,
+                                      const std::set<std::string> &collect_headers) {
+    return this->start(url, "POST", body, request_headers, collect_headers);
   }
 
-  virtual std::shared_ptr<HttpContainer> start(std::string url, std::string method, std::string body,
+  std::shared_ptr<HttpContainer> start(const std::string &url, const std::string &method, const std::string &body,
-                                               std::list<Header> headers) = 0;
+                                       const std::list<Header> &request_headers) {
+    return this->start(url, method, body, request_headers, {});
+  }
+
+  std::shared_ptr<HttpContainer> start(const std::string &url, const std::string &method, const std::string &body,
+                                       const std::list<Header> &request_headers,
+                                       const std::set<std::string> &collect_headers) {
+    std::set<std::string> lower_case_collect_headers;
+    for (const std::string &collect_header : collect_headers) {
+      lower_case_collect_headers.insert(str_lower_case(collect_header));
+    }
+    return this->perform(url, method, body, request_headers, lower_case_collect_headers);
+  }
 
  protected:
+  virtual std::shared_ptr<HttpContainer> perform(std::string url, std::string method, std::string body,
+                                                 std::list<Header> request_headers,
+                                                 std::set<std::string> collect_headers) = 0;
   const char *useragent_{nullptr};
   bool follow_redirects_{};
   uint16_t redirect_limit_{};
@@ -149,7 +185,11 @@ template<typename... Ts> class HttpRequestSendAction : public Action<Ts...> {
   TEMPLATABLE_VALUE(std::string, body)
   TEMPLATABLE_VALUE(bool, capture_response)
 
-  void add_header(const char *key, TemplatableValue<const char *, Ts...> value) { this->headers_.insert({key, value}); }
+  void add_request_header(const char *key, TemplatableValue<const char *, Ts...> value) {
+    this->request_headers_.insert({key, value});
+  }
+
+  void add_collect_header(const char *value) { this->collect_headers_.insert(value); }
 
   void add_json(const char *key, TemplatableValue<std::string, Ts...> value) { this->json_.insert({key, value}); }
 
@@ -176,16 +216,17 @@ template<typename... Ts> class HttpRequestSendAction : public Action<Ts...> {
       auto f = std::bind(&HttpRequestSendAction<Ts...>::encode_json_func_, this, x..., std::placeholders::_1);
       body = json::build_json(f);
     }
-    std::list<Header> headers;
+    std::list<Header> request_headers;
-    for (const auto &item : this->headers_) {
+    for (const auto &item : this->request_headers_) {
       auto val = item.second;
       Header header;
       header.name = item.first;
       header.value = val.value(x...);
-      headers.push_back(header);
+      request_headers.push_back(header);
     }
 
-    auto container = this->parent_->start(this->url_.value(x...), this->method_.value(x...), body, headers);
+    auto container = this->parent_->start(this->url_.value(x...), this->method_.value(x...), body, request_headers,
+                                          this->collect_headers_);
 
     if (container == nullptr) {
       for (auto *trigger : this->error_triggers_)
@@ -238,7 +279,8 @@ template<typename... Ts> class HttpRequestSendAction : public Action<Ts...> {
   }
   void encode_json_func_(Ts... x, JsonObject root) { this->json_func_(x..., root); }
   HttpRequestComponent *parent_;
-  std::map<const char *, TemplatableValue<const char *, Ts...>> headers_{};
+  std::map<const char *, TemplatableValue<const char *, Ts...>> request_headers_{};
+  std::set<std::string> collect_headers_{"content-type", "content-length"};
   std::map<const char *, TemplatableValue<std::string, Ts...>> json_{};
   std::function<void(Ts..., JsonObject)> json_func_{nullptr};
   std::vector<HttpRequestResponseTrigger *> response_triggers_{};

esphome/components/http_request/http_request_arduino.cpp

@@ -14,8 +14,9 @@ namespace http_request {
 
 static const char *const TAG = "http_request.arduino";
 
-std::shared_ptr<HttpContainer> HttpRequestArduino::start(std::string url, std::string method, std::string body,
+std::shared_ptr<HttpContainer> HttpRequestArduino::perform(std::string url, std::string method, std::string body,
-                                                         std::list<Header> headers) {
+                                                           std::list<Header> request_headers,
+                                                           std::set<std::string> collect_headers) {
   if (!network::is_connected()) {
     this->status_momentary_error("failed", 1000);
     ESP_LOGW(TAG, "HTTP Request failed; Not connected to network");
@@ -95,14 +96,17 @@ std::shared_ptr<HttpContainer> HttpRequestArduino::start(std::string url, std::s
   if (this->useragent_ != nullptr) {
     container->client_.setUserAgent(this->useragent_);
   }
-  for (const auto &header : headers) {
+  for (const auto &header : request_headers) {
     container->client_.addHeader(header.name.c_str(), header.value.c_str(), false, true);
   }
 
   // returned needed headers must be collected before the requests
-  static const char *header_keys[] = {"Content-Length", "Content-Type"};
+  const char *header_keys[collect_headers.size()];
-  static const size_t HEADER_COUNT = sizeof(header_keys) / sizeof(header_keys[0]);
+  int index = 0;
-  container->client_.collectHeaders(header_keys, HEADER_COUNT);
+  for (auto const &header_name : collect_headers) {
+    header_keys[index++] = header_name.c_str();
+  }
+  container->client_.collectHeaders(header_keys, index);
 
   App.feed_wdt();
   container->status_code = container->client_.sendRequest(method.c_str(), body.c_str());
@@ -121,6 +125,18 @@ std::shared_ptr<HttpContainer> HttpRequestArduino::start(std::string url, std::s
     // Still return the container, so it can be used to get the status code and error message
   }
 
+  container->response_headers_ = {};
+  auto header_count = container->client_.headers();
+  for (int i = 0; i < header_count; i++) {
+    const std::string header_name = str_lower_case(container->client_.headerName(i).c_str());
+    if (collect_headers.count(header_name) > 0) {
+      std::string header_value = container->client_.header(i).c_str();
+      ESP_LOGD(TAG, "Received response header, name: %s, value: %s", header_name.c_str(), header_value.c_str());
+      container->response_headers_[header_name].push_back(header_value);
+      break;
+    }
+  }
+
   int content_length = container->client_.getSize();
   ESP_LOGD(TAG, "Content-Length: %d", content_length);
   container->content_length = (size_t) content_length;

esphome/components/http_request/http_request_arduino.h

@@ -29,9 +29,10 @@ class HttpContainerArduino : public HttpContainer {
 };
 
 class HttpRequestArduino : public HttpRequestComponent {
- public:
+ protected:
-  std::shared_ptr<HttpContainer> start(std::string url, std::string method, std::string body,
+  std::shared_ptr<HttpContainer> perform(std::string url, std::string method, std::string body,
-                                       std::list<Header> headers) override;
+                                         std::list<Header> request_headers,
+                                         std::set<std::string> collect_headers) override;
 };
 
 }  // namespace http_request

esphome/components/http_request/http_request_idf.cpp

@@ -19,14 +19,41 @@ namespace http_request {
 
 static const char *const TAG = "http_request.idf";
 
+struct UserData {
+  const std::set<std::string> &collect_headers;
+  std::map<std::string, std::list<std::string>> response_headers;
+};
+
 void HttpRequestIDF::dump_config() {
   HttpRequestComponent::dump_config();
   ESP_LOGCONFIG(TAG, "  Buffer Size RX: %u", this->buffer_size_rx_);
   ESP_LOGCONFIG(TAG, "  Buffer Size TX: %u", this->buffer_size_tx_);
 }
 
-std::shared_ptr<HttpContainer> HttpRequestIDF::start(std::string url, std::string method, std::string body,
+esp_err_t HttpRequestIDF::http_event_handler(esp_http_client_event_t *evt) {
-                                                     std::list<Header> headers) {
+  UserData *user_data = (UserData *) evt->user_data;
+
+  switch (evt->event_id) {
+    case HTTP_EVENT_ON_HEADER: {
+      const std::string header_name = str_lower_case(evt->header_key);
+      if (user_data->collect_headers.count(header_name)) {
+        const std::string header_value = evt->header_value;
+        ESP_LOGD(TAG, "Received response header, name: %s, value: %s", header_name.c_str(), header_value.c_str());
+        user_data->response_headers[header_name].push_back(header_value);
+        break;
+      }
+      break;
+    }
+    default: {
+      break;
+    }
+  }
+  return ESP_OK;
+}
+
+std::shared_ptr<HttpContainer> HttpRequestIDF::perform(std::string url, std::string method, std::string body,
+                                                       std::list<Header> request_headers,
+                                                       std::set<std::string> collect_headers) {
   if (!network::is_connected()) {
     this->status_momentary_error("failed", 1000);
     ESP_LOGE(TAG, "HTTP Request failed; Not connected to network");
@@ -76,6 +103,10 @@ std::shared_ptr<HttpContainer> HttpRequestIDF::start(std::string url, std::strin
   const uint32_t start = millis();
   watchdog::WatchdogManager wdm(this->get_watchdog_timeout());
 
+  config.event_handler = http_event_handler;
+  auto user_data = UserData{collect_headers, {}};
+  config.user_data = static_cast<void *>(&user_data);
+
   esp_http_client_handle_t client = esp_http_client_init(&config);
 
   std::shared_ptr<HttpContainerIDF> container = std::make_shared<HttpContainerIDF>(client);
@@ -83,7 +114,7 @@ std::shared_ptr<HttpContainer> HttpRequestIDF::start(std::string url, std::strin
 
   container->set_secure(secure);
 
-  for (const auto &header : headers) {
+  for (const auto &header : request_headers) {
     esp_http_client_set_header(client, header.name.c_str(), header.value.c_str());
   }
 
@@ -124,6 +155,7 @@ std::shared_ptr<HttpContainer> HttpRequestIDF::start(std::string url, std::strin
   container->feed_wdt();
   container->status_code = esp_http_client_get_status_code(client);
   container->feed_wdt();
+  container->set_response_headers(user_data.response_headers);
   if (is_success(container->status_code)) {
     container->duration_ms = millis() - start;
     return container;

esphome/components/http_request/http_request_idf.h

@@ -21,6 +21,10 @@ class HttpContainerIDF : public HttpContainer {
   /// @brief Feeds the watchdog timer if the executing task has one attached
   void feed_wdt();
 
+  void set_response_headers(std::map<std::string, std::list<std::string>> &response_headers) {
+    this->response_headers_ = std::move(response_headers);
+  }
+
  protected:
   esp_http_client_handle_t client_;
 };
@@ -29,16 +33,19 @@ class HttpRequestIDF : public HttpRequestComponent {
  public:
   void dump_config() override;
 
-  std::shared_ptr<HttpContainer> start(std::string url, std::string method, std::string body,
-                                       std::list<Header> headers) override;
-
   void set_buffer_size_rx(uint16_t buffer_size_rx) { this->buffer_size_rx_ = buffer_size_rx; }
   void set_buffer_size_tx(uint16_t buffer_size_tx) { this->buffer_size_tx_ = buffer_size_tx; }
 
  protected:
+  std::shared_ptr<HttpContainer> perform(std::string url, std::string method, std::string body,
+                                         std::list<Header> request_headers,
+                                         std::set<std::string> collect_headers) override;
   // if zero ESP-IDF will use DEFAULT_HTTP_BUF_SIZE
   uint16_t buffer_size_rx_{};
   uint16_t buffer_size_tx_{};
+
+  /// @brief Monitors the http client events to gather response headers
+  static esp_err_t http_event_handler(esp_http_client_event_t *evt);
 };
 
 }  // namespace http_request

tests/components/http_request/common.yaml

@@ -10,27 +10,30 @@ esphome:
     then:
       - http_request.get:
           url: https://esphome.io
-          headers:
+          request_headers:
             Content-Type: application/json
+          collect_headers:
+            - age
           on_error:
             logger.log: "Request failed"
           on_response:
             then:
               - logger.log:
-                  format: "Response status: %d, Duration: %lu ms"
+                  format: "Response status: %d, Duration: %lu ms, age: %s"
                   args:
                     - response->status_code
                     - (long) response->duration_ms
+                    - response->get_response_header("age").c_str()
       - http_request.post:
           url: https://esphome.io
-          headers:
+          request_headers:
             Content-Type: application/json
           json:
             key: value
       - http_request.send:
           method: PUT
           url: https://esphome.io
-          headers:
+          request_headers:
             Content-Type: application/json
           body: "Some data"