[http_request] Implement for host platform (#8040)

esphome/components/http_request/__init__.py

@@ -10,9 +10,11 @@ from esphome.const import (
     CONF_TIMEOUT,
     CONF_TRIGGER_ID,
     CONF_URL,
+    PLATFORM_HOST,
     __version__,
 )
 from esphome.core import CORE, Lambda
+from esphome.helpers import IS_MACOS
 
 DEPENDENCIES = ["network"]
 AUTO_LOAD = ["json", "watchdog"]
@@ -21,6 +23,7 @@ http_request_ns = cg.esphome_ns.namespace("http_request")
 HttpRequestComponent = http_request_ns.class_("HttpRequestComponent", cg.Component)
 HttpRequestArduino = http_request_ns.class_("HttpRequestArduino", HttpRequestComponent)
 HttpRequestIDF = http_request_ns.class_("HttpRequestIDF", HttpRequestComponent)
+HttpRequestHost = http_request_ns.class_("HttpRequestHost", HttpRequestComponent)
 
 HttpContainer = http_request_ns.class_("HttpContainer")
 
@@ -43,6 +46,7 @@ CONF_REDIRECT_LIMIT = "redirect_limit"
 CONF_WATCHDOG_TIMEOUT = "watchdog_timeout"
 CONF_BUFFER_SIZE_RX = "buffer_size_rx"
 CONF_BUFFER_SIZE_TX = "buffer_size_tx"
+CONF_CA_CERTIFICATE_PATH = "ca_certificate_path"
 
 CONF_MAX_RESPONSE_BUFFER_SIZE = "max_response_buffer_size"
 CONF_ON_RESPONSE = "on_response"
@@ -87,6 +91,8 @@ def validate_ssl_verification(config):
 
 
 def _declare_request_class(value):
+    if CORE.is_host:
+        return cv.declare_id(HttpRequestHost)(value)
     if CORE.using_esp_idf:
         return cv.declare_id(HttpRequestIDF)(value)
     if CORE.is_esp8266 or CORE.is_esp32 or CORE.is_rp2040:
@@ -121,6 +127,10 @@ CONFIG_SCHEMA = cv.All(
             cv.SplitDefault(CONF_BUFFER_SIZE_TX, esp32_idf=512): cv.All(
                 cv.uint16_t, cv.only_with_esp_idf
             ),
+            cv.Optional(CONF_CA_CERTIFICATE_PATH): cv.All(
+                cv.file_,
+                cv.only_on(PLATFORM_HOST),
+            ),
         }
     ).extend(cv.COMPONENT_SCHEMA),
     cv.require_framework_version(
@@ -128,6 +138,7 @@ CONFIG_SCHEMA = cv.All(
         esp32_arduino=cv.Version(0, 0, 0),
         esp_idf=cv.Version(0, 0, 0),
         rp2040_arduino=cv.Version(0, 0, 0),
+        host=cv.Version(0, 0, 0),
     ),
     validate_ssl_verification,
 )
@@ -170,6 +181,21 @@ async def to_code(config):
         cg.add_library("ESP8266HTTPClient", None)
     if CORE.is_rp2040 and CORE.using_arduino:
         cg.add_library("HTTPClient", None)
+    if CORE.is_host:
+        if IS_MACOS:
+            cg.add_build_flag("-I/opt/homebrew/opt/openssl/include")
+            cg.add_build_flag("-L/opt/homebrew/opt/openssl/lib")
+            cg.add_build_flag("-lssl")
+            cg.add_build_flag("-lcrypto")
+            cg.add_build_flag("-Wl,-framework,CoreFoundation")
+            cg.add_build_flag("-Wl,-framework,Security")
+            cg.add_define("CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN")
+            cg.add_define("CPPHTTPLIB_OPENSSL_SUPPORT")
+        elif path := config.get(CONF_CA_CERTIFICATE_PATH):
+            cg.add_define("CPPHTTPLIB_OPENSSL_SUPPORT")
+            cg.add(var.set_ca_path(path))
+            cg.add_build_flag("-lssl")
+            cg.add_build_flag("-lcrypto")
 
     await cg.register_component(var, config)
 

esphome/components/http_request/http_request_host.cpp

@@ -0,0 +1,141 @@
+#include "http_request_host.h"
+
+#ifdef USE_HOST
+
+#include <regex>
+#include "esphome/components/network/util.h"
+#include "esphome/components/watchdog/watchdog.h"
+
+#include "esphome/core/application.h"
+#include "esphome/core/log.h"
+
+namespace esphome {
+namespace http_request {
+
+static const char *const TAG = "http_request.host";
+
+std::shared_ptr<HttpContainer> HttpRequestHost::perform(std::string url, std::string method, std::string body,
+                                                        std::list<Header> request_headers,
+                                                        std::set<std::string> response_headers) {
+  if (!network::is_connected()) {
+    this->status_momentary_error("failed", 1000);
+    ESP_LOGW(TAG, "HTTP Request failed; Not connected to network");
+    return nullptr;
+  }
+
+  std::regex url_regex(R"(^(([^:\/?#]+):)?(//([^\/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?)", std::regex::extended);
+  std::smatch url_match_result;
+
+  if (!std::regex_match(url, url_match_result, url_regex) || url_match_result.length() < 7) {
+    ESP_LOGE(TAG, "HTTP Request failed; Malformed URL: %s", url.c_str());
+    return nullptr;
+  }
+  auto host = url_match_result[4].str();
+  auto scheme_host = url_match_result[1].str() + url_match_result[3].str();
+  auto path = url_match_result[5].str() + url_match_result[6].str();
+  if (path.empty())
+    path = "/";
+
+  std::shared_ptr<HttpContainerHost> container = std::make_shared<HttpContainerHost>();
+  container->set_parent(this);
+
+  const uint32_t start = millis();
+
+  watchdog::WatchdogManager wdm(this->get_watchdog_timeout());
+
+  httplib::Headers h_headers;
+  h_headers.emplace("Host", host.c_str());
+  h_headers.emplace("User-Agent", this->useragent_);
+  for (const auto &[name, value] : request_headers) {
+    h_headers.emplace(name, value);
+  }
+  httplib::Client client(scheme_host.c_str());
+  if (!client.is_valid()) {
+    ESP_LOGE(TAG, "HTTP Request failed; Invalid URL: %s", url.c_str());
+    return nullptr;
+  }
+  client.set_follow_location(this->follow_redirects_);
+#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+  if (this->ca_path_ != nullptr)
+    client.set_ca_cert_path(this->ca_path_);
+#endif
+
+  httplib::Result result;
+  if (method == "GET") {
+    result = client.Get(path, h_headers, [&](const char *data, size_t data_length) {
+      ESP_LOGV(TAG, "Got data length: %zu", data_length);
+      container->response_body_.insert(container->response_body_.end(), (const uint8_t *) data,
+                                       (const uint8_t *) data + data_length);
+      return true;
+    });
+  } else if (method == "HEAD") {
+    result = client.Head(path, h_headers);
+  } else if (method == "PUT") {
+    result = client.Put(path, h_headers, body, "");
+    if (result) {
+      auto data = std::vector<uint8_t>(result->body.begin(), result->body.end());
+      container->response_body_.insert(container->response_body_.end(), data.begin(), data.end());
+    }
+  } else if (method == "PATCH") {
+    result = client.Patch(path, h_headers, body, "");
+    if (result) {
+      auto data = std::vector<uint8_t>(result->body.begin(), result->body.end());
+      container->response_body_.insert(container->response_body_.end(), data.begin(), data.end());
+    }
+  } else if (method == "POST") {
+    result = client.Post(path, h_headers, body, "");
+    if (result) {
+      auto data = std::vector<uint8_t>(result->body.begin(), result->body.end());
+      container->response_body_.insert(container->response_body_.end(), data.begin(), data.end());
+    }
+  } else {
+    ESP_LOGW(TAG, "HTTP Request failed - unsupported method %s; URL: %s", method.c_str(), url.c_str());
+    container->end();
+    return nullptr;
+  }
+  App.feed_wdt();
+  if (!result) {
+    ESP_LOGW(TAG, "HTTP Request failed; URL: %s, error code: %u", url.c_str(), (unsigned) result.error());
+    container->end();
+    this->status_momentary_error("failed", 1000);
+    return nullptr;
+  }
+  App.feed_wdt();
+  auto response = *result;
+  container->status_code = response.status;
+  if (!is_success(response.status)) {
+    ESP_LOGE(TAG, "HTTP Request failed; URL: %s; Code: %d", url.c_str(), response.status);
+    this->status_momentary_error("failed", 1000);
+    // Still return the container, so it can be used to get the status code and error message
+  }
+
+  container->content_length = container->response_body_.size();
+  for (auto header : response.headers) {
+    ESP_LOGD(TAG, "Header: %s: %s", header.first.c_str(), header.second.c_str());
+    auto lower_name = str_lower_case(header.first);
+    if (response_headers.find(lower_name) != response_headers.end()) {
+      container->response_headers_[lower_name].emplace_back(header.second);
+    }
+  }
+  container->duration_ms = millis() - start;
+  return container;
+}
+
+int HttpContainerHost::read(uint8_t *buf, size_t max_len) {
+  auto bytes_remaining = this->response_body_.size() - this->bytes_read_;
+  auto read_len = std::min(max_len, bytes_remaining);
+  memcpy(buf, this->response_body_.data() + this->bytes_read_, read_len);
+  this->bytes_read_ += read_len;
+  return read_len;
+}
+
+void HttpContainerHost::end() {
+  watchdog::WatchdogManager wdm(this->parent_->get_watchdog_timeout());
+  this->response_body_ = std::vector<uint8_t>();
+  this->bytes_read_ = 0;
+}
+
+}  // namespace http_request
+}  // namespace esphome
+
+#endif  // USE_HOST

esphome/components/http_request/http_request_host.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "http_request.h"
+
+#ifdef USE_HOST
+
+#define CPPHTTPLIB_NO_EXCEPTIONS
+#include "httplib.h"
+namespace esphome {
+namespace http_request {
+
+class HttpRequestHost;
+class HttpContainerHost : public HttpContainer {
+ public:
+  int read(uint8_t *buf, size_t max_len) override;
+  void end() override;
+
+ protected:
+  friend class HttpRequestHost;
+  std::vector<uint8_t> response_body_{};
+};
+
+class HttpRequestHost : public HttpRequestComponent {
+ public:
+  std::shared_ptr<HttpContainer> perform(std::string url, std::string method, std::string body,
+                                         std::list<Header> request_headers,
+                                         std::set<std::string> response_headers) override;
+  void set_ca_path(const char *ca_path) { this->ca_path_ = ca_path; }
+
+ protected:
+  const char *ca_path_{};
+};
+
+}  // namespace http_request
+}  // namespace esphome
+
+#endif  // USE_HOST

script/ci-custom.py

@@ -292,6 +292,7 @@ def highlight(s):
         "esphome/core/log.h",
         "esphome/components/socket/headers.h",
         "esphome/core/defines.h",
+        "esphome/components/http_request/httplib.h",
     ],
 )
 def lint_no_defines(fname, match):
@@ -552,6 +553,7 @@ def lint_relative_py_import(fname):
         "esphome/components/rp2040/core.cpp",
         "esphome/components/libretiny/core.cpp",
         "esphome/components/host/core.cpp",
+        "esphome/components/http_request/httplib.h",
     ],
 )
 def lint_namespace(fname, content):

tests/components/http_request/common.yaml

@@ -1,5 +1,4 @@
-substitutions:
-  verify_ssl: "true"
+<<: !include http_request.yaml
 
 wifi:
   ssid: MySSID

tests/components/http_request/http_request.yaml

@@ -0,0 +1,46 @@
+substitutions:
+  verify_ssl: "true"
+
+network:
+
+esphome:
+  on_boot:
+    then:
+      - http_request.get:
+          url: https://esphome.io
+          request_headers:
+            Content-Type: application/json
+          on_error:
+            logger.log: "Request failed"
+          on_response:
+            then:
+              - logger.log:
+                  format: "Response status: %d, Duration: %lu ms"
+                  args:
+                    - response->status_code
+                    - (long) response->duration_ms
+      - http_request.post:
+          url: https://esphome.io
+          request_headers:
+            Content-Type: application/json
+          json:
+            key: value
+      - http_request.send:
+          method: PUT
+          url: https://esphome.io
+          request_headers:
+            Content-Type: application/json
+          body: "Some data"
+
+http_request:
+  useragent: esphome/tagreader
+  timeout: 10s
+  verify_ssl: ${verify_ssl}
+
+script:
+  - id: does_not_compile
+    parameters:
+      api_url: string
+    then:
+      - http_request.get:
+          url: "http://google.com"

tests/components/http_request/test.host.yaml

@@ -0,0 +1,7 @@
+substitutions:
+  verify_ssl: "true"
+http_request:
+  # Just a file we can be sure exists
+  ca_certificate_path: /etc/passwd
+
+<<: !include http_request.yaml