Merge pull request #2613 from esphome/bump-2021.10.2

2021.10.2

CODEOWNERS

@@ -89,6 +89,7 @@ esphome/components/mcp23x17_base/* @jesserockz
 esphome/components/mcp23xxx_base/* @jesserockz
 esphome/components/mcp2515/* @danielschramm @mvturnho
 esphome/components/mcp9808/* @k7hpn
+esphome/components/md5/* @esphome/core
 esphome/components/mdns/* @esphome/core
 esphome/components/midea/* @dudanov
 esphome/components/mitsubishi/* @RubyBailey

docker/Dockerfile

@@ -43,7 +43,7 @@ RUN \
     # Ubuntu python3-pip is missing wheel
     pip3 install --no-cache-dir \
         wheel==0.36.2 \
-        platformio==5.2.0 \
+        platformio==5.2.1 \
     # Change some platformio settings
     && platformio settings set enable_telemetry No \
     && platformio settings set check_libraries_interval 1000000 \

docker/platformio_install_deps.py

@@ -8,6 +8,23 @@ import sys
 
 config = configparser.ConfigParser(inline_comment_prefixes=(';', ))
 config.read(sys.argv[1])
-libs = [x for x in config['common']['lib_deps'].splitlines() if len(x) != 0]
+
+libs = []
+# Extract from every lib_deps key in all sections
+for section in config.sections():
+    conf = config[section]
+    if "lib_deps" not in conf:
+        continue
+    for lib_dep in conf["lib_deps"].splitlines():
+        if not lib_dep:
+            # Empty line or comment
+            continue
+        if lib_dep.startswith("${"):
+            # Extending from another section
+            continue
+        if "@" not in lib_dep:
+            # No version pinned, this is an internal lib
+            continue
+        libs.append(lib_dep)
 
 subprocess.check_call(['platformio', 'lib', '-g', 'install', *libs])

esphome/__main__.py

@@ -180,7 +180,11 @@ def compile_program(args, config):
     from esphome import platformio_api
 
     _LOGGER.info("Compiling app...")
-    return platformio_api.run_compile(config, CORE.verbose)
+    rc = platformio_api.run_compile(config, CORE.verbose)
+    if rc != 0:
+        return rc
+    idedata = platformio_api.get_idedata(config)
+    return 0 if idedata is not None else 1
 
 
 def upload_using_esptool(config, port):
@@ -458,6 +462,21 @@ def command_update_all(args):
     return failed
 
 
+def command_idedata(args, config):
+    from esphome import platformio_api
+    import json
+
+    logging.disable(logging.INFO)
+    logging.disable(logging.WARNING)
+
+    idedata = platformio_api.get_idedata(config)
+    if idedata is None:
+        return 1
+
+    print(json.dumps(idedata.raw, indent=2) + "\n")
+    return 0
+
+
 PRE_CONFIG_ACTIONS = {
     "wizard": command_wizard,
     "version": command_version,
@@ -475,6 +494,7 @@ POST_CONFIG_ACTIONS = {
     "clean-mqtt": command_clean_mqtt,
     "mqtt-fingerprint": command_mqtt_fingerprint,
     "clean": command_clean,
+    "idedata": command_idedata,
 }
 
 
@@ -650,6 +670,11 @@ def parse_args(argv):
         "configuration", help="Your YAML configuration file directories.", nargs="+"
     )
 
+    parser_idedata = subparsers.add_parser("idedata")
+    parser_idedata.add_argument(
+        "configuration", help="Your YAML configuration file(s).", nargs=1
+    )
+
     # Keep backward compatibility with the old command line format of
     # esphome <config> <command>.
     #
@@ -733,7 +758,12 @@ def run_esphome(argv):
     args = parse_args(argv)
     CORE.dashboard = args.dashboard
 
-    setup_log(args.verbose, args.quiet)
+    setup_log(
+        args.verbose,
+        args.quiet,
+        # Show timestamp for dashboard access logs
+        args.command == "dashboard",
+    )
     if args.deprecated_argv_suggestion is not None and args.command != "vscode":
         _LOGGER.warning(
             "Calling ESPHome with the configuration before the command is deprecated "
@@ -762,7 +792,7 @@ def run_esphome(argv):
 
         config = read_config(dict(args.substitution) if args.substitution else {})
         if config is None:
-            return 1
+            return 2
         CORE.config = config
 
         if args.command not in POST_CONFIG_ACTIONS:

esphome/components/adc/sensor.py

@@ -35,7 +35,7 @@ def validate_adc_pin(value):
         if is_esp32c3():
             if not (0 <= value <= 4):  # ADC1
                 raise cv.Invalid("ESP32-C3: Only pins 0 though 4 support ADC.")
-        if not (32 <= value <= 39):  # ADC1
+        elif not (32 <= value <= 39):  # ADC1
             raise cv.Invalid("ESP32: Only pins 32 though 39 support ADC.")
     elif CORE.is_esp8266:
         from esphome.components.esp8266.gpio import CONF_ANALOG

esphome/components/ade7953/ade7953.h

@@ -76,9 +76,9 @@ class ADE7953 : public i2c::I2CDevice, public PollingComponent {
       return err;
     *value = 0;
     *value |= ((uint32_t) recv[0]) << 24;
-    *value |= ((uint32_t) recv[1]) << 24;
-    *value |= ((uint32_t) recv[2]) << 24;
-    *value |= ((uint32_t) recv[3]) << 24;
+    *value |= ((uint32_t) recv[1]) << 16;
+    *value |= ((uint32_t) recv[2]) << 8;
+    *value |= ((uint32_t) recv[3]);
     return i2c::ERROR_OK;
   }
 

esphome/components/api/__init__.py

@@ -121,7 +121,7 @@ async def to_code(config):
         decoded = base64.b64decode(conf[CONF_KEY])
         cg.add(var.set_noise_psk(list(decoded)))
         cg.add_define("USE_API_NOISE")
-        cg.add_library("esphome/noise-c", "0.1.3")
+        cg.add_library("esphome/noise-c", "0.1.4")
     else:
         cg.add_define("USE_API_PLAINTEXT")
 

esphome/components/api/api_connection.cpp

@@ -78,6 +78,8 @@ void APIConnection::loop() {
     on_fatal_error();
     if (err == APIError::SOCKET_READ_FAILED && errno == ECONNRESET) {
       ESP_LOGW(TAG, "%s: Connection reset", client_info_.c_str());
+    } else if (err == APIError::CONNECTION_CLOSED) {
+      ESP_LOGW(TAG, "%s: Connection closed", client_info_.c_str());
     } else {
       ESP_LOGW(TAG, "%s: Reading failed: %s errno=%d", client_info_.c_str(), api_error_to_str(err), errno);
     }

esphome/components/api/api_frame_helper.cpp

@@ -10,7 +10,7 @@ namespace api {
 
 static const char *const TAG = "api.socket";
 
-/// Is the given return value (from read/write syscalls) a wouldblock error?
+/// Is the given return value (from write syscalls) a wouldblock error?
 bool is_would_block(ssize_t ret) {
   if (ret == -1) {
     return errno == EWOULDBLOCK || errno == EAGAIN;
@@ -64,6 +64,8 @@ const char *api_error_to_str(APIError err) {
     return "HANDSHAKESTATE_SPLIT_FAILED";
   } else if (err == APIError::BAD_HANDSHAKE_ERROR_BYTE) {
     return "BAD_HANDSHAKE_ERROR_BYTE";
+  } else if (err == APIError::CONNECTION_CLOSED) {
+    return "CONNECTION_CLOSED";
   }
   return "UNKNOWN";
 }
@@ -185,12 +187,17 @@ APIError APINoiseFrameHelper::try_read_frame_(ParsedFrame *frame) {
     // no header information yet
     size_t to_read = 3 - rx_header_buf_len_;
     ssize_t received = socket_->read(&rx_header_buf_[rx_header_buf_len_], to_read);
-    if (is_would_block(received)) {
-      return APIError::WOULD_BLOCK;
-    } else if (received == -1) {
+    if (received == -1) {
+      if (errno == EWOULDBLOCK || errno == EAGAIN) {
+        return APIError::WOULD_BLOCK;
+      }
       state_ = State::FAILED;
       HELPER_LOG("Socket read failed with errno %d", errno);
       return APIError::SOCKET_READ_FAILED;
+    } else if (received == 0) {
+      state_ = State::FAILED;
+      HELPER_LOG("Connection closed");
+      return APIError::CONNECTION_CLOSED;
     }
     rx_header_buf_len_ += received;
     if (received != to_read) {
@@ -227,12 +234,17 @@ APIError APINoiseFrameHelper::try_read_frame_(ParsedFrame *frame) {
     // more data to read
     size_t to_read = msg_size - rx_buf_len_;
     ssize_t received = socket_->read(&rx_buf_[rx_buf_len_], to_read);
-    if (is_would_block(received)) {
-      return APIError::WOULD_BLOCK;
-    } else if (received == -1) {
+    if (received == -1) {
+      if (errno == EWOULDBLOCK || errno == EAGAIN) {
+        return APIError::WOULD_BLOCK;
+      }
       state_ = State::FAILED;
       HELPER_LOG("Socket read failed with errno %d", errno);
       return APIError::SOCKET_READ_FAILED;
+    } else if (received == 0) {
+      state_ = State::FAILED;
+      HELPER_LOG("Connection closed");
+      return APIError::CONNECTION_CLOSED;
     }
     rx_buf_len_ += received;
     if (received != to_read) {
@@ -778,12 +790,17 @@ APIError APIPlaintextFrameHelper::try_read_frame_(ParsedFrame *frame) {
   while (!rx_header_parsed_) {
     uint8_t data;
     ssize_t received = socket_->read(&data, 1);
-    if (is_would_block(received)) {
-      return APIError::WOULD_BLOCK;
-    } else if (received == -1) {
+    if (received == -1) {
+      if (errno == EWOULDBLOCK || errno == EAGAIN) {
+        return APIError::WOULD_BLOCK;
+      }
       state_ = State::FAILED;
       HELPER_LOG("Socket read failed with errno %d", errno);
       return APIError::SOCKET_READ_FAILED;
+    } else if (received == 0) {
+      state_ = State::FAILED;
+      HELPER_LOG("Connection closed");
+      return APIError::CONNECTION_CLOSED;
     }
     rx_header_buf_.push_back(data);
 
@@ -824,12 +841,17 @@ APIError APIPlaintextFrameHelper::try_read_frame_(ParsedFrame *frame) {
     // more data to read
     size_t to_read = rx_header_parsed_len_ - rx_buf_len_;
     ssize_t received = socket_->read(&rx_buf_[rx_buf_len_], to_read);
-    if (is_would_block(received)) {
-      return APIError::WOULD_BLOCK;
-    } else if (received == -1) {
+    if (received == -1) {
+      if (errno == EWOULDBLOCK || errno == EAGAIN) {
+        return APIError::WOULD_BLOCK;
+      }
       state_ = State::FAILED;
       HELPER_LOG("Socket read failed with errno %d", errno);
       return APIError::SOCKET_READ_FAILED;
+    } else if (received == 0) {
+      state_ = State::FAILED;
+      HELPER_LOG("Connection closed");
+      return APIError::CONNECTION_CLOSED;
     }
     rx_buf_len_ += received;
     if (received != to_read) {

esphome/components/api/api_frame_helper.h

@@ -53,6 +53,7 @@ enum class APIError : int {
   HANDSHAKESTATE_SETUP_FAILED = 1019,
   HANDSHAKESTATE_SPLIT_FAILED = 1020,
   BAD_HANDSHAKE_ERROR_BYTE = 1021,
+  CONNECTION_CLOSED = 1022,
 };
 
 const char *api_error_to_str(APIError err);

esphome/components/ble_client/ble_client.cpp

@@ -11,6 +11,8 @@ namespace ble_client {
 
 static const char *const TAG = "ble_client";
 
+float BLEClient::get_setup_priority() const { return setup_priority::AFTER_BLUETOOTH; }
+
 void BLEClient::setup() {
   auto ret = esp_ble_gattc_app_register(this->app_id);
   if (ret) {

esphome/components/ble_client/ble_client.h

@@ -81,6 +81,7 @@ class BLEClient : public espbt::ESPBTClient, public Component {
   void setup() override;
   void dump_config() override;
   void loop() override;
+  float get_setup_priority() const override;
 
   void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if,
                            esp_ble_gattc_cb_param_t *param) override;

esphome/components/bme680_bsec/__init__.py

@@ -2,6 +2,7 @@ import esphome.codegen as cg
 import esphome.config_validation as cv
 from esphome.components import i2c
 from esphome.const import CONF_ID
+from esphome.core import CORE
 
 CODEOWNERS = ["@trvrnrth"]
 DEPENDENCIES = ["i2c"]
@@ -44,7 +45,8 @@ CONFIG_SCHEMA = cv.Schema(
         cv.Optional(
             CONF_STATE_SAVE_INTERVAL, default="6hours"
         ): cv.positive_time_period_minutes,
-    }
+    },
+    cv.only_with_arduino,
 ).extend(i2c.i2c_device_schema(0x76))
 
 
@@ -60,5 +62,9 @@ async def to_code(config):
         var.set_state_save_interval(config[CONF_STATE_SAVE_INTERVAL].total_milliseconds)
     )
 
+    if CORE.is_esp32:
+        # Although this component does not use SPI, the BSEC library requires the SPI library
+        cg.add_library("SPI", None)
+
     cg.add_define("USE_BSEC")
-    cg.add_library("BSEC Software Library", "1.6.1480")
+    cg.add_library("boschsensortec/BSEC Software Library", "1.6.1480")

esphome/components/climate/climate.cpp

@@ -440,7 +440,11 @@ void Climate::set_visual_max_temperature_override(float visual_max_temperature_o
 void Climate::set_visual_temperature_step_override(float visual_temperature_step_override) {
   this->visual_temperature_step_override_ = visual_temperature_step_override;
 }
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 Climate::Climate(const std::string &name) : EntityBase(name) {}
+#pragma GCC diagnostic pop
+
 Climate::Climate() : Climate("") {}
 ClimateCall Climate::make_call() { return ClimateCall(this); }
 

esphome/components/esp32/__init__.py

@@ -7,6 +7,7 @@ from esphome.helpers import write_file_if_changed
 from esphome.const import (
     CONF_BOARD,
     CONF_FRAMEWORK,
+    CONF_SOURCE,
     CONF_TYPE,
     CONF_VARIANT,
     CONF_VERSION,
@@ -21,7 +22,7 @@ from esphome.core import CORE, HexInt
 import esphome.config_validation as cv
 import esphome.codegen as cg
 
-from .const import (
+from .const import (  # noqa
     KEY_BOARD,
     KEY_ESP32,
     KEY_SDKCONFIG_OPTIONS,
@@ -29,6 +30,7 @@ from .const import (
     VARIANT_ESP32C3,
     VARIANTS,
 )
+from .boards import BOARD_TO_VARIANT
 
 # force import gpio to register pin schema
 from .gpio import esp32_pin_to_code  # noqa
@@ -49,7 +51,7 @@ def set_core_data(config):
     elif conf[CONF_TYPE] == FRAMEWORK_ARDUINO:
         CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = "arduino"
     CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = cv.Version.parse(
-        config[CONF_FRAMEWORK][CONF_VERSION_HINT]
+        config[CONF_FRAMEWORK][CONF_VERSION]
     )
     CORE.data[KEY_ESP32][KEY_BOARD] = config[CONF_BOARD]
     CORE.data[KEY_ESP32][KEY_VARIANT] = config[CONF_VARIANT]
@@ -90,6 +92,13 @@ def _format_framework_arduino_version(ver: cv.Version) -> str:
     return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
 
 
+def _format_framework_espidf_version(ver: cv.Version) -> str:
+    # format the given arduino (https://github.com/espressif/esp-idf/releases) version to
+    # a PIO platformio/framework-espidf value
+    # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-espidf
+    return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
+
+
 # NOTE: Keep this in mind when updating the recommended version:
 #  * New framework historically have had some regressions, especially for WiFi.
 #    The new version needs to be thoroughly validated before changing the
@@ -119,119 +128,112 @@ ESP_IDF_PLATFORM_VERSION = cv.Version(3, 3, 2)
 def _arduino_check_versions(value):
     value = value.copy()
     lookups = {
-        "dev": ("https://github.com/espressif/arduino-esp32.git", cv.Version(2, 0, 0)),
-        "latest": ("", cv.Version(1, 0, 3)),
-        "recommended": (
-            _format_framework_arduino_version(RECOMMENDED_ARDUINO_FRAMEWORK_VERSION),
-            RECOMMENDED_ARDUINO_FRAMEWORK_VERSION,
-        ),
+        "dev": (cv.Version(2, 0, 0), "https://github.com/espressif/arduino-esp32.git"),
+        "latest": (cv.Version(1, 0, 6), None),
+        "recommended": (RECOMMENDED_ARDUINO_FRAMEWORK_VERSION, None),
     }
-    ver_value = value[CONF_VERSION]
-    default_ver_hint = None
-    if ver_value.lower() in lookups:
-        default_ver_hint = str(lookups[ver_value.lower()][1])
-        ver_value = lookups[ver_value.lower()][0]
+
+    if value[CONF_VERSION] in lookups:
+        if CONF_SOURCE in value:
+            raise cv.Invalid(
+                "Framework version needs to be explicitly specified when custom source is used."
+            )
+
+        version, source = lookups[value[CONF_VERSION]]
     else:
-        with cv.suppress_invalid():
-            ver = cv.Version.parse(cv.version_number(value))
-            if ver <= cv.Version(1, 0, 3):
-                ver_value = f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
-            else:
-                ver_value = f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
-            default_ver_hint = str(ver)
-    value[CONF_VERSION] = ver_value
+        version = cv.Version.parse(cv.version_number(value[CONF_VERSION]))
+        source = value.get(CONF_SOURCE, None)
 
-    if CONF_VERSION_HINT not in value and default_ver_hint is None:
-        raise cv.Invalid("Needs a version hint to understand the framework version")
+    value[CONF_VERSION] = str(version)
+    value[CONF_SOURCE] = source or _format_framework_arduino_version(version)
 
-    ver_hint_s = value.get(CONF_VERSION_HINT, default_ver_hint)
-    value[CONF_VERSION_HINT] = ver_hint_s
-    plat_ver = value.get(CONF_PLATFORM_VERSION, ARDUINO_PLATFORM_VERSION)
-    value[CONF_PLATFORM_VERSION] = str(plat_ver)
+    platform_version = value.get(CONF_PLATFORM_VERSION, ARDUINO_PLATFORM_VERSION)
+    value[CONF_PLATFORM_VERSION] = str(platform_version)
 
-    if cv.Version.parse(ver_hint_s) != RECOMMENDED_ARDUINO_FRAMEWORK_VERSION:
+    if version != RECOMMENDED_ARDUINO_FRAMEWORK_VERSION:
         _LOGGER.warning(
-            "The selected arduino framework version is not the recommended one"
-        )
-        _LOGGER.warning(
-            "If there are connectivity or build issues please remove the manual version"
+            "The selected Arduino framework version is not the recommended one. "
+            "If there are connectivity or build issues please remove the manual version."
         )
 
     return value
 
 
-def _format_framework_espidf_version(ver: cv.Version) -> str:
-    # format the given arduino (https://github.com/espressif/esp-idf/releases) version to
-    # a PIO platformio/framework-espidf value
-    # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-espidf
-    return f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
-
-
 def _esp_idf_check_versions(value):
     value = value.copy()
     lookups = {
-        "dev": ("https://github.com/espressif/esp-idf.git", cv.Version(4, 3, 1)),
-        "latest": ("", cv.Version(4, 3, 0)),
-        "recommended": (
-            _format_framework_espidf_version(RECOMMENDED_ESP_IDF_FRAMEWORK_VERSION),
-            RECOMMENDED_ESP_IDF_FRAMEWORK_VERSION,
-        ),
+        "dev": (cv.Version(4, 3, 1), "https://github.com/espressif/esp-idf.git"),
+        "latest": (cv.Version(4, 3, 0), None),
+        "recommended": (RECOMMENDED_ESP_IDF_FRAMEWORK_VERSION, None),
     }
-    ver_value = value[CONF_VERSION]
-    default_ver_hint = None
-    if ver_value.lower() in lookups:
-        default_ver_hint = str(lookups[ver_value.lower()][1])
-        ver_value = lookups[ver_value.lower()][0]
+
+    if value[CONF_VERSION] in lookups:
+        if CONF_SOURCE in value:
+            raise cv.Invalid(
+                "Framework version needs to be explicitly specified when custom source is used."
+            )
+
+        version, source = lookups[value[CONF_VERSION]]
     else:
-        with cv.suppress_invalid():
-            ver = cv.Version.parse(cv.version_number(value))
-            ver_value = f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
-            default_ver_hint = str(ver)
-    value[CONF_VERSION] = ver_value
+        version = cv.Version.parse(cv.version_number(value[CONF_VERSION]))
+        source = value.get(CONF_SOURCE, None)
 
-    if CONF_VERSION_HINT not in value and default_ver_hint is None:
-        raise cv.Invalid("Needs a version hint to understand the framework version")
+    if version < cv.Version(4, 0, 0):
+        raise cv.Invalid("Only ESP-IDF 4.0+ is supported.")
 
-    ver_hint_s = value.get(CONF_VERSION_HINT, default_ver_hint)
-    value[CONF_VERSION_HINT] = ver_hint_s
-    if cv.Version.parse(ver_hint_s) < cv.Version(4, 0, 0):
-        raise cv.Invalid("Only ESP-IDF 4.0+ is supported")
-    if cv.Version.parse(ver_hint_s) != RECOMMENDED_ESP_IDF_FRAMEWORK_VERSION:
+    value[CONF_VERSION] = str(version)
+    value[CONF_SOURCE] = source or _format_framework_espidf_version(version)
+
+    platform_version = value.get(CONF_PLATFORM_VERSION, ESP_IDF_PLATFORM_VERSION)
+    value[CONF_PLATFORM_VERSION] = str(platform_version)
+
+    if version != RECOMMENDED_ESP_IDF_FRAMEWORK_VERSION:
         _LOGGER.warning(
-            "The selected esp-idf framework version is not the recommended one"
+            "The selected ESP-IDF framework version is not the recommended one. "
+            "If there are connectivity or build issues please remove the manual version."
         )
-        _LOGGER.warning(
-            "If there are connectivity or build issues please remove the manual version"
-        )
-
-    plat_ver = value.get(CONF_PLATFORM_VERSION, ESP_IDF_PLATFORM_VERSION)
-    value[CONF_PLATFORM_VERSION] = str(plat_ver)
 
     return value
 
 
-CONF_VERSION_HINT = "version_hint"
+def _detect_variant(value):
+    if CONF_VARIANT not in value:
+        board = value[CONF_BOARD]
+        if board not in BOARD_TO_VARIANT:
+            raise cv.Invalid(
+                "This board is unknown, please set the variant manually",
+                path=[CONF_BOARD],
+            )
+
+        value = value.copy()
+        value[CONF_VARIANT] = BOARD_TO_VARIANT[board]
+
+    return value
+
+
 CONF_PLATFORM_VERSION = "platform_version"
+
 ARDUINO_FRAMEWORK_SCHEMA = cv.All(
     cv.Schema(
         {
             cv.Optional(CONF_VERSION, default="recommended"): cv.string_strict,
-            cv.Optional(CONF_VERSION_HINT): cv.version_number,
+            cv.Optional(CONF_SOURCE): cv.string_strict,
             cv.Optional(CONF_PLATFORM_VERSION): cv.string_strict,
         }
     ),
     _arduino_check_versions,
 )
+
 CONF_SDKCONFIG_OPTIONS = "sdkconfig_options"
 ESP_IDF_FRAMEWORK_SCHEMA = cv.All(
     cv.Schema(
         {
             cv.Optional(CONF_VERSION, default="recommended"): cv.string_strict,
-            cv.Optional(CONF_VERSION_HINT): cv.version_number,
+            cv.Optional(CONF_SOURCE): cv.string_strict,
+            cv.Optional(CONF_PLATFORM_VERSION): cv.string_strict,
             cv.Optional(CONF_SDKCONFIG_OPTIONS, default={}): {
                 cv.string_strict: cv.string_strict
             },
-            cv.Optional(CONF_PLATFORM_VERSION): cv.string_strict,
             cv.Optional(CONF_ADVANCED, default={}): cv.Schema(
                 {
                     cv.Optional(CONF_IGNORE_EFUSE_MAC_CRC, default=False): cv.boolean,
@@ -260,12 +262,11 @@ CONFIG_SCHEMA = cv.All(
     cv.Schema(
         {
             cv.Required(CONF_BOARD): cv.string_strict,
-            cv.Optional(CONF_VARIANT, default="ESP32"): cv.one_of(
-                *VARIANTS, upper=True
-            ),
+            cv.Optional(CONF_VARIANT): cv.one_of(*VARIANTS, upper=True),
             cv.Optional(CONF_FRAMEWORK, default={}): FRAMEWORK_SCHEMA,
         }
     ),
+    _detect_variant,
     set_core_data,
 )
 
@@ -289,7 +290,7 @@ async def to_code(config):
         cg.add_build_flag("-Wno-nonnull-compare")
         cg.add_platformio_option(
             "platform_packages",
-            [f"platformio/framework-espidf @ {conf[CONF_VERSION]}"],
+            [f"platformio/framework-espidf @ {conf[CONF_SOURCE]}"],
         )
         add_idf_sdkconfig_option("CONFIG_PARTITION_TABLE_SINGLE_APP", False)
         add_idf_sdkconfig_option("CONFIG_PARTITION_TABLE_CUSTOM", True)
@@ -319,7 +320,7 @@ async def to_code(config):
         cg.add_build_flag("-DUSE_ESP32_FRAMEWORK_ARDUINO")
         cg.add_platformio_option(
             "platform_packages",
-            [f"platformio/framework-arduinoespressif32 @ {conf[CONF_VERSION]}"],
+            [f"platformio/framework-arduinoespressif32 @ {conf[CONF_SOURCE]}"],
         )
 
         cg.add_platformio_option("board_build.partitions", "partitions.csv")

esphome/components/esp32/boards.py

@@ -1,3 +1,5 @@
+from .const import VARIANT_ESP32, VARIANT_ESP32S2, VARIANT_ESP32C3
+
 ESP32_BASE_PINS = {
     "TX": 1,
     "RX": 3,
@@ -925,3 +927,125 @@ ESP32_BOARD_PINS = {
     },
     "xinabox_cw02": {"LED": 27},
 }
+
+"""
+BOARD_TO_VARIANT generated with:
+
+git clone https://github.com/platformio/platform-espressif32
+for x in platform-espressif32/boards/*.json; do
+  mcu=$(jq -r .build.mcu <"$x");
+  fname=$(basename "$x")
+  board="${fname%.*}"
+  variant=$(echo "$mcu" | tr '[:lower:]' '[:upper:]')
+  echo "    \"$board\": VARIANT_${variant},"
+done | sort
+"""
+
+BOARD_TO_VARIANT = {
+    "alksesp32": VARIANT_ESP32,
+    "az-delivery-devkit-v4": VARIANT_ESP32,
+    "bpi-bit": VARIANT_ESP32,
+    "briki_abc_esp32": VARIANT_ESP32,
+    "briki_mbc-wb_esp32": VARIANT_ESP32,
+    "d-duino-32": VARIANT_ESP32,
+    "esp320": VARIANT_ESP32,
+    "esp32-c3-devkitm-1": VARIANT_ESP32C3,
+    "esp32cam": VARIANT_ESP32,
+    "esp32-devkitlipo": VARIANT_ESP32,
+    "esp32dev": VARIANT_ESP32,
+    "esp32doit-devkit-v1": VARIANT_ESP32,
+    "esp32doit-espduino": VARIANT_ESP32,
+    "esp32-evb": VARIANT_ESP32,
+    "esp32-gateway": VARIANT_ESP32,
+    "esp32-poe-iso": VARIANT_ESP32,
+    "esp32-poe": VARIANT_ESP32,
+    "esp32-pro": VARIANT_ESP32,
+    "esp32-s2-kaluga-1": VARIANT_ESP32S2,
+    "esp32-s2-saola-1": VARIANT_ESP32S2,
+    "esp32thing_plus": VARIANT_ESP32,
+    "esp32thing": VARIANT_ESP32,
+    "esp32vn-iot-uno": VARIANT_ESP32,
+    "espea32": VARIANT_ESP32,
+    "espectro32": VARIANT_ESP32,
+    "espino32": VARIANT_ESP32,
+    "esp-wrover-kit": VARIANT_ESP32,
+    "etboard": VARIANT_ESP32,
+    "featheresp32-s2": VARIANT_ESP32S2,
+    "featheresp32": VARIANT_ESP32,
+    "firebeetle32": VARIANT_ESP32,
+    "fm-devkit": VARIANT_ESP32,
+    "frogboard": VARIANT_ESP32,
+    "healthypi4": VARIANT_ESP32,
+    "heltec_wifi_kit_32_v2": VARIANT_ESP32,
+    "heltec_wifi_kit_32": VARIANT_ESP32,
+    "heltec_wifi_lora_32_V2": VARIANT_ESP32,
+    "heltec_wifi_lora_32": VARIANT_ESP32,
+    "heltec_wireless_stick_lite": VARIANT_ESP32,
+    "heltec_wireless_stick": VARIANT_ESP32,
+    "honeylemon": VARIANT_ESP32,
+    "hornbill32dev": VARIANT_ESP32,
+    "hornbill32minima": VARIANT_ESP32,
+    "imbrios-logsens-v1p1": VARIANT_ESP32,
+    "inex_openkb": VARIANT_ESP32,
+    "intorobot": VARIANT_ESP32,
+    "iotaap_magnolia": VARIANT_ESP32,
+    "iotbusio": VARIANT_ESP32,
+    "iotbusproteus": VARIANT_ESP32,
+    "kits-edu": VARIANT_ESP32,
+    "labplus_mpython": VARIANT_ESP32,
+    "lolin32_lite": VARIANT_ESP32,
+    "lolin32": VARIANT_ESP32,
+    "lolin_d32_pro": VARIANT_ESP32,
+    "lolin_d32": VARIANT_ESP32,
+    "lopy4": VARIANT_ESP32,
+    "lopy": VARIANT_ESP32,
+    "m5stack-atom": VARIANT_ESP32,
+    "m5stack-core2": VARIANT_ESP32,
+    "m5stack-core-esp32": VARIANT_ESP32,
+    "m5stack-coreink": VARIANT_ESP32,
+    "m5stack-fire": VARIANT_ESP32,
+    "m5stack-grey": VARIANT_ESP32,
+    "m5stack-timer-cam": VARIANT_ESP32,
+    "m5stick-c": VARIANT_ESP32,
+    "magicbit": VARIANT_ESP32,
+    "mgbot-iotik32a": VARIANT_ESP32,
+    "mgbot-iotik32b": VARIANT_ESP32,
+    "mhetesp32devkit": VARIANT_ESP32,
+    "mhetesp32minikit": VARIANT_ESP32,
+    "microduino-core-esp32": VARIANT_ESP32,
+    "nano32": VARIANT_ESP32,
+    "nina_w10": VARIANT_ESP32,
+    "node32s": VARIANT_ESP32,
+    "nodemcu-32s": VARIANT_ESP32,
+    "nscreen-32": VARIANT_ESP32,
+    "odroid_esp32": VARIANT_ESP32,
+    "onehorse32dev": VARIANT_ESP32,
+    "oroca_edubot": VARIANT_ESP32,
+    "pico32": VARIANT_ESP32,
+    "piranha_esp32": VARIANT_ESP32,
+    "pocket_32": VARIANT_ESP32,
+    "pycom_gpy": VARIANT_ESP32,
+    "qchip": VARIANT_ESP32,
+    "quantum": VARIANT_ESP32,
+    "sensesiot_weizen": VARIANT_ESP32,
+    "sg-o_airMon": VARIANT_ESP32,
+    "s_odi_ultra": VARIANT_ESP32,
+    "sparkfun_lora_gateway_1-channel": VARIANT_ESP32,
+    "tinypico": VARIANT_ESP32,
+    "ttgo-lora32-v1": VARIANT_ESP32,
+    "ttgo-lora32-v21": VARIANT_ESP32,
+    "ttgo-lora32-v2": VARIANT_ESP32,
+    "ttgo-t1": VARIANT_ESP32,
+    "ttgo-t7-v13-mini32": VARIANT_ESP32,
+    "ttgo-t7-v14-mini32": VARIANT_ESP32,
+    "ttgo-t-beam": VARIANT_ESP32,
+    "ttgo-t-watch": VARIANT_ESP32,
+    "turta_iot_node": VARIANT_ESP32,
+    "vintlabs-devkit-v1": VARIANT_ESP32,
+    "wemosbat": VARIANT_ESP32,
+    "wemos_d1_mini32": VARIANT_ESP32,
+    "wesp32": VARIANT_ESP32,
+    "widora-air": VARIANT_ESP32,
+    "wifiduino32": VARIANT_ESP32,
+    "xinabox_cw02": VARIANT_ESP32,
+}

esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp

@@ -57,7 +57,7 @@ void ESP32BLEBeacon::setup() {
   );
 }
 
-float ESP32BLEBeacon::get_setup_priority() const { return setup_priority::DATA; }
+float ESP32BLEBeacon::get_setup_priority() const { return setup_priority::BLUETOOTH; }
 void ESP32BLEBeacon::ble_core_task(void *params) {
   ble_setup();
 

esphome/components/esp32_ble_server/ble_server.cpp

@@ -98,19 +98,20 @@ bool BLEServer::create_device_characteristics_() {
   return true;
 }
 
-BLEService *BLEServer::create_service(const uint8_t *uuid, bool advertise) {
+std::shared_ptr<BLEService> BLEServer::create_service(const uint8_t *uuid, bool advertise) {
   return this->create_service(ESPBTUUID::from_raw(uuid), advertise);
 }
-BLEService *BLEServer::create_service(uint16_t uuid, bool advertise) {
+std::shared_ptr<BLEService> BLEServer::create_service(uint16_t uuid, bool advertise) {
   return this->create_service(ESPBTUUID::from_uint16(uuid), advertise);
 }
-BLEService *BLEServer::create_service(const std::string &uuid, bool advertise) {
+std::shared_ptr<BLEService> BLEServer::create_service(const std::string &uuid, bool advertise) {
   return this->create_service(ESPBTUUID::from_raw(uuid), advertise);
 }
-BLEService *BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t num_handles, uint8_t inst_id) {
+std::shared_ptr<BLEService> BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t num_handles,
+                                                      uint8_t inst_id) {
   ESP_LOGV(TAG, "Creating service - %s", uuid.to_string().c_str());
-  BLEService *service = new BLEService(uuid, num_handles, inst_id);  // NOLINT(cppcoreguidelines-owning-memory)
-  this->services_.push_back(service);
+  std::shared_ptr<BLEService> service = std::make_shared<BLEService>(uuid, num_handles, inst_id);
+  this->services_.emplace_back(service);
   if (advertise) {
     esp32_ble::global_ble->get_advertising()->add_service_uuid(uuid);
   }
@@ -149,12 +150,12 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga
       break;
   }
 
-  for (auto *service : this->services_) {
+  for (const auto &service : this->services_) {
     service->gatts_event_handler(event, gatts_if, param);
   }
 }
 
-float BLEServer::get_setup_priority() const { return setup_priority::BLUETOOTH - 10; }
+float BLEServer::get_setup_priority() const { return setup_priority::AFTER_BLUETOOTH + 10; }
 
 void BLEServer::dump_config() { ESP_LOGCONFIG(TAG, "ESP32 BLE Server:"); }
 

esphome/components/esp32_ble_server/ble_server.h

@@ -11,6 +11,7 @@
 #include "esphome/core/preferences.h"
 
 #include <map>
+#include <memory>
 
 #ifdef USE_ESP32
 
@@ -43,10 +44,11 @@ class BLEServer : public Component {
   void set_manufacturer(const std::string &manufacturer) { this->manufacturer_ = manufacturer; }
   void set_model(const std::string &model) { this->model_ = model; }
 
-  BLEService *create_service(const uint8_t *uuid, bool advertise = false);
-  BLEService *create_service(uint16_t uuid, bool advertise = false);
-  BLEService *create_service(const std::string &uuid, bool advertise = false);
-  BLEService *create_service(ESPBTUUID uuid, bool advertise = false, uint16_t num_handles = 15, uint8_t inst_id = 0);
+  std::shared_ptr<BLEService> create_service(const uint8_t *uuid, bool advertise = false);
+  std::shared_ptr<BLEService> create_service(uint16_t uuid, bool advertise = false);
+  std::shared_ptr<BLEService> create_service(const std::string &uuid, bool advertise = false);
+  std::shared_ptr<BLEService> create_service(ESPBTUUID uuid, bool advertise = false, uint16_t num_handles = 15,
+                                             uint8_t inst_id = 0);
 
   esp_gatt_if_t get_gatts_if() { return this->gatts_if_; }
   uint32_t get_connected_client_count() { return this->connected_clients_; }
@@ -74,8 +76,8 @@ class BLEServer : public Component {
   uint32_t connected_clients_{0};
   std::map<uint16_t, void *> clients_;
 
-  std::vector<BLEService *> services_;
-  BLEService *device_information_service_;
+  std::vector<std::shared_ptr<BLEService>> services_;
+  std::shared_ptr<BLEService> device_information_service_;
 
   std::vector<BLEServiceComponent *> service_components_;
 

esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp

@@ -40,6 +40,8 @@ uint64_t ble_addr_to_uint64(const esp_bd_addr_t address) {
   return u;
 }
 
+float ESP32BLETracker::get_setup_priority() const { return setup_priority::BLUETOOTH; }
+
 void ESP32BLETracker::setup() {
   global_esp32_ble_tracker = this;
   this->scan_result_lock_ = xSemaphoreCreateMutex();

esphome/components/esp32_ble_tracker/esp32_ble_tracker.h

@@ -171,6 +171,7 @@ class ESP32BLETracker : public Component {
   /// Setup the FreeRTOS task and the Bluetooth stack.
   void setup() override;
   void dump_config() override;
+  float get_setup_priority() const override;
 
   void loop() override;
 

esphome/components/esp32_improv/esp32_improv_component.h

@@ -48,7 +48,7 @@ class ESP32ImprovComponent : public Component, public BLEServiceComponent {
   std::vector<uint8_t> incoming_data_;
   wifi::WiFiAP connecting_sta_;
 
-  BLEService *service_;
+  std::shared_ptr<BLEService> service_;
   BLECharacteristic *status_;
   BLECharacteristic *error_;
   BLECharacteristic *rpc_;

esphome/components/esp8266/__init__.py

@@ -4,6 +4,7 @@ from esphome.const import (
     CONF_BOARD,
     CONF_BOARD_FLASH_MODE,
     CONF_FRAMEWORK,
+    CONF_SOURCE,
     CONF_VERSION,
     KEY_CORE,
     KEY_FRAMEWORK_VERSION,
@@ -31,7 +32,7 @@ def set_core_data(config):
     CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = "esp8266"
     CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = "arduino"
     CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = cv.Version.parse(
-        config[CONF_FRAMEWORK][CONF_VERSION_HINT]
+        config[CONF_FRAMEWORK][CONF_VERSION]
     )
     CORE.data[KEY_ESP8266][KEY_BOARD] = config[CONF_BOARD]
     return config
@@ -64,72 +65,56 @@ RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(2, 7, 4)
 #  - https://api.registry.platformio.org/v3/packages/platformio/platform/espressif8266
 ARDUINO_2_PLATFORM_VERSION = cv.Version(2, 6, 2)
 # for arduino 3 framework versions
-ARDUINO_3_PLATFORM_VERSION = cv.Version(3, 0, 2)
+ARDUINO_3_PLATFORM_VERSION = cv.Version(3, 2, 0)
 
 
 def _arduino_check_versions(value):
     value = value.copy()
     lookups = {
-        "dev": ("https://github.com/esp8266/Arduino.git", cv.Version(3, 0, 2)),
-        "latest": ("", cv.Version(3, 0, 2)),
-        "recommended": (
-            _format_framework_arduino_version(RECOMMENDED_ARDUINO_FRAMEWORK_VERSION),
-            RECOMMENDED_ARDUINO_FRAMEWORK_VERSION,
-        ),
+        "dev": (cv.Version(3, 0, 2), "https://github.com/esp8266/Arduino.git"),
+        "latest": (cv.Version(3, 0, 2), None),
+        "recommended": (RECOMMENDED_ARDUINO_FRAMEWORK_VERSION, None),
     }
-    ver_value = value[CONF_VERSION]
-    default_ver_hint = None
-    if ver_value.lower() in lookups:
-        default_ver_hint = str(lookups[ver_value.lower()][1])
-        ver_value = lookups[ver_value.lower()][0]
+
+    if value[CONF_VERSION] in lookups:
+        if CONF_SOURCE in value:
+            raise cv.Invalid(
+                "Framework version needs to be explicitly specified when custom source is used."
+            )
+
+        version, source = lookups[value[CONF_VERSION]]
     else:
-        with cv.suppress_invalid():
-            ver = cv.Version.parse(cv.version_number(value))
-            if ver <= cv.Version(2, 4, 1):
-                ver_value = f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
-            elif ver <= cv.Version(2, 6, 2):
-                ver_value = f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
-            else:
-                ver_value = f"~3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
-            default_ver_hint = str(ver)
+        version = cv.Version.parse(cv.version_number(value[CONF_VERSION]))
+        source = value.get(CONF_SOURCE, None)
 
-    value[CONF_VERSION] = ver_value
+    value[CONF_VERSION] = str(version)
+    value[CONF_SOURCE] = source or _format_framework_arduino_version(version)
 
-    if CONF_VERSION_HINT not in value and default_ver_hint is None:
-        raise cv.Invalid("Needs a version hint to understand the framework version")
-
-    ver_hint_s = value.get(CONF_VERSION_HINT, default_ver_hint)
-    value[CONF_VERSION_HINT] = ver_hint_s
-    plat_ver = value.get(CONF_PLATFORM_VERSION)
-
-    if plat_ver is None:
-        ver_hint = cv.Version.parse(ver_hint_s)
-        if ver_hint >= cv.Version(3, 0, 0):
-            plat_ver = ARDUINO_3_PLATFORM_VERSION
-        elif ver_hint >= cv.Version(2, 5, 0):
-            plat_ver = ARDUINO_2_PLATFORM_VERSION
+    platform_version = value.get(CONF_PLATFORM_VERSION)
+    if platform_version is None:
+        if version >= cv.Version(3, 0, 0):
+            platform_version = ARDUINO_3_PLATFORM_VERSION
+        elif version >= cv.Version(2, 5, 0):
+            platform_version = ARDUINO_2_PLATFORM_VERSION
         else:
-            plat_ver = cv.Version(1, 8, 0)
-    value[CONF_PLATFORM_VERSION] = str(plat_ver)
+            platform_version = cv.Version(1, 8, 0)
+    value[CONF_PLATFORM_VERSION] = str(platform_version)
 
-    if cv.Version.parse(ver_hint_s) != RECOMMENDED_ARDUINO_FRAMEWORK_VERSION:
+    if version != RECOMMENDED_ARDUINO_FRAMEWORK_VERSION:
         _LOGGER.warning(
-            "The selected arduino framework version is not the recommended one"
-        )
-        _LOGGER.warning(
-            "If there are connectivity or build issues please remove the manual version"
+            "The selected Arduino framework version is not the recommended one. "
+            "If there are connectivity or build issues please remove the manual version."
         )
 
     return value
 
 
-CONF_VERSION_HINT = "version_hint"
 CONF_PLATFORM_VERSION = "platform_version"
 ARDUINO_FRAMEWORK_SCHEMA = cv.All(
     cv.Schema(
         {
             cv.Optional(CONF_VERSION, default="recommended"): cv.string_strict,
-            cv.Optional(CONF_VERSION_HINT): cv.version_number,
+            cv.Optional(CONF_SOURCE): cv.string_strict,
             cv.Optional(CONF_PLATFORM_VERSION): cv.string_strict,
         }
     ),
@@ -167,7 +152,7 @@ async def to_code(config):
     cg.add_build_flag("-DUSE_ESP8266_FRAMEWORK_ARDUINO")
     cg.add_platformio_option(
         "platform_packages",
-        [f"platformio/framework-arduinoespressif8266 @ {conf[CONF_VERSION]}"],
+        [f"platformio/framework-arduinoespressif8266 @ {conf[CONF_SOURCE]}"],
     )
     cg.add_platformio_option(
         "platform", f"platformio/espressif8266 @ {conf[CONF_PLATFORM_VERSION]}"

esphome/components/esp8266/gpio.cpp

@@ -50,6 +50,13 @@ void ESP8266GPIOPin::pin_mode(gpio::Flags flags) {
     mode = OUTPUT;
   } else if (flags == (gpio::FLAG_INPUT | gpio::FLAG_PULLUP)) {
     mode = INPUT_PULLUP;
+    if (pin_ == 16) {
+      // GPIO16 doesn't have a pullup, so pinMode would fail.
+      // However, sometimes this method is called with pullup mode anyway
+      // for example from dallas one_wire. For those cases convert this
+      // to a INPUT mode.
+      mode = INPUT;
+    }
   } else if (flags == (gpio::FLAG_INPUT | gpio::FLAG_PULLDOWN)) {
     mode = INPUT_PULLDOWN_16;
   } else if (flags == (gpio::FLAG_OUTPUT | gpio::FLAG_OPEN_DRAIN)) {

esphome/components/esp8266/gpio.py

@@ -107,9 +107,9 @@ def validate_supports(value):
         raise cv.Invalid(
             "Open-drain only works with output mode", [CONF_MODE, CONF_OPEN_DRAIN]
         )
-    if is_pullup and num == 0:
+    if is_pullup and num == 16:
         raise cv.Invalid(
-            "GPIO Pin 0 does not support pullup pin mode. "
+            "GPIO Pin 16 does not support pullup pin mode. "
             "Please choose another pin.",
             [CONF_MODE, CONF_PULLUP],
         )

esphome/components/light/addressable_light.cpp

@@ -79,7 +79,7 @@ optional<LightColorValues> AddressableLightTransformer::apply() {
   // dynamically-calculated alpha values to match the look.
 
   float denom = (1.0f - smoothed_progress);
-  float alpha = denom == 0.0f ? 0.0f : (smoothed_progress - this->last_transition_progress_) / denom;
+  float alpha = denom == 0.0f ? 1.0f : (smoothed_progress - this->last_transition_progress_) / denom;
 
   // We need to use a low-resolution alpha here which makes the transition set in only after ~half of the length
   // We solve this by accumulating the fractional part of the alpha over time.

esphome/components/light/addressable_light.h

@@ -22,6 +22,12 @@ using ESPColor ESPDEPRECATED("esphome::light::ESPColor is deprecated, use esphom
 /// Convert the color information from a `LightColorValues` object to a `Color` object (does not apply brightness).
 Color color_from_light_color_values(LightColorValues val);
 
+/// Use a custom state class for addressable lights, to allow type system to discriminate between addressable and
+/// non-addressable lights.
+class AddressableLightState : public LightState {
+  using LightState::LightState;
+};
+
 class AddressableLight : public LightOutput, public Component {
  public:
   virtual int32_t size() const = 0;

esphome/components/light/addressable_light_wrapper.h

@@ -16,24 +16,94 @@ class AddressableLightWrapper : public light::AddressableLight {
 
   void clear_effect_data() override { this->wrapper_state_[4] = 0; }
 
-  light::LightTraits get_traits() override { return this->light_state_->get_traits(); }
+  light::LightTraits get_traits() override {
+    LightTraits traits;
+
+    // Choose which color mode to use.
+    // This is ordered by how closely each color mode matches the underlying RGBW data structure used in LightPartition.
+    ColorMode color_mode_precedence[] = {ColorMode::RGB_WHITE,
+                                         ColorMode::RGB_COLD_WARM_WHITE,
+                                         ColorMode::RGB_COLOR_TEMPERATURE,
+                                         ColorMode::RGB,
+                                         ColorMode::WHITE,
+                                         ColorMode::COLD_WARM_WHITE,
+                                         ColorMode::COLOR_TEMPERATURE,
+                                         ColorMode::BRIGHTNESS,
+                                         ColorMode::ON_OFF,
+                                         ColorMode::UNKNOWN};
+
+    LightTraits parent_traits = this->light_state_->get_traits();
+    for (auto cm : color_mode_precedence) {
+      if (parent_traits.supports_color_mode(cm)) {
+        this->color_mode_ = cm;
+        break;
+      }
+    }
+
+    // Report a color mode that's compatible with both the partition and the underlying light
+    switch (this->color_mode_) {
+      case ColorMode::RGB_WHITE:
+      case ColorMode::RGB_COLD_WARM_WHITE:
+      case ColorMode::RGB_COLOR_TEMPERATURE:
+        traits.set_supported_color_modes({light::ColorMode::RGB_WHITE});
+        break;
+
+      case ColorMode::RGB:
+        traits.set_supported_color_modes({light::ColorMode::RGB});
+        break;
+
+      case ColorMode::WHITE:
+      case ColorMode::COLD_WARM_WHITE:
+      case ColorMode::COLOR_TEMPERATURE:
+      case ColorMode::BRIGHTNESS:
+        traits.set_supported_color_modes({light::ColorMode::BRIGHTNESS});
+        break;
+
+      case ColorMode::ON_OFF:
+        traits.set_supported_color_modes({light::ColorMode::ON_OFF});
+        break;
+
+      default:
+        traits.set_supported_color_modes({light::ColorMode::UNKNOWN});
+    }
+
+    return traits;
+  }
 
   void write_state(light::LightState *state) override {
+    // Don't overwrite state if the underlying light is turned on
+    if (this->light_state_->remote_values.is_on()) {
+      this->mark_shown_();
+      return;
+    }
+
     float gamma = this->light_state_->get_gamma_correct();
     float r = gamma_uncorrect(this->wrapper_state_[0] / 255.0f, gamma);
     float g = gamma_uncorrect(this->wrapper_state_[1] / 255.0f, gamma);
     float b = gamma_uncorrect(this->wrapper_state_[2] / 255.0f, gamma);
     float w = gamma_uncorrect(this->wrapper_state_[3] / 255.0f, gamma);
-    float brightness = fmaxf(r, fmaxf(g, b));
 
     auto call = this->light_state_->make_call();
-    call.set_state(true);
-    call.set_brightness_if_supported(1.0f);
-    call.set_color_brightness_if_supported(brightness);
-    call.set_red_if_supported(r);
-    call.set_green_if_supported(g);
-    call.set_blue_if_supported(b);
-    call.set_white_if_supported(w);
+
+    float color_brightness = fmaxf(r, fmaxf(g, b));
+    float brightness = fmaxf(color_brightness, w);
+    if (brightness == 0.0f) {
+      call.set_state(false);
+    } else {
+      color_brightness /= brightness;
+      w /= brightness;
+
+      call.set_state(true);
+      call.set_color_mode_if_supported(this->color_mode_);
+      call.set_brightness_if_supported(brightness);
+      call.set_color_brightness_if_supported(color_brightness);
+      call.set_red_if_supported(r);
+      call.set_green_if_supported(g);
+      call.set_blue_if_supported(b);
+      call.set_white_if_supported(w);
+      call.set_warm_white_if_supported(w);
+      call.set_cold_white_if_supported(w);
+    }
     call.set_transition_length_if_supported(0);
     call.set_publish(false);
     call.set_save(false);
@@ -50,6 +120,7 @@ class AddressableLightWrapper : public light::AddressableLight {
 
   light::LightState *light_state_;
   uint8_t *wrapper_state_;
+  ColorMode color_mode_{ColorMode::UNKNOWN};
 };
 
 }  // namespace light

esphome/components/light/light_transformer.h

@@ -39,7 +39,15 @@ class LightTransformer {
 
  protected:
   /// The progress of this transition, on a scale of 0 to 1.
-  float get_progress_() { return clamp((millis() - this->start_time_) / float(this->length_), 0.0f, 1.0f); }
+  float get_progress_() {
+    uint32_t now = esphome::millis();
+    if (now < this->start_time_)
+      return 0.0f;
+    if (now >= this->start_time_ + this->length_)
+      return 1.0f;
+
+    return clamp((now - this->start_time_) / float(this->length_), 0.0f, 1.0f);
+  }
 
   uint32_t start_time_;
   uint32_t length_;

esphome/components/light/transformers.h

@@ -18,10 +18,13 @@ class LightTransitionTransformer : public LightTransformer {
       this->start_values_.set_brightness(0.0f);
     }
 
-    // When turning light off from on state, use source state and only decrease brightness to zero.
+    // When turning light off from on state, use source state and only decrease brightness to zero. Use a second
+    // variable for transition end state, as overwriting target_values breaks LightState logic.
     if (this->start_values_.is_on() && !this->target_values_.is_on()) {
-      this->target_values_ = LightColorValues(this->start_values_);
-      this->target_values_.set_brightness(0.0f);
+      this->end_values_ = LightColorValues(this->start_values_);
+      this->end_values_.set_brightness(0.0f);
+    } else {
+      this->end_values_ = LightColorValues(this->target_values_);
     }
 
     // When changing color mode, go through off state, as color modes are orthogonal and there can't be two active.
@@ -43,7 +46,7 @@ class LightTransitionTransformer : public LightTransformer {
     }
 
     LightColorValues &start = this->changing_color_mode_ && p > 0.5f ? this->intermediate_values_ : this->start_values_;
-    LightColorValues &end = this->changing_color_mode_ && p < 0.5f ? this->intermediate_values_ : this->target_values_;
+    LightColorValues &end = this->changing_color_mode_ && p < 0.5f ? this->intermediate_values_ : this->end_values_;
     if (this->changing_color_mode_)
       p = p < 0.5f ? p * 2 : (p - 0.5) * 2;
 
@@ -57,6 +60,7 @@ class LightTransitionTransformer : public LightTransformer {
   static float smoothed_progress(float x) { return x * x * x * (x * (x * 6.0f - 15.0f) + 10.0f); }
 
   bool changing_color_mode_{false};
+  LightColorValues end_values_{};
   LightColorValues intermediate_values_{};
 };
 
@@ -69,9 +73,7 @@ class LightFlashTransformer : public LightTransformer {
     if (this->transition_length_ * 2 > this->length_)
       this->transition_length_ = this->length_ / 2;
 
-    // do not create transition if length is 0
-    if (this->transition_length_ == 0)
-      return;
+    this->begun_lightstate_restore_ = false;
 
     // first transition to original target
     this->transformer_ = this->state_.get_output()->create_default_transition();
@@ -79,40 +81,45 @@ class LightFlashTransformer : public LightTransformer {
   }
 
   optional<LightColorValues> apply() override {
-    // transition transformer does not handle 0 length as progress returns nan
-    if (this->transition_length_ == 0)
-      return this->target_values_;
+    optional<LightColorValues> result = {};
+
+    if (this->transformer_ == nullptr && millis() > this->start_time_ + this->length_ - this->transition_length_) {
+      // second transition back to start value
+      this->transformer_ = this->state_.get_output()->create_default_transition();
+      this->transformer_->setup(this->state_.current_values, this->get_start_values(), this->transition_length_);
+      this->begun_lightstate_restore_ = true;
+    }
 
     if (this->transformer_ != nullptr) {
-      if (!this->transformer_->is_finished()) {
-        return this->transformer_->apply();
-      } else {
+      result = this->transformer_->apply();
+
+      if (this->transformer_->is_finished()) {
         this->transformer_->stop();
         this->transformer_ = nullptr;
       }
     }
 
-    if (millis() > this->start_time_ + this->length_ - this->transition_length_) {
-      // second transition back to start value
-      this->transformer_ = this->state_.get_output()->create_default_transition();
-      this->transformer_->setup(this->state_.current_values, this->get_start_values(), this->transition_length_);
-    }
-
-    // once transition is complete, don't change states until next transition
-    return optional<LightColorValues>();
+    return result;
   }
 
   // Restore the original values after the flash.
   void stop() override {
+    if (this->transformer_ != nullptr) {
+      this->transformer_->stop();
+      this->transformer_ = nullptr;
+    }
     this->state_.current_values = this->get_start_values();
     this->state_.remote_values = this->get_start_values();
     this->state_.publish_state();
   }
 
+  bool is_finished() override { return this->begun_lightstate_restore_ && LightTransformer::is_finished(); }
+
  protected:
   LightState &state_;
   uint32_t transition_length_;
   std::unique_ptr<LightTransformer> transformer_{nullptr};
+  bool begun_lightstate_restore_;
 };
 
 }  // namespace light

esphome/components/light/types.py

@@ -4,10 +4,9 @@ from esphome import automation
 # Base
 light_ns = cg.esphome_ns.namespace("light")
 LightState = light_ns.class_("LightState", cg.EntityBase, cg.Component)
-# Fake class for addressable lights
-AddressableLightState = light_ns.class_("LightState", LightState)
+AddressableLightState = light_ns.class_("AddressableLightState", LightState)
 LightOutput = light_ns.class_("LightOutput")
-AddressableLight = light_ns.class_("AddressableLight", cg.Component)
+AddressableLight = light_ns.class_("AddressableLight", LightOutput, cg.Component)
 AddressableLightRef = AddressableLight.operator("ref")
 
 Color = cg.esphome_ns.class_("Color")

esphome/components/logger/__init__.py

@@ -19,6 +19,8 @@ from esphome.const import (
     CONF_TX_BUFFER_SIZE,
 )
 from esphome.core import CORE, EsphomeError, Lambda, coroutine_with_priority
+from esphome.components.esp32 import get_esp32_variant
+from esphome.components.esp32.const import VARIANT_ESP32S2, VARIANT_ESP32C3
 
 CODEOWNERS = ["@esphome/core"]
 logger_ns = cg.esphome_ns.namespace("logger")
@@ -52,6 +54,10 @@ LOG_LEVEL_SEVERITY = [
     "VERY_VERBOSE",
 ]
 
+ESP32_REDUCED_VARIANTS = [VARIANT_ESP32C3, VARIANT_ESP32S2]
+
+UART_SELECTION_ESP32_REDUCED = ["UART0", "UART1"]
+
 UART_SELECTION_ESP32 = ["UART0", "UART1", "UART2"]
 
 UART_SELECTION_ESP8266 = ["UART0", "UART0_SWAP", "UART1"]
@@ -75,6 +81,8 @@ is_log_level = cv.one_of(*LOG_LEVELS, upper=True)
 
 def uart_selection(value):
     if CORE.is_esp32:
+        if get_esp32_variant() in ESP32_REDUCED_VARIANTS:
+            return cv.one_of(*UART_SELECTION_ESP32_REDUCED, upper=True)(value)
         return cv.one_of(*UART_SELECTION_ESP32, upper=True)(value)
     if CORE.is_esp8266:
         return cv.one_of(*UART_SELECTION_ESP8266, upper=True)(value)

esphome/components/logger/logger.cpp

@@ -153,13 +153,9 @@ void Logger::pre_setup() {
       case UART_SELECTION_UART1:
         this->hw_serial_ = &Serial1;
         break;
-#ifdef USE_ESP32
+#if defined(USE_ESP32) && !defined(USE_ESP32_VARIANT_ESP32C3) && !defined(USE_ESP32_VARIANT_ESP32S2)
       case UART_SELECTION_UART2:
-#if !CONFIG_IDF_TARGET_ESP32S2 && !CONFIG_IDF_TARGET_ESP32C3
-        // FIXME: Validate in config that UART2 can't be set for ESP32-S2 (only has
-        // UART0-UART1)
         this->hw_serial_ = &Serial2;
-#endif
         break;
 #endif
     }
@@ -173,9 +169,11 @@ void Logger::pre_setup() {
       case UART_SELECTION_UART1:
         uart_num_ = UART_NUM_1;
         break;
+#if defined(USE_ESP32) && !defined(USE_ESP32_VARIANT_ESP32C3) && !defined(USE_ESP32_VARIANT_ESP32S2)
       case UART_SELECTION_UART2:
         uart_num_ = UART_NUM_2;
         break;
+#endif
     }
     uart_config_t uart_config{};
     uart_config.baud_rate = (int) baud_rate_;

esphome/components/logger/logger.h

@@ -24,7 +24,7 @@ namespace logger {
 enum UARTSelection {
   UART_SELECTION_UART0 = 0,
   UART_SELECTION_UART1,
-#ifdef USE_ESP32
+#if defined(USE_ESP32) && !defined(USE_ESP32_VARIANT_ESP32C3) && !defined(USE_ESP32_VARIANT_ESP32S2)
   UART_SELECTION_UART2,
 #endif
 #ifdef USE_ESP8266

esphome/components/md5/__init__.py

@@ -0,0 +1 @@
+CODEOWNERS = ["@esphome/core"]

esphome/components/md5/md5.cpp

@@ -0,0 +1,51 @@
+#include <cstdio>
+#include <cstring>
+#include "md5.h"
+#include "esphome/core/helpers.h"
+
+namespace esphome {
+namespace md5 {
+
+void MD5Digest::init() {
+  memset(this->digest_, 0, 16);
+  MD5Init(&this->ctx_);
+}
+
+void MD5Digest::add(const uint8_t *data, size_t len) { MD5Update(&this->ctx_, data, len); }
+
+void MD5Digest::calculate() { MD5Final(this->digest_, &this->ctx_); }
+
+void MD5Digest::get_bytes(uint8_t *output) { memcpy(output, this->digest_, 16); }
+
+void MD5Digest::get_hex(char *output) {
+  for (size_t i = 0; i < 16; i++) {
+    sprintf(output + i * 2, "%02x", this->digest_[i]);
+  }
+}
+
+bool MD5Digest::equals_bytes(const char *expected) {
+  for (size_t i = 0; i < 16; i++) {
+    if (expected[i] != this->digest_[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+bool MD5Digest::equals_hex(const char *expected) {
+  for (size_t i = 0; i < 16; i++) {
+    auto high = parse_hex(expected[i * 2]);
+    auto low = parse_hex(expected[i * 2 + 1]);
+    if (!high.has_value() || !low.has_value()) {
+      return false;
+    }
+    auto value = (*high << 4) | *low;
+    if (value != this->digest_[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+}  // namespace md5
+}  // namespace esphome

esphome/components/md5/md5.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include "esphome/core/defines.h"
+
+#ifdef USE_ESP_IDF
+#include "esp32/rom/md5_hash.h"
+#define MD5_CTX_TYPE MD5Context
+#endif
+
+#if defined(USE_ARDUINO) && defined(USE_ESP32)
+#include "rom/md5_hash.h"
+#define MD5_CTX_TYPE MD5Context
+#endif
+
+#if defined(USE_ARDUINO) && defined(USE_ESP8266)
+#include <md5.h>
+#define MD5_CTX_TYPE md5_context_t
+#endif
+
+namespace esphome {
+namespace md5 {
+
+class MD5Digest {
+ public:
+  MD5Digest() = default;
+  ~MD5Digest() = default;
+
+  /// Initialize a new MD5 digest computation.
+  void init();
+
+  /// Add bytes of data for the digest.
+  void add(const uint8_t *data, size_t len);
+  void add(const char *data, size_t len) { this->add((const uint8_t *) data, len); }
+
+  /// Compute the digest, based on the provided data.
+  void calculate();
+
+  /// Retrieve the MD5 digest as bytes.
+  /// The output must be able to hold 16 bytes or more.
+  void get_bytes(uint8_t *output);
+
+  /// Retrieve the MD5 digest as hex characters.
+  /// The output must be able to hold 32 bytes or more.
+  void get_hex(char *output);
+
+  /// Compare the digest against a provided byte-encoded digest (16 bytes).
+  bool equals_bytes(const char *expected);
+
+  /// Compare the digest against a provided hex-encoded digest (32 bytes).
+  bool equals_hex(const char *expected);
+
+ protected:
+  MD5_CTX_TYPE ctx_{};
+  uint8_t digest_[16];
+};
+
+}  // namespace md5
+}  // namespace esphome

esphome/components/mdns/mdns_component.cpp

@@ -23,7 +23,7 @@ std::vector<MDNSService> MDNSComponent::compile_services_() {
 #ifdef USE_API
   if (api::global_api_server != nullptr) {
     MDNSService service{};
-    service.service_type = "esphomelib";
+    service.service_type = "_esphomelib";
     service.proto = "_tcp";
     service.port = api::global_api_server->get_port();
     service.txt_records.push_back({"version", ESPHOME_VERSION});
@@ -57,7 +57,7 @@ std::vector<MDNSService> MDNSComponent::compile_services_() {
 #ifdef USE_PROMETHEUS
   {
     MDNSService service{};
-    service.service_type = "prometheus-http";
+    service.service_type = "_prometheus-http";
     service.proto = "_tcp";
     service.port = WEBSERVER_PORT;
     res.push_back(service);
@@ -68,7 +68,7 @@ std::vector<MDNSService> MDNSComponent::compile_services_() {
     // Publish "http" service if not using native API
     // This is just to have *some* mDNS service so that .local resolution works
     MDNSService service{};
-    service.service_type = "http";
+    service.service_type = "_http";
     service.proto = "_tcp";
     service.port = WEBSERVER_PORT;
     service.txt_records.push_back({"version", ESPHOME_VERSION});

esphome/components/mdns/mdns_component.h

@@ -13,7 +13,11 @@ struct MDNSTXTRecord {
 };
 
 struct MDNSService {
+  // service name _including_ underscore character prefix
+  // as defined in RFC6763 Section 7
   std::string service_type;
+  // second label indicating protocol _including_ underscore character prefix
+  // as defined in RFC6763 Section 7, like "_tcp" or "_udp"
   std::string proto;
   uint16_t port;
   std::vector<MDNSTXTRecord> txt_records;

esphome/components/mdns/mdns_esp8266.cpp

@@ -17,9 +17,21 @@ void MDNSComponent::setup() {
 
   auto services = compile_services_();
   for (const auto &service : services) {
-    MDNS.addService(service.service_type.c_str(), service.proto.c_str(), service.port);
+    // Strip the leading underscore from the proto and service_type. While it is
+    // part of the wire protocol to have an underscore, and for example ESP-IDF
+    // expects the underscore to be there, the ESP8266 implementation always adds
+    // the underscore itself.
+    auto proto = service.proto.c_str();
+    while (*proto == '_') {
+      proto++;
+    }
+    auto service_type = service.service_type.c_str();
+    while (*service_type == '_') {
+      service_type++;
+    }
+    MDNS.addService(service_type, proto, service.port);
     for (const auto &record : service.txt_records) {
-      MDNS.addServiceTxt(service.service_type.c_str(), service.proto.c_str(), record.key.c_str(), record.value.c_str());
+      MDNS.addServiceTxt(service_type, proto, record.key.c_str(), record.value.c_str());
     }
   }
 }

esphome/components/modbus/modbus.cpp

@@ -96,23 +96,27 @@ bool Modbus::parse_modbus_byte_(uint8_t byte) {
     ESP_LOGW(TAG, "Modbus CRC Check failed! %02X!=%02X", computed_crc, remote_crc);
     return false;
   }
-
-  waiting_for_response = 0;
   std::vector<uint8_t> data(this->rx_buffer_.begin() + data_offset, this->rx_buffer_.begin() + data_offset + data_len);
-
   bool found = false;
   for (auto *device : this->devices_) {
     if (device->address_ == address) {
       // Is it an error response?
       if ((function_code & 0x80) == 0x80) {
-        ESP_LOGW(TAG, "Modbus error function code: 0x%X exception: %d", function_code, raw[2]);
-        device->on_modbus_error(function_code & 0x7F, raw[2]);
+        ESP_LOGD(TAG, "Modbus error function code: 0x%X exception: %d", function_code, raw[2]);
+        if (waiting_for_response != 0) {
+          device->on_modbus_error(function_code & 0x7F, raw[2]);
+        } else {
+          // Ignore modbus exception not related to a pending command
+          ESP_LOGD(TAG, "Ignoring Modbus error - not expecting a response");
+        }
       } else {
         device->on_modbus_data(data);
       }
       found = true;
     }
   }
+  waiting_for_response = 0;
+
   if (!found) {
     ESP_LOGW(TAG, "Got Modbus frame from unknown address 0x%02X! ", address);
   }
@@ -196,6 +200,7 @@ void Modbus::send_raw(const std::vector<uint8_t> &payload) {
   if (this->flow_control_pin_ != nullptr)
     this->flow_control_pin_->digital_write(false);
   waiting_for_response = payload[0];
+  ESP_LOGV(TAG, "Modbus write raw: %s", hexencode(payload).c_str());
   last_send_ = millis();
 }
 

esphome/components/modbus_controller/__init__.py

@@ -38,7 +38,7 @@ ModbusRegisterType_ns = modbus_controller_ns.namespace("ModbusRegisterType")
 ModbusRegisterType = ModbusRegisterType_ns.enum("ModbusRegisterType")
 MODBUS_REGISTER_TYPE = {
     "coil": ModbusRegisterType.COIL,
-    "discrete_input": ModbusRegisterType.DISCRETE,
+    "discrete_input": ModbusRegisterType.DISCRETE_INPUT,
     "holding": ModbusRegisterType.HOLDING,
     "read": ModbusRegisterType.READ,
 }

esphome/components/mqtt/__init__.py

@@ -215,7 +215,7 @@ async def to_code(config):
     await cg.register_component(var, config)
 
     # https://github.com/OttoWinter/async-mqtt-client/blob/master/library.json
-    cg.add_library("ottowinter/AsyncMqttClient-esphome", "0.8.4")
+    cg.add_library("ottowinter/AsyncMqttClient-esphome", "0.8.6")
     cg.add_define("USE_MQTT")
     cg.add_global(mqtt_ns.using)
 

esphome/components/mqtt/mqtt_fan.cpp

@@ -88,9 +88,12 @@ void MQTTFanComponent::setup() {
 
   if (this->state_->get_traits().supports_speed()) {
     this->subscribe(this->get_speed_command_topic(), [this](const std::string &topic, const std::string &payload) {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
       this->state_->make_call()
           .set_speed(payload.c_str())  // NOLINT(clang-diagnostic-deprecated-declarations)
           .perform();
+#pragma GCC diagnostic pop
     });
   }
 
@@ -145,6 +148,8 @@ bool MQTTFanComponent::publish_state() {
   }
   if (traits.supports_speed()) {
     const char *payload;
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     // NOLINTNEXTLINE(clang-diagnostic-deprecated-declarations)
     switch (fan::speed_level_to_enum(this->state_->speed, traits.supported_speed_count())) {
       case FAN_SPEED_LOW: {  // NOLINT(clang-diagnostic-deprecated-declarations)
@@ -161,6 +166,7 @@ bool MQTTFanComponent::publish_state() {
         break;
       }
     }
+#pragma GCC diagnostic pop
     bool success = this->publish(this->get_speed_state_topic(), payload);
     failed = failed || !success;
   }

esphome/components/nextion/binary_sensor/nextion_binarysensor.cpp

@@ -33,7 +33,7 @@ void NextionBinarySensor::update() {
   if (this->variable_name_.empty())  // This is a touch component
     return;
 
-  this->nextion_->add_to_get_queue(shared_from_this());
+  this->nextion_->add_to_get_queue(this);
 }
 
 void NextionBinarySensor::set_state(bool state, bool publish, bool send_to_nextion) {
@@ -48,7 +48,7 @@ void NextionBinarySensor::set_state(bool state, bool publish, bool send_to_nexti
       this->needs_to_send_update_ = true;
     } else {
       this->needs_to_send_update_ = false;
-      this->nextion_->add_no_result_to_queue_with_set(shared_from_this(), (int) state);
+      this->nextion_->add_no_result_to_queue_with_set(this, (int) state);
     }
   }
 

esphome/components/nextion/binary_sensor/nextion_binarysensor.h

@@ -10,8 +10,7 @@ class NextionBinarySensor;
 
 class NextionBinarySensor : public NextionComponent,
                             public binary_sensor::BinarySensorInitiallyOff,
-                            public PollingComponent,
-                            public std::enable_shared_from_this<NextionBinarySensor> {
+                            public PollingComponent {
  public:
   NextionBinarySensor(NextionBase *nextion) { this->nextion_ = nextion; }
 

esphome/components/nextion/display.py

@@ -8,7 +8,7 @@ from esphome.const import (
     CONF_BRIGHTNESS,
     CONF_TRIGGER_ID,
 )
-
+from esphome.core import CORE
 from . import Nextion, nextion_ns, nextion_ref
 from .base_component import (
     CONF_ON_SLEEP,
@@ -76,6 +76,9 @@ async def to_code(config):
     if CONF_TFT_URL in config:
         cg.add_define("USE_NEXTION_TFT_UPLOAD")
         cg.add(var.set_tft_url(config[CONF_TFT_URL]))
+        if CORE.is_esp32:
+            cg.add_library("WiFiClientSecure", None)
+            cg.add_library("HTTPClient", None)
 
     if CONF_TOUCH_SLEEP_TIMEOUT in config:
         cg.add(var.set_touch_sleep_timeout_internal(config[CONF_TOUCH_SLEEP_TIMEOUT]))

esphome/components/nextion/nextion.cpp

@@ -196,7 +196,7 @@ void Nextion::print_queue_members_() {
   ESP_LOGN(TAG, "print_queue_members_ (top 10) size %zu", this->nextion_queue_.size());
   ESP_LOGN(TAG, "*******************************************");
   int count = 0;
-  for (auto &i : this->nextion_queue_) {
+  for (auto *i : this->nextion_queue_) {
     if (count++ == 10)
       break;
 
@@ -257,9 +257,8 @@ bool Nextion::remove_from_q_(bool report_empty) {
     return false;
   }
 
-  auto nb = std::move(this->nextion_queue_.front());
-  this->nextion_queue_.pop_front();
-  auto &component = nb->component;
+  NextionQueue *nb = this->nextion_queue_.front();
+  NextionComponentBase *component = nb->component;
 
   ESP_LOGN(TAG, "Removing %s from the queue", component->get_variable_name().c_str());
 
@@ -267,8 +266,10 @@ bool Nextion::remove_from_q_(bool report_empty) {
     if (component->get_variable_name() == "sleep_wake") {
       this->is_sleeping_ = false;
     }
+    delete component;  // NOLINT(cppcoreguidelines-owning-memory)
   }
-
+  delete nb;  // NOLINT(cppcoreguidelines-owning-memory)
+  this->nextion_queue_.pop_front();
   return true;
 }
 
@@ -357,7 +358,7 @@ void Nextion::process_nextion_commands_() {
           int index = 0;
           int found = -1;
           for (auto &nb : this->nextion_queue_) {
-            auto &component = nb->component;
+            NextionComponentBase *component = nb->component;
 
             if (component->get_queue_type() == NextionQueueType::WAVEFORM_SENSOR) {
               ESP_LOGW(TAG, "Nextion reported invalid Waveform ID %d or Channel # %d was used!",
@@ -368,6 +369,9 @@ void Nextion::process_nextion_commands_() {
 
               found = index;
 
+              delete component;  // NOLINT(cppcoreguidelines-owning-memory)
+              delete nb;         // NOLINT(cppcoreguidelines-owning-memory)
+
               break;
             }
             ++index;
@@ -464,9 +468,8 @@ void Nextion::process_nextion_commands_() {
           break;
         }
 
-        auto nb = std::move(this->nextion_queue_.front());
-        this->nextion_queue_.pop_front();
-        auto &component = nb->component;
+        NextionQueue *nb = this->nextion_queue_.front();
+        NextionComponentBase *component = nb->component;
 
         if (component->get_queue_type() != NextionQueueType::TEXT_SENSOR) {
           ESP_LOGE(TAG, "ERROR: Received string return but next in queue \"%s\" is not a text sensor",
@@ -477,6 +480,9 @@ void Nextion::process_nextion_commands_() {
           component->set_state_from_string(to_process, true, false);
         }
 
+        delete nb;  // NOLINT(cppcoreguidelines-owning-memory)
+        this->nextion_queue_.pop_front();
+
         break;
       }
         //  0x71 0x01 0x02 0x03 0x04 0xFF 0xFF 0xFF
@@ -505,9 +511,8 @@ void Nextion::process_nextion_commands_() {
           ++dataindex;
         }
 
-        auto nb = std::move(this->nextion_queue_.front());
-        this->nextion_queue_.pop_front();
-        auto &component = nb->component;
+        NextionQueue *nb = this->nextion_queue_.front();
+        NextionComponentBase *component = nb->component;
 
         if (component->get_queue_type() != NextionQueueType::SENSOR &&
             component->get_queue_type() != NextionQueueType::BINARY_SENSOR &&
@@ -521,6 +526,9 @@ void Nextion::process_nextion_commands_() {
           component->set_state_from_int(value, true, false);
         }
 
+        delete nb;  // NOLINT(cppcoreguidelines-owning-memory)
+        this->nextion_queue_.pop_front();
+
         break;
       }
 
@@ -682,7 +690,7 @@ void Nextion::process_nextion_commands_() {
         int index = 0;
         int found = -1;
         for (auto &nb : this->nextion_queue_) {
-          auto &component = nb->component;
+          auto component = nb->component;
           if (component->get_queue_type() == NextionQueueType::WAVEFORM_SENSOR) {
             size_t buffer_to_send = component->get_wave_buffer().size() < 255 ? component->get_wave_buffer().size()
                                                                               : 255;  // ADDT command can only send 255
@@ -699,6 +707,8 @@ void Nextion::process_nextion_commands_() {
                                                  component->get_wave_buffer().begin() + buffer_to_send);
             }
             found = index;
+            delete component;  // NOLINT(cppcoreguidelines-owning-memory)
+            delete nb;         // NOLINT(cppcoreguidelines-owning-memory)
             break;
           }
           ++index;
@@ -727,7 +737,7 @@ void Nextion::process_nextion_commands_() {
 
   if (!this->nextion_queue_.empty() && this->nextion_queue_.front()->queue_time + this->max_q_age_ms_ < ms) {
     for (int i = 0; i < this->nextion_queue_.size(); i++) {
-      auto &component = this->nextion_queue_[i]->component;
+      NextionComponentBase *component = this->nextion_queue_[i]->component;
       if (this->nextion_queue_[i]->queue_time + this->max_q_age_ms_ < ms) {
         if (this->nextion_queue_[i]->queue_time == 0)
           ESP_LOGD(TAG, "Removing old queue type \"%s\" name \"%s\" queue_time 0",
@@ -744,8 +754,11 @@ void Nextion::process_nextion_commands_() {
           if (component->get_variable_name() == "sleep_wake") {
             this->is_sleeping_ = false;
           }
+          delete component;  // NOLINT(cppcoreguidelines-owning-memory)
         }
 
+        delete this->nextion_queue_[i];  // NOLINT(cppcoreguidelines-owning-memory)
+
         this->nextion_queue_.erase(this->nextion_queue_.begin() + i);
         i--;
 
@@ -899,16 +912,18 @@ uint16_t Nextion::recv_ret_string_(std::string &response, uint32_t timeout, bool
  * @param variable_name Name for the queue
  */
 void Nextion::add_no_result_to_queue_(const std::string &variable_name) {
-  auto nextion_queue = make_unique<nextion::NextionQueue>();
+  // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+  nextion::NextionQueue *nextion_queue = new nextion::NextionQueue;
 
-  nextion_queue->component = make_unique<nextion::NextionComponentBase>();
+  // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+  nextion_queue->component = new nextion::NextionComponentBase;
   nextion_queue->component->set_variable_name(variable_name);
 
   nextion_queue->queue_time = millis();
 
-  ESP_LOGN(TAG, "Add to queue type: NORESULT component %s", nextion_queue->component->get_variable_name().c_str());
+  this->nextion_queue_.push_back(nextion_queue);
 
-  this->nextion_queue_.push_back(std::move(nextion_queue));
+  ESP_LOGN(TAG, "Add to queue type: NORESULT component %s", nextion_queue->component->get_variable_name().c_str());
 }
 
 /**
@@ -979,7 +994,7 @@ bool Nextion::add_no_result_to_queue_with_printf_(const std::string &variable_na
  * @param is_sleep_safe The command is safe to send when the Nextion is sleeping
  */
 
-void Nextion::add_no_result_to_queue_with_set(std::shared_ptr<NextionComponentBase> component, int state_value) {
+void Nextion::add_no_result_to_queue_with_set(NextionComponentBase *component, int state_value) {
   this->add_no_result_to_queue_with_set(component->get_variable_name(), component->get_variable_name_to_send(),
                                         state_value);
 }
@@ -1007,8 +1022,7 @@ void Nextion::add_no_result_to_queue_with_set_internal_(const std::string &varia
  * @param state_value String value to set
  * @param is_sleep_safe The command is safe to send when the Nextion is sleeping
  */
-void Nextion::add_no_result_to_queue_with_set(std::shared_ptr<NextionComponentBase> component,
-                                              const std::string &state_value) {
+void Nextion::add_no_result_to_queue_with_set(NextionComponentBase *component, const std::string &state_value) {
   this->add_no_result_to_queue_with_set(component->get_variable_name(), component->get_variable_name_to_send(),
                                         state_value);
 }
@@ -1028,11 +1042,12 @@ void Nextion::add_no_result_to_queue_with_set_internal_(const std::string &varia
                                             state_value.c_str());
 }
 
-void Nextion::add_to_get_queue(std::shared_ptr<NextionComponentBase> component) {
+void Nextion::add_to_get_queue(NextionComponentBase *component) {
   if ((!this->is_setup() && !this->ignore_is_setup_))
     return;
 
-  auto nextion_queue = make_unique<nextion::NextionQueue>();
+  // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+  nextion::NextionQueue *nextion_queue = new nextion::NextionQueue;
 
   nextion_queue->component = component;
   nextion_queue->queue_time = millis();
@@ -1043,7 +1058,7 @@ void Nextion::add_to_get_queue(std::shared_ptr<NextionComponentBase> component)
   std::string command = "get " + component->get_variable_name_to_send();
 
   if (this->send_command_(command)) {
-    this->nextion_queue_.push_back(std::move(nextion_queue));
+    this->nextion_queue_.push_back(nextion_queue);
   }
 }
 
@@ -1055,13 +1070,15 @@ void Nextion::add_to_get_queue(std::shared_ptr<NextionComponentBase> component)
  * @param buffer_to_send The buffer size
  * @param buffer_size The buffer data
  */
-void Nextion::add_addt_command_to_queue(std::shared_ptr<NextionComponentBase> component) {
+void Nextion::add_addt_command_to_queue(NextionComponentBase *component) {
   if ((!this->is_setup() && !this->ignore_is_setup_) || this->is_sleeping())
     return;
 
-  auto nextion_queue = make_unique<nextion::NextionQueue>();
+  // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+  nextion::NextionQueue *nextion_queue = new nextion::NextionQueue;
 
-  nextion_queue->component = std::make_shared<nextion::NextionComponentBase>();
+  // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+  nextion_queue->component = new nextion::NextionComponentBase;
   nextion_queue->queue_time = millis();
 
   size_t buffer_to_send = component->get_wave_buffer_size() < 255 ? component->get_wave_buffer_size()
@@ -1070,7 +1087,7 @@ void Nextion::add_addt_command_to_queue(std::shared_ptr<NextionComponentBase> co
   std::string command = "addt " + to_string(component->get_component_id()) + "," +
                         to_string(component->get_wave_channel_id()) + "," + to_string(buffer_to_send);
   if (this->send_command_(command)) {
-    this->nextion_queue_.push_back(std::move(nextion_queue));
+    this->nextion_queue_.push_back(nextion_queue);
   }
 }
 

esphome/components/nextion/nextion.h

@@ -707,18 +707,17 @@ class Nextion : public NextionBase, public PollingComponent, public uart::UARTDe
   void set_nextion_sensor_state(NextionQueueType queue_type, const std::string &name, float state);
   void set_nextion_text_state(const std::string &name, const std::string &state);
 
-  void add_no_result_to_queue_with_set(std::shared_ptr<NextionComponentBase> component, int state_value) override;
+  void add_no_result_to_queue_with_set(NextionComponentBase *component, int state_value) override;
   void add_no_result_to_queue_with_set(const std::string &variable_name, const std::string &variable_name_to_send,
                                        int state_value) override;
 
-  void add_no_result_to_queue_with_set(std::shared_ptr<NextionComponentBase> component,
-                                       const std::string &state_value) override;
+  void add_no_result_to_queue_with_set(NextionComponentBase *component, const std::string &state_value) override;
   void add_no_result_to_queue_with_set(const std::string &variable_name, const std::string &variable_name_to_send,
                                        const std::string &state_value) override;
 
-  void add_to_get_queue(std::shared_ptr<NextionComponentBase> component) override;
+  void add_to_get_queue(NextionComponentBase *component) override;
 
-  void add_addt_command_to_queue(std::shared_ptr<NextionComponentBase> component) override;
+  void add_addt_command_to_queue(NextionComponentBase *component) override;
 
   void update_components_by_prefix(const std::string &prefix);
 
@@ -729,7 +728,7 @@ class Nextion : public NextionBase, public PollingComponent, public uart::UARTDe
   void set_auto_wake_on_touch_internal(bool auto_wake_on_touch) { this->auto_wake_on_touch_ = auto_wake_on_touch; }
 
  protected:
-  std::deque<std::unique_ptr<NextionQueue>> nextion_queue_;
+  std::deque<NextionQueue *> nextion_queue_;
   uint16_t recv_ret_string_(std::string &response, uint32_t timeout, bool recv_flag);
   void all_components_send_state_(bool force_update = false);
   uint64_t comok_sent_ = 0;

esphome/components/nextion/nextion_base.h

@@ -24,19 +24,18 @@ class NextionBase;
 
 class NextionBase {
  public:
-  virtual void add_no_result_to_queue_with_set(std::shared_ptr<NextionComponentBase> component, int state_value) = 0;
+  virtual void add_no_result_to_queue_with_set(NextionComponentBase *component, int state_value) = 0;
   virtual void add_no_result_to_queue_with_set(const std::string &variable_name,
                                                const std::string &variable_name_to_send, int state_value) = 0;
 
-  virtual void add_no_result_to_queue_with_set(std::shared_ptr<NextionComponentBase> component,
-                                               const std::string &state_value) = 0;
+  virtual void add_no_result_to_queue_with_set(NextionComponentBase *component, const std::string &state_value) = 0;
   virtual void add_no_result_to_queue_with_set(const std::string &variable_name,
                                                const std::string &variable_name_to_send,
                                                const std::string &state_value) = 0;
 
-  virtual void add_addt_command_to_queue(std::shared_ptr<NextionComponentBase> component) = 0;
+  virtual void add_addt_command_to_queue(NextionComponentBase *component) = 0;
 
-  virtual void add_to_get_queue(std::shared_ptr<NextionComponentBase> component) = 0;
+  virtual void add_to_get_queue(NextionComponentBase *component) = 0;
 
   virtual void set_component_background_color(const char *component, Color color) = 0;
   virtual void set_component_pressed_background_color(const char *component, Color color) = 0;

esphome/components/nextion/nextion_component_base.h

@@ -1,6 +1,5 @@
 #pragma once
 #include <utility>
-#include <memory>
 #include "esphome/core/defines.h"
 
 namespace esphome {
@@ -23,7 +22,7 @@ class NextionComponentBase;
 class NextionQueue {
  public:
   virtual ~NextionQueue() = default;
-  std::shared_ptr<NextionComponentBase> component;
+  NextionComponentBase *component;
   uint32_t queue_time = 0;
 };
 

esphome/components/nextion/nextion_upload.cpp

@@ -281,12 +281,14 @@ void Nextion::upload_tft() {
 #endif
       // NOLINTNEXTLINE(readability-static-accessed-through-instance)
       ESP_LOGD(TAG, "Allocating buffer size %d, Heap size is %u", chunk_size, ESP.getFreeHeap());
-      this->transfer_buffer_ = new (std::nothrow) uint8_t[chunk_size];  // NOLINT(cppcoreguidelines-owning-memory)
-      if (this->transfer_buffer_ == nullptr) {                          // Try a smaller size
+      // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+      this->transfer_buffer_ = new (std::nothrow) uint8_t[chunk_size];
+      if (this->transfer_buffer_ == nullptr) {  // Try a smaller size
         ESP_LOGD(TAG, "Could not allocate buffer size: %d trying 4096 instead", chunk_size);
         chunk_size = 4096;
         ESP_LOGD(TAG, "Allocating %d buffer", chunk_size);
-        this->transfer_buffer_ = new (std::nothrow) uint8_t[chunk_size];  // NOLINT(cppcoreguidelines-owning-memory)
+        // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+        this->transfer_buffer_ = new uint8_t[chunk_size];
 
         if (!this->transfer_buffer_)
           this->upload_end_();
@@ -330,7 +332,8 @@ void Nextion::upload_end_() {
 WiFiClient *Nextion::get_wifi_client_() {
   if (this->tft_url_.compare(0, 6, "https:") == 0) {
     if (this->wifi_client_secure_ == nullptr) {
-      this->wifi_client_secure_ = new BearSSL::WiFiClientSecure();  // NOLINT(cppcoreguidelines-owning-memory)
+      // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+      this->wifi_client_secure_ = new BearSSL::WiFiClientSecure();
       this->wifi_client_secure_->setInsecure();
       this->wifi_client_secure_->setBufferSizes(512, 512);
     }
@@ -338,7 +341,8 @@ WiFiClient *Nextion::get_wifi_client_() {
   }
 
   if (this->wifi_client_ == nullptr) {
-    this->wifi_client_ = new WiFiClient();  // NOLINT(cppcoreguidelines-owning-memory)
+    // NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
+    this->wifi_client_ = new WiFiClient();
   }
   return this->wifi_client_;
 }

esphome/components/nextion/sensor/nextion_sensor.cpp

@@ -34,7 +34,7 @@ void NextionSensor::update() {
     return;
 
   if (this->wave_chan_id_ == UINT8_MAX) {
-    this->nextion_->add_to_get_queue(shared_from_this());
+    this->nextion_->add_to_get_queue(this);
   } else {
     if (this->send_last_value_) {
       this->add_to_wave_buffer(this->last_value_);
@@ -62,9 +62,9 @@ void NextionSensor::set_state(float state, bool publish, bool send_to_nextion) {
           double to_multiply = pow(10, this->precision_);
           int state_value = (int) (state * to_multiply);
 
-          this->nextion_->add_no_result_to_queue_with_set(shared_from_this(), (int) state_value);
+          this->nextion_->add_no_result_to_queue_with_set(this, (int) state_value);
         } else {
-          this->nextion_->add_no_result_to_queue_with_set(shared_from_this(), (int) state);
+          this->nextion_->add_no_result_to_queue_with_set(this, (int) state);
         }
       }
     }
@@ -103,7 +103,7 @@ void NextionSensor::wave_update_() {
            buffer_to_send, this->wave_buffer_.size(), this->component_id_, this->wave_chan_id_);
 #endif
 
-  this->nextion_->add_addt_command_to_queue(shared_from_this());
+  this->nextion_->add_addt_command_to_queue(this);
 }
 
 }  // namespace nextion

esphome/components/nextion/sensor/nextion_sensor.h

@@ -8,10 +8,7 @@ namespace esphome {
 namespace nextion {
 class NextionSensor;
 
-class NextionSensor : public NextionComponent,
-                      public sensor::Sensor,
-                      public PollingComponent,
-                      public std::enable_shared_from_this<NextionSensor> {
+class NextionSensor : public NextionComponent, public sensor::Sensor, public PollingComponent {
  public:
   NextionSensor(NextionBase *nextion) { this->nextion_ = nextion; }
   void send_state_to_nextion() override { this->set_state(this->state, false, true); };

esphome/components/nextion/switch/nextion_switch.cpp

@@ -20,7 +20,7 @@ void NextionSwitch::process_bool(const std::string &variable_name, bool on) {
 void NextionSwitch::update() {
   if (!this->nextion_->is_setup())
     return;
-  this->nextion_->add_to_get_queue(shared_from_this());
+  this->nextion_->add_to_get_queue(this);
 }
 
 void NextionSwitch::set_state(bool state, bool publish, bool send_to_nextion) {
@@ -32,7 +32,7 @@ void NextionSwitch::set_state(bool state, bool publish, bool send_to_nextion) {
       this->needs_to_send_update_ = true;
     } else {
       this->needs_to_send_update_ = false;
-      this->nextion_->add_no_result_to_queue_with_set(shared_from_this(), (int) state);
+      this->nextion_->add_no_result_to_queue_with_set(this, (int) state);
     }
   }
   if (publish) {

esphome/components/nextion/switch/nextion_switch.h

@@ -8,10 +8,7 @@ namespace esphome {
 namespace nextion {
 class NextionSwitch;
 
-class NextionSwitch : public NextionComponent,
-                      public switch_::Switch,
-                      public PollingComponent,
-                      public std::enable_shared_from_this<NextionSwitch> {
+class NextionSwitch : public NextionComponent, public switch_::Switch, public PollingComponent {
  public:
   NextionSwitch(NextionBase *nextion) { this->nextion_ = nextion; }
 

esphome/components/nextion/text_sensor/nextion_textsensor.cpp

@@ -18,7 +18,7 @@ void NextionTextSensor::process_text(const std::string &variable_name, const std
 void NextionTextSensor::update() {
   if (!this->nextion_->is_setup())
     return;
-  this->nextion_->add_to_get_queue(shared_from_this());
+  this->nextion_->add_to_get_queue(this);
 }
 
 void NextionTextSensor::set_state(const std::string &state, bool publish, bool send_to_nextion) {
@@ -29,7 +29,7 @@ void NextionTextSensor::set_state(const std::string &state, bool publish, bool s
     if (this->nextion_->is_sleeping() || !this->visible_) {
       this->needs_to_send_update_ = true;
     } else {
-      this->nextion_->add_no_result_to_queue_with_set(shared_from_this(), state);
+      this->nextion_->add_no_result_to_queue_with_set(this, state);
     }
   }
 

esphome/components/nextion/text_sensor/nextion_textsensor.h

@@ -8,10 +8,7 @@ namespace esphome {
 namespace nextion {
 class NextionTextSensor;
 
-class NextionTextSensor : public NextionComponent,
-                          public text_sensor::TextSensor,
-                          public PollingComponent,
-                          public std::enable_shared_from_this<NextionTextSensor> {
+class NextionTextSensor : public NextionComponent, public text_sensor::TextSensor, public PollingComponent {
  public:
   NextionTextSensor(NextionBase *nextion) { this->nextion_ = nextion; }
   void update() override;

esphome/components/ota/__init__.py

@@ -15,7 +15,7 @@ from esphome.core import CORE, coroutine_with_priority
 
 CODEOWNERS = ["@esphome/core"]
 DEPENDENCIES = ["network"]
-AUTO_LOAD = ["socket"]
+AUTO_LOAD = ["socket", "md5"]
 
 CONF_ON_STATE_CHANGE = "on_state_change"
 CONF_ON_BEGIN = "on_begin"
@@ -35,20 +35,12 @@ OTAEndTrigger = ota_ns.class_("OTAEndTrigger", automation.Trigger.template())
 OTAErrorTrigger = ota_ns.class_("OTAErrorTrigger", automation.Trigger.template())
 
 
-def validate_password_support(value):
-    if CORE.using_arduino:
-        return value
-    if CORE.using_esp_idf:
-        raise cv.Invalid("Password support is not implemented yet for ESP-IDF")
-    raise NotImplementedError
-
-
 CONFIG_SCHEMA = cv.Schema(
     {
         cv.GenerateID(): cv.declare_id(OTAComponent),
         cv.Optional(CONF_SAFE_MODE, default=True): cv.boolean,
         cv.SplitDefault(CONF_PORT, esp8266=8266, esp32=3232): cv.port,
-        cv.Optional(CONF_PASSWORD): cv.All(cv.string, validate_password_support),
+        cv.Optional(CONF_PASSWORD): cv.string,
         cv.Optional(
             CONF_REBOOT_TIMEOUT, default="5min"
         ): cv.positive_time_period_milliseconds,
@@ -98,9 +90,7 @@ async def to_code(config):
         )
         cg.add(RawExpression(f"if ({condition}) return"))
 
-    if CORE.is_esp8266:
-        cg.add_library("Update", None)
-    elif CORE.is_esp32 and CORE.using_arduino:
+    if CORE.is_esp32 and CORE.using_arduino:
         cg.add_library("Update", None)
 
     use_state_callback = False

esphome/components/ota/ota_backend.h

@@ -12,6 +12,7 @@ class OTABackend {
   virtual OTAResponseTypes write(uint8_t *data, size_t len) = 0;
   virtual OTAResponseTypes end() = 0;
   virtual void abort() = 0;
+  virtual bool supports_compression() = 0;
 };
 
 }  // namespace ota

esphome/components/ota/ota_backend_arduino_esp32.h

@@ -9,11 +9,13 @@ namespace esphome {
 namespace ota {
 
 class ArduinoESP32OTABackend : public OTABackend {
+ public:
   OTAResponseTypes begin(size_t image_size) override;
   void set_update_md5(const char *md5) override;
   OTAResponseTypes write(uint8_t *data, size_t len) override;
   OTAResponseTypes end() override;
   void abort() override;
+  bool supports_compression() override { return false; }
 };
 
 }  // namespace ota

esphome/components/ota/ota_backend_arduino_esp8266.h

@@ -5,6 +5,7 @@
 
 #include "ota_component.h"
 #include "ota_backend.h"
+#include "esphome/core/macros.h"
 
 namespace esphome {
 namespace ota {
@@ -16,6 +17,11 @@ class ArduinoESP8266OTABackend : public OTABackend {
   OTAResponseTypes write(uint8_t *data, size_t len) override;
   OTAResponseTypes end() override;
   void abort() override;
+#if ARDUINO_VERSION_CODE >= VERSION_CODE(2, 7, 0)
+  bool supports_compression() override { return true; }
+#else
+  bool supports_compression() override { return false; }
+#endif
 };
 
 }  // namespace ota

esphome/components/ota/ota_backend_esp_idf.cpp

@@ -4,6 +4,7 @@
 #include "ota_backend_esp_idf.h"
 #include "ota_component.h"
 #include <esp_ota_ops.h>
+#include "esphome/components/md5/md5.h"
 
 namespace esphome {
 namespace ota {
@@ -24,15 +25,15 @@ OTAResponseTypes IDFOTABackend::begin(size_t image_size) {
     }
     return OTA_RESPONSE_ERROR_UNKNOWN;
   }
+  this->md5_.init();
   return OTA_RESPONSE_OK;
 }
 
-void IDFOTABackend::set_update_md5(const char *md5) {
-  // pass
-}
+void IDFOTABackend::set_update_md5(const char *expected_md5) { memcpy(this->expected_bin_md5_, expected_md5, 32); }
 
 OTAResponseTypes IDFOTABackend::write(uint8_t *data, size_t len) {
   esp_err_t err = esp_ota_write(this->update_handle_, data, len);
+  this->md5_.add(data, len);
   if (err != ESP_OK) {
     if (err == ESP_ERR_OTA_VALIDATE_FAILED) {
       return OTA_RESPONSE_ERROR_MAGIC;
@@ -45,6 +46,11 @@ OTAResponseTypes IDFOTABackend::write(uint8_t *data, size_t len) {
 }
 
 OTAResponseTypes IDFOTABackend::end() {
+  this->md5_.calculate();
+  if (!this->md5_.equals_hex(this->expected_bin_md5_)) {
+    this->abort();
+    return OTA_RESPONSE_ERROR_UPDATE_END;
+  }
   esp_err_t err = esp_ota_end(this->update_handle_);
   this->update_handle_ = 0;
   if (err == ESP_OK) {

esphome/components/ota/ota_backend_esp_idf.h

@@ -5,6 +5,7 @@
 #include "ota_component.h"
 #include "ota_backend.h"
 #include <esp_ota_ops.h>
+#include "esphome/components/md5/md5.h"
 
 namespace esphome {
 namespace ota {
@@ -16,10 +17,13 @@ class IDFOTABackend : public OTABackend {
   OTAResponseTypes write(uint8_t *data, size_t len) override;
   OTAResponseTypes end() override;
   void abort() override;
+  bool supports_compression() override { return false; }
 
  private:
   esp_ota_handle_t update_handle_{0};
   const esp_partition_t *partition_;
+  md5::MD5Digest md5_{};
+  char expected_bin_md5_[32];
 };
 
 }  // namespace ota

esphome/components/ota/ota_component.cpp

@@ -8,15 +8,12 @@
 #include "esphome/core/application.h"
 #include "esphome/core/hal.h"
 #include "esphome/core/util.h"
+#include "esphome/components/md5/md5.h"
 #include "esphome/components/network/util.h"
 
 #include <cerrno>
 #include <cstdio>
 
-#ifdef USE_OTA_PASSWORD
-#include <MD5Builder.h>
-#endif
-
 namespace esphome {
 namespace ota {
 
@@ -107,6 +104,8 @@ void OTAComponent::loop() {
   }
 }
 
+static const uint8_t FEATURE_SUPPORTS_COMPRESSION = 0x01;
+
 void OTAComponent::handle_() {
   OTAResponseTypes error_code = OTA_RESPONSE_ERROR_UNKNOWN;
   bool update_started = false;
@@ -157,6 +156,8 @@ void OTAComponent::handle_() {
   buf[1] = OTA_VERSION_1_0;
   this->writeall_(buf, 2);
 
+  backend = make_ota_backend();
+
   // Read features - 1 byte
   if (!this->readall_(buf, 1)) {
     ESP_LOGW(TAG, "Reading features failed!");
@@ -167,18 +168,22 @@ void OTAComponent::handle_() {
 
   // Acknowledge header - 1 byte
   buf[0] = OTA_RESPONSE_HEADER_OK;
+  if ((ota_features & FEATURE_SUPPORTS_COMPRESSION) != 0 && backend->supports_compression()) {
+    buf[0] = OTA_RESPONSE_SUPPORTS_COMPRESSION;
+  }
+
   this->writeall_(buf, 1);
 
 #ifdef USE_OTA_PASSWORD
   if (!this->password_.empty()) {
     buf[0] = OTA_RESPONSE_REQUEST_AUTH;
     this->writeall_(buf, 1);
-    MD5Builder md5_builder{};
-    md5_builder.begin();
+    md5::MD5Digest md5{};
+    md5.init();
     sprintf(sbuf, "%08X", random_uint32());
-    md5_builder.add(sbuf);
-    md5_builder.calculate();
-    md5_builder.getChars(sbuf);
+    md5.add(sbuf, 8);
+    md5.calculate();
+    md5.get_hex(sbuf);
     ESP_LOGV(TAG, "Auth: Nonce is %s", sbuf);
 
     // Send nonce, 32 bytes hex MD5
@@ -188,10 +193,10 @@ void OTAComponent::handle_() {
     }
 
     // prepare challenge
-    md5_builder.begin();
-    md5_builder.add(this->password_.c_str());
+    md5.init();
+    md5.add(this->password_.c_str(), this->password_.length());
     // add nonce
-    md5_builder.add(sbuf);
+    md5.add(sbuf, 32);
 
     // Receive cnonce, 32 bytes hex MD5
     if (!this->readall_(buf, 32)) {
@@ -201,11 +206,11 @@ void OTAComponent::handle_() {
     sbuf[32] = '\0';
     ESP_LOGV(TAG, "Auth: CNonce is %s", sbuf);
     // add cnonce
-    md5_builder.add(sbuf);
+    md5.add(sbuf, 32);
 
     // calculate result
-    md5_builder.calculate();
-    md5_builder.getChars(sbuf);
+    md5.calculate();
+    md5.get_hex(sbuf);
     ESP_LOGV(TAG, "Auth: Result is %s", sbuf);
 
     // Receive result, 32 bytes hex MD5
@@ -244,7 +249,6 @@ void OTAComponent::handle_() {
   }
   ESP_LOGV(TAG, "OTA size is %u bytes", ota_size);
 
-  backend = make_ota_backend();
   error_code = backend->begin(ota_size);
   if (error_code != OTA_RESPONSE_OK)
     goto error;
@@ -278,6 +282,12 @@ void OTAComponent::handle_() {
       }
       ESP_LOGW(TAG, "Error receiving data for update, errno: %d", errno);
       goto error;
+    } else if (read == 0) {
+      // $ man recv
+      // "When  a  stream socket peer has performed an orderly shutdown, the return value will
+      // be 0 (the traditional "end-of-file" return)."
+      ESP_LOGW(TAG, "Remote end closed connection");
+      goto error;
     }
 
     error_code = backend->write(buf, read);
@@ -365,6 +375,9 @@ bool OTAComponent::readall_(uint8_t *buf, size_t len) {
       }
       ESP_LOGW(TAG, "Failed to read %d bytes of data, errno: %d", len, errno);
       return false;
+    } else if (read == 0) {
+      ESP_LOGW(TAG, "Remote closed connection");
+      return false;
     } else {
       at += read;
     }

esphome/components/ota/ota_component.h

@@ -19,6 +19,7 @@ enum OTAResponseTypes {
   OTA_RESPONSE_BIN_MD5_OK = 67,
   OTA_RESPONSE_RECEIVE_OK = 68,
   OTA_RESPONSE_UPDATE_END_OK = 69,
+  OTA_RESPONSE_SUPPORTS_COMPRESSION = 70,
 
   OTA_RESPONSE_ERROR_MAGIC = 128,
   OTA_RESPONSE_ERROR_UPDATE_PREPARE = 129,

esphome/components/scd4x/scd4x.cpp

@@ -1,4 +1,5 @@
 #include "scd4x.h"
+#include "esphome/core/hal.h"
 #include "esphome/core/log.h"
 
 namespace esphome {
@@ -38,6 +39,7 @@ void SCD4XComponent::setup() {
       return;
     }
 
+    uint32_t stop_measurement_delay = 0;
     // In order to query the device periodic measurement must be ceased
     if (raw_read_status[0]) {
       ESP_LOGD(TAG, "Sensor has data available, stopping periodic measurement");
@@ -46,68 +48,72 @@ void SCD4XComponent::setup() {
         this->mark_failed();
         return;
       }
+      // According to the SCD4x datasheet the sensor will only respond to other commands after waiting 500 ms after
+      // issuing the stop_periodic_measurement command
+      stop_measurement_delay = 500;
     }
+    this->set_timeout(stop_measurement_delay, [this]() {
+      if (!this->write_command_(SCD4X_CMD_GET_SERIAL_NUMBER)) {
+        ESP_LOGE(TAG, "Failed to write get serial command");
+        this->error_code_ = COMMUNICATION_FAILED;
+        this->mark_failed();
+        return;
+      }
 
-    if (!this->write_command_(SCD4X_CMD_GET_SERIAL_NUMBER)) {
-      ESP_LOGE(TAG, "Failed to write get serial command");
-      this->error_code_ = COMMUNICATION_FAILED;
-      this->mark_failed();
-      return;
-    }
+      uint16_t raw_serial_number[3];
+      if (!this->read_data_(raw_serial_number, 3)) {
+        ESP_LOGE(TAG, "Failed to read serial number");
+        this->error_code_ = SERIAL_NUMBER_IDENTIFICATION_FAILED;
+        this->mark_failed();
+        return;
+      }
+      ESP_LOGD(TAG, "Serial number %02d.%02d.%02d", (uint16_t(raw_serial_number[0]) >> 8),
+               uint16_t(raw_serial_number[0] & 0xFF), (uint16_t(raw_serial_number[1]) >> 8));
 
-    uint16_t raw_serial_number[3];
-    if (!this->read_data_(raw_serial_number, 3)) {
-      ESP_LOGE(TAG, "Failed to read serial number");
-      this->error_code_ = SERIAL_NUMBER_IDENTIFICATION_FAILED;
-      this->mark_failed();
-      return;
-    }
-    ESP_LOGD(TAG, "Serial number %02d.%02d.%02d", (uint16_t(raw_serial_number[0]) >> 8),
-             uint16_t(raw_serial_number[0] & 0xFF), (uint16_t(raw_serial_number[1]) >> 8));
-
-    if (!this->write_command_(SCD4X_CMD_TEMPERATURE_OFFSET,
-                              (uint16_t)(temperature_offset_ * SCD4X_TEMPERATURE_OFFSET_MULTIPLIER))) {
-      ESP_LOGE(TAG, "Error setting temperature offset.");
-      this->error_code_ = MEASUREMENT_INIT_FAILED;
-      this->mark_failed();
-      return;
-    }
-
-    // If pressure compensation available use it
-    // else use altitude
-    if (ambient_pressure_compensation_) {
-      if (!this->write_command_(SCD4X_CMD_AMBIENT_PRESSURE_COMPENSATION, ambient_pressure_compensation_)) {
-        ESP_LOGE(TAG, "Error setting ambient pressure compensation.");
+      if (!this->write_command_(SCD4X_CMD_TEMPERATURE_OFFSET,
+                                (uint16_t)(temperature_offset_ * SCD4X_TEMPERATURE_OFFSET_MULTIPLIER))) {
+        ESP_LOGE(TAG, "Error setting temperature offset.");
         this->error_code_ = MEASUREMENT_INIT_FAILED;
         this->mark_failed();
         return;
       }
-    } else {
-      if (!this->write_command_(SCD4X_CMD_ALTITUDE_COMPENSATION, altitude_compensation_)) {
-        ESP_LOGE(TAG, "Error setting altitude compensation.");
+
+      // If pressure compensation available use it
+      // else use altitude
+      if (ambient_pressure_compensation_) {
+        if (!this->update_ambient_pressure_compensation_(ambient_pressure_)) {
+          ESP_LOGE(TAG, "Error setting ambient pressure compensation.");
+          this->error_code_ = MEASUREMENT_INIT_FAILED;
+          this->mark_failed();
+          return;
+        }
+      } else {
+        if (!this->write_command_(SCD4X_CMD_ALTITUDE_COMPENSATION, altitude_compensation_)) {
+          ESP_LOGE(TAG, "Error setting altitude compensation.");
+          this->error_code_ = MEASUREMENT_INIT_FAILED;
+          this->mark_failed();
+          return;
+        }
+      }
+
+      if (!this->write_command_(SCD4X_CMD_AUTOMATIC_SELF_CALIBRATION, enable_asc_ ? 1 : 0)) {
+        ESP_LOGE(TAG, "Error setting automatic self calibration.");
         this->error_code_ = MEASUREMENT_INIT_FAILED;
         this->mark_failed();
         return;
       }
-    }
 
-    if (!this->write_command_(SCD4X_CMD_AUTOMATIC_SELF_CALIBRATION, enable_asc_ ? 1 : 0)) {
-      ESP_LOGE(TAG, "Error setting automatic self calibration.");
-      this->error_code_ = MEASUREMENT_INIT_FAILED;
-      this->mark_failed();
-      return;
-    }
+      // Finally start sensor measurements
+      if (!this->write_command_(SCD4X_CMD_START_CONTINUOUS_MEASUREMENTS)) {
+        ESP_LOGE(TAG, "Error starting continuous measurements.");
+        this->error_code_ = MEASUREMENT_INIT_FAILED;
+        this->mark_failed();
+        return;
+      }
 
-    // Finally start sensor measurements
-    if (!this->write_command_(SCD4X_CMD_START_CONTINUOUS_MEASUREMENTS)) {
-      ESP_LOGE(TAG, "Error starting continuous measurements.");
-      this->error_code_ = MEASUREMENT_INIT_FAILED;
-      this->mark_failed();
-      return;
-    }
-
-    initialized_ = true;
-    ESP_LOGD(TAG, "Sensor initialized");
+      initialized_ = true;
+      ESP_LOGD(TAG, "Sensor initialized");
+    });
   });
 }
 
@@ -150,6 +156,13 @@ void SCD4XComponent::update() {
     return;
   }
 
+  if (this->ambient_pressure_source_ != nullptr) {
+    float pressure = this->ambient_pressure_source_->state / 1000.0f;
+    if (!std::isnan(pressure)) {
+      set_ambient_pressure_compensation(this->ambient_pressure_source_->state / 1000.0f);
+    }
+  }
+
   // Check if data is ready
   if (!this->write_command_(SCD4X_CMD_GET_DATA_READY_STATUS)) {
     this->status_set_warning();
@@ -191,6 +204,28 @@ void SCD4XComponent::update() {
 
   this->status_clear_warning();
 }
+// Note pressure in bar here. Convert to hPa
+void SCD4XComponent::set_ambient_pressure_compensation(float pressure_in_bar) {
+  ambient_pressure_compensation_ = true;
+  uint16_t new_ambient_pressure = (uint16_t)(pressure_in_bar * 1000);
+  // remove millibar from comparison to avoid frequent updates +/- 10 millibar doesn't matter
+  if (initialized_ && (new_ambient_pressure / 10 != ambient_pressure_ / 10)) {
+    update_ambient_pressure_compensation_(new_ambient_pressure);
+    ambient_pressure_ = new_ambient_pressure;
+  } else {
+    ESP_LOGD(TAG, "ambient pressure compensation skipped - no change required");
+  }
+}
+
+bool SCD4XComponent::update_ambient_pressure_compensation_(uint16_t pressure_in_hpa) {
+  if (this->write_command_(SCD4X_CMD_AMBIENT_PRESSURE_COMPENSATION, pressure_in_hpa)) {
+    ESP_LOGD(TAG, "setting ambient pressure compensation to %d hPa", pressure_in_hpa);
+    return true;
+  } else {
+    ESP_LOGE(TAG, "Error setting ambient pressure compensation.");
+    return false;
+  }
+}
 
 uint8_t SCD4XComponent::sht_crc_(uint8_t data1, uint8_t data2) {
   uint8_t bit;

esphome/components/scd4x/scd4x.h

@@ -18,10 +18,8 @@ class SCD4XComponent : public PollingComponent, public i2c::I2CDevice {
 
   void set_automatic_self_calibration(bool asc) { enable_asc_ = asc; }
   void set_altitude_compensation(uint16_t altitude) { altitude_compensation_ = altitude; }
-  void set_ambient_pressure_compensation(float pressure) {
-    ambient_pressure_compensation_ = true;
-    ambient_pressure_ = (uint16_t)(pressure * 1000);
-  }
+  void set_ambient_pressure_compensation(float pressure_in_bar);
+  void set_ambient_pressure_source(sensor::Sensor *pressure) { ambient_pressure_source_ = pressure; }
   void set_temperature_offset(float offset) { temperature_offset_ = offset; };
 
   void set_co2_sensor(sensor::Sensor *co2) { co2_sensor_ = co2; }
@@ -33,6 +31,7 @@ class SCD4XComponent : public PollingComponent, public i2c::I2CDevice {
   bool read_data_(uint16_t *data, uint8_t len);
   bool write_command_(uint16_t command);
   bool write_command_(uint16_t command, uint16_t data);
+  bool update_ambient_pressure_compensation_(uint16_t pressure_in_hpa);
 
   ERRORCODE error_code_;
 
@@ -47,6 +46,8 @@ class SCD4XComponent : public PollingComponent, public i2c::I2CDevice {
   sensor::Sensor *co2_sensor_{nullptr};
   sensor::Sensor *temperature_sensor_{nullptr};
   sensor::Sensor *humidity_sensor_{nullptr};
+  // used for compensation
+  sensor::Sensor *ambient_pressure_source_{nullptr};
 };
 
 }  // namespace scd4x

esphome/components/scd4x/sensor.py

@@ -29,6 +29,7 @@ CONF_AUTOMATIC_SELF_CALIBRATION = "automatic_self_calibration"
 CONF_ALTITUDE_COMPENSATION = "altitude_compensation"
 CONF_AMBIENT_PRESSURE_COMPENSATION = "ambient_pressure_compensation"
 CONF_TEMPERATURE_OFFSET = "temperature_offset"
+CONF_AMBIENT_PRESSURE_COMPENSATION_SOURCE = "ambient_pressure_compensation_source"
 
 CONFIG_SCHEMA = (
     cv.Schema(
@@ -62,6 +63,9 @@ CONFIG_SCHEMA = (
             ),
             cv.Optional(CONF_AMBIENT_PRESSURE_COMPENSATION): cv.pressure,
             cv.Optional(CONF_TEMPERATURE_OFFSET, default="4°C"): cv.temperature,
+            cv.Optional(CONF_AMBIENT_PRESSURE_COMPENSATION_SOURCE): cv.use_id(
+                sensor.Sensor
+            ),
         }
     )
     .extend(cv.polling_component_schema("60s"))
@@ -92,7 +96,10 @@ async def to_code(config):
             cg.add(getattr(var, funcName)(config[key]))
 
     for key, funcName in SENSOR_MAP.items():
-
         if key in config:
             sens = await sensor.new_sensor(config[key])
             cg.add(getattr(var, funcName)(sens))
+
+    if CONF_AMBIENT_PRESSURE_COMPENSATION_SOURCE in config:
+        sens = await cg.get_variable(config[CONF_AMBIENT_PRESSURE_COMPENSATION_SOURCE])
+        cg.add(var.set_ambient_pressure_source(sens))

esphome/components/sgp30/sgp30.cpp

@@ -1,4 +1,5 @@
 #include "sgp30.h"
+#include "esphome/core/hal.h"
 #include "esphome/core/log.h"
 #include "esphome/core/application.h"
 #include <cinttypes>

esphome/components/socket/lwip_raw_tcp_impl.cpp

@@ -320,8 +320,7 @@ class LWIPRawImpl : public Socket {
       return -1;
     }
     if (rx_closed_ && rx_buf_ == nullptr) {
-      errno = ECONNRESET;
-      return -1;
+      return 0;
     }
     if (len == 0) {
       return 0;
@@ -366,6 +365,11 @@ class LWIPRawImpl : public Socket {
       read += copysize;
     }
 
+    if (read == 0) {
+      errno = EWOULDBLOCK;
+      return -1;
+    }
+
     return read;
   }
   ssize_t readv(const struct iovec *iov, int iovcnt) override {

esphome/components/sx1509/__init__.py

@@ -80,8 +80,8 @@ def validate_mode(value):
 CONF_SX1509 = "sx1509"
 SX1509_PIN_SCHEMA = cv.All(
     {
-        cv.GenerateID(): cv.declare_id(SX1509Component),
-        cv.Required(CONF_SX1509): cv.use_id(SX1509GPIOPin),
+        cv.GenerateID(): cv.declare_id(SX1509GPIOPin),
+        cv.Required(CONF_SX1509): cv.use_id(SX1509Component),
         cv.Required(CONF_NUMBER): cv.int_range(min=0, max=15),
         cv.Optional(CONF_MODE, default={}): cv.All(
             {

esphome/components/tca9548a/tca9548a.cpp

@@ -22,7 +22,7 @@ i2c::ErrorCode TCA9548AChannel::writev(uint8_t address, i2c::WriteBuffer *buffer
 void TCA9548AComponent::setup() {
   ESP_LOGCONFIG(TAG, "Setting up TCA9548A...");
   uint8_t status = 0;
-  if (!this->read_register(0x00, &status, 1)) {
+  if (this->read_register(0x00, &status, 1) != i2c::ERROR_OK) {
     ESP_LOGI(TAG, "TCA9548A failed");
     this->mark_failed();
     return;

esphome/components/web_server/web_server.cpp

@@ -414,6 +414,8 @@ std::string WebServer::fan_json(fan::FanState *obj) {
     const auto traits = obj->get_traits();
     if (traits.supports_speed()) {
       root["speed_level"] = obj->speed;
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
       // NOLINTNEXTLINE(clang-diagnostic-deprecated-declarations)
       switch (fan::speed_level_to_enum(obj->speed, traits.supported_speed_count())) {
         case fan::FAN_SPEED_LOW:  // NOLINT(clang-diagnostic-deprecated-declarations)
@@ -426,6 +428,7 @@ std::string WebServer::fan_json(fan::FanState *obj) {
           root["speed"] = "high";
           break;
       }
+#pragma GCC diagnostic pop
     }
     if (obj->get_traits().supports_oscillation())
       root["oscillation"] = obj->oscillating;
@@ -448,7 +451,10 @@ void WebServer::handle_fan_request(AsyncWebServerRequest *request, const UrlMatc
       auto call = obj->turn_on();
       if (request->hasParam("speed")) {
         String speed = request->getParam("speed")->value();
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
         call.set_speed(speed.c_str());  // NOLINT(clang-diagnostic-deprecated-declarations)
+#pragma GCC diagnostic pop
       }
       if (request->hasParam("speed_level")) {
         String speed_level = request->getParam("speed_level")->value();

esphome/components/web_server_base/__init__.py

@@ -28,4 +28,4 @@ async def to_code(config):
         cg.add_library("FS", None)
         cg.add_library("Update", None)
     # https://github.com/esphome/ESPAsyncWebServer/blob/master/library.json
-    cg.add_library("esphome/ESPAsyncWebServer-esphome", "1.3.0")
+    cg.add_library("esphome/ESPAsyncWebServer-esphome", "2.0.0")

esphome/components/wifi/__init__.py

@@ -159,8 +159,15 @@ def final_validate_power_esp32_ble(value):
         "esp32_ble_server",
         "esp32_ble_tracker",
     ]:
+        if conflicting not in fv.full_config.get():
+            continue
+
         try:
-            cv.require_framework_version(esp32_arduino=cv.Version(1, 0, 5))(None)
+            # Only arduino 1.0.5+ and esp-idf impacted
+            cv.require_framework_version(
+                esp32_arduino=cv.Version(1, 0, 5),
+                esp_idf=cv.Version(4, 0, 0),
+            )(None)
         except cv.Invalid:
             pass
         else:

esphome/const.py

@@ -1,10 +1,13 @@
 """Constants used by esphome."""
 
-__version__ = "2021.11.0-dev"
+__version__ = "2021.10.2"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 
-TARGET_PLATFORMS = ["esp32", "esp8266"]
+PLATFORM_ESP32 = "esp32"
+PLATFORM_ESP8266 = "esp8266"
+
+TARGET_PLATFORMS = [PLATFORM_ESP32, PLATFORM_ESP8266]
 TARGET_FRAMEWORKS = ["arduino", "esp-idf"]
 
 # See also https://github.com/platformio/platform-espressif8266/releases

esphome/core/config.py

@@ -23,11 +23,13 @@ from esphome.const import (
     CONF_PLATFORMIO_OPTIONS,
     CONF_PRIORITY,
     CONF_PROJECT,
+    CONF_SOURCE,
     CONF_TRIGGER_ID,
     CONF_TYPE,
     CONF_VERSION,
     KEY_CORE,
     TARGET_PLATFORMS,
+    PLATFORM_ESP8266,
 )
 from esphome.core import CORE, coroutine_with_priority
 from esphome.helpers import copy_file_if_changed, walk_files
@@ -181,10 +183,16 @@ def preload_core_config(config, result):
         if CONF_BOARD_FLASH_MODE in conf:
             plat_conf[CONF_BOARD_FLASH_MODE] = conf.pop(CONF_BOARD_FLASH_MODE)
         if CONF_ARDUINO_VERSION in conf:
-            plat_conf[CONF_FRAMEWORK] = {
-                CONF_TYPE: "arduino",
-                CONF_VERSION: conf.pop(CONF_ARDUINO_VERSION),
-            }
+            plat_conf[CONF_FRAMEWORK] = {}
+            if plat != PLATFORM_ESP8266:
+                plat_conf[CONF_FRAMEWORK][CONF_TYPE] = "arduino"
+
+            try:
+                if conf[CONF_ARDUINO_VERSION] not in ("recommended", "latest", "dev"):
+                    cv.Version.parse(conf[CONF_ARDUINO_VERSION])
+                plat_conf[CONF_FRAMEWORK][CONF_VERSION] = conf.pop(CONF_ARDUINO_VERSION)
+            except ValueError:
+                plat_conf[CONF_FRAMEWORK][CONF_SOURCE] = conf.pop(CONF_ARDUINO_VERSION)
         if CONF_BOARD in conf:
             plat_conf[CONF_BOARD] = conf.pop(CONF_BOARD)
         # Insert generated target platform config to main config
@@ -203,6 +211,31 @@ def include_file(path, basename):
         cg.add_global(cg.RawStatement(f'#include "{basename}"'))
 
 
+ARDUINO_GLUE_CODE = """\
+#define yield() esphome::yield()
+#define millis() esphome::millis()
+#define delay(x) esphome::delay(x)
+#define delayMicroseconds(x) esphome::delayMicroseconds(x)
+"""
+
+
+@coroutine_with_priority(-999.0)
+async def add_arduino_global_workaround():
+    # The Arduino framework defined these itself in the global
+    # namespace. For the esphome codebase that is not a problem,
+    # but when custom code
+    #   1. writes `millis()` for example AND
+    #   2. has `using namespace esphome;` like our guides suggest
+    # Then the compiler will complain that the call is ambiguous
+    # Define a hacky macro so that the call is never ambiguous
+    # and always uses the esphome namespace one.
+    # See also https://github.com/esphome/issues/issues/2510
+    # Priority -999 so that it runs before adding includes, as those
+    # also might reference these symbols
+    for line in ARDUINO_GLUE_CODE.splitlines():
+        cg.add_global(cg.RawStatement(line))
+
+
 @coroutine_with_priority(-1000.0)
 async def add_includes(includes):
     # Add includes at the very end, so that the included files can access global variables
@@ -284,6 +317,9 @@ async def to_code(config):
     cg.add_build_flag("-Wno-unused-but-set-variable")
     cg.add_build_flag("-Wno-sign-compare")
 
+    if CORE.using_arduino:
+        CORE.add_job(add_arduino_global_workaround)
+
     if config[CONF_INCLUDES]:
         CORE.add_job(add_includes, config[CONF_INCLUDES])
 

esphome/core/defines.h

@@ -26,6 +26,7 @@
 #define USE_LOGGER
 #define USE_MDNS
 #define USE_NUMBER
+#define USE_OTA_PASSWORD
 #define USE_OTA_STATE_CALLBACK
 #define USE_POWER_SUPPLY
 #define USE_PROMETHEUS

esphome/core/helpers.cpp

@@ -358,6 +358,7 @@ template<typename T> T clamp(const T val, const T min, const T max) {
     return max;
   return val;
 }
+template uint8_t clamp(uint8_t, uint8_t, uint8_t);
 template float clamp(float, float, float);
 template int clamp(int, int, int);
 

esphome/dashboard/dashboard.py

@@ -9,6 +9,7 @@ import json
 import logging
 import multiprocessing
 import os
+from pathlib import Path
 import secrets
 import shutil
 import subprocess
@@ -26,7 +27,7 @@ import tornado.process
 import tornado.web
 import tornado.websocket
 
-from esphome import const, util
+from esphome import const, platformio_api, util
 from esphome.helpers import mkdir_p, get_bool_env, run_system_command
 from esphome.storage_json import (
     EsphomeStorageJSON,
@@ -398,17 +399,45 @@ class DownloadBinaryRequestHandler(BaseHandler):
     @authenticated
     @bind_config
     def get(self, configuration=None):
-        # pylint: disable=no-value-for-parameter
-        storage_path = ext_storage_path(settings.config_dir, configuration)
-        storage_json = StorageJSON.load(storage_path)
-        if storage_json is None:
-            self.send_error()
+        type = self.get_argument("type", "firmware.bin")
+
+        if type == "firmware.bin":
+            storage_path = ext_storage_path(settings.config_dir, configuration)
+            storage_json = StorageJSON.load(storage_path)
+            if storage_json is None:
+                self.send_error(404)
+                return
+            filename = f"{storage_json.name}.bin"
+            path = storage_json.firmware_bin_path
+
+        else:
+            args = ["esphome", "idedata", settings.rel_path(configuration)]
+            rc, stdout, _ = run_system_command(*args)
+
+            if rc != 0:
+                self.send_error(404 if rc == 2 else 500)
+                return
+
+            idedata = platformio_api.IDEData(json.loads(stdout))
+
+            found = False
+            for image in idedata.extra_flash_images:
+                if image.path.endswith(type):
+                    path = image.path
+                    filename = type
+                    found = True
+                    break
+
+            if not found:
+                self.send_error(404)
+                return
+
+        self.set_header("Content-Type", "application/octet-stream")
+        self.set_header("Content-Disposition", f'attachment; filename="{filename}"')
+        if not Path(path).is_file():
+            self.send_error(404)
             return
 
-        path = storage_json.firmware_bin_path
-        self.set_header("Content-Type", "application/octet-stream")
-        filename = f"{storage_json.name}.bin"
-        self.set_header("Content-Disposition", f'attachment; filename="{filename}"')
         with open(path, "rb") as f:
             while True:
                 data = f.read(16384)
@@ -418,6 +447,38 @@ class DownloadBinaryRequestHandler(BaseHandler):
         self.finish()
 
 
+class ManifestRequestHandler(BaseHandler):
+    @authenticated
+    @bind_config
+    def get(self, configuration=None):
+        args = ["esphome", "idedata", settings.rel_path(configuration)]
+        rc, stdout, _ = run_system_command(*args)
+
+        if rc != 0:
+            self.send_error(404 if rc == 2 else 500)
+            return
+
+        idedata = platformio_api.IDEData(json.loads(stdout))
+
+        firmware_offset = "0x10000" if idedata.extra_flash_images else "0x0"
+        flash_images = [
+            {
+                "path": f"./download.bin?configuration={configuration}&type=firmware.bin",
+                "offset": firmware_offset,
+            }
+        ] + [
+            {
+                "path": f"./download.bin?configuration={configuration}&type={os.path.basename(image.path)}",
+                "offset": image.offset,
+            }
+            for image in idedata.extra_flash_images
+        ]
+
+        self.set_header("Content-Type", "application/json")
+        self.write(json.dumps(flash_images))
+        self.finish()
+
+
 def _list_dashboard_entries():
     files = settings.list_yaml_files()
     return [DashboardEntry(file) for file in files]
@@ -536,7 +597,7 @@ class MainRequestHandler(BaseHandler):
             get_template_path("index"),
             begin=begin,
             **template_args(),
-            login_enabled=settings.using_auth,
+            login_enabled=settings.using_password,
         )
 
 
@@ -862,6 +923,7 @@ def make_app(debug=get_bool_env(ENV_DEV)):
             (f"{rel}info", InfoRequestHandler),
             (f"{rel}edit", EditRequestHandler),
             (f"{rel}download.bin", DownloadBinaryRequestHandler),
+            (f"{rel}manifest.json", ManifestRequestHandler),
             (f"{rel}serial-ports", SerialPortRequestHandler),
             (f"{rel}ping", PingRequestHandler),
             (f"{rel}delete", DeleteRequestHandler),

esphome/espota2.py

@@ -4,6 +4,7 @@ import random
 import socket
 import sys
 import time
+import gzip
 
 from esphome.core import EsphomeError
 from esphome.helpers import is_ip_address, resolve_ip_address
@@ -17,6 +18,7 @@ RESPONSE_UPDATE_PREPARE_OK = 66
 RESPONSE_BIN_MD5_OK = 67
 RESPONSE_RECEIVE_OK = 68
 RESPONSE_UPDATE_END_OK = 69
+RESPONSE_SUPPORTS_COMPRESSION = 70
 
 RESPONSE_ERROR_MAGIC = 128
 RESPONSE_ERROR_UPDATE_PREPARE = 129
@@ -34,6 +36,8 @@ OTA_VERSION_1_0 = 1
 
 MAGIC_BYTES = [0x6C, 0x26, 0xF7, 0x5C, 0x45]
 
+FEATURE_SUPPORTS_COMPRESSION = 0x01
+
 _LOGGER = logging.getLogger(__name__)
 
 
@@ -170,11 +174,9 @@ def send_check(sock, data, msg):
 
 
 def perform_ota(sock, password, file_handle, filename):
-    file_md5 = hashlib.md5(file_handle.read()).hexdigest()
-    file_size = file_handle.tell()
+    file_contents = file_handle.read()
+    file_size = len(file_contents)
     _LOGGER.info("Uploading %s (%s bytes)", filename, file_size)
-    file_handle.seek(0)
-    _LOGGER.debug("MD5 of binary is %s", file_md5)
 
     # Enable nodelay, we need it for phase 1
     sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
@@ -185,8 +187,16 @@ def perform_ota(sock, password, file_handle, filename):
         raise OTAError(f"Unsupported OTA version {version}")
 
     # Features
-    send_check(sock, 0x00, "features")
-    receive_exactly(sock, 1, "features", RESPONSE_HEADER_OK)
+    send_check(sock, FEATURE_SUPPORTS_COMPRESSION, "features")
+    features = receive_exactly(
+        sock, 1, "features", [RESPONSE_HEADER_OK, RESPONSE_SUPPORTS_COMPRESSION]
+    )[0]
+
+    if features == RESPONSE_SUPPORTS_COMPRESSION:
+        upload_contents = gzip.compress(file_contents, compresslevel=9)
+        _LOGGER.info("Compressed to %s bytes", len(upload_contents))
+    else:
+        upload_contents = file_contents
 
     (auth,) = receive_exactly(
         sock, 1, "auth", [RESPONSE_REQUEST_AUTH, RESPONSE_AUTH_OK]
@@ -213,16 +223,20 @@ def perform_ota(sock, password, file_handle, filename):
         send_check(sock, result, "auth result")
         receive_exactly(sock, 1, "auth result", RESPONSE_AUTH_OK)
 
-    file_size_encoded = [
-        (file_size >> 24) & 0xFF,
-        (file_size >> 16) & 0xFF,
-        (file_size >> 8) & 0xFF,
-        (file_size >> 0) & 0xFF,
+    upload_size = len(upload_contents)
+    upload_size_encoded = [
+        (upload_size >> 24) & 0xFF,
+        (upload_size >> 16) & 0xFF,
+        (upload_size >> 8) & 0xFF,
+        (upload_size >> 0) & 0xFF,
     ]
-    send_check(sock, file_size_encoded, "binary size")
+    send_check(sock, upload_size_encoded, "binary size")
     receive_exactly(sock, 1, "binary size", RESPONSE_UPDATE_PREPARE_OK)
 
-    send_check(sock, file_md5, "file checksum")
+    upload_md5 = hashlib.md5(upload_contents).hexdigest()
+    _LOGGER.debug("MD5 of upload is %s", upload_md5)
+
+    send_check(sock, upload_md5, "file checksum")
     receive_exactly(sock, 1, "file checksum", RESPONSE_BIN_MD5_OK)
 
     # Disable nodelay for transfer
@@ -236,7 +250,7 @@ def perform_ota(sock, password, file_handle, filename):
     offset = 0
     progress = ProgressBar()
     while True:
-        chunk = file_handle.read(1024)
+        chunk = upload_contents[offset : offset + 1024]
         if not chunk:
             break
         offset += len(chunk)
@@ -247,7 +261,7 @@ def perform_ota(sock, password, file_handle, filename):
             sys.stderr.write("\n")
             raise OTAError(f"Error sending data: {err}") from err
 
-        progress.update(offset / float(file_size))
+        progress.update(offset / upload_size)
     progress.done()
 
     # Enable nodelay for last checks

esphome/log.py

@@ -49,8 +49,10 @@ def color(col: str, msg: str, reset: bool = True) -> bool:
 
 
 class ESPHomeLogFormatter(logging.Formatter):
-    def __init__(self):
-        super().__init__(fmt="%(asctime)s %(levelname)s %(message)s", style="%")
+    def __init__(self, *, include_timestamp: bool):
+        fmt = "%(asctime)s " if include_timestamp else ""
+        fmt += "%(levelname)s %(message)s"
+        super().__init__(fmt=fmt, style="%")
 
     def format(self, record):
         formatted = super().format(record)
@@ -64,7 +66,9 @@ class ESPHomeLogFormatter(logging.Formatter):
         return f"{prefix}{formatted}{Style.RESET_ALL}"
 
 
-def setup_log(debug=False, quiet=False):
+def setup_log(
+    debug: bool = False, quiet: bool = False, include_timestamp: bool = False
+) -> None:
     import colorama
 
     if debug:
@@ -79,4 +83,6 @@ def setup_log(debug=False, quiet=False):
     logging.getLogger("urllib3").setLevel(logging.WARNING)
 
     colorama.init()
-    logging.getLogger().handlers[0].setFormatter(ESPHomeLogFormatter())
+    logging.getLogger().handlers[0].setFormatter(
+        ESPHomeLogFormatter(include_timestamp=include_timestamp)
+    )

esphome/platformio_api.py

@@ -46,24 +46,31 @@ IGNORE_LIB_WARNINGS = f"(?:{'|'.join(['Hash', 'Update'])})"
 FILTER_PLATFORMIO_LINES = [
     r"Verbose mode can be enabled via `-v, --verbose` option.*",
     r"CONFIGURATION: https://docs.platformio.org/.*",
-    r"PLATFORM: .*",
     r"DEBUG: Current.*",
-    r"PACKAGES: .*",
+    r"LDF Modes:.*",
     r"LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf.*",
-    r"LDF Modes: Finder ~ chain, Compatibility ~ soft.*",
     f"Looking for {IGNORE_LIB_WARNINGS} library in registry",
     f"Warning! Library `.*'{IGNORE_LIB_WARNINGS}.*` has not been found in PlatformIO Registry.",
     f"You can ignore this message, if `.*{IGNORE_LIB_WARNINGS}.*` is a built-in library.*",
     r"Scanning dependencies...",
     r"Found \d+ compatible libraries",
     r"Memory Usage -> http://bit.ly/pio-memory-usage",
-    r"esptool.py v.*",
     r"Found: https://platformio.org/lib/show/.*",
     r"Using cache: .*",
     r"Installing dependencies",
-    r".* @ .* is already installed",
+    r"Library Manager: Already installed, built-in library",
     r"Building in .* mode",
     r"Advanced Memory Usage is available via .*",
+    r"Merged .* ELF section",
+    r"esptool.py v.*",
+    r"Checking size .*",
+    r"Retrieving maximum program size .*",
+    r"PLATFORM: .*",
+    r"PACKAGES:.*",
+    r" - framework-arduinoespressif.* \(.*\)",
+    r" - tool-esptool.* \(.*\)",
+    r" - toolchain-.* \(.*\)",
+    r"Creating BIN file .*",
 ]
 
 

esphome/util.py

@@ -178,7 +178,7 @@ def run_external_command(
     orig_argv = sys.argv
     orig_exit = sys.exit  # mock sys.exit
     full_cmd = " ".join(shlex_quote(x) for x in cmd)
-    _LOGGER.info("Running:  %s", full_cmd)
+    _LOGGER.debug("Running:  %s", full_cmd)
 
     orig_stdout = sys.stdout
     sys.stdout = RedirectText(sys.stdout, filter_lines=filter_lines)
@@ -192,8 +192,8 @@ def run_external_command(
         sys.argv = list(cmd)
         sys.exit = mock_exit
         return func() or 0
-    except KeyboardInterrupt:
-        return 1
+    except KeyboardInterrupt:  # pylint: disable=try-except-raise
+        raise
     except SystemExit as err:
         return err.args[0]
     except Exception as err:  # pylint: disable=broad-except
@@ -214,7 +214,7 @@ def run_external_command(
 
 def run_external_process(*cmd, **kwargs):
     full_cmd = " ".join(shlex_quote(x) for x in cmd)
-    _LOGGER.info("Running:  %s", full_cmd)
+    _LOGGER.debug("Running:  %s", full_cmd)
     filter_lines = kwargs.get("filter_lines")
 
     capture_stdout = kwargs.get("capture_stdout", False)
@@ -227,6 +227,8 @@ def run_external_process(*cmd, **kwargs):
 
     try:
         return subprocess.call(cmd, stdout=sub_stdout, stderr=sub_stderr)
+    except KeyboardInterrupt:  # pylint: disable=try-except-raise
+        raise
     except Exception as err:  # pylint: disable=broad-except
         _LOGGER.error("Running command failed: %s", err)
         _LOGGER.error("Please try running %s locally.", full_cmd)

platformio.ini

@@ -26,7 +26,7 @@ build_flags =
 
 [common]
 lib_deps =
-    esphome/noise-c@0.1.3     ; api
+    esphome/noise-c@0.1.4     ; api
     makuna/NeoPixelBus@2.6.7  ; neopixelbus
 build_flags =
     -DESPHOME_LOG_LEVEL=ESPHOME_LOG_LEVEL_VERY_VERBOSE
@@ -39,9 +39,9 @@ src_filter =
 extends = common
 lib_deps =
     ${common.lib_deps}
-    ottowinter/AsyncMqttClient-esphome@0.8.4              ; mqtt
+    ottowinter/AsyncMqttClient-esphome@0.8.6              ; mqtt
     ottowinter/ArduinoJson-esphomelib@5.13.3              ; json
-    esphome/ESPAsyncWebServer-esphome@1.3.0               ; web_server_base
+    esphome/ESPAsyncWebServer-esphome@2.0.0               ; web_server_base
     fastled/FastLED@3.3.2                                 ; fastled_base
     mikalhart/TinyGPSPlus@1.0.2                           ; gps
     freekode/TM1651@1.0.1                                 ; tm1651
@@ -49,7 +49,7 @@ lib_deps =
     glmnet/Dsmr@0.5                                       ; dsmr
     rweather/Crypto@0.2.0                                 ; dsmr
     dudanov/MideaUART@1.1.8                               ; midea
-    tonia/HeatpumpIR@^1.0.15                              ; heatpumpir
+    tonia/HeatpumpIR@1.0.15                               ; heatpumpir
 build_flags =
     ${common.build_flags}
     -DUSE_ARDUINO

requirements.txt

@@ -6,10 +6,10 @@ tornado==6.1
 tzlocal==3.0    # from time
 tzdata>=2021.1  # from time
 pyserial==3.5
-platformio==5.2.1
+platformio==5.2.1  # When updating platformio, also update Dockerfile
 esptool==3.1
 click==8.0.3
-esphome-dashboard==20211011.1
+esphome-dashboard==20211021.1
 aioesphomeapi==9.1.5
 
 # esp-idf requires this, but doesn't bundle it by default

tests/test3.yaml

@@ -1150,7 +1150,7 @@ servo:
 ttp229_lsf:
 
 ttp229_bsf:
-  sdo_pin: D0
+  sdo_pin: D2
   scl_pin: D1
 
 sim800l:

tests/test4.yaml

@@ -59,6 +59,10 @@ tuya:
 pipsolar:
     id: inverter0
 
+sx1509:
+  - id: sx1509_hub
+    address: 0x3E
+
 sensor:
   - platform: homeassistant
     entity_id: sensor.hello_world
@@ -209,6 +213,7 @@ sensor:
       - or:
         - throttle: "20min"
         - delta: 0.02
+
 #
 # platform sensor.apds9960 requires component apds9960
 #
@@ -308,6 +313,11 @@ binary_sensor:
     y_max: 212
     on_state:
       - lambda: 'ESP_LOGI("main", "key0: %s", (x ? "touch" : "release"));'
+  - platform: gpio
+    name: GPIO SX1509 test
+    pin:
+      sx1509: sx1509_hub
+      number: 3
 
 climate:
   - platform: tuya