Replace asyncio.wait with asyncio.gather since wait ignores exceptions (#33380)

* replace asyncio.wait with asyncio.gather since wait ignores exceptions
fix for test_entity_platform so it expects the exception

* changed to log on failed domains

* discovered that this fix actually removes other uncaught exceptions

* fix in the list of ignored exceptions

* replaced a few ignores on dyson tests that work locally but fail in the CI

* two more tests that are failing on the CI and not locally

* removed assertion on multiple entries with same unique_id - replaced with log and return
reverted test_entity_platform to its original since now there is no exception thrown

* entered all the dyson tests. they all pass locally and probabilistically fail in the CI

* removed unnecessary str() for exception

* added log message for duplicate entity_id / unique_id

* removed log in case of False return value

* added exc_info

* change the use of exc_info

homeassistant/bootstrap.py

@@ -325,15 +325,30 @@ async def _async_set_up_integrations(
     hass: core.HomeAssistant, config: Dict[str, Any]
 ) -> None:
     """Set up all the integrations."""
+
+    async def async_setup_multi_components(domains: Set[str]) -> None:
+        """Set up multiple domains. Log on failure."""
+        futures = {
+            domain: hass.async_create_task(async_setup_component(hass, domain, config))
+            for domain in domains
+        }
+        await asyncio.wait(futures.values())
+        errors = [domain for domain in domains if futures[domain].exception()]
+        for domain in errors:
+            exception = futures[domain].exception()
+            _LOGGER.error(
+                "Error setting up integration %s - received exception",
+                domain,
+                exc_info=(type(exception), exception, exception.__traceback__),
+            )
+
     domains = _get_domains(hass, config)
 
     # Start up debuggers. Start these first in case they want to wait.
     debuggers = domains & DEBUGGER_INTEGRATIONS
     if debuggers:
         _LOGGER.debug("Starting up debuggers %s", debuggers)
-        await asyncio.gather(
+        await async_setup_multi_components(debuggers)
-            *(async_setup_component(hass, domain, config) for domain in debuggers)
-        )
         domains -= DEBUGGER_INTEGRATIONS
 
     # Resolve all dependencies of all components so we can find the logging
@@ -358,9 +373,7 @@ async def _async_set_up_integrations(
     if logging_domains:
         _LOGGER.info("Setting up %s", logging_domains)
 
-        await asyncio.gather(
+        await async_setup_multi_components(logging_domains)
-            *(async_setup_component(hass, domain, config) for domain in logging_domains)
-        )
 
     # Kick off loading the registries. They don't need to be awaited.
     asyncio.gather(
@@ -370,9 +383,7 @@ async def _async_set_up_integrations(
     )
 
     if stage_1_domains:
-        await asyncio.gather(
+        await async_setup_multi_components(stage_1_domains)
-            *(async_setup_component(hass, domain, config) for domain in stage_1_domains)
-        )
 
     # Load all integrations
     after_dependencies: Dict[str, Set[str]] = {}
@@ -401,9 +412,7 @@ async def _async_set_up_integrations(
 
         _LOGGER.debug("Setting up %s", domains_to_load)
 
-        await asyncio.gather(
+        await async_setup_multi_components(domains_to_load)
-            *(async_setup_component(hass, domain, config) for domain in domains_to_load)
-        )
 
         last_load = domains_to_load
         stage_2_domains -= domains_to_load
@@ -413,9 +422,7 @@ async def _async_set_up_integrations(
     if stage_2_domains:
         _LOGGER.debug("Final set up: %s", stage_2_domains)
 
-        await asyncio.gather(
+        await async_setup_multi_components(stage_2_domains)
-            *(async_setup_component(hass, domain, config) for domain in stage_2_domains)
-        )
 
     # Wrap up startup
     await hass.async_block_till_done()

homeassistant/helpers/entity_component.py

@@ -128,7 +128,7 @@ class EntityComponent:
             tasks.append(self.async_setup_platform(p_type, p_config))
 
         if tasks:
-            await asyncio.wait(tasks)
+            await asyncio.gather(*tasks)
 
         # Generic discovery listener for loading platform dynamically
         # Refer to: homeassistant.components.discovery.load_platform()
@@ -263,7 +263,7 @@ class EntityComponent:
                 tasks.append(platform.async_destroy())
 
         if tasks:
-            await asyncio.wait(tasks)
+            await asyncio.gather(*tasks)
 
         self._platforms = {self.domain: self._platforms[self.domain]}
         self.config = None

homeassistant/helpers/entity_platform.py

@@ -183,7 +183,7 @@ class EntityPlatform:
                 self._tasks.clear()
 
                 if pending:
-                    await asyncio.wait(pending)
+                    await asyncio.gather(*pending)
 
             hass.config.components.add(full_name)
             return True
@@ -292,7 +292,7 @@ class EntityPlatform:
         if not tasks:
             return
 
-        await asyncio.wait(tasks)
+        await asyncio.gather(*tasks)
 
         if self._async_unsub_polling is not None or not any(
             entity.should_poll for entity in self.entities.values()
@@ -431,10 +431,11 @@ class EntityPlatform:
                 already_exists = True
 
         if already_exists:
-            msg = f"Entity id already exists: {entity.entity_id}"
+            msg = f"Entity id already exists - ignoring: {entity.entity_id}"
             if entity.unique_id is not None:
                 msg += f". Platform {self.platform_name} does not generate unique IDs"
-            raise HomeAssistantError(msg)
+            self.logger.error(msg)
+            return
 
         entity_id = entity.entity_id
         self.entities[entity_id] = entity
@@ -459,7 +460,7 @@ class EntityPlatform:
 
         tasks = [self.async_remove_entity(entity_id) for entity_id in self.entities]
 
-        await asyncio.wait(tasks)
+        await asyncio.gather(*tasks)
 
         if self._async_unsub_polling is not None:
             self._async_unsub_polling()
@@ -548,7 +549,7 @@ class EntityPlatform:
                 tasks.append(entity.async_update_ha_state(True))  # type: ignore
 
             if tasks:
-                await asyncio.wait(tasks)
+                await asyncio.gather(*tasks)
 
 
 current_platform: ContextVar[Optional[EntityPlatform]] = ContextVar(

tests/helpers/test_entity_platform.py

@@ -168,7 +168,7 @@ async def test_adding_entities_with_generator_and_thread_callback(hass):
 
     def create_entity(number):
         """Create entity helper."""
-        entity = MockEntity()
+        entity = MockEntity(unique_id=f"unique{number}")
         entity.entity_id = async_generate_entity_id(DOMAIN + ".{}", "Number", hass=hass)
         return entity
 

tests/ignore_uncaught_exceptions.py

@@ -4,7 +4,6 @@ IGNORE_UNCAUGHT_EXCEPTIONS = [
     ("tests.components.cast.test_media_player", "test_entry_setup_single_config"),
     ("tests.components.cast.test_media_player", "test_entry_setup_list_config"),
     ("tests.components.cast.test_media_player", "test_entry_setup_platform_not_ready"),
-    ("tests.components.config.test_automation", "test_delete_automation"),
     ("tests.components.config.test_group", "test_update_device_config"),
     ("tests.components.default_config.test_init", "test_setup"),
     ("tests.components.demo.test_init", "test_setting_up_demo"),
@@ -46,20 +45,9 @@ IGNORE_UNCAUGHT_EXCEPTIONS = [
     ("tests.components.dyson.test_fan", "test_purecool_update_state_filter_inv"),
     ("tests.components.dyson.test_fan", "test_purecool_component_setup_only_once"),
     ("tests.components.dyson.test_sensor", "test_purecool_component_setup_only_once"),
-    ("test_homeassistant_bridge", "test_homeassistant_bridge_fan_setup"),
     ("tests.components.ios.test_init", "test_creating_entry_sets_up_sensor"),
     ("tests.components.ios.test_init", "test_not_configuring_ios_not_creates_entry"),
     ("tests.components.local_file.test_camera", "test_file_not_readable"),
-    ("tests.components.meteo_france.test_config_flow", "test_user"),
-    ("tests.components.meteo_france.test_config_flow", "test_import"),
-    ("tests.components.mikrotik.test_device_tracker", "test_restoring_devices"),
-    ("tests.components.mikrotik.test_hub", "test_arp_ping"),
-    ("tests.components.mqtt.test_alarm_control_panel", "test_unique_id"),
-    ("tests.components.mqtt.test_binary_sensor", "test_unique_id"),
-    ("tests.components.mqtt.test_camera", "test_unique_id"),
-    ("tests.components.mqtt.test_climate", "test_unique_id"),
-    ("tests.components.mqtt.test_cover", "test_unique_id"),
-    ("tests.components.mqtt.test_fan", "test_unique_id"),
     (
         "tests.components.mqtt.test_init",
         "test_setup_uses_certificate_on_certificate_set_to_auto",
@@ -80,22 +68,14 @@ IGNORE_UNCAUGHT_EXCEPTIONS = [
         "tests.components.mqtt.test_init",
         "test_setup_with_tls_config_of_v1_under_python36_only_uses_v1",
     ),
-    ("tests.components.mqtt.test_legacy_vacuum", "test_unique_id"),
-    ("tests.components.mqtt.test_light", "test_unique_id"),
     ("tests.components.mqtt.test_light", "test_entity_device_info_remove"),
-    ("tests.components.mqtt.test_light_json", "test_unique_id"),
     ("tests.components.mqtt.test_light_json", "test_entity_device_info_remove"),
     ("tests.components.mqtt.test_light_template", "test_entity_device_info_remove"),
-    ("tests.components.mqtt.test_lock", "test_unique_id"),
-    ("tests.components.mqtt.test_sensor", "test_unique_id"),
-    ("tests.components.mqtt.test_state_vacuum", "test_unique_id"),
-    ("tests.components.mqtt.test_switch", "test_unique_id"),
     ("tests.components.mqtt.test_switch", "test_entity_device_info_remove"),
     ("tests.components.qwikswitch.test_init", "test_binary_sensor_device"),
     ("tests.components.qwikswitch.test_init", "test_sensor_device"),
     ("tests.components.rflink.test_init", "test_send_command_invalid_arguments"),
     ("tests.components.samsungtv.test_media_player", "test_update_connection_failure"),
-    ("tests.components.tplink.test_init", "test_configuring_device_types"),
     (
         "tests.components.tplink.test_init",
         "test_configuring_devices_from_multiple_sources",
@@ -108,18 +88,8 @@ IGNORE_UNCAUGHT_EXCEPTIONS = [
     ("tests.components.unifi_direct.test_device_tracker", "test_get_scanner"),
     ("tests.components.upnp.test_init", "test_async_setup_entry_default"),
     ("tests.components.upnp.test_init", "test_async_setup_entry_port_mapping"),
-    ("tests.components.vera.test_init", "test_init"),
-    ("tests.components.wunderground.test_sensor", "test_fails_because_of_unique_id"),
     ("tests.components.yr.test_sensor", "test_default_setup"),
     ("tests.components.yr.test_sensor", "test_custom_setup"),
     ("tests.components.yr.test_sensor", "test_forecast_setup"),
     ("tests.components.zwave.test_init", "test_power_schemes"),
-    (
-        "tests.helpers.test_entity_platform",
-        "test_adding_entities_with_generator_and_thread_callback",
-    ),
-    (
-        "tests.helpers.test_entity_platform",
-        "test_not_adding_duplicate_entities_with_unique_id",
-    ),
 ]