Improve isoformat timestamp performance (#36991)

* adj

* time_fired_isoformat

* remove unused code

* tests for processing timestamps

* restore missing import lost in merge conflict

* test for None case

homeassistant/components/history/__init__.py

@@ -13,7 +13,11 @@ import voluptuous as vol
 
 from homeassistant.components import recorder
 from homeassistant.components.http import HomeAssistantView
-from homeassistant.components.recorder.models import States, process_timestamp
+from homeassistant.components.recorder.models import (
+    States,
+    process_timestamp,
+    process_timestamp_to_utc_isoformat,
+)
 from homeassistant.components.recorder.util import execute, session_scope
 from homeassistant.const import (
     ATTR_HIDDEN,
@@ -318,7 +322,7 @@ def _sorted_states_to_json(
 
     # Called in a tight loop so cache the function
     # here
-    _process_timestamp = process_timestamp
+    _process_timestamp_to_utc_isoformat = process_timestamp_to_utc_isoformat
 
     # Append all changes to it
     for ent_id, group in groupby(states, lambda state: state.entity_id):
@@ -362,9 +366,9 @@ def _sorted_states_to_json(
             ent_results.append(
                 {
                     STATE_KEY: db_state.state,
-                    LAST_CHANGED_KEY: _process_timestamp(
+                    LAST_CHANGED_KEY: _process_timestamp_to_utc_isoformat(
                         db_state.last_changed
-                    ).isoformat(),
+                    ),
                 }
             )
             prev_state = db_state

homeassistant/components/logbook/__init__.py

@@ -11,7 +11,12 @@ import voluptuous as vol
 
 from homeassistant.components import sun
 from homeassistant.components.http import HomeAssistantView
-from homeassistant.components.recorder.models import Events, States, process_timestamp
+from homeassistant.components.recorder.models import (
+    Events,
+    States,
+    process_timestamp,
+    process_timestamp_to_utc_isoformat,
+)
 from homeassistant.components.recorder.util import (
     QUERY_RETRY_WAIT,
     RETRIES,
@@ -248,7 +253,7 @@ def humanify(hass, events, entity_attr_cache, prev_states=None):
             if event.event_type in external_events:
                 domain, describe_event = external_events[event.event_type]
                 data = describe_event(event)
-                data["when"] = event.time_fired
+                data["when"] = event.time_fired_isoformat
                 data["domain"] = domain
                 data["context_user_id"] = event.context_user_id
                 yield data
@@ -275,7 +280,7 @@ def humanify(hass, events, entity_attr_cache, prev_states=None):
                 ) or split_entity_id(entity_id)[1].replace("_", " ")
 
                 yield {
-                    "when": event.time_fired,
+                    "when": event.time_fired_isoformat,
                     "name": name,
                     "message": _entry_message_from_event(
                         hass, entity_id, domain, event, entity_attr_cache
@@ -290,7 +295,7 @@ def humanify(hass, events, entity_attr_cache, prev_states=None):
                     continue
 
                 yield {
-                    "when": event.time_fired,
+                    "when": event.time_fired_isoformat,
                     "name": "Home Assistant",
                     "message": "started",
                     "domain": HA_DOMAIN,
@@ -304,7 +309,7 @@ def humanify(hass, events, entity_attr_cache, prev_states=None):
                     action = "stopped"
 
                 yield {
-                    "when": event.time_fired,
+                    "when": event.time_fired_isoformat,
                     "name": "Home Assistant",
                     "message": action,
                     "domain": HA_DOMAIN,
@@ -322,7 +327,7 @@ def humanify(hass, events, entity_attr_cache, prev_states=None):
                         pass
 
                 yield {
-                    "when": event.time_fired,
+                    "when": event.time_fired_isoformat,
                     "name": event_data.get(ATTR_NAME),
                     "message": event_data.get(ATTR_MESSAGE),
                     "domain": domain,
@@ -601,6 +606,7 @@ class LazyEventPartialState:
         "_row",
         "_event_data",
         "_time_fired",
+        "_time_fired_isoformat",
         "_attributes",
         "event_type",
         "entity_id",
@@ -613,6 +619,7 @@ class LazyEventPartialState:
         self._row = row
         self._event_data = None
         self._time_fired = None
+        self._time_fired_isoformat = None
         self._attributes = None
         self.event_type = self._row.event_type
         self.entity_id = self._row.entity_id
@@ -662,6 +669,18 @@ class LazyEventPartialState:
             )
         return self._time_fired
 
+    @property
+    def time_fired_isoformat(self):
+        """Time event was fired in utc isoformat."""
+        if not self._time_fired_isoformat:
+            if self._time_fired:
+                self._time_fired_isoformat = self._time_fired.isoformat()
+            else:
+                self._time_fired_isoformat = process_timestamp_to_utc_isoformat(
+                    self._row.time_fired or dt_util.utcnow()
+                )
+        return self._time_fired_isoformat
+
     @property
     def has_old_and_new_state(self):
         """Check the json data to see if new_state and old_state is present without decoding."""

homeassistant/components/recorder/models.py

@@ -28,7 +28,7 @@ SCHEMA_VERSION = 8
 
 _LOGGER = logging.getLogger(__name__)
 
-DB_TIMEZONE = "Z"
+DB_TIMEZONE = "+00:00"
 
 
 class Events(Base):  # type: ignore
@@ -202,3 +202,13 @@ def process_timestamp(ts):
         return ts.replace(tzinfo=dt_util.UTC)
 
     return dt_util.as_utc(ts)
+
+
+def process_timestamp_to_utc_isoformat(ts):
+    """Process a timestamp into UTC isotime."""
+    if ts is None:
+        return None
+    if ts.tzinfo is None:
+        return f"{ts.isoformat()}{DB_TIMEZONE}"
+
+    return dt_util.as_utc(ts).isoformat()

tests/components/logbook/test_init.py

@@ -13,6 +13,7 @@ import voluptuous as vol
 from homeassistant.components import logbook, recorder, sun
 from homeassistant.components.alexa.smart_home import EVENT_ALEXA_SMART_HOME
 from homeassistant.components.automation import EVENT_AUTOMATION_TRIGGERED
+from homeassistant.components.recorder.models import process_timestamp_to_utc_isoformat
 from homeassistant.components.script import EVENT_SCRIPT_STARTED
 from homeassistant.const import (
     ATTR_ENTITY_ID,
@@ -1230,7 +1231,7 @@ class TestComponentLogbook(unittest.TestCase):
     ):
         """Assert an entry is what is expected."""
         if when:
-            assert when == entry["when"]
+            assert when.isoformat() == entry["when"]
 
         if name:
             assert name == entry["name"]
@@ -1639,3 +1640,8 @@ class MockLazyEventPartialState(ha.Event):
     def context_user_id(self):
         """Context user id of event."""
         return self.context.user_id
+
+    @property
+    def time_fired_isoformat(self):
+        """Time event was fired in utc isoformat."""
+        return process_timestamp_to_utc_isoformat(self.time_fired)

tests/components/recorder/test_models.py

@@ -3,10 +3,18 @@ from datetime import datetime
 import unittest
 
 import pytest
+import pytz
 from sqlalchemy import create_engine
 from sqlalchemy.orm import scoped_session, sessionmaker
 
-from homeassistant.components.recorder.models import Base, Events, RecorderRuns, States
+from homeassistant.components.recorder.models import (
+    Base,
+    Events,
+    RecorderRuns,
+    States,
+    process_timestamp,
+    process_timestamp_to_utc_isoformat,
+)
 from homeassistant.const import EVENT_STATE_CHANGED
 import homeassistant.core as ha
 from homeassistant.exceptions import InvalidEntityFormatError
@@ -165,3 +173,68 @@ def test_states_from_native_invalid_entity_id():
 
     state = state.to_native(validate_entity_id=False)
     assert state.entity_id == "test.invalid__id"
+
+
+async def test_process_timestamp():
+    """Test processing time stamp to UTC."""
+    datetime_with_tzinfo = datetime(2016, 7, 9, 11, 0, 0, tzinfo=dt.UTC)
+    datetime_without_tzinfo = datetime(2016, 7, 9, 11, 0, 0)
+    est = pytz.timezone("US/Eastern")
+    datetime_est_timezone = datetime(2016, 7, 9, 11, 0, 0, tzinfo=est)
+    nst = pytz.timezone("Canada/Newfoundland")
+    datetime_nst_timezone = datetime(2016, 7, 9, 11, 0, 0, tzinfo=nst)
+    hst = pytz.timezone("US/Hawaii")
+    datetime_hst_timezone = datetime(2016, 7, 9, 11, 0, 0, tzinfo=hst)
+
+    assert process_timestamp(datetime_with_tzinfo) == datetime(
+        2016, 7, 9, 11, 0, 0, tzinfo=dt.UTC
+    )
+    assert process_timestamp(datetime_without_tzinfo) == datetime(
+        2016, 7, 9, 11, 0, 0, tzinfo=dt.UTC
+    )
+    assert process_timestamp(datetime_est_timezone) == datetime(
+        2016, 7, 9, 15, 56, tzinfo=dt.UTC
+    )
+    assert process_timestamp(datetime_nst_timezone) == datetime(
+        2016, 7, 9, 14, 31, tzinfo=dt.UTC
+    )
+    assert process_timestamp(datetime_hst_timezone) == datetime(
+        2016, 7, 9, 21, 31, tzinfo=dt.UTC
+    )
+    assert process_timestamp(None) is None
+
+
+async def test_process_timestamp_to_utc_isoformat():
+    """Test processing time stamp to UTC isoformat."""
+    datetime_with_tzinfo = datetime(2016, 7, 9, 11, 0, 0, tzinfo=dt.UTC)
+    datetime_without_tzinfo = datetime(2016, 7, 9, 11, 0, 0)
+    est = pytz.timezone("US/Eastern")
+    datetime_est_timezone = datetime(2016, 7, 9, 11, 0, 0, tzinfo=est)
+    est = pytz.timezone("US/Eastern")
+    datetime_est_timezone = datetime(2016, 7, 9, 11, 0, 0, tzinfo=est)
+    nst = pytz.timezone("Canada/Newfoundland")
+    datetime_nst_timezone = datetime(2016, 7, 9, 11, 0, 0, tzinfo=nst)
+    hst = pytz.timezone("US/Hawaii")
+    datetime_hst_timezone = datetime(2016, 7, 9, 11, 0, 0, tzinfo=hst)
+
+    assert (
+        process_timestamp_to_utc_isoformat(datetime_with_tzinfo)
+        == "2016-07-09T11:00:00+00:00"
+    )
+    assert (
+        process_timestamp_to_utc_isoformat(datetime_without_tzinfo)
+        == "2016-07-09T11:00:00+00:00"
+    )
+    assert (
+        process_timestamp_to_utc_isoformat(datetime_est_timezone)
+        == "2016-07-09T15:56:00+00:00"
+    )
+    assert (
+        process_timestamp_to_utc_isoformat(datetime_nst_timezone)
+        == "2016-07-09T14:31:00+00:00"
+    )
+    assert (
+        process_timestamp_to_utc_isoformat(datetime_hst_timezone)
+        == "2016-07-09T21:31:00+00:00"
+    )
+    assert process_timestamp_to_utc_isoformat(None) is None