Use orjson to parse json faster (#32153)

* [recorder] Use orjson to parse json faster

* Remove from http manifest

* Bump to orjson 2.5.1

* Empty commit to trigger CI

Co-authored-by: Paulus Schoutsen <paulus@home-assistant.io>

homeassistant/components/recorder/models.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import json
 import logging
 
+import orjson
 from sqlalchemy import (
     Boolean,
     Column,
@@ -63,7 +64,7 @@ class Events(Base):  # type: ignore
         try:
             return Event(
                 self.event_type,
-                json.loads(self.event_data),
+                orjson.loads(self.event_data),
                 EventOrigin(self.origin),
                 _process_timestamp(self.time_fired),
                 context=context,
@@ -133,7 +134,7 @@ class States(Base):  # type: ignore
             return State(
                 self.entity_id,
                 self.state,
-                json.loads(self.attributes),
+                orjson.loads(self.attributes),
                 _process_timestamp(self.last_changed),
                 _process_timestamp(self.last_updated),
                 context=context,

homeassistant/helpers/aiohttp_client.py

@@ -9,6 +9,7 @@ from aiohttp import web
 from aiohttp.hdrs import CONTENT_TYPE, USER_AGENT
 from aiohttp.web_exceptions import HTTPBadGateway, HTTPGatewayTimeout
 import async_timeout
+import orjson
 
 from homeassistant.const import EVENT_HOMEASSISTANT_CLOSE, __version__
 from homeassistant.core import Event, callback
@@ -67,6 +68,7 @@ def async_create_clientsession(
         loop=hass.loop,
         connector=connector,
         headers={USER_AGENT: SERVER_SOFTWARE},
+        json_serialize=lambda x: orjson.dumps(x).decode(),
         **kwargs,
     )
 

homeassistant/package_constraints.txt

@@ -16,6 +16,7 @@ home-assistant-frontend==20200220.1
 importlib-metadata==1.5.0
 jinja2>=2.10.3
 netdisco==2.6.0
+orjson==2.5.1
 pip>=8.0.3
 python-slugify==4.0.0
 pytz>=2019.03

homeassistant/util/json.py

@@ -6,6 +6,8 @@ import os
 import tempfile
 from typing import Any, Dict, List, Optional, Type, Union
 
+import orjson
+
 from homeassistant.exceptions import HomeAssistantError
 
 _LOGGER = logging.getLogger(__name__)
@@ -28,7 +30,7 @@ def load_json(
     """
     try:
         with open(filename, encoding="utf-8") as fdesc:
-            return json.loads(fdesc.read())  # type: ignore
+            return orjson.loads(fdesc.read())  # type: ignore
     except FileNotFoundError:
         # This is not a fatal error
         _LOGGER.debug("JSON file not found: %s", filename)
@@ -97,7 +99,7 @@ def find_paths_unserializable_data(bad_data: Any) -> List[str]:
         obj, obj_path = to_process.popleft()
 
         try:
-            json.dumps(obj)
+            orjson.dumps(obj)
             continue
         except TypeError:
             pass
@@ -106,7 +108,7 @@ def find_paths_unserializable_data(bad_data: Any) -> List[str]:
             for key, value in obj.items():
                 try:
                     # Is key valid?
-                    json.dumps({key: None})
+                    orjson.dumps({key: None})
                 except TypeError:
                     invalid.append(f"{obj_path}<key: {key}>")
                 else:

pylintrc

@@ -5,7 +5,7 @@ ignore=tests
 jobs=2
 load-plugins=pylint_strict_informational
 persistent=no
-extension-pkg-whitelist=ciso8601
+extension-pkg-whitelist=ciso8601,orjson
 
 [BASIC]
 good-names=id,i,j,k,ex,Run,_,fp

requirements_all.txt

@@ -10,6 +10,7 @@ importlib-metadata==1.5.0
 jinja2>=2.10.3
 PyJWT==1.7.1
 cryptography==2.8
+orjson==2.5.1
 pip>=8.0.3
 python-slugify==4.0.0
 pytz>=2019.03

setup.py

@@ -44,6 +44,7 @@ REQUIRES = [
     "PyJWT==1.7.1",
     # PyJWT has loose dependency. We want the latest one.
     "cryptography==2.8",
+    "orjson==2.5.1",
     "pip>=8.0.3",
     "python-slugify==4.0.0",
     "pytz>=2019.03",