Use statistics data in History graph card to fill gaps (#23612)

* Use statistics data in History graph card to fill gaps

* add convertStatisticsToHistory utility

src/components/chart/state-history-charts.ts

@@ -45,7 +45,7 @@ declare global {
 export class StateHistoryCharts extends LitElement {
   @property({ attribute: false }) public hass!: HomeAssistant;
 
-  @property({ attribute: false }) public historyData!: HistoryResult;
+  @property({ attribute: false }) public historyData?: HistoryResult;
 
   @property({ type: Boolean }) public narrow = false;
 
@@ -119,12 +119,12 @@ export class StateHistoryCharts extends LitElement {
         ${this.hass.localize("ui.components.history_charts.no_history_found")}
       </div>`;
     }
-    const combinedItems = this.historyData.timeline.length
+    const combinedItems = this.historyData!.timeline.length
       ? (this.virtualize
-          ? chunkData(this.historyData.timeline, CANVAS_TIMELINE_ROWS_CHUNK)
+          ? chunkData(this.historyData!.timeline, CANVAS_TIMELINE_ROWS_CHUNK)
-          : [this.historyData.timeline]
+          : [this.historyData!.timeline]
-        ).concat(this.historyData.line)
+        ).concat(this.historyData!.line)
-      : this.historyData.line;
+      : this.historyData!.line;
 
     // eslint-disable-next-line lit/no-this-assign-in-render
     this._chartCount = combinedItems.length;

src/data/history.ts

@@ -10,6 +10,7 @@ import { computeStateNameFromEntityAttributes } from "../common/entity/compute_s
 import type { LocalizeFunc } from "../common/translations/localize";
 import type { HomeAssistant } from "../types";
 import type { FrontendLocaleData } from "./translation";
+import type { Statistics } from "./recorder";
 
 const DOMAINS_USE_LAST_UPDATED = ["climate", "humidifier", "water_heater"];
 const NEED_ATTRIBUTE_DOMAINS = [
@@ -417,6 +418,54 @@ const isNumericSensorEntity = (
 
 const BLANK_UNIT = " ";
 
+export const convertStatisticsToHistory = (
+  hass: HomeAssistant,
+  statistics: Statistics,
+  statisticIds: string[],
+  sensorNumericDeviceClasses: string[],
+  splitDeviceClasses = false
+): HistoryResult => {
+  // Maintain the statistic id ordering
+  const orderedStatistics: Statistics = {};
+  statisticIds.forEach((id) => {
+    if (id in statistics) {
+      orderedStatistics[id] = statistics[id];
+    }
+  });
+
+  // Convert statistics to HistoryResult format
+  const statsHistoryStates: HistoryStates = {};
+  Object.entries(orderedStatistics).forEach(([key, value]) => {
+    const entityHistoryStates: EntityHistoryState[] = value.map((e) => ({
+      s: e.mean != null ? e.mean.toString() : e.state!.toString(),
+      lc: e.start / 1000,
+      a: {},
+      lu: e.start / 1000,
+    }));
+    statsHistoryStates[key] = entityHistoryStates;
+  });
+
+  const statisticsHistory = computeHistory(
+    hass,
+    statsHistoryStates,
+    [],
+    hass.localize,
+    sensorNumericDeviceClasses,
+    splitDeviceClasses,
+    true
+  );
+
+  // remap states array to statistics array
+  (statisticsHistory?.line || []).forEach((item) => {
+    item.data.forEach((data) => {
+      data.statistics = data.states;
+      data.states = [];
+    });
+  });
+
+  return statisticsHistory;
+};
+
 export const computeHistory = (
   hass: HomeAssistant,
   stateHistory: HistoryStates,
@@ -564,3 +613,101 @@ export const isNumericEntity = (
     domain === "sensor" &&
     isNumericSensorEntity(currentState, sensorNumericalDeviceClasses)) ||
   numericStateFromHistory != null;
+
+export const mergeHistoryResults = (
+  historyResult: HistoryResult,
+  ltsResult?: HistoryResult,
+  splitDeviceClasses = true
+): HistoryResult => {
+  if (!ltsResult) {
+    return historyResult;
+  }
+  const result: HistoryResult = { ...historyResult, line: [] };
+
+  const lookup: Record<
+    string,
+    { historyItem?: LineChartUnit; ltsItem?: LineChartUnit }
+  > = {};
+
+  for (const item of historyResult.line) {
+    const key = computeGroupKey(
+      item.unit,
+      item.device_class,
+      splitDeviceClasses
+    );
+    if (key) {
+      lookup[key] = {
+        historyItem: item,
+      };
+    }
+  }
+
+  for (const item of ltsResult.line) {
+    const key = computeGroupKey(
+      item.unit,
+      item.device_class,
+      splitDeviceClasses
+    );
+    if (!key) {
+      continue;
+    }
+    if (key in lookup) {
+      lookup[key].ltsItem = item;
+    } else {
+      lookup[key] = { ltsItem: item };
+    }
+  }
+
+  for (const { historyItem, ltsItem } of Object.values(lookup)) {
+    if (!historyItem || !ltsItem) {
+      // Only one result has data for this item, so just push it directly instead of merging.
+      result.line.push(historyItem || ltsItem!);
+      continue;
+    }
+
+    const newLineItem: LineChartUnit = { ...historyItem, data: [] };
+    const entities = new Set([
+      ...historyItem.data.map((d) => d.entity_id),
+      ...ltsItem.data.map((d) => d.entity_id),
+    ]);
+
+    for (const entity of entities) {
+      const historyDataItem = historyItem.data.find(
+        (d) => d.entity_id === entity
+      );
+      const ltsDataItem = ltsItem.data.find((d) => d.entity_id === entity);
+
+      if (!historyDataItem || !ltsDataItem) {
+        newLineItem.data.push(historyDataItem || ltsDataItem!);
+        continue;
+      }
+
+      // Remove statistics that overlap with states
+      const oldestState =
+        historyDataItem.states[0]?.last_changed ||
+        // If no state, fall back to the max last changed of the last statistics (so approve all)
+        ltsDataItem.statistics![ltsDataItem.statistics!.length - 1]
+          .last_changed + 1;
+
+      const statistics: LineChartState[] = [];
+      for (const s of ltsDataItem.statistics!) {
+        if (s.last_changed >= oldestState) {
+          break;
+        }
+        statistics.push(s);
+      }
+
+      newLineItem.data.push(
+        statistics.length === 0
+          ? // All statistics overlapped with states, so just push the states
+            historyDataItem
+          : {
+              ...historyDataItem,
+              statistics,
+            }
+      );
+    }
+    result.line.push(newLineItem);
+  }
+  return result;
+};

src/panels/history/ha-panel-history.ts

@@ -36,19 +36,13 @@ import "../../components/ha-icon-button-arrow-prev";
 import "../../components/ha-menu-button";
 import "../../components/ha-target-picker";
 import "../../components/ha-top-app-bar-fixed";
-import type {
+import type { HistoryResult } from "../../data/history";
-  EntityHistoryState,
-  HistoryResult,
-  HistoryStates,
-  LineChartState,
-  LineChartUnit,
-} from "../../data/history";
 import {
-  computeGroupKey,
   computeHistory,
   subscribeHistory,
+  mergeHistoryResults,
+  convertStatisticsToHistory,
 } from "../../data/history";
-import type { Statistics } from "../../data/recorder";
 import { fetchStatistics } from "../../data/recorder";
 import { resolveEntityIDs } from "../../data/selector";
 import { getSensorNumericDeviceClasses } from "../../data/sensor";
@@ -210,92 +204,6 @@ class HaPanelHistory extends LitElement {
     `;
   }
 
-  private _mergeHistoryResults(
-    ltsResult: HistoryResult,
-    historyResult: HistoryResult
-  ): HistoryResult {
-    const result: HistoryResult = { ...historyResult, line: [] };
-
-    const lookup: Record<
-      string,
-      { historyItem?: LineChartUnit; ltsItem?: LineChartUnit }
-    > = {};
-
-    for (const item of historyResult.line) {
-      const key = computeGroupKey(item.unit, item.device_class, true);
-      if (key) {
-        lookup[key] = {
-          historyItem: item,
-        };
-      }
-    }
-
-    for (const item of ltsResult.line) {
-      const key = computeGroupKey(item.unit, item.device_class, true);
-      if (!key) {
-        continue;
-      }
-      if (key in lookup) {
-        lookup[key].ltsItem = item;
-      } else {
-        lookup[key] = { ltsItem: item };
-      }
-    }
-
-    for (const { historyItem, ltsItem } of Object.values(lookup)) {
-      if (!historyItem || !ltsItem) {
-        // Only one result has data for this item, so just push it directly instead of merging.
-        result.line.push(historyItem || ltsItem!);
-        continue;
-      }
-
-      const newLineItem: LineChartUnit = { ...historyItem, data: [] };
-      const entities = new Set([
-        ...historyItem.data.map((d) => d.entity_id),
-        ...ltsItem.data.map((d) => d.entity_id),
-      ]);
-
-      for (const entity of entities) {
-        const historyDataItem = historyItem.data.find(
-          (d) => d.entity_id === entity
-        );
-        const ltsDataItem = ltsItem.data.find((d) => d.entity_id === entity);
-
-        if (!historyDataItem || !ltsDataItem) {
-          newLineItem.data.push(historyDataItem || ltsDataItem!);
-          continue;
-        }
-
-        // Remove statistics that overlap with states
-        const oldestState =
-          historyDataItem.states[0]?.last_changed ||
-          // If no state, fall back to the max last changed of the last statistics (so approve all)
-          ltsDataItem.statistics![ltsDataItem.statistics!.length - 1]
-            .last_changed + 1;
-
-        const statistics: LineChartState[] = [];
-        for (const s of ltsDataItem.statistics!) {
-          if (s.last_changed >= oldestState) {
-            break;
-          }
-          statistics.push(s);
-        }
-
-        newLineItem.data.push(
-          statistics.length === 0
-            ? // All statistics overlapped with states, so just push the states
-              historyDataItem
-            : {
-                ...historyDataItem,
-                statistics,
-              }
-        );
-      }
-      result.line.push(newLineItem);
-    }
-    return result;
-  }
-
   public willUpdate(changedProps: PropertyValues) {
     super.willUpdate(changedProps);
 
@@ -307,9 +215,9 @@ class HaPanelHistory extends LitElement {
       changedProps.has("_targetPickerValue")
     ) {
       if (this._statisticsHistory && this._stateHistory) {
-        this._mungedStateHistory = this._mergeHistoryResults(
+        this._mungedStateHistory = mergeHistoryResults(
-          this._statisticsHistory,
+          this._stateHistory,
-          this._stateHistory
+          this._statisticsHistory
         );
       } else {
         this._mungedStateHistory =
@@ -410,45 +318,16 @@ class HaPanelHistory extends LitElement {
       ["mean", "state"]
     );
 
-    // Maintain the statistic id ordering
-    const orderedStatistics: Statistics = {};
-    statisticIds.forEach((id) => {
-      if (id in statistics) {
-        orderedStatistics[id] = statistics[id];
-      }
-    });
-
-    // Convert statistics to HistoryResult format
-    const statsHistoryStates: HistoryStates = {};
-    Object.entries(orderedStatistics).forEach(([key, value]) => {
-      const entityHistoryStates: EntityHistoryState[] = value.map((e) => ({
-        s: e.mean != null ? e.mean.toString() : e.state!.toString(),
-        lc: e.start / 1000,
-        a: {},
-        lu: e.start / 1000,
-      }));
-      statsHistoryStates[key] = entityHistoryStates;
-    });
-
     const { numeric_device_classes: sensorNumericDeviceClasses } =
       await getSensorNumericDeviceClasses(this.hass);
 
-    this._statisticsHistory = computeHistory(
+    this._statisticsHistory = convertStatisticsToHistory(
-      this.hass,
+      this.hass!,
-      statsHistoryStates,
+      statistics,
-      [],
+      statisticIds,
-      this.hass.localize,
       sensorNumericDeviceClasses,
-      true,
       true
     );
-    // remap states array to statistics array
-    (this._statisticsHistory?.line || []).forEach((item) => {
-      item.data.forEach((data) => {
-        data.statistics = data.states;
-        data.states = [];
-      });
-    });
   }
 
   private async _getHistory() {

src/panels/lovelace/cards/hui-history-graph-card.ts

@@ -7,10 +7,12 @@ import "../../../components/chart/state-history-charts";
 import "../../../components/ha-alert";
 import "../../../components/ha-card";
 import "../../../components/ha-icon-next";
-import type { HistoryResult } from "../../../data/history";
 import {
   computeHistory,
   subscribeHistoryStatesTimeWindow,
+  type HistoryResult,
+  convertStatisticsToHistory,
+  mergeHistoryResults,
 } from "../../../data/history";
 import { getSensorNumericDeviceClasses } from "../../../data/sensor";
 import type { HomeAssistant } from "../../../types";
@@ -19,6 +21,7 @@ import { processConfigEntities } from "../common/process-config-entities";
 import type { LovelaceCard, LovelaceGridOptions } from "../types";
 import type { HistoryGraphCardConfig } from "./types";
 import { createSearchParam } from "../../../common/url/search-params";
+import { fetchStatistics } from "../../../data/recorder";
 
 export const DEFAULT_HOURS_TO_SHOW = 24;
 
@@ -36,7 +39,9 @@ export class HuiHistoryGraphCard extends LitElement implements LovelaceCard {
 
   @property({ attribute: false }) public hass?: HomeAssistant;
 
-  @state() private _stateHistory?: HistoryResult;
+  @state() private _history?: HistoryResult;
+
+  @state() private _statisticsHistory?: HistoryResult;
 
   @state() private _config?: HistoryGraphCardConfig;
 
@@ -118,7 +123,7 @@ export class HuiHistoryGraphCard extends LitElement implements LovelaceCard {
           return;
         }
 
-        this._stateHistory = computeHistory(
+        const stateHistory = computeHistory(
           this.hass!,
           combinedHistory,
           this._entityIds,
@@ -126,6 +131,12 @@ export class HuiHistoryGraphCard extends LitElement implements LovelaceCard {
           sensorNumericDeviceClasses,
           this._config?.split_device_classes
         );
+
+        this._history = mergeHistoryResults(
+          stateHistory,
+          this._statisticsHistory,
+          this._config?.split_device_classes
+        );
       },
       this._hoursToShow,
       this._entityIds
@@ -133,12 +144,39 @@ export class HuiHistoryGraphCard extends LitElement implements LovelaceCard {
       this._subscribed = undefined;
       this._error = err;
     });
+
+    await this._fetchStatistics(sensorNumericDeviceClasses);
+
     this._setRedrawTimer();
   }
 
+  private async _fetchStatistics(sensorNumericDeviceClasses: string[]) {
+    const now = new Date();
+    const start = new Date();
+    start.setHours(start.getHours() - this._hoursToShow);
+
+    const statistics = await fetchStatistics(
+      this.hass!,
+      start,
+      now,
+      this._entityIds,
+      "hour",
+      undefined,
+      ["mean", "state"]
+    );
+
+    this._statisticsHistory = convertStatisticsToHistory(
+      this.hass!,
+      statistics,
+      this._entityIds,
+      sensorNumericDeviceClasses,
+      this._config?.split_device_classes
+    );
+  }
+
   private _redrawGraph() {
-    if (this._stateHistory) {
+    if (this._history) {
-      this._stateHistory = { ...this._stateHistory };
+      this._history = { ...this._history };
     }
   }
 
@@ -229,8 +267,8 @@ export class HuiHistoryGraphCard extends LitElement implements LovelaceCard {
             : html`
                 <state-history-charts
                   .hass=${this.hass}
-                  .isLoadingData=${!this._stateHistory}
+                  .isLoadingData=${!this._history}
-                  .historyData=${this._stateHistory}
+                  .historyData=${this._history}
                   .names=${this._names}
                   up-to-now
                   .hoursToShow=${this._hoursToShow}