Clean up the overlapping history in the CSV download (#19622)

* Clean up the overlapping history in the CSV download

* Speed up merge

* undefined unit

* Fix targetPickerValue handling

src/panels/history/ha-panel-history.ts

@@ -6,6 +6,7 @@ import {
 } from "home-assistant-js-websocket/dist/types";
 import { LitElement, PropertyValues, css, html } from "lit";
 import { property, query, state } from "lit/decorators";
+import memoizeOne from "memoize-one";
 import { ensureArray } from "../../common/array/ensure-array";
 import { storage } from "../../common/decorators/storage";
 import { navigate } from "../../common/navigate";
@@ -44,8 +45,8 @@ import {
   HistoryStates,
   EntityHistoryState,
   LineChartUnit,
-  LineChartEntity,
   computeGroupKey,
+  LineChartState,
 } from "../../data/history";
 import { fetchStatistics, Statistics } from "../../data/recorder";
 import { getSensorNumericDeviceClasses } from "../../data/sensor";
@@ -71,7 +72,7 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
     state: true,
     subscribe: false,
   })
-  private _targetPickerValue?: HassServiceTarget;
+  private _targetPickerValue: HassServiceTarget = {};
 
   @state() private _isLoading = false;
 
@@ -138,6 +139,7 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
   }
 
   protected render() {
+    const entitiesSelected = this._getEntityIds().length > 0;
     return html`
       <ha-top-app-bar-fixed>
         ${this._showBack
@@ -155,7 +157,7 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
               ></ha-menu-button>
             `}
         <div slot="title">${this.hass.localize("panel.history")}</div>
-        ${this._targetPickerValue
+        ${entitiesSelected
           ? html`
               <ha-icon-button
                 slot="actionItems"
@@ -196,7 +198,7 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
             ? html`<div class="progress-wrapper">
                 <ha-circular-progress indeterminate></ha-circular-progress>
               </div>`
-            : !this._targetPickerValue
+            : !entitiesSelected
               ? html`<div class="start-search">
                   ${this.hass.localize("ui.panel.history.start_search")}
                 </div>`
@@ -220,50 +222,83 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
   ): HistoryResult {
     const result: HistoryResult = { ...historyResult, line: [] };
 
-    const keys = new Set(
+    const lookup: Record<
-      historyResult.line
+      string,
-        .map((i) => computeGroupKey(i.unit, i.device_class, true))
+      { historyItem?: LineChartUnit; ltsItem?: LineChartUnit }
-        .concat(
+    > = {};
-          ltsResult.line.map((i) =>
+
-            computeGroupKey(i.unit, i.device_class, true)
+    for (const item of historyResult.line) {
-          )
+      const key = computeGroupKey(item.unit, item.device_class, true);
-        )
+      if (key) {
-    );
+        lookup[key] = {
-    keys.forEach((key) => {
+          historyItem: item,
-      const historyItem = historyResult.line.find(
+        };
-        (i) => computeGroupKey(i.unit, i.device_class, true) === key
+      }
-      );
+    }
-      const ltsItem = ltsResult.line.find(
+
-        (i) => computeGroupKey(i.unit, i.device_class, true) === key
+    for (const item of ltsResult.line) {
-      );
+      const key = computeGroupKey(item.unit, item.device_class, true);
-      if (historyItem && ltsItem) {
+      if (!key) {
-        const newLineItem: LineChartUnit = { ...historyItem, data: [] };
+        continue;
-        const entities = new Set(
+      }
-          historyItem.data
+      if (key in lookup) {
-            .map((d) => d.entity_id)
+        lookup[key].ltsItem = item;
-            .concat(ltsItem.data.map((d) => d.entity_id))
-        );
-        entities.forEach((entity) => {
-          const historyDataItem = historyItem.data.find(
-            (d) => d.entity_id === entity
-          );
-          const ltsDataItem = ltsItem.data.find((d) => d.entity_id === entity);
-          if (historyDataItem && ltsDataItem) {
-            const newDataItem: LineChartEntity = {
-              ...historyDataItem,
-              statistics: ltsDataItem.statistics,
-            };
-            newLineItem.data.push(newDataItem);
-          } else {
-            newLineItem.data.push(historyDataItem || ltsDataItem!);
-          }
-        });
-        result.line.push(newLineItem);
       } else {
+        lookup[key] = { ltsItem: item };
+      }
+    }
+
+    for (const { historyItem, ltsItem } of Object.values(lookup)) {
+      if (!historyItem || !ltsItem) {
         // Only one result has data for this item, so just push it directly instead of merging.
         result.line.push(historyItem || ltsItem!);
+        continue;
       }
-    });
+
+      const newLineItem: LineChartUnit = { ...historyItem, data: [] };
+      const entities = new Set([
+        ...historyItem.data.map((d) => d.entity_id),
+        ...ltsItem.data.map((d) => d.entity_id),
+      ]);
+
+      for (const entity of entities) {
+        const historyDataItem = historyItem.data.find(
+          (d) => d.entity_id === entity
+        );
+        const ltsDataItem = ltsItem.data.find((d) => d.entity_id === entity);
+
+        if (!historyDataItem || !ltsDataItem) {
+          newLineItem.data.push(historyDataItem || ltsDataItem!);
+          continue;
+        }
+
+        // Remove statistics that overlap with states
+        const oldestState =
+          historyDataItem.states[0]?.last_changed ||
+          // If no state, fall back to the max last changed of the last statistics (so approve all)
+          ltsDataItem.statistics![ltsDataItem.statistics!.length - 1]
+            .last_changed + 1;
+
+        const statistics: LineChartState[] = [];
+        for (const s of ltsDataItem.statistics!) {
+          if (s.last_changed >= oldestState) {
+            break;
+          }
+          statistics.push(s);
+        }
+
+        newLineItem.data.push(
+          statistics.length === 0
+            ? // All statistics overlapped with states, so just push the states
+              historyDataItem
+            : {
+                ...historyDataItem,
+                statistics,
+              }
+        );
+      }
+      result.line.push(newLineItem);
+    }
     return result;
   }
 
@@ -335,14 +370,13 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
 
   protected updated(changedProps: PropertyValues) {
     if (
-      this._targetPickerValue &&
+      changedProps.has("_startDate") ||
-      (changedProps.has("_startDate") ||
+      changedProps.has("_endDate") ||
-        changedProps.has("_endDate") ||
+      changedProps.has("_targetPickerValue") ||
-        changedProps.has("_targetPickerValue") ||
+      (!this._stateHistory &&
-        (!this._stateHistory &&
+        (changedProps.has("_deviceEntityLookup") ||
-          (changedProps.has("_deviceEntityLookup") ||
+          changedProps.has("_areaEntityLookup") ||
-            changedProps.has("_areaEntityLookup") ||
+          changedProps.has("_areaDeviceLookup")))
-            changedProps.has("_areaDeviceLookup"))))
     ) {
       this._getHistory();
       this._getStats();
@@ -350,74 +384,71 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
   }
 
   private _removeAll() {
-    this._targetPickerValue = undefined;
+    this._targetPickerValue = {};
     this._updatePath();
   }
 
   private async _getStats() {
     const statisticIds = this._getEntityIds();
-    if (!statisticIds) {
+
+    if (statisticIds.length === 0) {
       this._statisticsHistory = undefined;
       return;
     }
 
-    try {
+    const statistics = await fetchStatistics(
-      const statistics = await fetchStatistics(
+      this.hass!,
-        this.hass!,
+      this._startDate,
-        this._startDate,
+      this._endDate,
-        this._endDate,
+      statisticIds,
-        statisticIds,
+      "hour",
-        "hour",
+      undefined,
-        undefined,
+      ["mean", "state"]
-        ["mean", "state"]
+    );
-      );
 
-      // Maintain the statistic id ordering
+    // Maintain the statistic id ordering
-      const orderedStatistics: Statistics = {};
+    const orderedStatistics: Statistics = {};
-      statisticIds.forEach((id) => {
+    statisticIds.forEach((id) => {
-        if (id in statistics) {
+      if (id in statistics) {
-          orderedStatistics[id] = statistics[id];
+        orderedStatistics[id] = statistics[id];
-        }
+      }
+    });
+
+    // Convert statistics to HistoryResult format
+    const statsHistoryStates: HistoryStates = {};
+    Object.entries(orderedStatistics).forEach(([key, value]) => {
+      const entityHistoryStates: EntityHistoryState[] = value.map((e) => ({
+        s: e.mean != null ? e.mean.toString() : e.state!.toString(),
+        lc: e.start / 1000,
+        a: {},
+        lu: e.start / 1000,
+      }));
+      statsHistoryStates[key] = entityHistoryStates;
+    });
+
+    const { numeric_device_classes: sensorNumericDeviceClasses } =
+      await getSensorNumericDeviceClasses(this.hass);
+
+    this._statisticsHistory = computeHistory(
+      this.hass,
+      statsHistoryStates,
+      this.hass.localize,
+      sensorNumericDeviceClasses,
+      true
+    );
+    // remap states array to statistics array
+    (this._statisticsHistory?.line || []).forEach((item) => {
+      item.data.forEach((data) => {
+        data.statistics = data.states;
+        data.states = [];
       });
-
+    });
-      // Convert statistics to HistoryResult format
-      const statsHistoryStates: HistoryStates = {};
-      Object.entries(orderedStatistics).forEach(([key, value]) => {
-        const entityHistoryStates: EntityHistoryState[] = value.map((e) => ({
-          s: e.mean != null ? e.mean.toString() : e.state!.toString(),
-          lc: e.start / 1000,
-          a: {},
-          lu: e.start / 1000,
-        }));
-        statsHistoryStates[key] = entityHistoryStates;
-      });
-
-      const { numeric_device_classes: sensorNumericDeviceClasses } =
-        await getSensorNumericDeviceClasses(this.hass);
-
-      this._statisticsHistory = computeHistory(
-        this.hass,
-        statsHistoryStates,
-        this.hass.localize,
-        sensorNumericDeviceClasses,
-        true
-      );
-      // remap states array to statistics array
-      (this._statisticsHistory?.line || []).forEach((item) => {
-        item.data.forEach((data) => {
-          data.statistics = data.states;
-          data.states = [];
-        });
-      });
-    } catch (err) {
-      this._statisticsHistory = undefined;
-    }
   }
 
   private async _getHistory() {
     const entityIds = this._getEntityIds();
 
-    if (!entityIds) {
+    if (entityIds.length === 0) {
       this._stateHistory = undefined;
       return;
     }
@@ -485,84 +516,100 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
     }
   }
 
-  private _getEntityIds(): string[] | undefined {
+  private _getEntityIds(): string[] {
-    if (
+    return this.__getEntityIds(
-      !this._targetPickerValue ||
+      this._targetPickerValue,
-      this._deviceEntityLookup === undefined ||
+      this._deviceEntityLookup,
-      this._areaEntityLookup === undefined ||
+      this._areaEntityLookup,
-      this._areaDeviceLookup === undefined
+      this._areaDeviceLookup
-    ) {
+    );
-      return undefined;
+  }
-    }
 
-    const entityIds = new Set<string>();
+  private __getEntityIds = memoizeOne(
-    let {
+    (
-      area_id: searchingAreaId,
+      targetPickerValue: HassServiceTarget,
-      device_id: searchingDeviceId,
+      deviceEntityLookup: DeviceEntityLookup | undefined,
-      entity_id: searchingEntityId,
+      areaEntityLookup: AreaEntityLookup | undefined,
-    } = this._targetPickerValue;
+      areaDeviceLookup: AreaDeviceLookup | undefined
+    ): string[] => {
+      if (
+        !targetPickerValue ||
+        deviceEntityLookup === undefined ||
+        areaEntityLookup === undefined ||
+        areaDeviceLookup === undefined
+      ) {
+        return [];
+      }
+
+      const entityIds = new Set<string>();
+      let {
+        area_id: searchingAreaId,
+        device_id: searchingDeviceId,
+        entity_id: searchingEntityId,
+      } = targetPickerValue;
+
+      if (searchingAreaId) {
+        searchingAreaId = ensureArray(searchingAreaId);
+        for (const singleSearchingAreaId of searchingAreaId) {
+          const foundEntities = areaEntityLookup[singleSearchingAreaId];
+          if (foundEntities?.length) {
+            for (const foundEntity of foundEntities) {
+              if (foundEntity.entity_category === null) {
+                entityIds.add(foundEntity.entity_id);
+              }
+            }
+          }
+
+          const foundDevices = areaDeviceLookup[singleSearchingAreaId];
+          if (!foundDevices?.length) {
+            continue;
+          }
+
+          for (const foundDevice of foundDevices) {
+            const foundDeviceEntities = deviceEntityLookup[foundDevice.id];
+            if (!foundDeviceEntities?.length) {
+              continue;
+            }
+
+            for (const foundDeviceEntity of foundDeviceEntities) {
+              if (
+                (!foundDeviceEntity.area_id ||
+                  foundDeviceEntity.area_id === singleSearchingAreaId) &&
+                foundDeviceEntity.entity_category === null
+              ) {
+                entityIds.add(foundDeviceEntity.entity_id);
+              }
+            }
+          }
+        }
+      }
+
+      if (searchingDeviceId) {
+        searchingDeviceId = ensureArray(searchingDeviceId);
+        for (const singleSearchingDeviceId of searchingDeviceId) {
+          const foundEntities = deviceEntityLookup[singleSearchingDeviceId];
+          if (!foundEntities?.length) {
+            continue;
+          }
 
-    if (searchingAreaId) {
-      searchingAreaId = ensureArray(searchingAreaId);
-      for (const singleSearchingAreaId of searchingAreaId) {
-        const foundEntities = this._areaEntityLookup[singleSearchingAreaId];
-        if (foundEntities?.length) {
           for (const foundEntity of foundEntities) {
             if (foundEntity.entity_category === null) {
               entityIds.add(foundEntity.entity_id);
             }
           }
         }
+      }
 
-        const foundDevices = this._areaDeviceLookup[singleSearchingAreaId];
+      if (searchingEntityId) {
-        if (!foundDevices?.length) {
+        searchingEntityId = ensureArray(searchingEntityId);
-          continue;
+        for (const singleSearchingEntityId of searchingEntityId) {
-        }
+          entityIds.add(singleSearchingEntityId);
-
-        for (const foundDevice of foundDevices) {
-          const foundDeviceEntities = this._deviceEntityLookup[foundDevice.id];
-          if (!foundDeviceEntities?.length) {
-            continue;
-          }
-
-          for (const foundDeviceEntity of foundDeviceEntities) {
-            if (
-              (!foundDeviceEntity.area_id ||
-                foundDeviceEntity.area_id === singleSearchingAreaId) &&
-              foundDeviceEntity.entity_category === null
-            ) {
-              entityIds.add(foundDeviceEntity.entity_id);
-            }
-          }
         }
       }
+
+      return [...entityIds];
     }
-
+  );
-    if (searchingDeviceId) {
-      searchingDeviceId = ensureArray(searchingDeviceId);
-      for (const singleSearchingDeviceId of searchingDeviceId) {
-        const foundEntities = this._deviceEntityLookup[singleSearchingDeviceId];
-        if (!foundEntities?.length) {
-          continue;
-        }
-
-        for (const foundEntity of foundEntities) {
-          if (foundEntity.entity_category === null) {
-            entityIds.add(foundEntity.entity_id);
-          }
-        }
-      }
-    }
-
-    if (searchingEntityId) {
-      searchingEntityId = ensureArray(searchingEntityId);
-      for (const singleSearchingEntityId of searchingEntityId) {
-        entityIds.add(singleSearchingEntityId);
-      }
-    }
-
-    return [...entityIds];
-  }
 
   private _dateRangeChanged(ev) {
     this._startDate = ev.detail.startDate;
@@ -584,20 +631,18 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
   private _updatePath() {
     const params: Record<string, string> = {};
 
-    if (this._targetPickerValue) {
+    if (this._targetPickerValue.entity_id) {
-      if (this._targetPickerValue.entity_id) {
+      params.entity_id = ensureArray(this._targetPickerValue.entity_id).join(
-        params.entity_id = ensureArray(this._targetPickerValue.entity_id).join(
+        ","
-          ","
+      );
-        );
+    }
-      }
+    if (this._targetPickerValue.area_id) {
-      if (this._targetPickerValue.area_id) {
+      params.area_id = ensureArray(this._targetPickerValue.area_id).join(",");
-        params.area_id = ensureArray(this._targetPickerValue.area_id).join(",");
+    }
-      }
+    if (this._targetPickerValue.device_id) {
-      if (this._targetPickerValue.device_id) {
+      params.device_id = ensureArray(this._targetPickerValue.device_id).join(
-        params.device_id = ensureArray(this._targetPickerValue.device_id).join(
+        ","
-          ","
+      );
-        );
-      }
     }
 
     if (this._startDate) {
@@ -613,7 +658,7 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
 
   private _downloadHistory() {
     const entities = this._getEntityIds();
-    if (!entities || !this._mungedStateHistory) {
+    if (entities.length === 0 || !this._mungedStateHistory) {
       showAlertDialog(this, {
         title: this.hass.localize("ui.panel.history.download_data_error"),
         text: this.hass.localize("ui.panel.history.error_no_data"),
@@ -628,14 +673,16 @@ class HaPanelHistory extends SubscribeMixin(LitElement) {
     for (const line of this._mungedStateHistory.line) {
       for (const entity of line.data) {
         const entityId = entity.entity_id;
-        for (const data of [entity.states, entity.statistics]) {
+
-          if (!data) {
+        if (entity.statistics) {
-            continue;
+          for (const s of entity.statistics) {
-          }
-          for (const s of data) {
             csv.push(`${entityId},${s.state},${formatDate(s.last_changed)}\n`);
           }
         }
+
+        for (const s of entity.states) {
+          csv.push(`${entityId},${s.state},${formatDate(s.last_changed)}\n`);
+        }
       }
     }
     for (const timeline of this._mungedStateHistory.timeline) {