From 682f9a0f047082031ec6b319e2b3ef31b243d152 Mon Sep 17 00:00:00 2001 From: Paulus Schoutsen Date: Fri, 2 Feb 2024 10:06:29 -0500 Subject: [PATCH] Clean up the overlapping history in the CSV download (#19622) * Clean up the overlapping history in the CSV download * Speed up merge * undefined unit * Fix targetPickerValue handling --- src/panels/history/ha-panel-history.ts | 427 ++++++++++++++----------- 1 file changed, 237 insertions(+), 190 deletions(-) diff --git a/src/panels/history/ha-panel-history.ts b/src/panels/history/ha-panel-history.ts index 6889f0369b..52f640e2c4 100644 --- a/src/panels/history/ha-panel-history.ts +++ b/src/panels/history/ha-panel-history.ts @@ -6,6 +6,7 @@ import { } from "home-assistant-js-websocket/dist/types"; import { LitElement, PropertyValues, css, html } from "lit"; import { property, query, state } from "lit/decorators"; +import memoizeOne from "memoize-one"; import { ensureArray } from "../../common/array/ensure-array"; import { storage } from "../../common/decorators/storage"; import { navigate } from "../../common/navigate"; @@ -44,8 +45,8 @@ import { HistoryStates, EntityHistoryState, LineChartUnit, - LineChartEntity, computeGroupKey, + LineChartState, } from "../../data/history"; import { fetchStatistics, Statistics } from "../../data/recorder"; import { getSensorNumericDeviceClasses } from "../../data/sensor"; @@ -71,7 +72,7 @@ class HaPanelHistory extends SubscribeMixin(LitElement) { state: true, subscribe: false, }) - private _targetPickerValue?: HassServiceTarget; + private _targetPickerValue: HassServiceTarget = {}; @state() private _isLoading = false; @@ -138,6 +139,7 @@ class HaPanelHistory extends SubscribeMixin(LitElement) { } protected render() { + const entitiesSelected = this._getEntityIds().length > 0; return html` ${this._showBack @@ -155,7 +157,7 @@ class HaPanelHistory extends SubscribeMixin(LitElement) { > `}
${this.hass.localize("panel.history")}
- ${this._targetPickerValue + ${entitiesSelected ? html` ` - : !this._targetPickerValue + : !entitiesSelected ? html`
${this.hass.localize("ui.panel.history.start_search")}
` @@ -220,50 +222,83 @@ class HaPanelHistory extends SubscribeMixin(LitElement) { ): HistoryResult { const result: HistoryResult = { ...historyResult, line: [] }; - const keys = new Set( - historyResult.line - .map((i) => computeGroupKey(i.unit, i.device_class, true)) - .concat( - ltsResult.line.map((i) => - computeGroupKey(i.unit, i.device_class, true) - ) - ) - ); - keys.forEach((key) => { - const historyItem = historyResult.line.find( - (i) => computeGroupKey(i.unit, i.device_class, true) === key - ); - const ltsItem = ltsResult.line.find( - (i) => computeGroupKey(i.unit, i.device_class, true) === key - ); - if (historyItem && ltsItem) { - const newLineItem: LineChartUnit = { ...historyItem, data: [] }; - const entities = new Set( - historyItem.data - .map((d) => d.entity_id) - .concat(ltsItem.data.map((d) => d.entity_id)) - ); - entities.forEach((entity) => { - const historyDataItem = historyItem.data.find( - (d) => d.entity_id === entity - ); - const ltsDataItem = ltsItem.data.find((d) => d.entity_id === entity); - if (historyDataItem && ltsDataItem) { - const newDataItem: LineChartEntity = { - ...historyDataItem, - statistics: ltsDataItem.statistics, - }; - newLineItem.data.push(newDataItem); - } else { - newLineItem.data.push(historyDataItem || ltsDataItem!); - } - }); - result.line.push(newLineItem); + const lookup: Record< + string, + { historyItem?: LineChartUnit; ltsItem?: LineChartUnit } + > = {}; + + for (const item of historyResult.line) { + const key = computeGroupKey(item.unit, item.device_class, true); + if (key) { + lookup[key] = { + historyItem: item, + }; + } + } + + for (const item of ltsResult.line) { + const key = computeGroupKey(item.unit, item.device_class, true); + if (!key) { + continue; + } + if (key in lookup) { + lookup[key].ltsItem = item; } else { + lookup[key] = { ltsItem: item }; + } + } + + for (const { historyItem, ltsItem } of Object.values(lookup)) { + if (!historyItem || !ltsItem) { // Only one result has data for this item, so just push it directly instead of merging. result.line.push(historyItem || ltsItem!); + continue; } - }); + + const newLineItem: LineChartUnit = { ...historyItem, data: [] }; + const entities = new Set([ + ...historyItem.data.map((d) => d.entity_id), + ...ltsItem.data.map((d) => d.entity_id), + ]); + + for (const entity of entities) { + const historyDataItem = historyItem.data.find( + (d) => d.entity_id === entity + ); + const ltsDataItem = ltsItem.data.find((d) => d.entity_id === entity); + + if (!historyDataItem || !ltsDataItem) { + newLineItem.data.push(historyDataItem || ltsDataItem!); + continue; + } + + // Remove statistics that overlap with states + const oldestState = + historyDataItem.states[0]?.last_changed || + // If no state, fall back to the max last changed of the last statistics (so approve all) + ltsDataItem.statistics![ltsDataItem.statistics!.length - 1] + .last_changed + 1; + + const statistics: LineChartState[] = []; + for (const s of ltsDataItem.statistics!) { + if (s.last_changed >= oldestState) { + break; + } + statistics.push(s); + } + + newLineItem.data.push( + statistics.length === 0 + ? // All statistics overlapped with states, so just push the states + historyDataItem + : { + ...historyDataItem, + statistics, + } + ); + } + result.line.push(newLineItem); + } return result; } @@ -335,14 +370,13 @@ class HaPanelHistory extends SubscribeMixin(LitElement) { protected updated(changedProps: PropertyValues) { if ( - this._targetPickerValue && - (changedProps.has("_startDate") || - changedProps.has("_endDate") || - changedProps.has("_targetPickerValue") || - (!this._stateHistory && - (changedProps.has("_deviceEntityLookup") || - changedProps.has("_areaEntityLookup") || - changedProps.has("_areaDeviceLookup")))) + changedProps.has("_startDate") || + changedProps.has("_endDate") || + changedProps.has("_targetPickerValue") || + (!this._stateHistory && + (changedProps.has("_deviceEntityLookup") || + changedProps.has("_areaEntityLookup") || + changedProps.has("_areaDeviceLookup"))) ) { this._getHistory(); this._getStats(); @@ -350,74 +384,71 @@ class HaPanelHistory extends SubscribeMixin(LitElement) { } private _removeAll() { - this._targetPickerValue = undefined; + this._targetPickerValue = {}; this._updatePath(); } private async _getStats() { const statisticIds = this._getEntityIds(); - if (!statisticIds) { + + if (statisticIds.length === 0) { this._statisticsHistory = undefined; return; } - try { - const statistics = await fetchStatistics( - this.hass!, - this._startDate, - this._endDate, - statisticIds, - "hour", - undefined, - ["mean", "state"] - ); + const statistics = await fetchStatistics( + this.hass!, + this._startDate, + this._endDate, + statisticIds, + "hour", + undefined, + ["mean", "state"] + ); - // Maintain the statistic id ordering - const orderedStatistics: Statistics = {}; - statisticIds.forEach((id) => { - if (id in statistics) { - orderedStatistics[id] = statistics[id]; - } + // Maintain the statistic id ordering + const orderedStatistics: Statistics = {}; + statisticIds.forEach((id) => { + if (id in statistics) { + orderedStatistics[id] = statistics[id]; + } + }); + + // Convert statistics to HistoryResult format + const statsHistoryStates: HistoryStates = {}; + Object.entries(orderedStatistics).forEach(([key, value]) => { + const entityHistoryStates: EntityHistoryState[] = value.map((e) => ({ + s: e.mean != null ? e.mean.toString() : e.state!.toString(), + lc: e.start / 1000, + a: {}, + lu: e.start / 1000, + })); + statsHistoryStates[key] = entityHistoryStates; + }); + + const { numeric_device_classes: sensorNumericDeviceClasses } = + await getSensorNumericDeviceClasses(this.hass); + + this._statisticsHistory = computeHistory( + this.hass, + statsHistoryStates, + this.hass.localize, + sensorNumericDeviceClasses, + true + ); + // remap states array to statistics array + (this._statisticsHistory?.line || []).forEach((item) => { + item.data.forEach((data) => { + data.statistics = data.states; + data.states = []; }); - - // Convert statistics to HistoryResult format - const statsHistoryStates: HistoryStates = {}; - Object.entries(orderedStatistics).forEach(([key, value]) => { - const entityHistoryStates: EntityHistoryState[] = value.map((e) => ({ - s: e.mean != null ? e.mean.toString() : e.state!.toString(), - lc: e.start / 1000, - a: {}, - lu: e.start / 1000, - })); - statsHistoryStates[key] = entityHistoryStates; - }); - - const { numeric_device_classes: sensorNumericDeviceClasses } = - await getSensorNumericDeviceClasses(this.hass); - - this._statisticsHistory = computeHistory( - this.hass, - statsHistoryStates, - this.hass.localize, - sensorNumericDeviceClasses, - true - ); - // remap states array to statistics array - (this._statisticsHistory?.line || []).forEach((item) => { - item.data.forEach((data) => { - data.statistics = data.states; - data.states = []; - }); - }); - } catch (err) { - this._statisticsHistory = undefined; - } + }); } private async _getHistory() { const entityIds = this._getEntityIds(); - if (!entityIds) { + if (entityIds.length === 0) { this._stateHistory = undefined; return; } @@ -485,84 +516,100 @@ class HaPanelHistory extends SubscribeMixin(LitElement) { } } - private _getEntityIds(): string[] | undefined { - if ( - !this._targetPickerValue || - this._deviceEntityLookup === undefined || - this._areaEntityLookup === undefined || - this._areaDeviceLookup === undefined - ) { - return undefined; - } + private _getEntityIds(): string[] { + return this.__getEntityIds( + this._targetPickerValue, + this._deviceEntityLookup, + this._areaEntityLookup, + this._areaDeviceLookup + ); + } - const entityIds = new Set(); - let { - area_id: searchingAreaId, - device_id: searchingDeviceId, - entity_id: searchingEntityId, - } = this._targetPickerValue; + private __getEntityIds = memoizeOne( + ( + targetPickerValue: HassServiceTarget, + deviceEntityLookup: DeviceEntityLookup | undefined, + areaEntityLookup: AreaEntityLookup | undefined, + areaDeviceLookup: AreaDeviceLookup | undefined + ): string[] => { + if ( + !targetPickerValue || + deviceEntityLookup === undefined || + areaEntityLookup === undefined || + areaDeviceLookup === undefined + ) { + return []; + } + + const entityIds = new Set(); + let { + area_id: searchingAreaId, + device_id: searchingDeviceId, + entity_id: searchingEntityId, + } = targetPickerValue; + + if (searchingAreaId) { + searchingAreaId = ensureArray(searchingAreaId); + for (const singleSearchingAreaId of searchingAreaId) { + const foundEntities = areaEntityLookup[singleSearchingAreaId]; + if (foundEntities?.length) { + for (const foundEntity of foundEntities) { + if (foundEntity.entity_category === null) { + entityIds.add(foundEntity.entity_id); + } + } + } + + const foundDevices = areaDeviceLookup[singleSearchingAreaId]; + if (!foundDevices?.length) { + continue; + } + + for (const foundDevice of foundDevices) { + const foundDeviceEntities = deviceEntityLookup[foundDevice.id]; + if (!foundDeviceEntities?.length) { + continue; + } + + for (const foundDeviceEntity of foundDeviceEntities) { + if ( + (!foundDeviceEntity.area_id || + foundDeviceEntity.area_id === singleSearchingAreaId) && + foundDeviceEntity.entity_category === null + ) { + entityIds.add(foundDeviceEntity.entity_id); + } + } + } + } + } + + if (searchingDeviceId) { + searchingDeviceId = ensureArray(searchingDeviceId); + for (const singleSearchingDeviceId of searchingDeviceId) { + const foundEntities = deviceEntityLookup[singleSearchingDeviceId]; + if (!foundEntities?.length) { + continue; + } - if (searchingAreaId) { - searchingAreaId = ensureArray(searchingAreaId); - for (const singleSearchingAreaId of searchingAreaId) { - const foundEntities = this._areaEntityLookup[singleSearchingAreaId]; - if (foundEntities?.length) { for (const foundEntity of foundEntities) { if (foundEntity.entity_category === null) { entityIds.add(foundEntity.entity_id); } } } + } - const foundDevices = this._areaDeviceLookup[singleSearchingAreaId]; - if (!foundDevices?.length) { - continue; - } - - for (const foundDevice of foundDevices) { - const foundDeviceEntities = this._deviceEntityLookup[foundDevice.id]; - if (!foundDeviceEntities?.length) { - continue; - } - - for (const foundDeviceEntity of foundDeviceEntities) { - if ( - (!foundDeviceEntity.area_id || - foundDeviceEntity.area_id === singleSearchingAreaId) && - foundDeviceEntity.entity_category === null - ) { - entityIds.add(foundDeviceEntity.entity_id); - } - } + if (searchingEntityId) { + searchingEntityId = ensureArray(searchingEntityId); + for (const singleSearchingEntityId of searchingEntityId) { + entityIds.add(singleSearchingEntityId); } } + + return [...entityIds]; } - - if (searchingDeviceId) { - searchingDeviceId = ensureArray(searchingDeviceId); - for (const singleSearchingDeviceId of searchingDeviceId) { - const foundEntities = this._deviceEntityLookup[singleSearchingDeviceId]; - if (!foundEntities?.length) { - continue; - } - - for (const foundEntity of foundEntities) { - if (foundEntity.entity_category === null) { - entityIds.add(foundEntity.entity_id); - } - } - } - } - - if (searchingEntityId) { - searchingEntityId = ensureArray(searchingEntityId); - for (const singleSearchingEntityId of searchingEntityId) { - entityIds.add(singleSearchingEntityId); - } - } - - return [...entityIds]; - } + ); private _dateRangeChanged(ev) { this._startDate = ev.detail.startDate; @@ -584,20 +631,18 @@ class HaPanelHistory extends SubscribeMixin(LitElement) { private _updatePath() { const params: Record = {}; - if (this._targetPickerValue) { - if (this._targetPickerValue.entity_id) { - params.entity_id = ensureArray(this._targetPickerValue.entity_id).join( - "," - ); - } - if (this._targetPickerValue.area_id) { - params.area_id = ensureArray(this._targetPickerValue.area_id).join(","); - } - if (this._targetPickerValue.device_id) { - params.device_id = ensureArray(this._targetPickerValue.device_id).join( - "," - ); - } + if (this._targetPickerValue.entity_id) { + params.entity_id = ensureArray(this._targetPickerValue.entity_id).join( + "," + ); + } + if (this._targetPickerValue.area_id) { + params.area_id = ensureArray(this._targetPickerValue.area_id).join(","); + } + if (this._targetPickerValue.device_id) { + params.device_id = ensureArray(this._targetPickerValue.device_id).join( + "," + ); } if (this._startDate) { @@ -613,7 +658,7 @@ class HaPanelHistory extends SubscribeMixin(LitElement) { private _downloadHistory() { const entities = this._getEntityIds(); - if (!entities || !this._mungedStateHistory) { + if (entities.length === 0 || !this._mungedStateHistory) { showAlertDialog(this, { title: this.hass.localize("ui.panel.history.download_data_error"), text: this.hass.localize("ui.panel.history.error_no_data"), @@ -628,14 +673,16 @@ class HaPanelHistory extends SubscribeMixin(LitElement) { for (const line of this._mungedStateHistory.line) { for (const entity of line.data) { const entityId = entity.entity_id; - for (const data of [entity.states, entity.statistics]) { - if (!data) { - continue; - } - for (const s of data) { + + if (entity.statistics) { + for (const s of entity.statistics) { csv.push(`${entityId},${s.state},${formatDate(s.last_changed)}\n`); } } + + for (const s of entity.states) { + csv.push(`${entityId},${s.state},${formatDate(s.last_changed)}\n`); + } } } for (const timeline of this._mungedStateHistory.timeline) {