Fix calculate sum growth + refreshing too often (#9699)

src/data/energy.ts

@@ -302,10 +302,10 @@ export const getEnergyDataCollection = (
         // Schedule a refresh for 20 minutes past the hour
         // If the end is larger than the current time.
         const nextFetch = new Date();
-        if (nextFetch.getMinutes() > 20) {
+        if (nextFetch.getMinutes() >= 20) {
           nextFetch.setHours(nextFetch.getHours() + 1);
         }
-        nextFetch.setMinutes(20);
+        nextFetch.setMinutes(20, 0, 0);
 
         collection._refreshTimeout = window.setTimeout(
           () => collection.refresh(),
@@ -362,15 +362,15 @@ export const getEnergyDataCollection = (
   collection.setPeriod = (newStart: Date, newEnd?: Date) => {
     collection.start = newStart;
     collection.end = newEnd;
-    if (collection._updatePeriodTimeout) {
-      clearTimeout(collection._updatePeriodTimeout);
-      collection._updatePeriodTimeout = undefined;
-    }
     if (
       collection.start.getTime() === startOfToday().getTime() &&
-      collection.end?.getTime() === endOfToday().getTime()
+      collection.end?.getTime() === endOfToday().getTime() &&
+      !collection._updatePeriodTimeout
     ) {
       scheduleUpdatePeriod();
+    } else if (collection._updatePeriodTimeout) {
+      clearTimeout(collection._updatePeriodTimeout);
+      collection._updatePeriodTimeout = undefined;
     }
   };
   return collection;

src/data/history.ts

@@ -346,64 +346,32 @@ export const statisticsHaveType = (
   type: StatisticType
 ) => stats.some((stat) => stat[type] !== null);
 
-/**
+// Merge the growth of multiple sum statistics into one
- * Get the earliest start from a list of statistics.
+const mergeSumGrowthStatistics = (stats: StatisticValue[][]) => {
- */
+  const result = {};
-const getMinStatisticStart = (stats: StatisticValue[][]): string | null => {
-  let earliestString: string | null = null;
-  let earliestTime: Date | null = null;
 
-  for (const stat of stats) {
+  stats.forEach((stat) => {
     if (stat.length === 0) {
-      continue;
+      return;
     }
-    const curTime = new Date(stat[0].start);
+    let prevSum: number | null = null;
-
+    stat.forEach((statVal) => {
-    if (earliestString === null) {
+      if (statVal.sum === null) {
-      earliestString = stat[0].start;
+        return;
-      earliestTime = curTime;
-      continue;
-    }
-
-    if (curTime < earliestTime!) {
-      earliestString = stat[0].start;
-      earliestTime = curTime;
-    }
-  }
-
-  return earliestString;
-};
-
-// Merge multiple sum statistics into one
-const mergeSumStatistics = (stats: StatisticValue[][]) => {
-  const result: { start: string; sum: number }[] = [];
-
-  const statsCopy: StatisticValue[][] = stats.map((stat) => [...stat]);
-
-  while (statsCopy.some((stat) => stat.length > 0)) {
-    const earliestStart = getMinStatisticStart(statsCopy)!;
-
-    let sum = 0;
-
-    for (const stat of statsCopy) {
-      if (stat.length === 0) {
-        continue;
       }
-      if (stat[0].start !== earliestStart) {
+      if (prevSum === null) {
-        continue;
+        prevSum = statVal.sum;
+        return;
       }
-      const statVal = stat.shift()!;
+      const growth = statVal.sum - prevSum;
-      if (!statVal.sum) {
+      if (statVal.start in result) {
-        continue;
+        result[statVal.start] += growth;
+      } else {
+        result[statVal.start] = growth;
       }
-      sum += statVal.sum;
+      prevSum = statVal.sum;
-    }
-
-    result.push({
-      start: earliestStart,
-      sum,
     });
-  }
+  });
 
   return result;
 };
@@ -418,55 +386,23 @@ export const calculateStatisticsSumGrowthWithPercentage = (
 ): number | null => {
   let sum: number | null = null;
 
-  if (sumStats.length === 0) {
+  if (sumStats.length === 0 || percentageStat.length === 0) {
     return null;
   }
 
-  const sumStatsToProcess = mergeSumStatistics(sumStats);
+  const sumGrowthToProcess = mergeSumGrowthStatistics(sumStats);
-  const percentageStatToProcess = [...percentageStat];
 
-  let lastSum: number | null = null;
+  percentageStat.forEach((percentageStatValue) => {
-
+    const sumGrowth = sumGrowthToProcess[percentageStatValue.start];
-  // pre-populate lastSum with last sum statistic _before_ the first percentage statistic
+    if (sumGrowth === undefined) {
-  for (const stat of sumStatsToProcess) {
+      return;
-    if (new Date(stat.start) >= new Date(percentageStat[0].start)) {
-      break;
     }
-    lastSum = stat.sum;
+    if (sum === null) {
-  }
+      sum = sumGrowth * (percentageStatValue.mean! / 100);
-
+    } else {
-  while (percentageStatToProcess.length > 0) {
+      sum += sumGrowth * (percentageStatValue.mean! / 100);
-    if (!sumStatsToProcess.length) {
-      return sum;
     }
-
+  });
-    // If they are not equal, pop the value that is earlier in time
-    if (sumStatsToProcess[0].start !== percentageStatToProcess[0].start) {
-      if (
-        new Date(sumStatsToProcess[0].start) <
-        new Date(percentageStatToProcess[0].start)
-      ) {
-        sumStatsToProcess.shift();
-      } else {
-        percentageStatToProcess.shift();
-      }
-      continue;
-    }
-
-    const sumStatValue = sumStatsToProcess.shift()!;
-    const percentageStatValue = percentageStatToProcess.shift()!;
-
-    if (lastSum !== null) {
-      const sumGrowth = sumStatValue.sum! - lastSum;
-      if (sum === null) {
-        sum = sumGrowth * (percentageStatValue.mean! / 100);
-      } else {
-        sum += sumGrowth * (percentageStatValue.mean! / 100);
-      }
-    }
-
-    lastSum = sumStatValue.sum;
-  }
 
   return sum;
 };

test-mocha/data/history.spec.ts

@@ -10,7 +10,81 @@ describe("calculateStatisticsSumGrowthWithPercentage", () => {
     );
   });
 
-  it("Returns null if not enough values", async () => {
+  it("Returns null if not enough sum stat values", async () => {
+    assert.strictEqual(
+      calculateStatisticsSumGrowthWithPercentage(
+        [
+          {
+            statistic_id: "sensor.carbon_intensity",
+            start: "2021-07-28T05:00:00Z",
+            last_reset: null,
+            max: 75,
+            mean: 50,
+            min: 25,
+            sum: null,
+            state: null,
+          },
+          {
+            statistic_id: "sensor.carbon_intensity",
+            start: "2021-07-28T07:00:00Z",
+            last_reset: null,
+            max: 100,
+            mean: 75,
+            min: 50,
+            sum: null,
+            state: null,
+          },
+        ],
+        []
+      ),
+      null
+    );
+  });
+
+  it("Returns null if not enough percentage stat values", async () => {
+    assert.strictEqual(
+      calculateStatisticsSumGrowthWithPercentage(
+        [],
+        [
+          [
+            {
+              statistic_id: "sensor.peak_consumption",
+              start: "2021-07-28T04:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 50,
+              state: null,
+            },
+            {
+              statistic_id: "sensor.peak_consumption",
+              start: "2021-07-28T05:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 100,
+              state: null,
+            },
+            {
+              statistic_id: "sensor.peak_consumption",
+              start: "2021-07-28T07:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 200,
+              state: null,
+            },
+          ],
+        ]
+      ),
+      null
+    );
+  });
+
+  it("Returns a percentage of the growth", async () => {
     assert.strictEqual(
       calculateStatisticsSumGrowthWithPercentage(
         [
@@ -68,10 +142,435 @@ describe("calculateStatisticsSumGrowthWithPercentage", () => {
               state: null,
             },
           ],
-          [],
+          [
+            {
+              statistic_id: "sensor.off_peak_consumption",
+              start: "2021-07-28T04:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 50,
+              state: null,
+            },
+            {
+              statistic_id: "sensor.off_peak_consumption",
+              start: "2021-07-28T05:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 100,
+              state: null,
+            },
+            {
+              statistic_id: "sensor.off_peak_consumption",
+              start: "2021-07-28T07:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 200,
+              state: null,
+            },
+          ],
+        ]
+      ),
+      200
+    );
+  });
+
+  it("It ignores sum data that doesnt match start", async () => {
+    assert.strictEqual(
+      calculateStatisticsSumGrowthWithPercentage(
+        [
+          {
+            statistic_id: "sensor.carbon_intensity",
+            start: "2021-07-28T05:00:00Z",
+            last_reset: null,
+            max: 75,
+            mean: 50,
+            min: 25,
+            sum: null,
+            state: null,
+          },
+          {
+            statistic_id: "sensor.carbon_intensity",
+            start: "2021-07-28T07:00:00Z",
+            last_reset: null,
+            max: 100,
+            mean: 75,
+            min: 50,
+            sum: null,
+            state: null,
+          },
+        ],
+        [
+          [
+            {
+              statistic_id: "sensor.peak_consumption",
+              start: "2021-07-28T04:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 50,
+              state: null,
+            },
+            {
+              statistic_id: "sensor.peak_consumption",
+              start: "2021-07-28T04:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 50,
+              state: null,
+            },
+            {
+              statistic_id: "sensor.peak_consumption",
+              start: "2021-07-28T05:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 100,
+              state: null,
+            },
+            {
+              statistic_id: "sensor.peak_consumption",
+              start: "2021-07-28T07:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 200,
+              state: null,
+            },
+          ],
         ]
       ),
       100
     );
   });
+
+  it("It ignores percentage data that doesnt match start", async () => {
+    assert.strictEqual(
+      calculateStatisticsSumGrowthWithPercentage(
+        [
+          {
+            statistic_id: "sensor.carbon_intensity",
+            start: "2021-07-28T04:00:00Z",
+            last_reset: null,
+            max: 25,
+            mean: 25,
+            min: 25,
+            sum: null,
+            state: null,
+          },
+          {
+            statistic_id: "sensor.carbon_intensity",
+            start: "2021-07-28T05:00:00Z",
+            last_reset: null,
+            max: 75,
+            mean: 50,
+            min: 25,
+            sum: null,
+            state: null,
+          },
+          {
+            statistic_id: "sensor.carbon_intensity",
+            start: "2021-07-28T07:00:00Z",
+            last_reset: null,
+            max: 100,
+            mean: 75,
+            min: 50,
+            sum: null,
+            state: null,
+          },
+        ],
+        [
+          [
+            {
+              statistic_id: "sensor.peak_consumption",
+              start: "2021-07-28T04:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 50,
+              state: null,
+            },
+            {
+              statistic_id: "sensor.peak_consumption",
+              start: "2021-07-28T05:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 100,
+              state: null,
+            },
+            {
+              statistic_id: "sensor.peak_consumption",
+              start: "2021-07-28T07:00:00Z",
+              last_reset: null,
+              max: null,
+              mean: null,
+              min: null,
+              sum: 200,
+              state: null,
+            },
+          ],
+        ]
+      ),
+      100
+    );
+  });
+
+  it("Returns a percentage of the growth", async () => {
+    assert.strictEqual(
+      calculateStatisticsSumGrowthWithPercentage(
+        [
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T06:00:00.000Z",
+            mean: 10,
+            min: 10,
+            max: 10,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 10,
+            sum: null,
+          },
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T07:00:00.000Z",
+            mean: 20,
+            min: 20,
+            max: 20,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 20,
+            sum: null,
+          },
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T08:00:00.000Z",
+            mean: 30,
+            min: 30,
+            max: 30,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 30,
+            sum: null,
+          },
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T09:00:00.000Z",
+            mean: 40,
+            min: 40,
+            max: 40,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 40,
+            sum: null,
+          },
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T10:00:00.000Z",
+            mean: 50,
+            min: 50,
+            max: 50,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 50,
+            sum: null,
+          },
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T11:00:00.000Z",
+            mean: 60,
+            min: 60,
+            max: 60,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 60,
+            sum: null,
+          },
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T12:00:00.000Z",
+            mean: 70,
+            min: 70,
+            max: 70,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 70,
+            sum: null,
+          },
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T13:00:00.000Z",
+            mean: 80,
+            min: 80,
+            max: 80,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 80,
+            sum: null,
+          },
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T14:00:00.000Z",
+            mean: 90,
+            min: 90,
+            max: 90,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 90,
+            sum: null,
+          },
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T15:00:00.000Z",
+            mean: 100,
+            min: 100,
+            max: 100,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 100,
+            sum: null,
+          },
+          {
+            statistic_id: "sensor.grid_fossil_fuel_percentage",
+            start: "2021-08-03T16:00:00.000Z",
+            mean: 110,
+            min: 110,
+            max: 110,
+            last_reset: "1970-01-01T00:00:00+00:00",
+            state: 120,
+            sum: null,
+          },
+        ],
+        [
+          [
+            {
+              statistic_id: "sensor.energy_consumption_tarif_1",
+              start: "2021-08-03T06:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 10,
+              sum: 10,
+            },
+            {
+              statistic_id: "sensor.energy_consumption_tarif_1",
+              start: "2021-08-03T07:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 20,
+              sum: 20,
+            },
+            {
+              statistic_id: "sensor.energy_consumption_tarif_1",
+              start: "2021-08-03T08:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 30,
+              sum: 30,
+            },
+            {
+              statistic_id: "sensor.energy_consumption_tarif_1",
+              start: "2021-08-03T09:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 40,
+              sum: 40,
+            },
+            {
+              statistic_id: "sensor.energy_consumption_tarif_1",
+              start: "2021-08-03T10:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 50,
+              sum: 50,
+            },
+            {
+              statistic_id: "sensor.energy_consumption_tarif_1",
+              start: "2021-08-03T11:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 60,
+              sum: 60,
+            },
+            {
+              statistic_id: "sensor.energy_consumption_tarif_1",
+              start: "2021-08-03T12:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 70,
+              sum: 70,
+            },
+            {
+              statistic_id: "sensor.energy_consumption_tarif_1",
+              start: "2021-08-03T13:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 80,
+              sum: 80,
+            },
+            {
+              statistic_id: "sensor.energy_consumption_tarif_1",
+              start: "2021-08-03T14:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 90,
+              sum: 90,
+            },
+            {
+              statistic_id: "sensor.energy_consumption_tarif_1",
+              start: "2021-08-03T15:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 100,
+              sum: 100,
+            },
+          ],
+          [
+            {
+              statistic_id: "sensor.energy_consumption_tarif_2",
+              start: "2021-08-03T15:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 10,
+              sum: 10,
+            },
+            {
+              statistic_id: "sensor.energy_consumption_tarif_2",
+              start: "2021-08-03T16:00:00.000Z",
+              mean: null,
+              min: null,
+              max: null,
+              last_reset: "1970-01-01T00:00:00+00:00",
+              state: 20,
+              sum: 20,
+            },
+          ],
+        ]
+      ),
+      65
+    );
+  });
 });