Merge branch 'development' into release

README.md

@@ -67,7 +67,9 @@ See [wiki migration path](https://tasmota.github.io/docs/#/Upgrading?id=migratio
 3. Migrate to **Sonoff-Tasmota 5.14**
 4. Migrate to **Sonoff-Tasmota 6.x**
 5. Migrate to **Tasmota 7.x**
-6. Migrate to **Tasmota 8.x**
+--- Major change in parameter storage layout ---
+6. Migrate to **Tasmota 8.1**
+7. Migrate to **Tasmota 8.x**
 
 ## Support Information
 

RELEASENOTES.md

@@ -11,7 +11,9 @@ See [migration path](https://tasmota.github.io/docs/#/Upgrading?id=migration-pat
 3. Migrate to **Sonoff-Tasmota 5.14**
 4. Migrate to **Sonoff-Tasmota 6.x**
 5. Migrate to **Tasmota 7.x**
-6. Migrate to **Tasmota 8.x**
+--- Major change in parameter storage layout ---
+6. Migrate to **Tasmota 8.1**
+7. Migrate to **Tasmota 8.x**
 
 ## Supported Core versions
 
@@ -50,6 +52,7 @@ The following binary downloads have been compiled with ESP8266/Arduino library c
 
 - Change Settings text handling allowing variable length text within a total text pool of 699 characters
 - Change Smoother ``Fade`` using 100Hz instead of 20Hz animation (#7179)
-- Change max number of rule ``Mem``s from 5 to 16 (#4933)
-- Change max number of rule ``Var``s from 5 to 16 (#4933)
+- Change number of rule ``Var``s and ``Mem``s from 5 to 16 (#4933)
 - Add support for max 150 characters in most command parameter strings (#3686, #4754)
+- Add Zigbee coalesce sensor attributes into a single message
+- Add Deepsleep start delay based on Teleperiod if ``Teleperiod`` differs from 10 or 300

tasmota/CHANGELOG.md

@@ -10,9 +10,10 @@
 
 - Change Settings text handling allowing variable length text within a total text pool of 699 characters
 - Change Smoother ``Fade`` using 100Hz instead of 20Hz animation (#7179)
-- Change max number of rule ``Mem``s from 5 to 16 (#4933)
-- Change max number of rule ``Var``s from 5 to 16 (#4933)
+- Change number of rule ``Var``s and ``Mem``s from 5 to 16 (#4933)
 - Add support for max 150 characters in most command parameter strings (#3686, #4754)
+- Add Zigbee coalesce sensor attributes into a single message
+- Add Deepsleep start delay based on Teleperiod if ``Teleperiod`` differs from 10 or 300
 
 ### 7.2.0 20191221
 

tasmota/my_user_config.h

@@ -530,6 +530,7 @@
   #define USE_ZIGBEE_PRECFGKEY_L 0x0F0D0B0907050301L  // note: changing requires to re-pair all devices
   #define USE_ZIGBEE_PRECFGKEY_H 0x0D0C0A0806040200L  // note: changing requires to re-pair all devices
   #define USE_ZIGBEE_PERMIT_JOIN false           // don't allow joining by default
+  #define USE_ZIGBEE_COALESCE_ATTR_TIMER 350     // timer to coalesce attribute values (in ms)
 
 // -- Other sensors/drivers -----------------------
 

tasmota/xdrv_23_zigbee_0_constants.ino

@@ -19,6 +19,8 @@
 
 #ifdef USE_ZIGBEE
 
+#define OCCUPANCY "Occupancy"             // global define for Aqara
+
 typedef uint64_t Z_IEEEAddress;
 typedef uint16_t Z_ShortAddress;
 

tasmota/xdrv_23_zigbee_1_headers.ino

@@ -23,4 +23,24 @@
 
 void ZigbeeZCLSend(uint16_t dtsAddr, uint16_t clusterId, uint8_t endpoint, uint8_t cmdId, bool clusterSpecific, const uint8_t *msg, size_t len, bool disableDefResp = true, uint8_t transacId = 1);
 
+
+// Get an JSON attribute, with case insensitive key search
+JsonVariant &getCaseInsensitive(const JsonObject &json, const char *needle) {
+  // key can be in PROGMEM
+  if ((nullptr == &json) || (nullptr == needle) || (0 == pgm_read_byte(needle))) {
+    return *(JsonVariant*)nullptr;
+  }
+
+  for (auto kv : json) {
+    const char *key = kv.key;
+    JsonVariant &value = kv.value;
+
+    if (0 == strcasecmp_P(key, needle)) {
+      return value;
+    }
+  }
+  // if not found
+  return *(JsonVariant*)nullptr;
+}
+
 #endif // USE_ZIGBEE

tasmota/xdrv_23_zigbee_3_devices.ino

@@ -42,6 +42,9 @@ typedef struct Z_Device {
   uint16_t              endpoint;       // endpoint to use for timer
   uint32_t              value;          // any raw value to use for the timer
   Z_DeviceTimer         func;           // function to call when timer occurs
+  // json buffer used for attribute reporting
+  DynamicJsonBuffer    *json_buffer;
+  JsonObject           *json;
 } Z_Device;
 
 // All devices are stored in a Vector
@@ -84,6 +87,11 @@ public:
   void setTimer(uint32_t shortaddr, uint32_t wait_ms, uint16_t cluster, uint16_t endpoint, uint32_t value, Z_DeviceTimer func);
   void runTimer(void);
 
+  // Append or clear attributes Json structure
+  void jsonClear(uint16_t shortaddr);
+  void jsonAppend(uint16_t shortaddr, JsonObject &values);
+  const JsonObject *jsonGet(uint16_t shortaddr);
+
 private:
   std::vector<Z_Device> _devices = {};
 
@@ -173,7 +181,9 @@ Z_Device & Z_Devices::createDeviceEntry(uint16_t shortaddr, uint64_t longaddr) {
                       std::vector<uint32_t>(),
                       std::vector<uint32_t>(),
                       0,0,0,0,
-                      nullptr };
+                      nullptr,
+                      nullptr, nullptr };
+  device.json_buffer = new DynamicJsonBuffer();
   _devices.push_back(device);
   return _devices.back();
 }
@@ -394,14 +404,55 @@ void Z_Devices::runTimer(void) {
 
     uint32_t timer = device.timer;
     if ((timer) && (timer <= now)) {
+      device.timer = 0;       // cancel the timer before calling, so the callback can set another timer
       // trigger the timer
       (*device.func)(device.shortaddr, device.cluster, device.endpoint, device.value);
-
-      device.timer = 0;       // cancel the timer
     }
   }
 }
 
+void Z_Devices::jsonClear(uint16_t shortaddr) {
+  Z_Device & device = getShortAddr(shortaddr);
+  if (&device == nullptr) { return; }                 // don't crash if not found
+
+  device.json = nullptr;
+  device.json_buffer->clear();
+}
+
+void Z_Devices::jsonAppend(uint16_t shortaddr, JsonObject &values) {
+  Z_Device & device = getShortAddr(shortaddr);
+  if (&device == nullptr) { return; }                 // don't crash if not found
+  if (&values == nullptr) { return; }
+
+  if (nullptr == device.json) {
+    device.json = &(device.json_buffer->createObject());
+  }
+  // copy all values from 'values' to 'json'
+  for (auto kv : values) {
+    String key_string = kv.key;
+    const char * key = key_string.c_str();
+    JsonVariant &val = kv.value;
+
+    device.json->remove(key_string);    // force remove to have metadata like LinkQuality at the end
+
+    if (val.is<char*>()) {
+      String sval = val.as<String>();       // force a copy of the String value
+      device.json->set(key_string, sval);
+    } else if (val.is<JsonArray>()) {
+      // todo
+    } else if (val.is<JsonObject>()) {
+      // todo
+    } else {
+      device.json->set(key_string, kv.value);
+    }
+  }
+}
+
+const JsonObject *Z_Devices::jsonGet(uint16_t shortaddr) {
+  Z_Device & device = getShortAddr(shortaddr);
+  if (&device == nullptr) { return nullptr; }                 // don't crash if not found
+  return device.json;
+}
 
 // Dump the internal memory of Zigbee devices
 // Mode = 1: simple dump of devices addresses and names

tasmota/xdrv_23_zigbee_5_converters.ino

@@ -480,8 +480,6 @@ typedef struct Z_AttributeConverter {
   Z_AttrConverter func;
 } Z_AttributeConverter;
 
-#define OCCUPANCY "Occupancy"             // global define for Aqara
-
 // list of post-processing directives
 const Z_AttributeConverter Z_PostProcess[] PROGMEM = {
   { 0x0000, 0x0000,  "ZCLVersion",           &Z_Copy },
@@ -751,7 +749,7 @@ const Z_AttributeConverter Z_PostProcess[] PROGMEM = {
   { 0x0405, 0xFFFF,  nullptr,                &Z_Remove },     // Remove all other values
 
   // Occupancy Sensing cluster
-  { 0x0406, 0x0000,  OCCUPANCY,              &Z_AqaraOccupancy },    // Occupancy (map8)
+  { 0x0406, 0x0000,  OCCUPANCY,              &Z_Copy },    // Occupancy (map8)
   { 0x0406, 0x0001,  "OccupancySensorType",  &Z_Copy },    // OccupancySensorType
   { 0x0406, 0xFFFF,  nullptr,                &Z_Remove },    // Remove all other values
 
@@ -818,11 +816,7 @@ int32_t Z_FloatDiv10(const class ZCLFrame *zcl, uint16_t shortaddr, JsonObject&
   return 1;   // remove original key
 }
 
-
-// Aqara Occupancy behavior: the Aqara device only sends Occupancy: true events every 60 seconds.
-// Here we add a timer so if we don't receive a Occupancy event for 90 seconds, we send Occupancy:false
-const uint32_t OCCUPANCY_TIMEOUT = 90 * 1000;  // 90 s
-
+// Publish a message for `"Occupancy":0` when the timer expired
 int32_t Z_OccupancyCallback(uint16_t shortaddr, uint16_t cluster, uint16_t endpoint, uint32_t value) {
   // send Occupancy:false message
   Response_P(PSTR("{\"" D_CMND_ZIGBEE_RECEIVED "\":{\"0x%04X\":{\"" OCCUPANCY "\":0}}}"), shortaddr);
@@ -830,18 +824,6 @@ int32_t Z_OccupancyCallback(uint16_t shortaddr, uint16_t cluster, uint16_t endpo
   XdrvRulesProcess();
 }
 
-int32_t Z_AqaraOccupancy(const class ZCLFrame *zcl, uint16_t shortaddr, JsonObject& json, const char *name, JsonVariant& value, const __FlashStringHelper *new_name, uint16_t cluster, uint16_t attr) {
-  json[new_name] = value;
-  uint32_t occupancy = value;
-
-  if (occupancy) {
-    zigbee_devices.setTimer(shortaddr, OCCUPANCY_TIMEOUT, cluster, zcl->getSrcEndpoint(), 0, &Z_OccupancyCallback);
-  } else {
-    zigbee_devices.resetTimer(shortaddr);
-  }
-  return 1;   // remove original key
-}
-
 // Aqara Vibration Sensor - special proprietary attributes
 int32_t Z_AqaraVibration(const class ZCLFrame *zcl, uint16_t shortaddr, JsonObject& json, const char *name, JsonVariant& value, const __FlashStringHelper *new_name, uint16_t cluster, uint16_t attr) {
   //json[new_name] = value;

tasmota/xdrv_23_zigbee_8_parsers.ino

@@ -357,6 +357,39 @@ int32_t Z_ReceiveEndDeviceAnnonce(int32_t res, const class SBuffer &buf) {
   return -1;
 }
 
+// Aqara Occupancy behavior: the Aqara device only sends Occupancy: true events every 60 seconds.
+// Here we add a timer so if we don't receive a Occupancy event for 90 seconds, we send Occupancy:false
+const uint32_t OCCUPANCY_TIMEOUT = 90 * 1000;  // 90 s
+
+void Z_AqaraOccupancy(uint16_t shortaddr, uint16_t cluster, uint16_t endpoint, const JsonObject *json) {
+  // Read OCCUPANCY value if any
+  const JsonVariant &val_endpoint = getCaseInsensitive(*json, PSTR(OCCUPANCY));
+  if (nullptr != &val_endpoint) {
+    uint32_t occupancy = strToUInt(val_endpoint);
+
+    if (occupancy) {
+      zigbee_devices.setTimer(shortaddr, OCCUPANCY_TIMEOUT, cluster, endpoint, 0, &Z_OccupancyCallback);
+    }
+  }
+}
+
+
+// Publish the received values once they have been coalesced
+int32_t Z_PublishAttributes(uint16_t shortaddr, uint16_t cluster, uint16_t endpoint, uint32_t value) {
+  const JsonObject *json = zigbee_devices.jsonGet(shortaddr);
+  if (json == nullptr) { return 0; }                 // don't crash if not found
+
+  // Post-provess for Aqara Presence Senson
+  Z_AqaraOccupancy(shortaddr, cluster, endpoint, json);
+
+  String msg = "";
+  json->printTo(msg);
+  zigbee_devices.jsonClear(shortaddr);
+  Response_P(PSTR("{\"" D_CMND_ZIGBEE_RECEIVED "\":{\"0x%04X\":%s}}"), shortaddr, msg.c_str());
+  MqttPublishPrefixTopic_P(TELE, PSTR(D_RSLT_SENSOR));
+  XdrvRulesProcess();
+}
+
 int32_t Z_ReceiveAfIncomingMessage(int32_t res, const class SBuffer &buf) {
   uint16_t        groupid = buf.get16(2);
   uint16_t        clusterid = buf.get16(4);
@@ -369,6 +402,8 @@ int32_t Z_ReceiveAfIncomingMessage(int32_t res, const class SBuffer &buf) {
   uint32_t        timestamp = buf.get32(13);
   uint8_t         seqnumber = buf.get8(17);
 
+  bool            defer_attributes = false;     // do we defer attributes reporting to coalesce
+
   zigbee_devices.updateLastSeen(srcaddr);
   ZCLFrame zcl_received = ZCLFrame::parseRawFrame(buf, 19, buf.get8(18), clusterid, groupid,
                               srcaddr,
@@ -384,13 +419,13 @@ int32_t Z_ReceiveAfIncomingMessage(int32_t res, const class SBuffer &buf) {
   JsonObject& json1 = json_root.createNestedObject(F(D_CMND_ZIGBEE_RECEIVED));
   JsonObject& json = json1.createNestedObject(shortaddr);
 
-  // TODO add name field if it is known
   if ( (!zcl_received.isClusterSpecificCommand()) && (ZCL_REPORT_ATTRIBUTES == zcl_received.getCmdId())) {
-   zcl_received.parseRawAttributes(json);
+    zcl_received.parseRawAttributes(json);
+    if (clusterid) { defer_attributes = true; }  // don't defer system Cluster=0 messages
   } else if ( (!zcl_received.isClusterSpecificCommand()) && (ZCL_READ_ATTRIBUTES_RESPONSE == zcl_received.getCmdId())) {
     zcl_received.parseReadAttributes(json);
   } else if (zcl_received.isClusterSpecificCommand()) {
-   zcl_received.parseClusterSpecificCommand(json);
+    zcl_received.parseClusterSpecificCommand(json);
   }
   String msg("");
   msg.reserve(100);
@@ -401,11 +436,18 @@ int32_t Z_ReceiveAfIncomingMessage(int32_t res, const class SBuffer &buf) {
   // Add linkquality
   json[F(D_CMND_ZIGBEE_LINKQUALITY)] = linkquality;
 
-  msg = "";
-  json_root.printTo(msg);
-  Response_P(PSTR("%s"), msg.c_str());
-  MqttPublishPrefixTopic_P(TELE, PSTR(D_RSLT_SENSOR));
-  XdrvRulesProcess();
+  if (defer_attributes) {
+    // Prepare for publish
+    zigbee_devices.jsonAppend(srcaddr, json);
+    zigbee_devices.setTimer(srcaddr, USE_ZIGBEE_COALESCE_ATTR_TIMER, clusterid, srcendpoint, 0, &Z_PublishAttributes);
+  } else {
+    // Publish immediately
+    msg = "";
+    json_root.printTo(msg);
+    Response_P(PSTR("%s"), msg.c_str());
+    MqttPublishPrefixTopic_P(TELE, PSTR(D_RSLT_SENSOR));
+    XdrvRulesProcess();
+  }
   return -1;
 }
 

tasmota/xdrv_23_zigbee_9_impl.ino

@@ -423,25 +423,6 @@ void zigbeeZCLSendStr(uint16_t dstAddr, uint8_t endpoint, const char *data) {
   ResponseCmndDone();
 }
 
-// Get an JSON attribute, with case insensitive key search
-JsonVariant &getCaseInsensitive(const JsonObject &json, const char *needle) {
-  // key can be in PROGMEM
-  if ((nullptr == &json) || (nullptr == needle) || (0 == pgm_read_byte(needle))) {
-    return *(JsonVariant*)nullptr;
-  }
-
-  for (auto kv : json) {
-    const char *key = kv.key;
-    JsonVariant &value = kv.value;
-
-    if (0 == strcasecmp_P(key, needle)) {
-      return value;
-    }
-  }
-  // if not found
-  return *(JsonVariant*)nullptr;
-}
-
 void CmndZigbeeSend(void) {
   // ZigbeeSend { "device":"0x1234", "endpoint":"0x03", "send":{"Power":1} }
   // ZigbeeSend { "device":"0x1234", "endpoint":"0x03", "send":{"Power":"3"} }

tasmota/xdrv_29_deepsleep.ino

@@ -192,7 +192,7 @@ bool Xdrv29(uint8_t function)
       DeepSleepEverySecond();
       break;
     case FUNC_AFTER_TELEPERIOD:
-      if (DeepSleepEnabled() && !deepsleep_flag) {
+        if (DeepSleepEnabled() && !deepsleep_flag && (Settings.tele_period == 10 || Settings.tele_period == 300 || UpTime() > Settings.tele_period)) {
         deepsleep_flag = DEEPSLEEP_START_COUNTDOWN;  // Start deepsleep in 4 seconds
       }
       break;