ESP-NOW packet modification

- include up to 5 segments in 1st packet
- header contains total number of packets (instead of segments)

web server code reorganise

wled00/fcn_declare.h

@@ -253,6 +253,7 @@ void setRealtimePixel(uint16_t i, byte r, byte g, byte b, byte w);
 void refreshNodeList();
 void sendSysInfoUDP();
 #ifndef WLED_DISABLE_ESPNOW
+void espNowSentCB(uint8_t* address, uint8_t status);
 void espNowReceiveCB(uint8_t* address, uint8_t* data, uint8_t len, signed int rssi, bool broadcast);
 #endif
 
@@ -428,15 +429,10 @@ void handleSerial();
 void updateBaudRate(uint32_t rate);
 
 //wled_server.cpp
-bool isIp(String str);
 void createEditHandler(bool enable);
-bool captivePortal(AsyncWebServerRequest *request);
 void initServer();
-void serveIndex(AsyncWebServerRequest* request);
-String msgProcessor(const String& var);
 void serveMessage(AsyncWebServerRequest* request, uint16_t code, const String& headl, const String& subl="", byte optionT=255);
 void serveJsonError(AsyncWebServerRequest* request, uint16_t code, uint16_t error);
-String dmxProcessor(const String& var);
 void serveSettings(AsyncWebServerRequest* request, bool post = false);
 void serveSettingsJS(AsyncWebServerRequest* request);
 
@@ -447,7 +443,6 @@ void sendDataWs(AsyncWebSocketClient * client = nullptr);
 
 //xml.cpp
 void XML_response(AsyncWebServerRequest *request, char* dest = nullptr);
-void URL_response(AsyncWebServerRequest *request);
 void getSettingsJS(byte subPage, char* dest);
 
 #endif

wled00/udp.cpp

@@ -13,7 +13,7 @@
 typedef struct PartialEspNowPacket {
   uint8_t magic;
   uint8_t packet;
-  uint8_t segs;
+  uint8_t noOfPackets;
   uint8_t data[247];
 } partial_packet_t;
 
@@ -151,20 +151,32 @@ void notify(byte callMode, bool followUp)
 
 #ifndef WLED_DISABLE_ESPNOW
   if (enableESPNow && useESPNowSync && statusESPNow == ESP_NOW_STATE_ON) {
-    partial_packet_t buffer = {'W', 0, (uint8_t)s, {0}};
+    partial_packet_t buffer = {'W', 0, 1, {0}};
     // send global data
-    DEBUG_PRINTLN(F("ESP-NOW sending first packet.")); 
+    DEBUG_PRINTLN(F("ESP-NOW sending first packet."));
-    memcpy(buffer.data, udpOut, 41);
+    const size_t bufferSize = sizeof(buffer.data)/sizeof(uint8_t);
-    auto err = quickEspNow.send(ESPNOW_BROADCAST_ADDRESS, reinterpret_cast<const uint8_t*>(&buffer), 41+3);
+    size_t packetSize = 41;
-    if (!err) {
+    size_t s0 = 0;
-      // send segment data
+    memcpy(buffer.data, udpOut, packetSize);
+    // stuff as many segments in first packet as possible (normally up to 5)
+    for (size_t i = 0; packetSize < bufferSize && i < s; i++) {
+      memcpy(buffer.data + packetSize, &udpOut[41+i*UDP_SEG_SIZE], UDP_SEG_SIZE);
+      packetSize += UDP_SEG_SIZE;
+      s0++;
+    }
+    if (s > s0) buffer.noOfPackets += 1 + ((s - s0) * UDP_SEG_SIZE) / bufferSize; // set number of packets
+    auto err = quickEspNow.send(ESPNOW_BROADCAST_ADDRESS, reinterpret_cast<const uint8_t*>(&buffer), packetSize+3);
+    if (!err && s0 < s) {
+      // send rest of the segments
       buffer.packet++;
-      size_t packetSize = 0;
+      packetSize = 0;
-      int32_t err = 0;
+      // WARNING: this will only work for up to 3 messages (~17 segments) as QuickESPNOW only has a ring buffer capable of holding 3 queued messages
-      for (size_t i = 0; i < s; i++) {
+      // to work around that limitation it is mandatory to utilize onDataSent() callback which should reduce number queued messages
+      // and wait until at least one space is available in the buffer
+      for (size_t i = s0; i < s; i++) {
         memcpy(buffer.data + packetSize, &udpOut[41+i*UDP_SEG_SIZE], UDP_SEG_SIZE);
         packetSize += UDP_SEG_SIZE;
-        if (packetSize + UDP_SEG_SIZE < sizeof(buffer.data)/sizeof(uint8_t)) continue;
+        if (packetSize + UDP_SEG_SIZE < bufferSize) continue;
         DEBUG_PRINTF("ESP-NOW sending packet: %d (%d)\n", (int)buffer.packet, packetSize+3);
         err = quickEspNow.send(ESPNOW_BROADCAST_ADDRESS, reinterpret_cast<const uint8_t*>(&buffer), packetSize+3);
         buffer.packet++;
@@ -175,9 +187,9 @@ void notify(byte callMode, bool followUp)
         DEBUG_PRINTF("ESP-NOW sending last packet: %d (%d)\n", (int)buffer.packet, packetSize+3);
         err = quickEspNow.send(ESPNOW_BROADCAST_ADDRESS, reinterpret_cast<const uint8_t*>(&buffer), packetSize+3);
       }
-      if (err) {
+    }
-        DEBUG_PRINTLN(F("ESP-NOW sending packet failed."));
+    if (err) {
-      }
+      DEBUG_PRINTLN(F("ESP-NOW sending packet failed."));
     }
   }
   if (udpConnected) 
@@ -942,6 +954,11 @@ uint8_t realtimeBroadcast(uint8_t type, IPAddress client, uint16_t length, uint8
 }
 
 #ifndef WLED_DISABLE_ESPNOW
+// ESP-NOW message sent callback function
+void espNowSentCB(uint8_t* address, uint8_t status) {
+    DEBUG_PRINTF("Message sent to " MACSTR ", status: %d\n", MAC2STR(address), status);
+}
+
 // ESP-NOW message receive callback function
 void espNowReceiveCB(uint8_t* address, uint8_t* data, uint8_t len, signed int rssi, bool broadcast) {
   sprintf_P(last_signal_src, PSTR("%02x%02x%02x%02x%02x%02x"), address[0], address[1], address[2], address[3], address[4], address[5]);
@@ -970,19 +987,35 @@ void espNowReceiveCB(uint8_t* address, uint8_t* data, uint8_t len, signed int rs
   }
 
   static uint8_t *udpIn = nullptr;
-  static uint8_t packetsReceived = 0; // bitfield (max 5 packets ATM)
+  static uint8_t packetsReceived = 0;
   static uint8_t segsReceived = 0;
   static unsigned long lastProcessed = 0;
 
   if (buffer->packet == 0) {
-    if (udpIn == nullptr) udpIn = (uint8_t *)malloc(WLEDPACKETSIZE); // we cannot use stack as we are in callback
+    packetsReceived = 0; // it will increment later (this is to make sure we start counting packets correctly)
-    DEBUG_PRINTLN(F("ESP-NOW inited UDP buffer."));
+    if (udpIn == nullptr) {
-    memcpy(udpIn, buffer->data, len-3); // global data (41 bytes)
+      udpIn = (uint8_t *)malloc(WLEDPACKETSIZE); // we cannot use stack as we are in callback
-    packetsReceived |= 0x01 << buffer->packet;
+      if (!udpIn) return; // memory alocation failed
-    segsReceived = 0;
+      DEBUG_PRINTLN(F("ESP-NOW inited UDP buffer."));
-    return;
+    }
-  } else if (((len-3)/UDP_SEG_SIZE)*UDP_SEG_SIZE != (len-3)) {
+    memcpy(udpIn, buffer->data, len-3); // global data (41 bytes + up to 5 segments)
-    DEBUG_PRINTF("ESP-NOW incorrect packet size: %d (%d) [%d]\n", (int)buffer->packet, (int)len-3, (int)UDP_SEG_SIZE);
+    segsReceived = (len - 3 - 41) / UDP_SEG_SIZE;
+  } else if (buffer->packet == packetsReceived && udpIn && ((len - 3) / UDP_SEG_SIZE) * UDP_SEG_SIZE == (len-3)) {
+    // we received a packet full of segments
+    if (segsReceived >= MAX_NUM_SEGMENTS) {
+      // we are already past max segments, just ignore
+      DEBUG_PRINTLN(F("ESP-NOW received segments past maximum."));
+      len = 3;
+    } else if ((segsReceived + ((len - 3) / UDP_SEG_SIZE)) >= MAX_NUM_SEGMENTS) {
+      len = ((MAX_NUM_SEGMENTS - segsReceived) * UDP_SEG_SIZE) + 3; // we have reached max number of segments
+    }
+    if (len > 3) {
+      memcpy(udpIn + 41 + (segsReceived * UDP_SEG_SIZE), buffer->data, len-3);
+      segsReceived += (len - 3) / UDP_SEG_SIZE;
+    }
+  } else {
+    // any out of order packet or incorrectly sized packet or if we have no UDP buffer will abort
+    DEBUG_PRINTF("ESP-NOW incorrect packet: %d (%d) [%d]\n", (int)buffer->packet, (int)len-3, (int)UDP_SEG_SIZE);
     if (udpIn) free(udpIn);
     udpIn = nullptr;
     packetsReceived = 0;
@@ -991,15 +1024,9 @@ void espNowReceiveCB(uint8_t* address, uint8_t* data, uint8_t len, signed int rs
   }
   if (!udpIn) return;
 
-  // TODO add verification if segsReceived > MAX_NUM_SEGMENTS or WLEDPACKETSIZE
+  packetsReceived++;
-
+  DEBUG_PRINTF("ESP-NOW packet received: %d (%d/%d) s:[%d/%d]\n", (int)buffer->packet, (int)packetsReceived, (int)buffer->noOfPackets, (int)segsReceived, MAX_NUM_SEGMENTS);
-  memcpy(udpIn+41+segsReceived, buffer->data, len-3);
+  if (packetsReceived >= buffer->noOfPackets) {
-  packetsReceived |= 0x01 << buffer->packet;
-  segsReceived += (len-3)/UDP_SEG_SIZE;
-
-  DEBUG_PRINTF("ESP-NOW packet received: %d (%d) [%d]\n", (int)buffer->packet, (int)len-3, (int)segsReceived);
-
-  if (segsReceived == buffer->segs) {
     // last packet received
     if (millis() - lastProcessed > 250) {
       DEBUG_PRINTLN(F("ESP-NOW processing complete message."));

wled00/wled.cpp

@@ -247,6 +247,9 @@ void WLED::loop()
     }
     #endif
     DEBUG_PRINT(F("Wifi state: "));      DEBUG_PRINTLN(WiFi.status());
+    #ifndef WLED_DISABLE_ESPNOW
+    DEBUG_PRINT(F("ESP-NOW state: "));   DEBUG_PRINTLN(statusESPNow);
+    #endif
 
     if (WiFi.status() != lastWifiState) {
       wifiStateChangedTime = millis();
@@ -834,7 +837,8 @@ void WLED::initConnection()
 
 #ifndef WLED_DISABLE_ESPNOW
   if (enableESPNow) {
-    quickEspNow.onDataRcvd(espNowReceiveCB);
+    quickEspNow.onDataSent(espNowSentCB);     // see udp.cpp
+    quickEspNow.onDataRcvd(espNowReceiveCB);  // see udp.cpp
     bool espNowOK;
     if (apActive) {
       DEBUG_PRINTLN(F("ESP-NOW initing in AP mode."));

wled00/wled_server.cpp

@@ -11,14 +11,6 @@
 #endif
 #include "html_cpal.h"
 
-/*
- * Integrated HTTP web server page declarations
- */
-
-bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest* request, int code, uint16_t eTagSuffix = 0);
-void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, uint16_t eTagSuffix = 0);
-void handleStaticContent(AsyncWebServerRequest *request, const String &path, int code, const String &contentType, const uint8_t *content, size_t len, bool gzip = true, uint16_t eTagSuffix = 0);
-
 // define flash strings once (saves flash memory)
 static const char s_redirecting[] PROGMEM = "Redirecting...";
 static const char s_content_enc[] PROGMEM = "Content-Encoding";
@@ -33,7 +25,7 @@ static const char s_plain[]          PROGMEM = "text/plain";
 static const char s_css[]            PROGMEM = "text/css";
 
 //Is this an IP?
-bool isIp(String str) {
+static bool isIp(String str) {
   for (size_t i = 0; i < str.length(); i++) {
     int c = str.charAt(i);
     if (c != '.' && (c < '0' || c > '9')) {
@@ -43,7 +35,128 @@ bool isIp(String str) {
   return true;
 }
 
-void handleUpload(AsyncWebServerRequest *request, const String& filename, size_t index, uint8_t *data, size_t len, bool final) {
+/*
+ * Integrated HTTP web server page declarations
+ */
+
+static void generateEtag(char *etag, uint16_t eTagSuffix) {
+  sprintf_P(etag, PSTR("%7d-%02x-%04x"), VERSION, cacheInvalidate, eTagSuffix);
+}
+
+static void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, uint16_t eTagSuffix = 0) {
+  // Only send ETag for 200 (OK) responses
+  if (code != 200) return;
+
+  // https://medium.com/@codebyamir/a-web-developers-guide-to-browser-caching-cc41f3b73e7c
+  #ifndef WLED_DEBUG
+  // this header name is misleading, "no-cache" will not disable cache,
+  // it just revalidates on every load using the "If-None-Match" header with the last ETag value
+  response->addHeader(F("Cache-Control"), F("no-cache"));
+  #else
+  response->addHeader(F("Cache-Control"), F("no-store,max-age=0"));  // prevent caching if debug build
+  #endif
+  char etag[32];
+  generateEtag(etag, eTagSuffix);
+  response->addHeader(F("ETag"), etag);
+}
+
+static bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest *request, int code, uint16_t eTagSuffix = 0) {
+  // Only send 304 (Not Modified) if response code is 200 (OK)
+  if (code != 200) return false;
+
+  AsyncWebHeader *header = request->getHeader(F("If-None-Match"));
+  char etag[32];
+  generateEtag(etag, eTagSuffix);
+  if (header && header->value() == etag) {
+    AsyncWebServerResponse *response = request->beginResponse(304);
+    setStaticContentCacheHeaders(response, code, eTagSuffix);
+    request->send(response);
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Handles the request for a static file.
+ * If the file was found in the filesystem, it will be sent to the client.
+ * Otherwise it will be checked if the browser cached the file and if so, a 304 response will be sent.
+ * If the file was not found in the filesystem and not in the browser cache, the request will be handled as a 200 response with the content of the page.
+ *
+ * @param request The request object
+ * @param path If a file with this path exists in the filesystem, it will be sent to the client. Set to "" to skip this check.
+ * @param code The HTTP status code
+ * @param contentType The content type of the web page
+ * @param content Content of the web page
+ * @param len Length of the content
+ * @param gzip Optional. Defaults to true. If false, the gzip header will not be added.
+ * @param eTagSuffix Optional. Defaults to 0. A suffix that will be added to the ETag header. This can be used to invalidate the cache for a specific page.
+ */
+static void handleStaticContent(AsyncWebServerRequest *request, const String &path, int code, const String &contentType, const uint8_t *content, size_t len, bool gzip = true, uint16_t eTagSuffix = 0) {
+  if (path != "" && handleFileRead(request, path)) return;
+  if (handleIfNoneMatchCacheHeader(request, code, eTagSuffix)) return;
+  AsyncWebServerResponse *response = request->beginResponse_P(code, contentType, content, len);
+  if (gzip) response->addHeader(FPSTR(s_content_enc), F("gzip"));
+  setStaticContentCacheHeaders(response, code, eTagSuffix);
+  request->send(response);
+}
+
+#ifdef WLED_ENABLE_DMX
+static String dmxProcessor(const String& var)
+{
+  String mapJS;
+  if (var == F("DMXVARS")) {
+    mapJS += F("\nCN=");
+    mapJS += String(DMXChannels);
+    mapJS += F(";\nCS=");
+    mapJS += String(DMXStart);
+    mapJS += F(";\nCG=");
+    mapJS += String(DMXGap);
+    mapJS += F(";\nLC=");
+    mapJS += String(strip.getLengthTotal());
+    mapJS += F(";\nvar CH=[");
+    for (int i=0; i<15; i++) {
+      mapJS += String(DMXFixtureMap[i]) + ',';
+    }
+    mapJS += F("0];");
+  }
+  return mapJS;
+}
+#endif
+
+static String msgProcessor(const String& var)
+{
+  if (var == "MSG") {
+    String messageBody = messageHead;
+    messageBody += F("</h2>");
+    messageBody += messageSub;
+    uint32_t optt = optionType;
+
+    if (optt < 60) //redirect to settings after optionType seconds
+    {
+      messageBody += F("<script>setTimeout(RS,");
+      messageBody +=String(optt*1000);
+      messageBody += F(")</script>");
+    } else if (optt < 120) //redirect back after optionType-60 seconds, unused
+    {
+      //messageBody += "<script>setTimeout(B," + String((optt-60)*1000) + ")</script>";
+    } else if (optt < 180) //reload parent after optionType-120 seconds
+    {
+      messageBody += F("<script>setTimeout(RP,");
+      messageBody += String((optt-120)*1000);
+      messageBody += F(")</script>");
+    } else if (optt == 253)
+    {
+      messageBody += F("<br><br><form action=/settings><button class=\"bt\" type=submit>Back</button></form>"); //button to settings
+    } else if (optt == 254)
+    {
+      messageBody += F("<br><br><button type=\"button\" class=\"bt\" onclick=\"B()\">Back</button>");
+    }
+    return messageBody;
+  }
+  return String();
+}
+
+static void handleUpload(AsyncWebServerRequest *request, const String& filename, size_t index, uint8_t *data, size_t len, bool final) {
   if (!correctPIN) {
     if (final) request->send(401, FPSTR(s_plain), FPSTR(s_unlock_cfg));
     return;
@@ -96,7 +209,7 @@ void createEditHandler(bool enable) {
   }
 }
 
-bool captivePortal(AsyncWebServerRequest *request)
+static bool captivePortal(AsyncWebServerRequest *request)
 {
   if (!apActive) return false; //only serve captive in AP mode
   if (!request->hasHeader("Host")) return false;
@@ -368,102 +481,6 @@ void initServer()
   });
 }
 
-void generateEtag(char *etag, uint16_t eTagSuffix) {
-  sprintf_P(etag, PSTR("%7d-%02x-%04x"), VERSION, cacheInvalidate, eTagSuffix);
-}
-
-bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest *request, int code, uint16_t eTagSuffix) {
-  // Only send 304 (Not Modified) if response code is 200 (OK)
-  if (code != 200) return false;
-
-  AsyncWebHeader *header = request->getHeader(F("If-None-Match"));
-  char etag[32];
-  generateEtag(etag, eTagSuffix);
-  if (header && header->value() == etag) {
-    AsyncWebServerResponse *response = request->beginResponse(304);
-    setStaticContentCacheHeaders(response, code, eTagSuffix);
-    request->send(response);
-    return true;
-  }
-  return false;
-}
-
-void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, uint16_t eTagSuffix) {
-  // Only send ETag for 200 (OK) responses
-  if (code != 200) return;
-
-  // https://medium.com/@codebyamir/a-web-developers-guide-to-browser-caching-cc41f3b73e7c
-  #ifndef WLED_DEBUG
-  // this header name is misleading, "no-cache" will not disable cache,
-  // it just revalidates on every load using the "If-None-Match" header with the last ETag value
-  response->addHeader(F("Cache-Control"), F("no-cache"));
-  #else
-  response->addHeader(F("Cache-Control"), F("no-store,max-age=0"));  // prevent caching if debug build
-  #endif
-  char etag[32];
-  generateEtag(etag, eTagSuffix);
-  response->addHeader(F("ETag"), etag);
-}
-
-/**
- * Handels the request for a static file.
- * If the file was found in the filesystem, it will be sent to the client.
- * Otherwise it will be checked if the browser cached the file and if so, a 304 response will be sent.
- * If the file was not found in the filesystem and not in the browser cache, the request will be handled as a 200 response with the content of the page.
- *
- * @param request The request object
- * @param path If a file with this path exists in the filesystem, it will be sent to the client. Set to "" to skip this check.
- * @param code The HTTP status code
- * @param contentType The content type of the web page
- * @param content Content of the web page
- * @param len Length of the content
- * @param gzip Optional. Defaults to true. If false, the gzip header will not be added.
- * @param eTagSuffix Optional. Defaults to 0. A suffix that will be added to the ETag header. This can be used to invalidate the cache for a specific page.
- */
-void handleStaticContent(AsyncWebServerRequest *request, const String &path, int code, const String &contentType, const uint8_t *content, size_t len, bool gzip, uint16_t eTagSuffix) {
-  if (path != "" && handleFileRead(request, path)) return;
-  if (handleIfNoneMatchCacheHeader(request, code, eTagSuffix)) return;
-  AsyncWebServerResponse *response = request->beginResponse_P(code, contentType, content, len);
-  if (gzip) response->addHeader(FPSTR(s_content_enc), F("gzip"));
-  setStaticContentCacheHeaders(response, code, eTagSuffix);
-  request->send(response);
-}
-
-
-
-String msgProcessor(const String& var)
-{
-  if (var == "MSG") {
-    String messageBody = messageHead;
-    messageBody += F("</h2>");
-    messageBody += messageSub;
-    uint32_t optt = optionType;
-
-    if (optt < 60) //redirect to settings after optionType seconds
-    {
-      messageBody += F("<script>setTimeout(RS,");
-      messageBody +=String(optt*1000);
-      messageBody += F(")</script>");
-    } else if (optt < 120) //redirect back after optionType-60 seconds, unused
-    {
-      //messageBody += "<script>setTimeout(B," + String((optt-60)*1000) + ")</script>";
-    } else if (optt < 180) //reload parent after optionType-120 seconds
-    {
-      messageBody += F("<script>setTimeout(RP,");
-      messageBody += String((optt-120)*1000);
-      messageBody += F(")</script>");
-    } else if (optt == 253)
-    {
-      messageBody += F("<br><br><form action=/settings><button class=\"bt\" type=submit>Back</button></form>"); //button to settings
-    } else if (optt == 254)
-    {
-      messageBody += F("<br><br><button type=\"button\" class=\"bt\" onclick=\"B()\">Back</button>");
-    }
-    return messageBody;
-  }
-  return String();
-}
-
 
 void serveMessage(AsyncWebServerRequest* request, uint16_t code, const String& headl, const String& subl, byte optionT)
 {
@@ -487,29 +504,6 @@ void serveJsonError(AsyncWebServerRequest* request, uint16_t code, uint16_t erro
     request->send(response);
 }
 
-#ifdef WLED_ENABLE_DMX
-String dmxProcessor(const String& var)
-{
-  String mapJS;
-  if (var == F("DMXVARS")) {
-    mapJS += F("\nCN=");
-    mapJS += String(DMXChannels);
-    mapJS += F(";\nCS=");
-    mapJS += String(DMXStart);
-    mapJS += F(";\nCG=");
-    mapJS += String(DMXGap);
-    mapJS += F(";\nLC=");
-    mapJS += String(strip.getLengthTotal());
-    mapJS += F(";\nvar CH=[");
-    for (int i=0; i<15; i++) {
-      mapJS += String(DMXFixtureMap[i]) + ',';
-    }
-    mapJS += F("0];");
-  }
-  return mapJS;
-}
-#endif
-
 
 void serveSettingsJS(AsyncWebServerRequest* request)
 {