From fd6ce570037c60cf83b5696d1e624e5d538e0ad9 Mon Sep 17 00:00:00 2001 From: Woody Date: Thu, 21 Dec 2023 02:21:43 +0100 Subject: [PATCH 01/15] Improve ETag Caching This also adds support for ETag caching for the settings pages Also fixed some issues with the previous caching not being RFC 7232 compliant. --- tools/cdata.js | 2 +- wled00/wled_server.cpp | 196 +++++++++++++++++++---------------------- 2 files changed, 94 insertions(+), 104 deletions(-) diff --git a/tools/cdata.js b/tools/cdata.js index bb9854608..fabe7a5bd 100644 --- a/tools/cdata.js +++ b/tools/cdata.js @@ -182,7 +182,7 @@ ${result} const result = hexdump(buf); const chunk = ` // Autogenerated from ${srcDir}/${s.file}, do not edit!! -const uint16_t ${s.name}_length = ${result.length}; +const uint16_t ${s.name}_length = ${buf.length}; const uint8_t ${s.name}[] PROGMEM = { ${result} }; diff --git a/wled00/wled_server.cpp b/wled00/wled_server.cpp index dcb21990e..00c30c4e5 100644 --- a/wled00/wled_server.cpp +++ b/wled00/wled_server.cpp @@ -15,8 +15,9 @@ * Integrated HTTP web server page declarations */ -bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest* request); -void setStaticContentCacheHeaders(AsyncWebServerResponse *response); +bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest* request, int code, const String &eTagSuffix = ""); +void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, const String &eTagSuffix = ""); +void handleStaticContent(AsyncWebServerRequest *request, const String &path, int code, const String &contentType, const uint8_t *content, size_t len, bool gzip = true, const String &eTagSuffix = ""); // define flash strings once (saves flash memory) static const char s_redirecting[] PROGMEM = "Redirecting..."; @@ -113,22 +114,14 @@ void initServer() DefaultHeaders::Instance().addHeader(F("Access-Control-Allow-Headers"), "*"); #ifdef WLED_ENABLE_WEBSOCKETS - #ifndef WLED_DISABLE_2D - server.on("/liveview2D", HTTP_GET, [](AsyncWebServerRequest *request){ - if (handleIfNoneMatchCacheHeader(request)) return; - AsyncWebServerResponse *response = request->beginResponse_P(200, "text/html", PAGE_liveviewws2D, PAGE_liveviewws2D_length); - response->addHeader(FPSTR(s_content_enc),"gzip"); - setStaticContentCacheHeaders(response); - request->send(response); + #ifndef WLED_DISABLE_2D + server.on("/liveview2D", HTTP_GET, [](AsyncWebServerRequest *request) { + handleStaticContent(request, "", 200, "text/html", PAGE_liveviewws2D, PAGE_liveviewws2D_length); }); #endif #endif - server.on("/liveview", HTTP_GET, [](AsyncWebServerRequest *request){ - if (handleIfNoneMatchCacheHeader(request)) return; - AsyncWebServerResponse *response = request->beginResponse_P(200, "text/html", PAGE_liveview, PAGE_liveview_length); - response->addHeader(FPSTR(s_content_enc),"gzip"); - setStaticContentCacheHeaders(response); - request->send(response); + server.on("/liveview", HTTP_GET, [](AsyncWebServerRequest *request) { + handleStaticContent(request, "", 200, "text/html", PAGE_liveview, PAGE_liveview_length); }); //settings page @@ -138,17 +131,12 @@ void initServer() // "/settings/settings.js&p=x" request also handled by serveSettings() - server.on("/style.css", HTTP_GET, [](AsyncWebServerRequest *request){ - if (handleIfNoneMatchCacheHeader(request)) return; - AsyncWebServerResponse *response = request->beginResponse_P(200, "text/css", PAGE_settingsCss, PAGE_settingsCss_length); - response->addHeader(FPSTR(s_content_enc),"gzip"); - setStaticContentCacheHeaders(response); - request->send(response); + server.on("/style.css", HTTP_GET, [](AsyncWebServerRequest *request) { + handleStaticContent(request, "/style.css", 200, "text/css", PAGE_settingsCss, PAGE_settingsCss_length); }); server.on("/favicon.ico", HTTP_GET, [](AsyncWebServerRequest *request) { - if (handleFileRead(request, "/favicon.ico")) return; - request->send_P(200, "image/x-icon", favicon, 156); + handleStaticContent(request, "/favicon.ico", 200, "image/x-icon", favicon, favicon_length, false); }); server.on("/skin.css", HTTP_GET, [](AsyncWebServerRequest *request) { @@ -236,12 +224,8 @@ void initServer() }); #ifdef WLED_ENABLE_USERMOD_PAGE - server.on("/u", HTTP_GET, [](AsyncWebServerRequest *request){ - if (handleIfNoneMatchCacheHeader(request)) return; - AsyncWebServerResponse *response = request->beginResponse_P(200, "text/html", PAGE_usermod, PAGE_usermod_length); - response->addHeader(FPSTR(s_content_enc),"gzip"); - setStaticContentCacheHeaders(response); - request->send(response); + server.on("/u", HTTP_GET, [](AsyncWebServerRequest *request) { + handleStaticContent(request, "", 200, "text/html", PAGE_usermod, PAGE_usermod_length); }); #endif @@ -325,44 +309,29 @@ void initServer() }); #endif - server.on("/", HTTP_GET, [](AsyncWebServerRequest *request){ + server.on("/", HTTP_GET, [](AsyncWebServerRequest *request) { if (captivePortal(request)) return; - if (!showWelcomePage || request->hasArg(F("sliders"))){ - serveIndex(request); + if (!showWelcomePage || request->hasArg(F("sliders"))) { + handleStaticContent(request, "/index.htm", 200, "text/html", PAGE_index, PAGE_index_L); } else { serveSettings(request); } }); - #ifdef WLED_ENABLE_PIXART - server.on("/pixart.htm", HTTP_GET, [](AsyncWebServerRequest *request){ - if (handleFileRead(request, "/pixart.htm")) return; - if (handleIfNoneMatchCacheHeader(request)) return; - AsyncWebServerResponse *response = request->beginResponse_P(200, "text/html", PAGE_pixart, PAGE_pixart_L); - response->addHeader(FPSTR(s_content_enc),"gzip"); - setStaticContentCacheHeaders(response); - request->send(response); +#ifdef WLED_ENABLE_PIXART + server.on("/pixart.htm", HTTP_GET, [](AsyncWebServerRequest *request) { + handleStaticContent(request, "/pixart.htm", 200, "text/html", PAGE_pixart, PAGE_pixart_L); }); #endif - #ifndef WLED_DISABLE_PXMAGIC - server.on("/pxmagic.htm", HTTP_GET, [](AsyncWebServerRequest *request){ - if (handleFileRead(request, "/pxmagic.htm")) return; - if (handleIfNoneMatchCacheHeader(request)) return; - AsyncWebServerResponse *response = request->beginResponse_P(200, "text/html", PAGE_pxmagic, PAGE_pxmagic_L); - response->addHeader(FPSTR(s_content_enc),"gzip"); - setStaticContentCacheHeaders(response); - request->send(response); +#ifndef WLED_DISABLE_PXMAGIC + server.on("/pxmagic.htm", HTTP_GET, [](AsyncWebServerRequest *request) { + handleStaticContent(request, "/pxmagic.htm", 200, "text/html", PAGE_pxmagic, PAGE_pxmagic_L); }); - #endif +#endif - server.on("/cpal.htm", HTTP_GET, [](AsyncWebServerRequest *request){ - if (handleFileRead(request, "/cpal.htm")) return; - if (handleIfNoneMatchCacheHeader(request)) return; - AsyncWebServerResponse *response = request->beginResponse_P(200, "text/html", PAGE_cpal, PAGE_cpal_L); - response->addHeader(FPSTR(s_content_enc),"gzip"); - setStaticContentCacheHeaders(response); - request->send(response); + server.on("/cpal.htm", HTTP_GET, [](AsyncWebServerRequest *request) { + handleStaticContent(request, "/cpal.htm", 200, "text/html", PAGE_cpal, PAGE_cpal_L); }); #ifdef WLED_ENABLE_WEBSOCKETS @@ -388,26 +357,36 @@ void initServer() #ifndef WLED_DISABLE_ALEXA if(espalexa.handleAlexaApiCall(request)) return; #endif - if(handleFileRead(request, request->url())) return; - AsyncWebServerResponse *response = request->beginResponse_P(404, "text/html", PAGE_404, PAGE_404_length); - response->addHeader(FPSTR(s_content_enc),"gzip"); - setStaticContentCacheHeaders(response); - request->send(response); + handleStaticContent(request, request->url(), 404, "text/html", PAGE_404, PAGE_404_length); }); } -bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest *request) { +void generateEtag(char *etag, const String &eTagSuffix) { + char eTagSuffixBuf[eTagSuffix.length() + 1]; + eTagSuffix.toCharArray(eTagSuffixBuf, eTagSuffix.length() + 1); + sprintf_P(etag, PSTR("%7d-%02x-%s"), VERSION, cacheInvalidate, eTagSuffixBuf); +} + +bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest *request, int code, const String &eTagSuffix) { + // Only send 304 (Not Modified) if response code is 200 (OK) + if (code != 200) return false; + AsyncWebHeader *header = request->getHeader("If-None-Match"); - char etag[11]; - sprintf_P(etag, PSTR("%7d-%02x"), VERSION, cacheInvalidate); + char etag[12 + eTagSuffix.length()]; + generateEtag(etag, eTagSuffix); if (header && header->value() == etag) { - request->send(304); + AsyncWebServerResponse *response = request->beginResponse(304); + setStaticContentCacheHeaders(response, code, eTagSuffix); + request->send(response); return true; } return false; } -void setStaticContentCacheHeaders(AsyncWebServerResponse *response) { +void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, const String &eTagSuffix) { + // Only send ETag for 200 (OK) responses + if (code != 200) return; + // https://medium.com/@codebyamir/a-web-developers-guide-to-browser-caching-cc41f3b73e7c #ifndef WLED_DEBUG // this header name is misleading, "no-cache" will not disable cache, @@ -416,25 +395,37 @@ void setStaticContentCacheHeaders(AsyncWebServerResponse *response) { #else response->addHeader(F("Cache-Control"), "no-store,max-age=0"); // prevent caching if debug build #endif - char etag[11]; - sprintf_P(etag, PSTR("%7d-%02x"), VERSION, cacheInvalidate); + char etag[12 + eTagSuffix.length()]; + generateEtag(etag, eTagSuffix); response->addHeader(F("ETag"), etag); } -void serveIndex(AsyncWebServerRequest* request) -{ - if (handleFileRead(request, "/index.htm")) return; - - if (handleIfNoneMatchCacheHeader(request)) return; - - AsyncWebServerResponse *response = request->beginResponse_P(200, "text/html", PAGE_index, PAGE_index_L); - - response->addHeader(FPSTR(s_content_enc),"gzip"); - setStaticContentCacheHeaders(response); +/** + * Handels the request for a static file. + * If the file was found in the filesystem, it will be sent to the client. + * Otherwise it will be checked if the browser cached the file and if so, a 304 response will be sent. + * If the file was not found in the filesystem and not in the browser cache, the request will be handled as a 200 response with the content of the page. + * + * @param request The request object + * @param path If a file with this path exists in the filesystem, it will be sent to the client. Set to "" to skip this check. + * @param code The HTTP status code + * @param contentType The content type of the web page + * @param content Content of the web page + * @param len Length of the content + * @param gzip Optional. Defaults to true. If false, the gzip header will not be added. + * @param eTagSuffix Optional. Defaults to "". A suffix that will be added to the ETag header. This can be used to invalidate the cache for a specific page. + */ +void handleStaticContent(AsyncWebServerRequest *request, const String &path, int code, const String &contentType, const uint8_t *content, size_t len, bool gzip, const String &eTagSuffix) { + if (path != "" && handleFileRead(request, path)) return; + if (handleIfNoneMatchCacheHeader(request, code, eTagSuffix)) return; + AsyncWebServerResponse *response = request->beginResponse_P(code, contentType, content, len); + if (gzip) response->addHeader(FPSTR(s_content_enc), "gzip"); + setStaticContentCacheHeaders(response, code, eTagSuffix); request->send(response); } + String msgProcessor(const String& var) { if (var == "MSG") { @@ -536,13 +527,11 @@ void serveSettingsJS(AsyncWebServerRequest* request) } -void serveSettings(AsyncWebServerRequest* request, bool post) -{ +void serveSettings(AsyncWebServerRequest* request, bool post) { byte subPage = 0, originalSubPage = 0; const String& url = request->url(); - if (url.indexOf("sett") >= 0) - { + if (url.indexOf("sett") >= 0) { if (url.indexOf(".js") > 0) subPage = SUBPAGE_JS; else if (url.indexOf(".css") > 0) subPage = SUBPAGE_CSS; else if (url.indexOf("wifi") > 0) subPage = SUBPAGE_WIFI; @@ -603,22 +592,25 @@ void serveSettings(AsyncWebServerRequest* request, bool post) } } - AsyncWebServerResponse *response; - switch (subPage) - { - case SUBPAGE_WIFI : response = request->beginResponse_P(200, "text/html", PAGE_settings_wifi, PAGE_settings_wifi_length); break; - case SUBPAGE_LEDS : response = request->beginResponse_P(200, "text/html", PAGE_settings_leds, PAGE_settings_leds_length); break; - case SUBPAGE_UI : response = request->beginResponse_P(200, "text/html", PAGE_settings_ui, PAGE_settings_ui_length); break; - case SUBPAGE_SYNC : response = request->beginResponse_P(200, "text/html", PAGE_settings_sync, PAGE_settings_sync_length); break; - case SUBPAGE_TIME : response = request->beginResponse_P(200, "text/html", PAGE_settings_time, PAGE_settings_time_length); break; - case SUBPAGE_SEC : response = request->beginResponse_P(200, "text/html", PAGE_settings_sec, PAGE_settings_sec_length); break; + int code = 200; + String contentType = "text/html"; + const uint8_t* content; + size_t len; + + switch (subPage) { + case SUBPAGE_WIFI : content = PAGE_settings_wifi; len = PAGE_settings_wifi_length; break; + case SUBPAGE_LEDS : content = PAGE_settings_leds; len = PAGE_settings_leds_length; break; + case SUBPAGE_UI : content = PAGE_settings_ui; len = PAGE_settings_ui_length; break; + case SUBPAGE_SYNC : content = PAGE_settings_sync; len = PAGE_settings_sync_length; break; + case SUBPAGE_TIME : content = PAGE_settings_time; len = PAGE_settings_time_length; break; + case SUBPAGE_SEC : content = PAGE_settings_sec; len = PAGE_settings_sec_length; break; #ifdef WLED_ENABLE_DMX - case SUBPAGE_DMX : response = request->beginResponse_P(200, "text/html", PAGE_settings_dmx, PAGE_settings_dmx_length); break; + case SUBPAGE_DMX : content = PAGE_settings_dmx; len = PAGE_settings_dmx_length; break; #endif - case SUBPAGE_UM : response = request->beginResponse_P(200, "text/html", PAGE_settings_um, PAGE_settings_um_length); break; - case SUBPAGE_UPDATE : response = request->beginResponse_P(200, "text/html", PAGE_update, PAGE_update_length); break; + case SUBPAGE_UM : content = PAGE_settings_um; len = PAGE_settings_um_length; break; + case SUBPAGE_UPDATE : content = PAGE_update; len = PAGE_update_length; break; #ifndef WLED_DISABLE_2D - case SUBPAGE_2D : response = request->beginResponse_P(200, "text/html", PAGE_settings_2D, PAGE_settings_2D_length); break; + case SUBPAGE_2D : content = PAGE_settings_2D; len = PAGE_settings_2D_length; break; #endif case SUBPAGE_LOCK : { correctPIN = !strlen(settingsPIN); // lock if a pin is set @@ -626,13 +618,11 @@ void serveSettings(AsyncWebServerRequest* request, bool post) serveMessage(request, 200, strlen(settingsPIN) > 0 ? PSTR("Settings locked") : PSTR("No PIN set"), FPSTR(s_redirecting), 1); return; } - case SUBPAGE_PINREQ : response = request->beginResponse_P(401, "text/html", PAGE_settings_pin, PAGE_settings_pin_length); break; - case SUBPAGE_CSS : response = request->beginResponse_P(200, "text/css", PAGE_settingsCss, PAGE_settingsCss_length); break; - case SUBPAGE_JS : serveSettingsJS(request); return; - case SUBPAGE_WELCOME : response = request->beginResponse_P(200, "text/html", PAGE_welcome, PAGE_welcome_length); break; - default: response = request->beginResponse_P(200, "text/html", PAGE_settings, PAGE_settings_length); break; + case SUBPAGE_PINREQ : content = PAGE_settings_pin; len = PAGE_settings_pin_length; code = 401; break; + case SUBPAGE_CSS : content = PAGE_settingsCss; len = PAGE_settingsCss_length; contentType = "text/css"; break; + case SUBPAGE_JS : serveSettingsJS(request); return; + case SUBPAGE_WELCOME : content = PAGE_welcome; len = PAGE_welcome_length; break; + default: content = PAGE_settings; len = PAGE_settings_length; break; } - response->addHeader(FPSTR(s_content_enc),"gzip"); - setStaticContentCacheHeaders(response); - request->send(response); + handleStaticContent(request, "", code, contentType, content, len); } From 59a725c52c4660dd1c80d1151dafb542b6680837 Mon Sep 17 00:00:00 2001 From: Woody Date: Wed, 3 Jan 2024 23:39:45 +0100 Subject: [PATCH 02/15] Use uint16_t eTagSuffix instead of String --- wled00/wled_server.cpp | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/wled00/wled_server.cpp b/wled00/wled_server.cpp index 00c30c4e5..7eb6a8b24 100644 --- a/wled00/wled_server.cpp +++ b/wled00/wled_server.cpp @@ -15,9 +15,9 @@ * Integrated HTTP web server page declarations */ -bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest* request, int code, const String &eTagSuffix = ""); -void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, const String &eTagSuffix = ""); -void handleStaticContent(AsyncWebServerRequest *request, const String &path, int code, const String &contentType, const uint8_t *content, size_t len, bool gzip = true, const String &eTagSuffix = ""); +bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest* request, int code, uint16_t eTagSuffix = 0); +void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, uint16_t eTagSuffix = 0); +void handleStaticContent(AsyncWebServerRequest *request, const String &path, int code, const String &contentType, const uint8_t *content, size_t len, bool gzip = true, uint16_t eTagSuffix = 0); // define flash strings once (saves flash memory) static const char s_redirecting[] PROGMEM = "Redirecting..."; @@ -361,18 +361,16 @@ void initServer() }); } -void generateEtag(char *etag, const String &eTagSuffix) { - char eTagSuffixBuf[eTagSuffix.length() + 1]; - eTagSuffix.toCharArray(eTagSuffixBuf, eTagSuffix.length() + 1); - sprintf_P(etag, PSTR("%7d-%02x-%s"), VERSION, cacheInvalidate, eTagSuffixBuf); +void generateEtag(char *etag, uint16_t eTagSuffix) { + sprintf_P(etag, PSTR("%7d-%02x-%04x"), VERSION, cacheInvalidate, eTagSuffix); } -bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest *request, int code, const String &eTagSuffix) { +bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest *request, int code, uint16_t eTagSuffix) { // Only send 304 (Not Modified) if response code is 200 (OK) if (code != 200) return false; AsyncWebHeader *header = request->getHeader("If-None-Match"); - char etag[12 + eTagSuffix.length()]; + char etag[14]; generateEtag(etag, eTagSuffix); if (header && header->value() == etag) { AsyncWebServerResponse *response = request->beginResponse(304); @@ -383,7 +381,7 @@ bool handleIfNoneMatchCacheHeader(AsyncWebServerRequest *request, int code, cons return false; } -void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, const String &eTagSuffix) { +void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, uint16_t eTagSuffix) { // Only send ETag for 200 (OK) responses if (code != 200) return; @@ -395,7 +393,7 @@ void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, co #else response->addHeader(F("Cache-Control"), "no-store,max-age=0"); // prevent caching if debug build #endif - char etag[12 + eTagSuffix.length()]; + char etag[14]; generateEtag(etag, eTagSuffix); response->addHeader(F("ETag"), etag); } @@ -413,9 +411,9 @@ void setStaticContentCacheHeaders(AsyncWebServerResponse *response, int code, co * @param content Content of the web page * @param len Length of the content * @param gzip Optional. Defaults to true. If false, the gzip header will not be added. - * @param eTagSuffix Optional. Defaults to "". A suffix that will be added to the ETag header. This can be used to invalidate the cache for a specific page. + * @param eTagSuffix Optional. Defaults to 0. A suffix that will be added to the ETag header. This can be used to invalidate the cache for a specific page. */ -void handleStaticContent(AsyncWebServerRequest *request, const String &path, int code, const String &contentType, const uint8_t *content, size_t len, bool gzip, const String &eTagSuffix) { +void handleStaticContent(AsyncWebServerRequest *request, const String &path, int code, const String &contentType, const uint8_t *content, size_t len, bool gzip, uint16_t eTagSuffix) { if (path != "" && handleFileRead(request, path)) return; if (handleIfNoneMatchCacheHeader(request, code, eTagSuffix)) return; AsyncWebServerResponse *response = request->beginResponse_P(code, contentType, content, len); From 6382d2b7302b1318d3180fe47b7e84bb1b02a9b2 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 6 Jan 2024 10:05:18 -0500 Subject: [PATCH 03/15] Add C++-style guard class for JSON Buffer lock Add JSONBufferGuard, an RAII lock guard class for JSONBufferLock analogous to std::lock_guard. This permits binding the guard to a scope, such as an object life cycle or function body, guaranteeing that the lock will be released when the scope exits. --- wled00/fcn_declare.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/wled00/fcn_declare.h b/wled00/fcn_declare.h index ec7527269..e256ceb5f 100644 --- a/wled00/fcn_declare.h +++ b/wled00/fcn_declare.h @@ -360,6 +360,22 @@ um_data_t* simulateSound(uint8_t simulationId); void enumerateLedmaps(); uint8_t get_random_wheel_index(uint8_t pos); +// RAII guard class for the JSON Buffer lock +// Modeled after std::lock_guard +class JSONBufferGuard { + bool holding_lock; + public: + inline JSONBufferGuard(uint8_t module=255) : holding_lock(requestJSONBufferLock(module)) {}; + inline ~JSONBufferGuard() { if (holding_lock) releaseJSONBufferLock(); }; + inline JSONBufferGuard(const JSONBufferGuard&) = delete; // Noncopyable + inline JSONBufferGuard& operator=(const JSONBufferGuard&) = delete; + inline JSONBufferGuard(JSONBufferGuard&& r) : holding_lock(r.holding_lock) { r.holding_lock = false; }; // but movable + inline JSONBufferGuard& operator=(JSONBufferGuard&& r) { holding_lock |= r.holding_lock; r.holding_lock = false; return *this; }; + inline bool owns_lock() const { return holding_lock; } + explicit inline operator bool() const { return owns_lock(); }; + inline void release() { if (holding_lock) releaseJSONBufferLock(); holding_lock = false; } +}; + #ifdef WLED_ADD_EEPROM_SUPPORT //wled_eeprom.cpp void applyMacro(byte index); From 9d3c0f4ff082d7f596c142e32e42eda607d6c1ae Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 6 Jan 2024 10:09:07 -0500 Subject: [PATCH 04/15] serveJson: Ensure semaphore scope lasts until reply is done Extend the JSON response class to hold the global JSON buffer lock until the transaction is completed. Fixes #3641 --- wled00/json.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/wled00/json.cpp b/wled00/json.cpp index ef2bdf346..d9bbb74ce 100644 --- a/wled00/json.cpp +++ b/wled00/json.cpp @@ -1009,6 +1009,17 @@ void serializeModeNames(JsonArray arr) } } + +// Global buffer locking response helper class +class GlobalBufferAsyncJsonResponse: public JSONBufferGuard, public AsyncJsonResponse { + public: + inline GlobalBufferAsyncJsonResponse(bool isArray) : JSONBufferGuard(17), AsyncJsonResponse(pDoc, isArray) {}; + virtual ~GlobalBufferAsyncJsonResponse() {}; + + // Other members are inherited +}; + + static volatile bool servingClient = false; void serveJson(AsyncWebServerRequest* request) { @@ -1050,12 +1061,12 @@ void serveJson(AsyncWebServerRequest* request) return; } - if (!requestJSONBufferLock(17)) { + GlobalBufferAsyncJsonResponse *response = new GlobalBufferAsyncJsonResponse(subJson==JSON_PATH_FXDATA || subJson==JSON_PATH_EFFECTS); // will clear and convert JsonDocument into JsonArray if necessary + if (!response->owns_lock()) { serveJsonError(request, 503, ERR_NOBUF); servingClient = false; return; } - AsyncJsonResponse *response = new AsyncJsonResponse(pDoc, subJson==JSON_PATH_FXDATA || subJson==JSON_PATH_EFFECTS); // will clear and convert JsonDocument into JsonArray if necessary JsonVariant lDoc = response->getRoot(); @@ -1098,7 +1109,6 @@ void serveJson(AsyncWebServerRequest* request) DEBUG_PRINT(F("JSON content length: ")); DEBUG_PRINTLN(len); request->send(response); - releaseJSONBufferLock(); servingClient = false; } From 77116172e466a0c33f9f460e0ff56a838e5c0ec4 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sat, 6 Jan 2024 10:24:05 -0500 Subject: [PATCH 05/15] serveJson: Fix possible memory leak Ensure we delete the response if it's not locked --- wled00/json.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/wled00/json.cpp b/wled00/json.cpp index d9bbb74ce..ccec74ed8 100644 --- a/wled00/json.cpp +++ b/wled00/json.cpp @@ -1065,6 +1065,7 @@ void serveJson(AsyncWebServerRequest* request) if (!response->owns_lock()) { serveJsonError(request, 503, ERR_NOBUF); servingClient = false; + delete response; return; } From 21f8d7a967017a4d0e4c44bdfa25e7f775898d84 Mon Sep 17 00:00:00 2001 From: Blaz Kristan Date: Sat, 6 Jan 2024 17:04:56 +0100 Subject: [PATCH 06/15] Bump --- wled00/FX.h | 4 ++-- wled00/wled.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/wled00/FX.h b/wled00/FX.h index c4169c209..ee6b397b4 100644 --- a/wled00/FX.h +++ b/wled00/FX.h @@ -282,7 +282,7 @@ #define FX_MODE_RIPPLEPEAK 148 #define FX_MODE_2DFIRENOISE 149 #define FX_MODE_2DSQUAREDSWIRL 150 -#define FX_MODE_2DFIRE2012 151 +// #define FX_MODE_2DFIRE2012 151 #define FX_MODE_2DDNA 152 #define FX_MODE_2DMATRIX 153 #define FX_MODE_2DMETABALLS 154 @@ -292,7 +292,7 @@ #define FX_MODE_GRAVFREQ 158 #define FX_MODE_DJLIGHT 159 #define FX_MODE_2DFUNKYPLANK 160 -#define FX_MODE_2DCENTERBARS 161 +//#define FX_MODE_2DCENTERBARS 161 #define FX_MODE_2DPULSER 162 #define FX_MODE_BLURZ 163 #define FX_MODE_2DDRIFT 164 diff --git a/wled00/wled.h b/wled00/wled.h index 76f492079..ac6fe72b3 100644 --- a/wled00/wled.h +++ b/wled00/wled.h @@ -8,7 +8,7 @@ */ // version code in format yymmddb (b = daily build) -#define VERSION 2401010 +#define VERSION 2401060 //uncomment this if you have a "my_config.h" file you'd like to use //#define WLED_USE_MY_CONFIG From 43f5e4360deb91ed2e1e2f43a29098fcaa4a1c5a Mon Sep 17 00:00:00 2001 From: Blaz Kristan Date: Sat, 6 Jan 2024 20:28:05 +0100 Subject: [PATCH 07/15] Changelog update Remove obsolete semaphore --- CHANGELOG.md | 9 +++++++-- wled00/json.cpp | 13 ------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae4121484..17c37c530 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,12 @@ ## WLED changelog -#### Build 2309120 till build 2312290 +#### Build 2309120 till build 2201060 - WLED version 0.15.0-a0 +- Global JSON buffer guarding (#3648 by @willmmiles, resolves #3641, #3312, #3367, #3637, #3646, #3447) +- Effect: Fireworks 1D (fix for matrix trailing strip) +- BREAKING: Reduced number of segments (12) on ESP8266 due to less available RAM +- Increased available effect data buffer (increases more if board has PSRAM) +- Custom palette editor mobile UI enhancement (by @imeszaros) - Per port Auto Brightness Limiter (ABL) - Use PSRAM for JSON buffer (double size, larger ledmaps, up to 2k) - Reduced heap fragmentation by allocating ledmap array only once and not deallocating effect buffer @@ -26,7 +31,7 @@ - Better reverse proxy support (nested paths) - Implement global JSON API boolean toggle (i.e. instead of "var":true or "var":false -> "var":"t"). - Sort presets by ID -- Fix for #3496, #2922, #3593, #3514, #3522, #3578 (partial), #3606 (@WoodyLetsCode) +- Fix for #3641, #3312, #3367, #3637, #3646, #3447, #3632, #3496, #2922, #3593, #3514, #3522, #3578 (partial), #3606 (@WoodyLetsCode) - Improved random bg image and added random bg image options (@WoodyLetsCode, #3481) - Audio palettes (Audioreactive usermod, credit @netmindz) - Better UI tooltips (@ajotnac, #3464) diff --git a/wled00/json.cpp b/wled00/json.cpp index ccec74ed8..8a3bfbe63 100644 --- a/wled00/json.cpp +++ b/wled00/json.cpp @@ -1020,15 +1020,8 @@ class GlobalBufferAsyncJsonResponse: public JSONBufferGuard, public AsyncJsonRes }; -static volatile bool servingClient = false; void serveJson(AsyncWebServerRequest* request) { - if (servingClient) { - serveJsonError(request, 503, ERR_CONCURRENCY); - return; - } - servingClient = true; - byte subJson = 0; const String& url = request->url(); if (url.indexOf("state") > 0) subJson = JSON_PATH_STATE; @@ -1042,29 +1035,24 @@ void serveJson(AsyncWebServerRequest* request) #ifdef WLED_ENABLE_JSONLIVE else if (url.indexOf("live") > 0) { serveLiveLeds(request); - servingClient = false; return; } #endif else if (url.indexOf("pal") > 0) { request->send_P(200, "application/json", JSON_palette_names); - servingClient = false; return; } else if (url.indexOf("cfg") > 0 && handleFileRead(request, "/cfg.json")) { - servingClient = false; return; } else if (url.length() > 6) { //not just /json serveJsonError(request, 501, ERR_NOT_IMPL); - servingClient = false; return; } GlobalBufferAsyncJsonResponse *response = new GlobalBufferAsyncJsonResponse(subJson==JSON_PATH_FXDATA || subJson==JSON_PATH_EFFECTS); // will clear and convert JsonDocument into JsonArray if necessary if (!response->owns_lock()) { serveJsonError(request, 503, ERR_NOBUF); - servingClient = false; delete response; return; } @@ -1110,7 +1098,6 @@ void serveJson(AsyncWebServerRequest* request) DEBUG_PRINT(F("JSON content length: ")); DEBUG_PRINTLN(len); request->send(response); - servingClient = false; } #ifdef WLED_ENABLE_JSONLIVE From d2dbaf52a11c2b0d734c9f86fa279c0206bc6728 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Wed, 3 Jan 2024 13:08:07 -0500 Subject: [PATCH 08/15] usermod_mpu6050: Fix debug prints The MPU6050 library happens to choose the same defines as WLED, which collide and result in nothing printing. Un- and re-define the macros to work around this. --- usermods/mpu6050_imu/usermod_mpu6050_imu.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/usermods/mpu6050_imu/usermod_mpu6050_imu.h b/usermods/mpu6050_imu/usermod_mpu6050_imu.h index 748ddf1a6..0f8d5a319 100644 --- a/usermods/mpu6050_imu/usermod_mpu6050_imu.h +++ b/usermods/mpu6050_imu/usermod_mpu6050_imu.h @@ -33,6 +33,9 @@ #include "I2Cdev.h" +#undef DEBUG_PRINT +#undef DEBUG_PRINTLN +#undef DEBUG_PRINTF #include "MPU6050_6Axis_MotionApps20.h" //#include "MPU6050.h" // not necessary if using MotionApps include file @@ -42,6 +45,23 @@ #include "Wire.h" #endif +// Restore debug macros +// MPU6050 unfortunately uses the same macro names as WLED :( +#undef DEBUG_PRINT +#undef DEBUG_PRINTLN +#undef DEBUG_PRINTF +#ifdef WLED_DEBUG + #define DEBUG_PRINT(x) DEBUGOUT.print(x) + #define DEBUG_PRINTLN(x) DEBUGOUT.println(x) + #define DEBUG_PRINTF(x...) DEBUGOUT.printf(x) +#else + #define DEBUG_PRINT(x) + #define DEBUG_PRINTLN(x) + #define DEBUG_PRINTF(x...) +#endif + + + // ================================================================ // === INTERRUPT DETECTION ROUTINE === // ================================================================ From 9a006dc84a9d3afc4a6cffb7d03ba5f8943dbfbd Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 7 Jan 2024 14:32:22 -0500 Subject: [PATCH 09/15] usermod_mpu6050: Add options and outputs Upgrade the MPU6050 usermod to provide configuration settings for interrupt pin selection and calibration, a polled mode, and data output for consumption by other usermods. --- usermods/mpu6050_imu/readme.md | 7 +- usermods/mpu6050_imu/usermod_mpu6050_imu.h | 284 +++++++++++++++------ wled00/pin_manager.h | 2 +- 3 files changed, 213 insertions(+), 80 deletions(-) diff --git a/usermods/mpu6050_imu/readme.md b/usermods/mpu6050_imu/readme.md index 412004151..b804ba602 100644 --- a/usermods/mpu6050_imu/readme.md +++ b/usermods/mpu6050_imu/readme.md @@ -20,14 +20,11 @@ react to the globes orientation. See the blog post on building it = 0) { + irqBound = pinManager.allocatePin(config.interruptPin, false, PinOwner::UM_IMU); + if (!irqBound) { DEBUG_PRINTLN(F("MPU6050: IRQ pin already in use.")); return; } + pinMode(config.interruptPin, INPUT); + }; + #if I2CDEV_IMPLEMENTATION == I2CDEV_ARDUINO_WIRE Wire.setClock(400000U); // 400kHz I2C clock. Comment this line if having compilation difficulties #elif I2CDEV_IMPLEMENTATION == I2CDEV_BUILTIN_FASTWIRE @@ -115,7 +175,6 @@ class MPU6050Driver : public Usermod { // initialize device DEBUG_PRINTLN(F("Initializing I2C devices...")); mpu.initialize(); - pinMode(INTERRUPT_PIN, INPUT); // verify connection DEBUG_PRINTLN(F("Testing device connections...")); @@ -125,11 +184,16 @@ class MPU6050Driver : public Usermod { DEBUG_PRINTLN(F("Initializing DMP...")); devStatus = mpu.dmpInitialize(); - // supply your own gyro offsets here, scaled for min sensitivity - mpu.setXGyroOffset(220); - mpu.setYGyroOffset(76); - mpu.setZGyroOffset(-85); - mpu.setZAccelOffset(1788); // 1688 factory default for my test chip + // set offsets (from config) + mpu.setXGyroOffset(config.gyro_offset[0]); + mpu.setYGyroOffset(config.gyro_offset[1]); + mpu.setZGyroOffset(config.gyro_offset[2]); + mpu.setXAccelOffset(config.accel_offset[0]); + mpu.setYAccelOffset(config.accel_offset[1]); + mpu.setZAccelOffset(config.accel_offset[2]); + + // set sample rate + mpu.setRate(16); // ~100Hz // make sure it worked (returns 0 if so) if (devStatus == 0) { @@ -137,17 +201,19 @@ class MPU6050Driver : public Usermod { DEBUG_PRINTLN(F("Enabling DMP...")); mpu.setDMPEnabled(true); - // enable Arduino interrupt detection - DEBUG_PRINTLN(F("Enabling interrupt detection (Arduino external interrupt 0)...")); - attachInterrupt(digitalPinToInterrupt(INTERRUPT_PIN), dmpDataReady, RISING); - mpuIntStatus = mpu.getIntStatus(); - - // set our DMP Ready flag so the main loop() function knows it's okay to use it - DEBUG_PRINTLN(F("DMP ready! Waiting for first interrupt...")); - dmpReady = true; + mpuInterrupt = true; + if (irqBound) { + // enable Arduino interrupt detection + DEBUG_PRINTLN(F("Enabling interrupt detection (Arduino external interrupt 0)...")); + attachInterrupt(digitalPinToInterrupt(config.interruptPin), dmpDataReady, RISING); + } // get expected DMP packet size for later comparison packetSize = mpu.dmpGetFIFOPacketSize(); + + // set our DMP Ready flag so the main loop() function knows it's okay to use it + DEBUG_PRINTLN(F("DMP ready!")); + dmpReady = true; } else { // ERROR! // 1 = initial memory load failed @@ -157,6 +223,9 @@ class MPU6050Driver : public Usermod { DEBUG_PRINT(devStatus); DEBUG_PRINTLN(")"); } + + fifoCount = 0; + sample_count = 0; } /* @@ -173,28 +242,31 @@ class MPU6050Driver : public Usermod { */ void loop() { // if programming failed, don't try to do anything - if (!enabled || !dmpReady || strip.isUpdating()) return; + if (!config.enabled || !dmpReady || strip.isUpdating()) return; // wait for MPU interrupt or extra packet(s) available + // mpuInterrupt is fixed on if interrupt pin is disabled if (!mpuInterrupt && fifoCount < packetSize) return; // reset interrupt flag and get INT_STATUS byte - mpuInterrupt = false; - mpuIntStatus = mpu.getIntStatus(); - - // get current FIFO count + auto mpuIntStatus = mpu.getIntStatus(); + // Update current FIFO count fifoCount = mpu.getFIFOCount(); // check for overflow (this should never happen unless our code is too inefficient) if ((mpuIntStatus & 0x10) || fifoCount == 1024) { // reset so we can continue cleanly mpu.resetFIFO(); - DEBUG_PRINTLN(F("FIFO overflow!")); + DEBUG_PRINTLN(F("MPU6050: FIFO overflow!")); - // otherwise, check for DMP data ready interrupt (this should happen frequently) - } else if (mpuIntStatus & 0x02) { - // wait for correct available data length, should be a VERY short wait - while (fifoCount < packetSize) fifoCount = mpu.getFIFOCount(); + // otherwise, check for data ready + } else if (fifoCount >= packetSize) { + // clear local interrupt pending status, if not polling + mpuInterrupt = !irqBound; + + // DEBUG_PRINT(F("MPU6050: Processing packet: ")); + // DEBUG_PRINT(fifoCount); + // DEBUG_PRINTLN(F(" bytes in FIFO")); // read a packet from FIFO mpu.getFIFOBytes(fifoBuffer, packetSize); @@ -203,7 +275,6 @@ class MPU6050Driver : public Usermod { // (this lets us immediately read more without waiting for an interrupt) fifoCount -= packetSize; - //NOTE: some of these can be removed to save memory, processing time // if the measurement isn't needed mpu.dmpGetQuaternion(&qat, fifoBuffer); @@ -214,87 +285,141 @@ class MPU6050Driver : public Usermod { mpu.dmpGetLinearAccel(&aaReal, &aa, &gravity); mpu.dmpGetLinearAccelInWorld(&aaWorld, &aaReal, &qat); mpu.dmpGetYawPitchRoll(ypr, &qat, &gravity); + ++sample_count; } } - - void addToJsonInfo(JsonObject& root) { - int reading = 20; - //this code adds "u":{"Light":[20," lux"]} to the info object JsonObject user = root["u"]; if (user.isNull()) user = root.createNestedObject("u"); - JsonObject imu_meas = user.createNestedObject("IMU"); - JsonArray quat_json = imu_meas.createNestedArray("Quat"); + // Unfortunately the web UI doesn't know how to print sub-objects: you just see '[object Object]' + // For now, we just put everything in the root userdata object. + //auto imu_meas = user.createNestedObject("IMU"); + auto& imu_meas = user; + + // If an element is an array, the UI expects two elements in the form [value, unit] + // Since our /value/ is an array, wrap it, eg. [[a, b, c]] + JsonArray quat_json = imu_meas.createNestedArray("Quat").createNestedArray(); quat_json.add(qat.w); quat_json.add(qat.x); quat_json.add(qat.y); quat_json.add(qat.z); - JsonArray euler_json = imu_meas.createNestedArray("Euler"); + JsonArray euler_json = imu_meas.createNestedArray("Euler").createNestedArray(); euler_json.add(euler[0]); euler_json.add(euler[1]); euler_json.add(euler[2]); - JsonArray accel_json = imu_meas.createNestedArray("Accel"); + JsonArray accel_json = imu_meas.createNestedArray("Accel").createNestedArray(); accel_json.add(aa.x); accel_json.add(aa.y); accel_json.add(aa.z); - JsonArray gyro_json = imu_meas.createNestedArray("Gyro"); + JsonArray gyro_json = imu_meas.createNestedArray("Gyro").createNestedArray(); gyro_json.add(gy.x); gyro_json.add(gy.y); gyro_json.add(gy.z); - JsonArray world_json = imu_meas.createNestedArray("WorldAccel"); + JsonArray world_json = imu_meas.createNestedArray("WorldAccel").createNestedArray(); world_json.add(aaWorld.x); world_json.add(aaWorld.y); world_json.add(aaWorld.z); - JsonArray real_json = imu_meas.createNestedArray("RealAccel"); + JsonArray real_json = imu_meas.createNestedArray("RealAccel").createNestedArray(); real_json.add(aaReal.x); real_json.add(aaReal.y); real_json.add(aaReal.z); - JsonArray grav_json = imu_meas.createNestedArray("Gravity"); + JsonArray grav_json = imu_meas.createNestedArray("Gravity").createNestedArray(); grav_json.add(gravity.x); grav_json.add(gravity.y); grav_json.add(gravity.z); - JsonArray orient_json = imu_meas.createNestedArray("Orientation"); + JsonArray orient_json = imu_meas.createNestedArray("Orientation").createNestedArray(); orient_json.add(ypr[0]); orient_json.add(ypr[1]); orient_json.add(ypr[2]); } - /* - * addToJsonState() can be used to add custom entries to the /json/state part of the JSON API (state object). - * Values in the state object may be modified by connected clients - */ - //void addToJsonState(JsonObject& root) - //{ - //root["user0"] = userVar0; - //} - - - /* - * readFromJsonState() can be used to receive data clients send to the /json/state part of the JSON API (state object). - * Values in the state object may be modified by connected clients - */ - //void readFromJsonState(JsonObject& root) - //{ - //if (root["bri"] == 255) DEBUG_PRINTLN(F("Don't burn down your garage!")); - //} - - /* * addToConfig() can be used to add custom persistent settings to the cfg.json file in the "um" (usermod) object. * It will be called by WLED when settings are actually saved (for example, LED settings are saved) * I highly recommend checking out the basics of ArduinoJson serialization and deserialization in order to use custom settings! */ -// void addToConfig(JsonObject& root) -// { -// JsonObject top = root.createNestedObject("MPU6050_IMU"); -// JsonArray pins = top.createNestedArray("pin"); -// pins.add(HW_PIN_SCL); -// pins.add(HW_PIN_SDA); -// } + void addToConfig(JsonObject& root) + { + JsonObject top = root.createNestedObject(FPSTR(_name)); + + //save these vars persistently whenever settings are saved + top[FPSTR(_enabled)] = config.enabled; + top[FPSTR(_interrupt_pin)] = config.interruptPin; + top[FPSTR(_x_acc_bias)] = config.accel_offset[0]; + top[FPSTR(_y_acc_bias)] = config.accel_offset[1]; + top[FPSTR(_z_acc_bias)] = config.accel_offset[2]; + top[FPSTR(_x_gyro_bias)] = config.gyro_offset[0]; + top[FPSTR(_y_gyro_bias)] = config.gyro_offset[1]; + top[FPSTR(_z_gyro_bias)] = config.gyro_offset[2]; + } + + /* + * readFromConfig() can be used to read back the custom settings you added with addToConfig(). + * This is called by WLED when settings are loaded (currently this only happens immediately after boot, or after saving on the Usermod Settings page) + * + * readFromConfig() is called BEFORE setup(). This means you can use your persistent values in setup() (e.g. pin assignments, buffer sizes), + * but also that if you want to write persistent values to a dynamic buffer, you'd need to allocate it here instead of in setup. + * If you don't know what that is, don't fret. It most likely doesn't affect your use case :) + * + * Return true in case the config values returned from Usermod Settings were complete, or false if you'd like WLED to save your defaults to disk (so any missing values are editable in Usermod Settings) + * + * getJsonValue() returns false if the value is missing, or copies the value into the variable provided and returns true if the value is present + * The configComplete variable is true only if the "exampleUsermod" object and all values are present. If any values are missing, WLED will know to call addToConfig() to save them + * + * This function is guaranteed to be called on boot, but could also be called every time settings are updated + */ + bool readFromConfig(JsonObject& root) + { + // default settings values could be set here (or below using the 3-argument getJsonValue()) instead of in the class definition or constructor + // setting them inside readFromConfig() is slightly more robust, handling the rare but plausible use case of single value being missing after boot (e.g. if the cfg.json was manually edited and a value was removed) + auto old_cfg = config; + + JsonObject top = root[FPSTR(_name)]; + + bool configComplete = top.isNull(); + // Ensure default configuration is loaded + configComplete &= getJsonValue(top[FPSTR(_enabled)], config.enabled, true); + configComplete &= getJsonValue(top[FPSTR(_interrupt_pin)], config.interruptPin, -1); + configComplete &= getJsonValue(top[FPSTR(_x_acc_bias)], config.accel_offset[0], 0); + configComplete &= getJsonValue(top[FPSTR(_y_acc_bias)], config.accel_offset[1], 0); + configComplete &= getJsonValue(top[FPSTR(_z_acc_bias)], config.accel_offset[2], 0); + configComplete &= getJsonValue(top[FPSTR(_x_gyro_bias)], config.gyro_offset[0], 0); + configComplete &= getJsonValue(top[FPSTR(_y_gyro_bias)], config.gyro_offset[1], 0); + configComplete &= getJsonValue(top[FPSTR(_z_gyro_bias)], config.gyro_offset[2], 0); + + DEBUG_PRINT(FPSTR(_name)); + if (top.isNull()) { + DEBUG_PRINTLN(F(": No config found. (Using defaults.)")); + } else if (!initDone()) { + DEBUG_PRINTLN(F(": config loaded.")); + } else if (memcmp(&config, &old_cfg, sizeof(config)) == 0) { + DEBUG_PRINTLN(F(": config unchanged.")); + } else { + DEBUG_PRINTLN(F(": config updated.")); + // Previously loaded and config changed + if (irqBound && ((old_cfg.interruptPin != config.interruptPin) || !config.enabled)) { + detachInterrupt(old_cfg.interruptPin); + pinManager.deallocatePin(old_cfg.interruptPin, PinOwner::UM_IMU); + irqBound = false; + } + + // Just re-init + setup(); + } + + return configComplete; + } + + bool getUMData(um_data_t **data) + { + if (!data || !config.enabled || !dmpReady) return false; // no pointer provided by caller or not enabled -> exit + *data = &um_data; + return true; + } /* * getId() allows you to optionally give your V2 usermod an unique ID (please define it in const.h!). @@ -305,3 +430,14 @@ class MPU6050Driver : public Usermod { } }; + + +const char MPU6050Driver::_name[] PROGMEM = "MPU6050_IMU"; +const char MPU6050Driver::_enabled[] PROGMEM = "enabled"; +const char MPU6050Driver::_interrupt_pin[] PROGMEM = "interrupt_pin"; +const char MPU6050Driver::_x_acc_bias[] PROGMEM = "x_acc_bias"; +const char MPU6050Driver::_y_acc_bias[] PROGMEM = "y_acc_bias"; +const char MPU6050Driver::_z_acc_bias[] PROGMEM = "z_acc_bias"; +const char MPU6050Driver::_x_gyro_bias[] PROGMEM = "x_gyro_bias"; +const char MPU6050Driver::_y_gyro_bias[] PROGMEM = "y_gyro_bias"; +const char MPU6050Driver::_z_gyro_bias[] PROGMEM = "z_gyro_bias"; diff --git a/wled00/pin_manager.h b/wled00/pin_manager.h index 6248a6066..1402de2bd 100644 --- a/wled00/pin_manager.h +++ b/wled00/pin_manager.h @@ -41,7 +41,7 @@ enum struct PinOwner : uint8_t { UM_Temperature = USERMOD_ID_TEMPERATURE, // 0x03 // Usermod "usermod_temperature.h" // #define USERMOD_ID_FIXNETSERVICES // 0x04 // Usermod "usermod_Fix_unreachable_netservices.h" -- Does not allocate pins UM_PIR = USERMOD_ID_PIRSWITCH, // 0x05 // Usermod "usermod_PIR_sensor_switch.h" - // #define USERMOD_ID_IMU // 0x06 // Usermod "usermod_mpu6050_imu.h" -- Uses "standard" HW_I2C pins + UM_IMU = USERMOD_ID_IMU, // 0x06 // Usermod "usermod_mpu6050_imu.h" -- Interrupt pin UM_FourLineDisplay = USERMOD_ID_FOUR_LINE_DISP, // 0x07 // Usermod "usermod_v2_four_line_display.h -- May use "standard" HW_I2C pins UM_RotaryEncoderUI = USERMOD_ID_ROTARY_ENC_UI, // 0x08 // Usermod "usermod_v2_rotary_encoder_ui.h" // #define USERMOD_ID_AUTO_SAVE // 0x09 // Usermod "usermod_v2_auto_save.h" -- Does not allocate pins From 6f8b3a6242a8080962d00ddf85feeaca6ccc6c3b Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 7 Jan 2024 14:25:47 -0500 Subject: [PATCH 10/15] usermod_list: Add IMU usermod --- wled00/usermods_list.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/wled00/usermods_list.cpp b/wled00/usermods_list.cpp index 6184571ac..ed4dfbb8d 100644 --- a/wled00/usermods_list.cpp +++ b/wled00/usermods_list.cpp @@ -197,6 +197,10 @@ #include "../usermods/pwm_outputs/usermod_pwm_outputs.h" #endif +#ifdef USERMOD_MPU6050_IMU +#include "../usermods/mpu6050_imu/usermod_mpu6050_imu.h" +#endif + void registerUsermods() { @@ -373,4 +377,9 @@ void registerUsermods() #ifdef USERMOD_INTERNAL_TEMPERATURE usermods.add(new InternalTemperatureUsermod()); #endif + + #ifdef USERMOD_MPU6050_IMU + static MPU6050Driver mpu6050; usermods.add(&mpu6050); + #endif + } From 0616d184e03ee7e439d428a05a55d3bfc1e572b8 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 7 Jan 2024 14:26:11 -0500 Subject: [PATCH 11/15] Add GyroSurge usermod This usermod applies the IMU data to create a "surge" effect when the device is moved. --- usermods/mpu6050_imu/usermod_gyro_surge.h | 219 ++++++++++++++++++++++ wled00/usermods_list.cpp | 7 + 2 files changed, 226 insertions(+) create mode 100644 usermods/mpu6050_imu/usermod_gyro_surge.h diff --git a/usermods/mpu6050_imu/usermod_gyro_surge.h b/usermods/mpu6050_imu/usermod_gyro_surge.h new file mode 100644 index 000000000..c49d930ff --- /dev/null +++ b/usermods/mpu6050_imu/usermod_gyro_surge.h @@ -0,0 +1,219 @@ +#pragma once + +/* This usermod uses gyro data to provide a "surge" effect on movement + +Requires lib_deps = bolderflight/Bolder Flight Systems Eigen@^3.0.0 + +*/ + +#include "wled.h" + +// Eigen include block +#ifdef A0 +namespace { constexpr size_t A0_temp {A0}; } +#undef A0 +static constexpr size_t A0 {A0_temp}; +#endif + +#ifdef A1 +namespace { constexpr size_t A1_temp {A1}; } +#undef A1 +static constexpr size_t A1 {A1_temp}; +#endif + +#ifdef B0 +namespace { constexpr size_t B0_temp {B0}; } +#undef B0 +static constexpr size_t B0 {B0_temp}; +#endif + +#ifdef B1 +namespace { constexpr size_t B1_temp {B1}; } +#undef B1 +static constexpr size_t B1 {B1_temp}; +#endif + +#ifdef D0 +namespace { constexpr size_t D0_temp {D0}; } +#undef D0 +static constexpr size_t D0 {D0_temp}; +#endif + +#ifdef D1 +namespace { constexpr size_t D1_temp {D1}; } +#undef D1 +static constexpr size_t D1 {D1_temp}; +#endif + +#ifdef D2 +namespace { constexpr size_t D2_temp {D2}; } +#undef D2 +static constexpr size_t D2 {D2_temp}; +#endif + +#ifdef D3 +namespace { constexpr size_t D3_temp {D3}; } +#undef D3 +static constexpr size_t D3 {D3_temp}; +#endif + +#include "eigen.h" +#include + +constexpr auto ESTIMATED_G = 9.801; // m/s^2 +constexpr auto ESTIMATED_G_COUNTS = 8350.; +constexpr auto ESTIMATED_ANGULAR_RATE = (M_PI * 2000) / (INT16_MAX * 180); // radians per second + +// Horribly lame digital filter code +// Currently implements a static IIR filter. +template +class xir_filter { + typedef Eigen::Array array_t; + const array_t a_coeff, b_coeff; + const T gain; + array_t x, y; + + public: + xir_filter(T gain_, array_t a, array_t b) : a_coeff(std::move(a)), b_coeff(std::move(b)), gain(gain_), x(array_t::Zero()), y(array_t::Zero()) {}; + + T operator()(T input) { + x.head(C-1) = x.tail(C-1); // shift by one + x(C-1) = input / gain; + y.head(C-1) = y.tail(C-1); // shift by one + y(C-1) = (x * b_coeff).sum(); + y(C-1) -= (y.head(C-1) * a_coeff.head(C-1)).sum(); + return y(C-1); + } + + T last() { return y(C-1); }; +}; + + + +class GyroSurge : public Usermod { + private: + static const char _name[]; + bool enabled = true; + + // Params + uint8_t max = 0; + float sensitivity = 0; + + // State + uint32_t last_sample; + // 100hz input + // butterworth low pass filter at 20hz + xir_filter filter = { 1., { -0.36952738, 0.19581571, 1.}, {0.20657208, 0.41314417, 0.20657208} }; + // { 1., { 0., 0., 1.}, { 0., 0., 1. } }; // no filter + + + public: + + /* + * setup() is called once at boot. WiFi is not yet connected at this point. + */ + void setup() {}; + + + /* + * addToConfig() can be used to add custom persistent settings to the cfg.json file in the "um" (usermod) object. + * It will be called by WLED when settings are actually saved (for example, LED settings are saved) + * I highly recommend checking out the basics of ArduinoJson serialization and deserialization in order to use custom settings! + */ + void addToConfig(JsonObject& root) + { + JsonObject top = root.createNestedObject(FPSTR(_name)); + + //save these vars persistently whenever settings are saved + top["max"] = max; + top["sensitivity"] = sensitivity; + } + + + /* + * readFromConfig() can be used to read back the custom settings you added with addToConfig(). + * This is called by WLED when settings are loaded (currently this only happens immediately after boot, or after saving on the Usermod Settings page) + * + * readFromConfig() is called BEFORE setup(). This means you can use your persistent values in setup() (e.g. pin assignments, buffer sizes), + * but also that if you want to write persistent values to a dynamic buffer, you'd need to allocate it here instead of in setup. + * If you don't know what that is, don't fret. It most likely doesn't affect your use case :) + * + * Return true in case the config values returned from Usermod Settings were complete, or false if you'd like WLED to save your defaults to disk (so any missing values are editable in Usermod Settings) + * + * getJsonValue() returns false if the value is missing, or copies the value into the variable provided and returns true if the value is present + * The configComplete variable is true only if the "exampleUsermod" object and all values are present. If any values are missing, WLED will know to call addToConfig() to save them + * + * This function is guaranteed to be called on boot, but could also be called every time settings are updated + */ + bool readFromConfig(JsonObject& root) + { + // default settings values could be set here (or below using the 3-argument getJsonValue()) instead of in the class definition or constructor + // setting them inside readFromConfig() is slightly more robust, handling the rare but plausible use case of single value being missing after boot (e.g. if the cfg.json was manually edited and a value was removed) + + JsonObject top = root[FPSTR(_name)]; + + bool configComplete = !top.isNull(); + + configComplete &= getJsonValue(top["max"], max, 0); + configComplete &= getJsonValue(top["sensitivity"], sensitivity, 10); + + return configComplete; + } + + void loop() { + // get IMU data + um_data_t *um_data; + if (!usermods.getUMData(&um_data, USERMOD_ID_IMU)) { + // Apply max + strip.getSegment(0).fadeToBlackBy(max); + return; + } + uint32_t sample_count = *(uint32_t*)(um_data->u_data[8]); + + if (sample_count != last_sample) { + last_sample = sample_count; + // Calculate based on new data + // We use the raw gyro data (angular rate) + auto gyros = (int16_t*)um_data->u_data[4]; // 16384 == 2000 deg/s + + // Compute the overall rotation rate + // For my application (a plasma sword) we ignore X axis rotations (eg. around the long axis) + auto gyro_q = Eigen::AngleAxis { + //Eigen::AngleAxis(ESTIMATED_ANGULAR_RATE * gyros[0], Eigen::Vector3f::UnitX()) * + Eigen::AngleAxis(ESTIMATED_ANGULAR_RATE * gyros[1], Eigen::Vector3f::UnitY()) * + Eigen::AngleAxis(ESTIMATED_ANGULAR_RATE * gyros[2], Eigen::Vector3f::UnitZ()) }; + + // Filter the results + filter(std::min(sensitivity * gyro_q.angle(), 1.0f)); // radians per second +/* + Serial.printf("[%lu] Gy: %d, %d, %d -- ", millis(), (int)gyros[0], (int)gyros[1], (int)gyros[2]); + Serial.print(gyro_q.angle()); + Serial.print(", "); + Serial.print(sensitivity * gyro_q.angle()); + Serial.print(" --> "); + Serial.println(filter.last()); +*/ + } + }; // noop + + /* + * handleOverlayDraw() is called just before every show() (LED strip update frame) after effects have set the colors. + * Use this to blank out some LEDs or set them to a different color regardless of the set effect mode. + * Commonly used for custom clocks (Cronixie, 7 segment) + */ + void handleOverlayDraw() + { + + // TODO: some kind of timing analysis for filtering ... + + // Calculate brightness boost + auto r_float = std::max(std::min(filter.last(), 1.0f), 0.f); + auto result = (uint8_t) (r_float * max); + //Serial.printf("[%lu] %d -- ", millis(), result); + //Serial.println(r_float); + // TODO - multiple segment handling?? + strip.getSegment(0).fadeToBlackBy(max - result); + } +}; + +const char GyroSurge::_name[] PROGMEM = "GyroSurge"; \ No newline at end of file diff --git a/wled00/usermods_list.cpp b/wled00/usermods_list.cpp index ed4dfbb8d..b8247b9a5 100644 --- a/wled00/usermods_list.cpp +++ b/wled00/usermods_list.cpp @@ -201,6 +201,9 @@ #include "../usermods/mpu6050_imu/usermod_mpu6050_imu.h" #endif +#ifdef USERMOD_MPU6050_IMU +#include "../usermods/mpu6050_imu/usermod_gyro_surge.h" +#endif void registerUsermods() { @@ -210,6 +213,7 @@ void registerUsermods() * \/ \/ \/ */ //usermods.add(new MyExampleUsermod()); + #ifdef USERMOD_BATTERY usermods.add(new UsermodBattery()); #endif @@ -382,4 +386,7 @@ void registerUsermods() static MPU6050Driver mpu6050; usermods.add(&mpu6050); #endif + #ifdef USERMOD_GYRO_SURGE + static GyroSurge gyro_surge; usermods.add(&gyro_surge); + #endif } From 9c4d3cd6e2918123959d8dd84fc0831055dec7d5 Mon Sep 17 00:00:00 2001 From: Will Miles Date: Sun, 7 Jan 2024 15:50:15 -0500 Subject: [PATCH 12/15] platformio.ini: Add MPU6050 library dep --- platformio.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio.ini b/platformio.ini index fa6bf3bb8..50362de30 100644 --- a/platformio.ini +++ b/platformio.ini @@ -198,6 +198,8 @@ lib_deps = #For ADS1115 sensor uncomment following ; adafruit/Adafruit BusIO @ 1.13.2 ; adafruit/Adafruit ADS1X15 @ 2.4.0 + #For MPU6050 IMU uncomment follwoing + ; electroniccats/MPU6050 @1.0.1 extra_scripts = ${scripts_defaults.extra_scripts} From 157debf8e41a4bedf6f669d7116b16193a11296c Mon Sep 17 00:00:00 2001 From: Woody Date: Sun, 7 Jan 2024 23:37:33 +0100 Subject: [PATCH 13/15] Make loadBg() more readable --- wled00/data/index.js | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/wled00/data/index.js b/wled00/data/index.js index f6c7c862c..ec8fcb23a 100644 --- a/wled00/data/index.js +++ b/wled00/data/index.js @@ -32,13 +32,14 @@ var cfg = { labels:true, pcmbot:false, pid:true, seglen:false, segpwr:false, segexp:false, css:true, hdays:false, fxdef:true, on:0, off:0, idsort: false} }; +// [year, month (0 -> January, 11 -> December), day, duration in days, image url] var hol = [ - [0,11,24,4,"https://aircoookie.github.io/xmas.png"], // christmas - [0,2,17,1,"https://images.alphacoders.com/491/491123.jpg"], // st. Patrick's day - [2025,3,20,2,"https://aircoookie.github.io/easter.png"], - [2024,2,31,2,"https://aircoookie.github.io/easter.png"], - [0,6,4,1,"https://images.alphacoders.com/516/516792.jpg"], // 4th of July - [0,0,1,1,"https://images.alphacoders.com/119/1198800.jpg"] // new year + [0, 11, 24, 4, "https://aircoookie.github.io/xmas.png"], // christmas + [0, 2, 17, 1, "https://images.alphacoders.com/491/491123.jpg"], // st. Patrick's day + [2025, 3, 20, 2, "https://aircoookie.github.io/easter.png"], // easter 2025 + [2024, 2, 31, 2, "https://aircoookie.github.io/easter.png"], // easter 2024 + [0, 6, 4, 1, "https://images.alphacoders.com/516/516792.jpg"], // 4th of July + [0, 0, 1, 1, "https://images.alphacoders.com/119/1198800.jpg"] // new year ]; var cpick = new iro.ColorPicker("#picker", { @@ -175,19 +176,19 @@ function cTheme(light) { } } -function loadBg(iUrl) -{ - let bg = gId('bg'); - let img = d.createElement("img"); +function loadBg() { + const { url: iUrl, rnd: iRnd } = cfg.theme.bg; + const bg = gId('bg'); + const img = d.createElement("img"); img.src = iUrl; - if (iUrl == "" || iUrl==="https://picsum.photos/1920/1080") { - var today = new Date(); - for (var h of (hol||[])) { - var yr = h[0]==0 ? today.getFullYear() : h[0]; - var hs = new Date(yr,h[1],h[2]); - var he = new Date(hs); - he.setDate(he.getDate() + h[3]); - if (today>=hs && today<=he) img.src = h[4]; + if (!iUrl || iRnd) { + const today = new Date(); + for (const holiday of (hol || [])) { + const year = holiday[0] == 0 ? today.getFullYear() : holiday[0]; + const holidayStart = new Date(year, holiday[1], holiday[2]); + const holidayEnd = new Date(holidayStart); + holidayEnd.setDate(holidayEnd.getDate() + holiday[3]); + if (today >= holidayStart && today <= holidayEnd) img.src = holiday[4]; } } img.addEventListener('load', (e) => { @@ -195,7 +196,6 @@ function loadBg(iUrl) if (isNaN(a)) a = 0.6; bg.style.opacity = a; bg.style.backgroundImage = `url(${img.src})`; - img = null; gId('namelabel').style.color = "var(--c-c)"; // improve namelabel legibility on background image }); } @@ -266,10 +266,10 @@ function onLoad() console.log("No array of holidays in holidays.json. Defaults loaded."); }) .finally(()=>{ - loadBg(cfg.theme.bg.url); + loadBg(); }); } else - loadBg(cfg.theme.bg.url); + loadBg(); selectSlot(0); updateTablinks(0); From a5b132dfd73169268df7aa009d839ffb85aea0a6 Mon Sep 17 00:00:00 2001 From: Woody Date: Mon, 8 Jan 2024 14:20:02 +0100 Subject: [PATCH 14/15] Fix that color picker is not centered Color Picker was not centerd on screens with width between 1024px and 1249px --- wled00/data/index.css | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/wled00/data/index.css b/wled00/data/index.css index 0ffdf68d3..095de572a 100644 --- a/wled00/data/index.css +++ b/wled00/data/index.css @@ -785,7 +785,7 @@ input[type=range]::-moz-range-thumb { #picker { margin-top: 8px !important; - max-width: 260px; + max-width: max-content; } /* buttons */ @@ -1566,9 +1566,6 @@ dialog { max-width: 280px; font-size: 18px; } - #picker { - width: 230px; - } #putil .btn-s { width: 114px; } From 220217561a361f3ba49aa9b0bbe97375396d816c Mon Sep 17 00:00:00 2001 From: Blaz Kristan Date: Tue, 9 Jan 2024 18:20:20 +0100 Subject: [PATCH 15/15] Possible fix for #3589 & partial fix for #3605 --- wled00/json.cpp | 9 ++++++++- wled00/udp.cpp | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/wled00/json.cpp b/wled00/json.cpp index 8a3bfbe63..8d8a0316c 100644 --- a/wled00/json.cpp +++ b/wled00/json.cpp @@ -210,7 +210,14 @@ bool deserializeSegment(JsonObject elem, byte it, byte presetId) #endif byte fx = seg.mode; - if (getVal(elem["fx"], &fx, 0, strip.getModeCount())) { //load effect ('r' random, '~' inc/dec, 0-255 exact value) + byte last = strip.getModeCount(); + // partial fix for #3605 + if (!elem["fx"].isNull() && elem["fx"].is()) { + const char *tmp = elem["fx"].as(); + if (strlen(tmp) > 3 && (strchr(tmp,'r') || strchr(tmp,'~') != strrchr(tmp,'~'))) last = 0; // we have "X~Y(r|[w]~[-])" form + } + // end fix + if (getVal(elem["fx"], &fx, 0, last)) { //load effect ('r' random, '~' inc/dec, 0-255 exact value, 5~10r pick random between 5 & 10) if (!presetId && currentPlaylist>=0) unloadPlaylist(); if (fx != seg.mode) seg.setMode(fx, elem[F("fxdef")]); } diff --git a/wled00/udp.cpp b/wled00/udp.cpp index 4c55a9727..6d161f5d9 100644 --- a/wled00/udp.cpp +++ b/wled00/udp.cpp @@ -434,6 +434,8 @@ void exitRealtime() { realtimeIP[0] = 0; if (useMainSegmentOnly) { // unfreeze live segment again strip.getMainSegment().freeze = false; + } else { + strip.show(); // possible fix for #3589 } updateInterfaces(CALL_MODE_WS_SEND); }