Merge pull request #3699 from WoodyLetsCode/cdata

Update dependencies for cdata.js and make some improvements

.github/workflows/wled-ci.yml

@@ -1,4 +1,4 @@
-name: PlatformIO CI
+name: WLED CI
 
 on: [push, pull_request]
 
@@ -94,3 +94,17 @@ jobs:
           *.bin
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+
+  testCdata:
+    name: Test cdata.js
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20.x'
+          cache: 'npm'
+      - run: npm ci
+      - run: npm test

package.json

@@ -9,6 +9,7 @@
   },
   "scripts": {
     "build": "node tools/cdata.js",
+    "test": "node --test",
     "dev": "nodemon -e js,html,htm,css,png,jpg,gif,ico,js -w tools/ -w wled00/data/ -x node tools/cdata.js"
   },
   "repository": {
@@ -22,10 +23,10 @@
   },
   "homepage": "https://github.com/Aircoookie/WLED#readme",
   "dependencies": {
-    "clean-css": "^4.2.3",
-    "html-minifier-terser": "^5.1.1",
+    "clean-css": "^5.3.3",
+    "html-minifier-terser": "^7.2.0",
     "inliner": "^1.13.1",
-    "nodemon": "^2.0.20",
+    "nodemon": "^3.0.2",
     "zlib": "^1.0.5"
   }
 }

tools/cdata-test.js

@@ -0,0 +1,205 @@
+'use strict';
+
+const assert = require('node:assert');
+const { describe, it, before, after } = require('node:test');
+const fs = require('fs');
+const path = require('path');
+const child_process = require('child_process');
+const util = require('util');
+const execPromise = util.promisify(child_process.exec);
+
+process.env.NODE_ENV = 'test'; // Set the environment to testing
+const cdata = require('./cdata.js');
+
+describe('Function', () => {
+  const testFolderPath = path.join(__dirname, 'testFolder');
+  const oldFilePath = path.join(testFolderPath, 'oldFile.txt');
+  const newFilePath = path.join(testFolderPath, 'newFile.txt');
+
+  // Create a temporary file before the test
+  before(() => {
+    // Create test folder
+    if (!fs.existsSync(testFolderPath)) {
+      fs.mkdirSync(testFolderPath);
+    }
+
+    // Create an old file
+    fs.writeFileSync(oldFilePath, 'This is an old file.');
+    // Modify the 'mtime' to simulate an old file
+    const oldTime = new Date();
+    oldTime.setFullYear(oldTime.getFullYear() - 1);
+    fs.utimesSync(oldFilePath, oldTime, oldTime);
+
+    // Create a new file
+    fs.writeFileSync(newFilePath, 'This is a new file.');
+  });
+
+  // delete the temporary files after the test
+  after(() => {
+    fs.rmSync(testFolderPath, { recursive: true });
+  });
+
+  describe('isFileNewerThan', async () => {
+    it('should return true if the file is newer than the provided time', async () => {
+      const pastTime = Date.now() - 10000; // 10 seconds ago
+      assert.strictEqual(cdata.isFileNewerThan(newFilePath, pastTime), true);
+    });
+
+    it('should return false if the file is older than the provided time', async () => {
+      assert.strictEqual(cdata.isFileNewerThan(oldFilePath, Date.now()), false);
+    });
+
+    it('should throw an exception if the file does not exist', async () => {
+      assert.throws(() => {
+        cdata.isFileNewerThan('nonexistent.txt', Date.now());
+      });
+    });
+  });
+
+  describe('isAnyFileInFolderNewerThan', async () => {
+    it('should return true if a file in the folder is newer than the given time', async () => {
+      const time = fs.statSync(path.join(testFolderPath, 'oldFile.txt')).mtime;
+      assert.strictEqual(cdata.isAnyFileInFolderNewerThan(testFolderPath, time), true);
+    });
+
+    it('should return false if no files in the folder are newer than the given time', async () => {
+      assert.strictEqual(cdata.isAnyFileInFolderNewerThan(testFolderPath, new Date()), false);
+    });
+
+    it('should throw an exception if the folder does not exist', async () => {
+      assert.throws(() => {
+        cdata.isAnyFileInFolderNewerThan('nonexistent', new Date());
+      });
+    });
+  });
+});
+
+describe('Script', () => {
+  const folderPath = 'wled00';
+  const dataPath = path.join(folderPath, 'data');
+
+  before(() => {
+    process.env.NODE_ENV = 'production';
+    // Backup files
+    fs.cpSync("wled00/data", "wled00Backup", { recursive: true });
+    fs.cpSync("tools/cdata.js", "cdata.bak.js");
+  });
+  after(() => {
+    // Restore backup
+    fs.rmSync("wled00/data", { recursive: true });
+    fs.renameSync("wled00Backup", "wled00/data");
+    fs.rmSync("tools/cdata.js");
+    fs.renameSync("cdata.bak.js", "tools/cdata.js");
+  });
+
+  // delete all html_*.h files
+  async function deleteBuiltFiles() {
+    const files = await fs.promises.readdir(folderPath);
+    await Promise.all(files.map(file => {
+      if (file.startsWith('html_') && path.extname(file) === '.h') {
+        return fs.promises.unlink(path.join(folderPath, file));
+      }
+    }));
+  }
+
+  // check if html_*.h files were created
+  async function checkIfBuiltFilesExist() {
+    const files = await fs.promises.readdir(folderPath);
+    const htmlFiles = files.filter(file => file.startsWith('html_') && path.extname(file) === '.h');
+    assert(htmlFiles.length > 0, 'html_*.h files were not created');
+  }
+
+  async function runAndCheckIfBuiltFilesExist() {
+    await execPromise('node tools/cdata.js');
+    await checkIfBuiltFilesExist();
+  }
+
+  async function checkIfFileWasNewlyCreated(file) {
+    const modifiedTime = fs.statSync(file).mtimeMs;
+    assert(Date.now() - modifiedTime < 500, file + ' was not modified');
+  }
+
+  async function testFileModification(sourceFilePath, resultFile) {
+    // run cdata.js to ensure html_*.h files are created
+    await execPromise('node tools/cdata.js');
+
+    // modify file
+    fs.appendFileSync(sourceFilePath, ' ');
+    // delay for 1 second to ensure the modified time is different
+    await new Promise(resolve => setTimeout(resolve, 1000));
+
+    // run script cdata.js again and wait for it to finish
+    await execPromise('node tools/cdata.js');
+
+    checkIfFileWasNewlyCreated(path.join(folderPath, resultFile));
+  }
+
+  describe('should build if', () => {
+    it('html_*.h files are missing', async () => {
+      await deleteBuiltFiles();
+      await runAndCheckIfBuiltFilesExist();
+    });
+
+    it('only one html_*.h file is missing', async () => {
+      // run script cdata.js and wait for it to finish
+      await execPromise('node tools/cdata.js');
+
+      // delete a random html_*.h file
+      let files = await fs.promises.readdir(folderPath);
+      let htmlFiles = files.filter(file => file.startsWith('html_') && path.extname(file) === '.h');
+      const randomFile = htmlFiles[Math.floor(Math.random() * htmlFiles.length)];
+      await fs.promises.unlink(path.join(folderPath, randomFile));
+
+      await runAndCheckIfBuiltFilesExist();
+    });
+
+    it('script was executed with -f or --force', async () => {
+      await execPromise('node tools/cdata.js');
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      await execPromise('node tools/cdata.js --force');
+      await checkIfFileWasNewlyCreated(path.join(folderPath, 'html_ui.h'));
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      await execPromise('node tools/cdata.js -f');
+      await checkIfFileWasNewlyCreated(path.join(folderPath, 'html_ui.h'));
+    });
+
+    it('a file changes', async () => {
+      await testFileModification(path.join(dataPath, 'index.htm'), 'html_ui.h');
+    });
+
+    it('a inlined file changes', async () => {
+      await testFileModification(path.join(dataPath, 'index.js'), 'html_ui.h');
+    });
+
+    it('a settings file changes', async () => {
+      await testFileModification(path.join(dataPath, 'settings_leds.htm'), 'html_ui.h');
+    });
+
+    it('the favicon changes', async () => {
+      await testFileModification(path.join(dataPath, 'favicon.ico'), 'html_ui.h');
+    });
+
+    it('cdata.js changes', async () => {
+      await testFileModification('tools/cdata.js', 'html_ui.h');
+    });
+  });
+
+  describe('should not build if', () => {
+    it('the files are already built', async () => {
+      await deleteBuiltFiles();
+
+      // run script cdata.js and wait for it to finish
+      let startTime = Date.now();
+      await execPromise('node tools/cdata.js');
+      const firstRunTime = Date.now() - startTime;
+
+      // run script cdata.js and wait for it to finish
+      startTime = Date.now();
+      await execPromise('node tools/cdata.js');
+      const secondRunTime = Date.now() - startTime;
+
+      // check if second run was faster than the first (must be at least 2x faster)
+      assert(secondRunTime < firstRunTime / 2, 'html_*.h files were rebuilt');
+    });
+  });
+});

tools/cdata.js

@@ -16,28 +16,59 @@
  */
 
 const fs = require("fs");
-const path = require('path');
+const path = require("path");
 const inliner = require("inliner");
 const zlib = require("zlib");
 const CleanCSS = require("clean-css");
-const MinifyHTML = require("html-minifier-terser").minify;
+const minifyHtml = require("html-minifier-terser").minify;
 const packageJson = require("../package.json");
 
+// Export functions for testing
+module.exports = { isFileNewerThan, isAnyFileInFolderNewerThan };
+
 const output = ["wled00/html_ui.h", "wled00/html_pixart.h", "wled00/html_cpal.h", "wled00/html_pxmagic.h", "wled00/html_settings.h", "wled00/html_other.h"]
 
-/**
- *
+// \x1b[34m is blue, \x1b[36m is cyan, \x1b[0m is reset
+const wledBanner = `
+\t\x1b[34m##      ## ##       ######## ########  
+\t\x1b[34m##  ##  ## ##       ##       ##     ## 
+\t\x1b[34m##  ##  ## ##       ##       ##     ## 
+\t\x1b[34m##  ##  ## ##       ######   ##     ## 
+\t\x1b[34m##  ##  ## ##       ##       ##     ## 
+\t\x1b[34m##  ##  ## ##       ##       ##     ## 
+\t\x1b[34m ###  ###  ######## ######## ########  
+\t\t\x1b[36mbuild script for web UI
+\x1b[0m`;
+
+const singleHeader = `/*
+ * Binary array for the Web UI.
+ * gzip is used for smaller size and improved speeds.
+ * 
+ * Please see https://kno.wled.ge/advanced/custom-features/#changing-web-ui
+ * to find out how to easily modify the web UI source!
  */
-function hexdump(buffer,isHex=false) {
+ 
+`;
+
+const multiHeader = `/*
+ * More web UI HTML source arrays.
+ * This file is auto generated, please don't make any changes manually.
+ *
+ * Instead, see https://kno.wled.ge/advanced/custom-features/#changing-web-ui
+ * to find out how to easily modify the web UI source!
+ */
+`;
+
+function hexdump(buffer, isHex = false) {
   let lines = [];
 
-  for (let i = 0; i < buffer.length; i +=(isHex?32:16)) {
+  for (let i = 0; i < buffer.length; i += (isHex ? 32 : 16)) {
     var block;
     let hexArray = [];
     if (isHex) {
       block = buffer.slice(i, i + 32)
-      for (let j = 0; j < block.length; j +=2 ) {
-        hexArray.push("0x" + block.slice(j,j+2))
+      for (let j = 0; j < block.length; j += 2) {
+        hexArray.push("0x" + block.slice(j, j + 2))
       }
     } else {
       block = buffer.slice(i, i + 16); // cut buffer into blocks of 16
@@ -54,183 +85,112 @@ function hexdump(buffer,isHex=false) {
   return lines.join(",\n");
 }
 
-function strReplace(str, search, replacement) {
-  return str.split(search).join(replacement);
-}
-
 function adoptVersionAndRepo(html) {
   let repoUrl = packageJson.repository ? packageJson.repository.url : undefined;
   if (repoUrl) {
     repoUrl = repoUrl.replace(/^git\+/, "");
     repoUrl = repoUrl.replace(/\.git$/, "");
-    // Replace we
-    html = strReplace(html, "https://github.com/atuline/WLED", repoUrl);
-    html = strReplace(html, "https://github.com/Aircoookie/WLED", repoUrl);
+    html = html.replaceAll("https://github.com/atuline/WLED", repoUrl);
+    html = html.replaceAll("https://github.com/Aircoookie/WLED", repoUrl);
   }
   let version = packageJson.version;
   if (version) {
-    html = strReplace(html, "##VERSION##", version);
+    html = html.replaceAll("##VERSION##", version);
   }
   return html;
 }
 
-function filter(str, type) {
-  str = adoptVersionAndRepo(str);
-  if (type === undefined) {
+async function minify(str, type = "plain") {
+  const options = {
+    collapseWhitespace: true,
+    collapseBooleanAttributes: true,
+    collapseInlineTagWhitespace: true,
+    minifyCSS: true,
+    minifyJS: true,
+    removeAttributeQuotes: true,
+    removeComments: true,
+    sortAttributes: true,
+    sortClassName: true,
+  };
+
+  if (type == "plain") {
     return str;
   } else if (type == "css-minify") {
     return new CleanCSS({}).minify(str).styles;
   } else if (type == "js-minify") {
-    return MinifyHTML('<script>' + str + '</script>', {
-      collapseWhitespace: true,
-      minifyJS: true, 
-      continueOnParseError: false,
-      removeComments: true,
-    }).replace(/<[\/]*script>/g,'');
+    return await minifyHtml('<script>' + str + '</script>', options).replace(/<[\/]*script>/g, '');
   } else if (type == "html-minify") {
-    return MinifyHTML(str, {
-      collapseWhitespace: true,
-      maxLineLength: 80,
-      minifyCSS: true,
-      minifyJS: true, 
-      continueOnParseError: false,
-      removeComments: true,
-    });
-  } else if (type == "html-minify-ui") {
-    return MinifyHTML(str, {
-      collapseWhitespace: true,
-      conservativeCollapse: true,
-      maxLineLength: 80,
-      minifyCSS: true,
-      minifyJS: true, 
-      continueOnParseError: false,
-      removeComments: true,
-    });
-  } else {
-    console.warn("Unknown filter: " + type);
-    return str;
+    return await minifyHtml(str, options);
   }
+
+  throw new Error("Unknown filter: " + type);
 }
 
-function writeHtmlGzipped(sourceFile, resultFile, page) {
+async function writeHtmlGzipped(sourceFile, resultFile, page) {
   console.info("Reading " + sourceFile);
-  new inliner(sourceFile, function (error, html) {
-    console.info("Inlined " + html.length + " characters");
-    html = filter(html, "html-minify-ui");
-    console.info("Minified to " + html.length + " characters");
-
-    if (error) {
-      console.warn(error);
-      throw error;
-    }
+  new inliner(sourceFile, async function (error, html) {
+    if (error) throw error;
 
     html = adoptVersionAndRepo(html);
-    zlib.gzip(html, { level: zlib.constants.Z_BEST_COMPRESSION }, function (error, result) {
-      if (error) {
-        console.warn(error);
-        throw error;
-      }
-
-      console.info("Compressed " + result.length + " bytes");
-      const array = hexdump(result);
-      const src = `/*
- * Binary array for the Web UI.
- * gzip is used for smaller size and improved speeds.
- * 
- * Please see https://kno.wled.ge/advanced/custom-features/#changing-web-ui
- * to find out how to easily modify the web UI source!
- */
- 
-// Autogenerated from ${sourceFile}, do not edit!!
-const uint16_t PAGE_${page}_L = ${result.length};
-const uint8_t PAGE_${page}[] PROGMEM = {
-${array}
-};
-`;
-      console.info("Writing " + resultFile);
-      fs.writeFileSync(resultFile, src);
-    });
+    const originalLength = html.length;
+    html = await minify(html, "html-minify");
+    const result = zlib.gzipSync(html, { level: zlib.constants.Z_BEST_COMPRESSION });
+    console.info("Minified and compressed " + sourceFile + " from " + originalLength + " to " + result.length + " bytes");
+    const array = hexdump(result);
+    let src = singleHeader;
+    src += `const uint16_t PAGE_${page}_L = ${result.length};\n`;
+    src += `const uint8_t PAGE_${page}[] PROGMEM = {\n${array}\n};\n\n`;
+    console.info("Writing " + resultFile);
+    fs.writeFileSync(resultFile, src);
   });
 }
 
-function specToChunk(srcDir, s) {
-  if (s.method == "plaintext") {
-    const buf = fs.readFileSync(srcDir + "/" + s.file);
-    const str = buf.toString("utf-8");
-    const chunk = `
-// Autogenerated from ${srcDir}/${s.file}, do not edit!!
-const char ${s.name}[] PROGMEM = R"${s.prepend || ""}${filter(str, s.filter)}${
-      s.append || ""
-    }";
+async function specToChunk(srcDir, s) {
+  const buf = fs.readFileSync(srcDir + "/" + s.file);
+  let chunk = `\n// Autogenerated from ${srcDir}/${s.file}, do not edit!!\n`
 
-`;
-    return s.mangle ? s.mangle(chunk) : chunk;
-  } else if (s.method == "gzip") {
-    const buf = fs.readFileSync(srcDir + "/" + s.file);
-    var str = buf.toString('utf-8');
-    if (s.mangle) str = s.mangle(str);
-    const zip = zlib.gzipSync(filter(str, s.filter), { level: zlib.constants.Z_BEST_COMPRESSION });
-    const result = hexdump(zip.toString('hex'), true);
-    const chunk = `
-// Autogenerated from ${srcDir}/${s.file}, do not edit!!
-const uint16_t ${s.name}_length = ${zip.length};
-const uint8_t ${s.name}[] PROGMEM = {
-${result}
-};
-
-`;
-    return chunk;
-  } else if (s.method == "binary") {
-    const buf = fs.readFileSync(srcDir + "/" + s.file);
-    const result = hexdump(buf);
-    const chunk = `
-// Autogenerated from ${srcDir}/${s.file}, do not edit!!
-const uint16_t ${s.name}_length = ${buf.length};
-const uint8_t ${s.name}[] PROGMEM = {
-${result}
-};
-
-`;
-    return chunk;
-  } else {
-    console.warn("Unknown method: " + s.method);
-    return undefined;
-  }
-}
-
-function writeChunks(srcDir, specs, resultFile) {
-  let src = `/*
- * More web UI HTML source arrays.
- * This file is auto generated, please don't make any changes manually.
- * Instead, see https://kno.wled.ge/advanced/custom-features/#changing-web-ui
- * to find out how to easily modify the web UI source!
- */ 
-`;
-  specs.forEach((s) => {
-    const file = srcDir + "/" + s.file;
-    try {
-      console.info("Reading " + file + " as " + s.name);
-      src += specToChunk(srcDir, s);
-    } catch (e) {
-      console.warn(
-        "Failed " + s.name + " from " + file,
-        e.message.length > 60 ? e.message.substring(0, 60) : e.message
-      );
+  if (s.method == "plaintext" || s.method == "gzip") {
+    let str = buf.toString("utf-8");
+    str = adoptVersionAndRepo(str);
+    const originalLength = str.length;
+    if (s.method == "gzip") {
+      if (s.mangle) str = s.mangle(str);
+      const zip = zlib.gzipSync(await minify(str, s.filter), { level: zlib.constants.Z_BEST_COMPRESSION });
+      console.info("Minified and compressed " + s.file + " from " + originalLength + " to " + zip.length + " bytes");
+      const result = hexdump(zip);
+      chunk += `const uint16_t ${s.name}_length = ${zip.length};\n`;
+      chunk += `const uint8_t ${s.name}[] PROGMEM = {\n${result}\n};\n\n`;
+      return chunk;
+    } else {
+      const minified = await minify(str, s.filter);
+      console.info("Minified " + s.file + " from " + originalLength + " to " + minified.length + " bytes");
+      chunk += `const char ${s.name}[] PROGMEM = R"${s.prepend || ""}${minified}${s.append || ""}";\n\n`;
+      return s.mangle ? s.mangle(chunk) : chunk;
     }
-  });
+  } else if (s.method == "binary") {
+    const result = hexdump(buf);
+    chunk += `const uint16_t ${s.name}_length = ${buf.length};\n`;
+    chunk += `const uint8_t ${s.name}[] PROGMEM = {\n${result}\n};\n\n`;
+    return chunk;
+  }
+
+  throw new Error("Unknown method: " + s.method);
+}
+
+async function writeChunks(srcDir, specs, resultFile) {
+  let src = multiHeader;
+  for (const s of specs) {
+    console.info("Reading " + srcDir + "/" + s.file + " as " + s.name);
+    src += await specToChunk(srcDir, s);
+  }
   console.info("Writing " + src.length + " characters into " + resultFile);
   fs.writeFileSync(resultFile, src);
 }
 
 // Check if a file is newer than a given time
 function isFileNewerThan(filePath, time) {
-  try {
-    const stats = fs.statSync(filePath);
-    return stats.mtimeMs > time;
-  } catch (e) {
-    console.error(`Failed to get stats for file ${filePath}:`, e);
-    return false;
-  }
+  const stats = fs.statSync(filePath);
+  return stats.mtimeMs > time;
 }
 
 // Check if any file in a folder (or its subfolders) is newer than a given time
@@ -248,21 +208,30 @@ function isAnyFileInFolderNewerThan(folderPath, time) {
   return false;
 }
 
+// Check if the web UI is already built
 function isAlreadyBuilt(folderPath) {
   let lastBuildTime = Infinity;
 
   for (const file of output) {
     try {
       lastBuildTime = Math.min(lastBuildTime, fs.statSync(file).mtimeMs);
-    }
-    catch (e) {
+    } catch (e) {
+      if (e.code !== 'ENOENT') throw e;
+      console.info("File " + file + " does not exist. Rebuilding...");
       return false;
     }
   }
 
-  return !isAnyFileInFolderNewerThan(folderPath, lastBuildTime);
+  return !isAnyFileInFolderNewerThan(folderPath, lastBuildTime) && !isFileNewerThan("tools/cdata.js", lastBuildTime);
 }
 
+// Don't run this script if we're in a test environment
+if (process.env.NODE_ENV === 'test') {
+  return;
+}
+
+console.info(wledBanner);
+
 if (isAlreadyBuilt("wled00/data") && process.argv[2] !== '--force' && process.argv[2] !== '-f') {
   console.info("Web UI is already built");
   return;
@@ -283,7 +252,7 @@ writeChunks(
       filter: "css-minify",
       mangle: (str) =>
         str
-          .replace("%%","%")
+          .replace("%%", "%")
     },
     {
       file: "settings.htm",