Update translation hashing (#5025)

* Update translation hashing

* Move gulp-rename

build-scripts/gulp/app.js

@@ -24,7 +24,7 @@ gulp.task(
       gulp.parallel("gen-icons-app", "gen-icons-mdi"),
       "gen-pages-dev",
       "gen-index-app-dev",
-      gulp.series("create-test-translation", "build-translations")
+      "build-translations"
     ),
     "copy-static",
     "webpack-watch-app"

build-scripts/gulp/gen-icons.js

@@ -2,6 +2,7 @@ const gulp = require("gulp");
 const path = require("path");
 const fs = require("fs");
 const paths = require("../paths");
+const { mapFiles } = require("../util");
 
 const ICON_PACKAGE_PATH = path.resolve(
   __dirname,
@@ -57,20 +58,6 @@ function generateIconset(iconsetName, iconNames) {
   return `<ha-iconset-svg name="${iconsetName}" size="24"><svg><defs>${iconDefs}</defs></svg></ha-iconset-svg>`;
 }
 
-// Helper function to map recursively over files in a folder and it's subfolders
-function mapFiles(startPath, filter, mapFunc) {
-  const files = fs.readdirSync(startPath);
-  for (let i = 0; i < files.length; i++) {
-    const filename = path.join(startPath, files[i]);
-    const stat = fs.lstatSync(filename);
-    if (stat.isDirectory()) {
-      mapFiles(filename, filter, mapFunc);
-    } else if (filename.indexOf(filter) >= 0) {
-      mapFunc(filename);
-    }
-  }
-}
-
 // Find all icons used by the project.
 function findIcons(searchPath, iconsetName) {
   const iconRegex = new RegExp(`${iconsetName}:[\\w-]+`, "g");

build-scripts/gulp/translations.js

@@ -1,14 +1,17 @@
+const crypto = require("crypto");
 const del = require("del");
 const path = require("path");
+const source = require("vinyl-source-stream");
+const vinylBuffer = require("vinyl-buffer");
 const gulp = require("gulp");
 const fs = require("fs");
 const foreach = require("gulp-foreach");
-const hash = require("gulp-hash");
-const hashFilename = require("gulp-hash-filename");
 const merge = require("gulp-merge-json");
 const minify = require("gulp-jsonminify");
 const rename = require("gulp-rename");
 const transform = require("gulp-json-transform");
+const { mapFiles } = require("../util");
+const env = require("../env");
 
 const inDir = "translations";
 const workDir = "build-translations";
@@ -39,8 +42,6 @@ const TRANSLATION_FRAGMENTS = [
   "developer-tools",
 ];
 
-const tasks = [];
-
 function recursiveFlatten(prefix, data) {
   let output = {};
   Object.keys(data).forEach(function(key) {
@@ -116,11 +117,9 @@ function lokaliseTransform(data, original, file) {
   return output;
 }
 
-let taskName = "clean-translations";
-gulp.task(taskName, function() {
-  return del([`${outDir}/**/*.json`]);
+gulp.task("clean-translations", function() {
+  return del([workDir]);
 });
-tasks.push(taskName);
 
 gulp.task("ensure-translations-build-dir", (done) => {
   if (!fs.existsSync(workDir)) {
@@ -129,27 +128,21 @@ gulp.task("ensure-translations-build-dir", (done) => {
   done();
 });
 
-taskName = "create-test-metadata";
-gulp.task(
-  taskName,
-  gulp.series("ensure-translations-build-dir", function writeTestMetaData(cb) {
-    fs.writeFile(
-      workDir + "/testMetadata.json",
-      JSON.stringify({
-        test: {
-          nativeName: "Test",
-        },
-      }),
-      cb
-    );
-  })
-);
-tasks.push(taskName);
+gulp.task("create-test-metadata", function(cb) {
+  fs.writeFile(
+    workDir + "/testMetadata.json",
+    JSON.stringify({
+      test: {
+        nativeName: "Test",
+      },
+    }),
+    cb
+  );
+});
 
-taskName = "create-test-translation";
 gulp.task(
-  taskName,
-  gulp.series("create-test-metadata", function() {
+  "create-test-translation",
+  gulp.series("create-test-metadata", function createTestTranslation() {
     return gulp
       .src("src/translations/en.json")
       .pipe(
@@ -161,7 +154,6 @@ gulp.task(
       .pipe(gulp.dest(workDir));
   })
 );
-tasks.push(taskName);
 
 /**
  * This task will build a master translation file, to be used as the base for
@@ -172,235 +164,215 @@ tasks.push(taskName);
  * project is buildable immediately after merging new translation keys, since
  * the Lokalise update to translations/en.json will not happen immediately.
  */
-taskName = "build-master-translation";
-gulp.task(
-  taskName,
-  gulp.series("clean-translations", function() {
-    return gulp
-      .src("src/translations/en.json")
-      .pipe(
-        transform(function(data, file) {
-          return lokaliseTransform(data, data, file);
-        })
-      )
-      .pipe(rename("translationMaster.json"))
-      .pipe(gulp.dest(workDir));
-  })
-);
-tasks.push(taskName);
+gulp.task("build-master-translation", function() {
+  return gulp
+    .src("src/translations/en.json")
+    .pipe(
+      transform(function(data, file) {
+        return lokaliseTransform(data, data, file);
+      })
+    )
+    .pipe(rename("translationMaster.json"))
+    .pipe(gulp.dest(workDir));
+});
 
-taskName = "build-merged-translations";
-gulp.task(
-  taskName,
-  gulp.series("build-master-translation", function() {
-    return gulp
-      .src([inDir + "/*.json", workDir + "/test.json"], { allowEmpty: true })
-      .pipe(
-        transform(function(data, file) {
-          return lokaliseTransform(data, data, file);
-        })
-      )
-      .pipe(
-        foreach(function(stream, file) {
-          // For each language generate a merged json file. It begins with the master
-          // translation as a failsafe for untranslated strings, and merges all parent
-          // tags into one file for each specific subtag
-          //
-          // TODO: This is a naive interpretation of BCP47 that should be improved.
-          //       Will be OK for now as long as we don't have anything more complicated
-          //       than a base translation + region.
-          const tr = path.basename(file.history[0], ".json");
-          const subtags = tr.split("-");
-          const src = [workDir + "/translationMaster.json"];
-          for (let i = 1; i <= subtags.length; i++) {
-            const lang = subtags.slice(0, i).join("-");
-            if (lang === "test") {
-              src.push(workDir + "/test.json");
-            } else if (lang !== "en") {
-              src.push(inDir + "/" + lang + ".json");
-            }
+gulp.task("build-merged-translations", function() {
+  return gulp
+    .src([inDir + "/*.json", workDir + "/test.json"], { allowEmpty: true })
+    .pipe(
+      transform(function(data, file) {
+        return lokaliseTransform(data, data, file);
+      })
+    )
+    .pipe(
+      foreach(function(stream, file) {
+        // For each language generate a merged json file. It begins with the master
+        // translation as a failsafe for untranslated strings, and merges all parent
+        // tags into one file for each specific subtag
+        //
+        // TODO: This is a naive interpretation of BCP47 that should be improved.
+        //       Will be OK for now as long as we don't have anything more complicated
+        //       than a base translation + region.
+        const tr = path.basename(file.history[0], ".json");
+        const subtags = tr.split("-");
+        const src = [workDir + "/translationMaster.json"];
+        for (let i = 1; i <= subtags.length; i++) {
+          const lang = subtags.slice(0, i).join("-");
+          if (lang === "test") {
+            src.push(workDir + "/test.json");
+          } else if (lang !== "en") {
+            src.push(inDir + "/" + lang + ".json");
           }
-          return gulp
-            .src(src, { allowEmpty: true })
-            .pipe(transform((data) => emptyFilter(data)))
-            .pipe(
-              merge({
-                fileName: tr + ".json",
-              })
-            )
-            .pipe(gulp.dest(fullDir));
-        })
-      );
-  })
-);
-tasks.push(taskName);
+        }
+        return gulp
+          .src(src, { allowEmpty: true })
+          .pipe(transform((data) => emptyFilter(data)))
+          .pipe(
+            merge({
+              fileName: tr + ".json",
+            })
+          )
+          .pipe(gulp.dest(fullDir));
+      })
+    );
+});
+
+var taskName;
 
 const splitTasks = [];
 TRANSLATION_FRAGMENTS.forEach((fragment) => {
   taskName = "build-translation-fragment-" + fragment;
-  gulp.task(
-    taskName,
-    gulp.series("build-merged-translations", function() {
-      // Return only the translations for this fragment.
-      return gulp
-        .src(fullDir + "/*.json")
-        .pipe(
-          transform((data) => ({
-            ui: {
-              panel: {
-                [fragment]: data.ui.panel[fragment],
-              },
+  gulp.task(taskName, function() {
+    // Return only the translations for this fragment.
+    return gulp
+      .src(fullDir + "/*.json")
+      .pipe(
+        transform((data) => ({
+          ui: {
+            panel: {
+              [fragment]: data.ui.panel[fragment],
             },
-          }))
-        )
-        .pipe(gulp.dest(workDir + "/" + fragment));
-    })
-  );
-  tasks.push(taskName);
+          },
+        }))
+      )
+      .pipe(gulp.dest(workDir + "/" + fragment));
+  });
   splitTasks.push(taskName);
 });
 
 taskName = "build-translation-core";
-gulp.task(
-  taskName,
-  gulp.series("build-merged-translations", function() {
-    // Remove the fragment translations from the core translation.
-    return gulp
-      .src(fullDir + "/*.json")
-      .pipe(
-        transform((data) => {
-          TRANSLATION_FRAGMENTS.forEach((fragment) => {
-            delete data.ui.panel[fragment];
-          });
-          return data;
-        })
-      )
-      .pipe(gulp.dest(coreDir));
-  })
-);
-tasks.push(taskName);
+gulp.task(taskName, function() {
+  // Remove the fragment translations from the core translation.
+  return gulp
+    .src(fullDir + "/*.json")
+    .pipe(
+      transform((data) => {
+        TRANSLATION_FRAGMENTS.forEach((fragment) => {
+          delete data.ui.panel[fragment];
+        });
+        return data;
+      })
+    )
+    .pipe(gulp.dest(coreDir));
+});
+
 splitTasks.push(taskName);
 
-taskName = "build-flattened-translations";
-gulp.task(
-  taskName,
-  gulp.series(...splitTasks, function() {
-    // Flatten the split versions of our translations, and move them into outDir
-    return gulp
-      .src(
-        TRANSLATION_FRAGMENTS.map(
-          (fragment) => workDir + "/" + fragment + "/*.json"
-        ).concat(coreDir + "/*.json"),
-        { base: workDir }
-      )
-      .pipe(
-        transform(function(data) {
-          // Polymer.AppLocalizeBehavior requires flattened json
-          return flatten(data);
-        })
-      )
-      .pipe(minify())
-      .pipe(hashFilename())
-      .pipe(
-        rename((filePath) => {
-          if (filePath.dirname === "core") {
-            filePath.dirname = "";
-          }
-        })
-      )
-      .pipe(gulp.dest(outDir));
-  })
-);
-tasks.push(taskName);
+gulp.task("build-flattened-translations", function() {
+  // Flatten the split versions of our translations, and move them into outDir
+  return gulp
+    .src(
+      TRANSLATION_FRAGMENTS.map(
+        (fragment) => workDir + "/" + fragment + "/*.json"
+      ).concat(coreDir + "/*.json"),
+      { base: workDir }
+    )
+    .pipe(
+      transform(function(data) {
+        // Polymer.AppLocalizeBehavior requires flattened json
+        return flatten(data);
+      })
+    )
+    .pipe(minify())
+    .pipe(
+      rename((filePath) => {
+        if (filePath.dirname === "core") {
+          filePath.dirname = "";
+        }
+      })
+    )
+    .pipe(gulp.dest(outDir));
+});
 
-taskName = "build-translation-fingerprints";
-gulp.task(
-  taskName,
-  gulp.series("build-flattened-translations", function() {
-    return gulp
-      .src(outDir + "/**/*.json")
-      .pipe(
-        rename({
-          extname: "",
-        })
-      )
-      .pipe(
-        hash({
-          algorithm: "md5",
-          hashLength: 32,
-          template: "<%= name %>.json",
-        })
-      )
-      .pipe(hash.manifest("translationFingerprints.json"))
-      .pipe(
-        transform(function(data) {
-          // After generating fingerprints of our translation files, consolidate
-          // all translation fragment fingerprints under the translation name key
-          const newData = {};
-          Object.entries(data).forEach(([key, value]) => {
-            const [path, _md5] = key.rsplit("-", 1);
-            // let translation = key;
-            let translation = path;
-            const parts = translation.split("/");
-            if (parts.length === 2) {
-              translation = parts[1];
-            }
-            if (!(translation in newData)) {
-              newData[translation] = {
-                fingerprints: {},
-              };
-            }
-            newData[translation].fingerprints[path] = value;
-          });
-          return newData;
-        })
-      )
-      .pipe(gulp.dest(workDir));
-  })
-);
-tasks.push(taskName);
+const fingerprints = {};
 
-taskName = "build-translations";
 gulp.task(
-  taskName,
-  gulp.series("build-translation-fingerprints", function() {
-    return gulp
-      .src(
-        [
-          "src/translations/translationMetadata.json",
-          workDir + "/testMetadata.json",
-          workDir + "/translationFingerprints.json",
-        ],
-        { allowEmpty: true }
-      )
-      .pipe(merge({}))
-      .pipe(
-        transform(function(data) {
-          const newData = {};
-          Object.entries(data).forEach(([key, value]) => {
-            // Filter out translations without native name.
-            if (data[key].nativeName) {
-              newData[key] = data[key];
-            } else {
-              console.warn(
-                `Skipping language ${key}. Native name was not translated.`
-              );
-            }
-            if (data[key]) newData[key] = value;
-          });
-          return newData;
-        })
-      )
-      .pipe(
-        transform((data) => ({
-          fragments: TRANSLATION_FRAGMENTS,
-          translations: data,
-        }))
-      )
-      .pipe(rename("translationMetadata.json"))
-      .pipe(gulp.dest(workDir));
-  })
-);
-tasks.push(taskName);
+  "build-translation-fingerprints",
+  function fingerprintTranslationFiles() {
+    // Fingerprint full file of each language
+    const files = fs.readdirSync(fullDir);
+    for (let i = 0; i < files.length; i++) {
+      fingerprints[files[i].split(".")[0]] = {
+        // In dev we create fake hashes
+        hash: env.isProdBuild
+          ? crypto
+              .createHash("md5")
+              .update(fs.readFileSync(path.join(fullDir, files[i]), "utf-8"))
+              .digest("hex")
+          : "dev",
+      };
+    }
 
-module.exports = tasks;
+    mapFiles(outDir, ".json", (filename) => {
+      const parsed = path.parse(filename);
+
+      // nl.json -> nl-<hash>.json
+      if (!(parsed.name in fingerprints)) {
+        throw new Error(`Unable to find hash for ${filename}`);
+      }
+
+      fs.renameSync(
+        filename,
+        `${parsed.dir}/${parsed.name}-${fingerprints[parsed.name].hash}${
+          parsed.ext
+        }`
+      );
+    });
+
+    const stream = source("translationFingerprints.json");
+    stream.write(JSON.stringify(fingerprints));
+    process.nextTick(() => stream.end());
+    return stream.pipe(vinylBuffer()).pipe(gulp.dest(workDir));
+  }
+);
+
+gulp.task(
+  "build-translations",
+  gulp.series(
+    "clean-translations",
+    "ensure-translations-build-dir",
+    env.isProdBuild ? (done) => done() : "create-test-translation",
+    "build-master-translation",
+    "build-merged-translations",
+    gulp.parallel(...splitTasks),
+    "build-flattened-translations",
+    "build-translation-fingerprints",
+    function writeMetadata() {
+      return gulp
+        .src(
+          [
+            "src/translations/translationMetadata.json",
+            workDir + "/testMetadata.json",
+            workDir + "/translationFingerprints.json",
+          ],
+          { allowEmpty: true }
+        )
+        .pipe(merge({}))
+        .pipe(
+          transform(function(data) {
+            const newData = {};
+            Object.entries(data).forEach(([key, value]) => {
+              // Filter out translations without native name.
+              if (data[key].nativeName) {
+                newData[key] = data[key];
+              } else {
+                console.warn(
+                  `Skipping language ${key}. Native name was not translated.`
+                );
+              }
+              if (data[key]) newData[key] = value;
+            });
+            return newData;
+          })
+        )
+        .pipe(
+          transform((data) => ({
+            fragments: TRANSLATION_FRAGMENTS,
+            translations: data,
+          }))
+        )
+        .pipe(rename("translationMetadata.json"))
+        .pipe(gulp.dest(workDir));
+    }
+  )
+);