Refactor translation merges to use native transform stream (#20666)

build-scripts/gulp/translations.js

@@ -1,8 +1,10 @@
+/* eslint-disable max-classes-per-file */
+
 import { deleteAsync } from "del";
 import { glob } from "glob";
 import gulp from "gulp";
-import merge from "gulp-merge-json";
 import rename from "gulp-rename";
+import merge from "lodash.merge";
 import { createHash } from "node:crypto";
 import { mkdir, readFile } from "node:fs/promises";
 import { basename, join } from "node:path";
@@ -54,6 +56,39 @@ class CustomJSON extends Transform {
   }
 }
 
+// Transform stream to merge Vinyl JSON files (buffer mode only).
+class MergeJSON extends Transform {
+  _objects = [];
+
+  constructor(stem, startObj = {}, reviver = null) {
+    super({ objectMode: true, allowHalfOpen: false });
+    this._stem = stem;
+    this._startObj = structuredClone(startObj);
+    this._reviver = reviver;
+  }
+
+  async _transform(file, _, callback) {
+    try {
+      this._objects.push(JSON.parse(file.contents.toString(), this._reviver));
+      if (!this._outFile) this._outFile = file.clone({ contents: false });
+      callback(null);
+    } catch (err) {
+      callback(err);
+    }
+  }
+
+  async _flush(callback) {
+    try {
+      const mergedObj = merge(this._startObj, ...this._objects);
+      this._outFile.contents = Buffer.from(JSON.stringify(mergedObj));
+      this._outFile.stem = this._stem;
+      callback(null, this._outFile);
+    } catch (err) {
+      callback(err);
+    }
+  }
+}
+
 // Utility to flatten object keys to single level using separator
 const flatten = (data, prefix = "", sep = ".") => {
   const output = {};
@@ -131,12 +166,7 @@ const createMasterTranslation = () =>
   gulp
     .src([EN_SRC, ...(mergeBackend ? [`${inBackendDir}/en.json`] : [])])
     .pipe(new CustomJSON(lokaliseTransform))
-    .pipe(
+    .pipe(new MergeJSON("en"))
-      merge({
-        fileName: "en.json",
-        jsonSpace: undefined,
-      })
-    )
     .pipe(gulp.dest(workDir));
 
 const FRAGMENTS = ["base"];
@@ -233,14 +263,9 @@ const createTranslations = async () => {
         }
       }
     }
-    const mergeStream = gulp.src(mergeFiles, { allowEmpty: true }).pipe(
+    const mergeStream = gulp
-      merge({
+      .src(mergeFiles, { allowEmpty: true })
-        fileName: `${locale}.json`,
+      .pipe(new MergeJSON(locale, enMaster, emptyReviver));
-        startObj: enMaster,
-        jsonReviver: emptyReviver,
-        jsonSpace: undefined,
-      })
-    );
     mergesFinished.push(finished(mergeStream));
     mergeStream.pipe(hashStream, { end: false });
   }

package.json

@@ -177,6 +177,7 @@
     "@types/js-yaml": "4.0.9",
     "@types/leaflet": "1.9.12",
     "@types/leaflet-draw": "1.0.11",
+    "@types/lodash.merge": "4.6.9",
     "@types/luxon": "3.4.2",
     "@types/mocha": "10.0.6",
     "@types/qrcode": "1.5.5",
@@ -209,7 +210,6 @@
     "glob": "10.3.12",
     "gulp": "5.0.0",
     "gulp-json-transform": "0.5.0",
-    "gulp-merge-json": "2.2.1",
     "gulp-rename": "2.0.0",
     "gulp-zopfli-green": "6.0.1",
     "html-minifier-terser": "7.2.0",
@@ -218,6 +218,7 @@
     "jszip": "3.10.1",
     "lint-staged": "15.2.2",
     "lit-analyzer": "2.0.3",
+    "lodash.merge": "4.6.2",
     "lodash.template": "4.5.0",
     "magic-string": "0.30.10",
     "map-stream": "0.0.7",

yarn.lock

@@ -4336,6 +4336,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/lodash.merge@npm:4.6.9":
+  version: 4.6.9
+  resolution: "@types/lodash.merge@npm:4.6.9"
+  dependencies:
+    "@types/lodash": "npm:*"
+  checksum: 10/d0dd6654547c9d8d905184d14aa5c2a37a1ed1c3204f5ab20b7d591a05f34859ef09d3b72c065e94ca1989abf9109eb8230f67c4d64a5768b1d65b9ed8baf8e7
+  languageName: node
+  linkType: hard
+
 "@types/lodash@npm:*":
   version: 4.17.0
   resolution: "@types/lodash@npm:4.17.0"
@@ -8779,20 +8788,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"gulp-merge-json@npm:2.2.1":
-  version: 2.2.1
-  resolution: "gulp-merge-json@npm:2.2.1"
-  dependencies:
-    json5: "npm:^2.2.3"
-    lodash.clonedeep: "npm:^4.5.0"
-    lodash.mergewith: "npm:^4.6.1"
-    plugin-error: "npm:^2.0.1"
-    through: "npm:^2.3.8"
-    vinyl: "npm:^3.0.0"
-  checksum: 10/fe87fd2101d0a35f2c5d488d546affaa6115d3be376fcf07b56029eed22ccfa8811206ac8bdf13ff79e5f518d0a65d886506204d2bdc3c0f871dc7cfa82c75eb
-  languageName: node
-  linkType: hard
-
 "gulp-rename@npm:2.0.0":
   version: 2.0.0
   resolution: "gulp-rename@npm:2.0.0"
@@ -9025,6 +9020,7 @@ __metadata:
     "@types/js-yaml": "npm:4.0.9"
     "@types/leaflet": "npm:1.9.12"
     "@types/leaflet-draw": "npm:1.0.11"
+    "@types/lodash.merge": "npm:4.6.9"
     "@types/luxon": "npm:3.4.2"
     "@types/mocha": "npm:10.0.6"
     "@types/qrcode": "npm:1.5.5"
@@ -9078,7 +9074,6 @@ __metadata:
     google-timezones-json: "npm:1.2.0"
     gulp: "npm:5.0.0"
     gulp-json-transform: "npm:0.5.0"
-    gulp-merge-json: "npm:2.2.1"
     gulp-rename: "npm:2.0.0"
     gulp-zopfli-green: "npm:6.0.1"
     hls.js: "patch:hls.js@npm%3A1.5.7#~/.yarn/patches/hls.js-npm-1.5.7-f5bbd3d060.patch"
@@ -9095,6 +9090,7 @@ __metadata:
     lint-staged: "npm:15.2.2"
     lit: "npm:2.8.0"
     lit-analyzer: "npm:2.0.3"
+    lodash.merge: "npm:4.6.2"
     lodash.template: "npm:4.5.0"
     luxon: "npm:3.4.4"
     magic-string: "npm:0.30.10"
@@ -10665,13 +10661,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lodash.clonedeep@npm:^4.5.0":
-  version: 4.5.0
-  resolution: "lodash.clonedeep@npm:4.5.0"
-  checksum: 10/957ed243f84ba6791d4992d5c222ffffca339a3b79dbe81d2eaf0c90504160b500641c5a0f56e27630030b18b8e971ea10b44f928a977d5ced3c8948841b555f
-  languageName: node
-  linkType: hard
-
 "lodash.debounce@npm:^4.0.8":
   version: 4.0.8
   resolution: "lodash.debounce@npm:4.0.8"
@@ -10693,20 +10682,13 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lodash.merge@npm:^4.6.2":
+"lodash.merge@npm:4.6.2, lodash.merge@npm:^4.6.2":
   version: 4.6.2
   resolution: "lodash.merge@npm:4.6.2"
   checksum: 10/d0ea2dd0097e6201be083865d50c3fb54fbfbdb247d9cc5950e086c991f448b7ab0cdab0d57eacccb43473d3f2acd21e134db39f22dac2d6c9ba6bf26978e3d6
   languageName: node
   linkType: hard
 
-"lodash.mergewith@npm:^4.6.1":
-  version: 4.6.2
-  resolution: "lodash.mergewith@npm:4.6.2"
-  checksum: 10/aea75a4492541a4902ac7e551dc6c54b722da0c187f84385d02e8fc33a7ae3454b837822446e5f63fcd5ad1671534ea408740b776670ea4d9c7890b10105fce0
-  languageName: node
-  linkType: hard
-
 "lodash.sortby@npm:^4.7.0":
   version: 4.7.0
   resolution: "lodash.sortby@npm:4.7.0"
@@ -12076,15 +12058,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"plugin-error@npm:^2.0.1":
-  version: 2.0.1
-  resolution: "plugin-error@npm:2.0.1"
-  dependencies:
-    ansi-colors: "npm:^1.0.1"
-  checksum: 10/9a4f91461cd24cce401112098969991d7aa6b4c94f78e0381234280c07da779570a8b21ab143292b534ec0117c09705a67e5d756c1c303d4706fdd7f861bf5bc
-  languageName: node
-  linkType: hard
-
 "pngjs@npm:^3.0.0, pngjs@npm:^3.3.3":
   version: 3.4.0
   resolution: "pngjs@npm:3.4.0"
@@ -13870,13 +13843,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"through@npm:^2.3.8":
-  version: 2.3.8
-  resolution: "through@npm:2.3.8"
-  checksum: 10/5da78346f70139a7d213b65a0106f3c398d6bc5301f9248b5275f420abc2c4b1e77c2abc72d218dedc28c41efb2e7c312cb76a7730d04f9c2d37d247da3f4198
-  languageName: node
-  linkType: hard
-
 "thunky@npm:^1.0.2":
   version: 1.1.0
   resolution: "thunky@npm:1.1.0"