From cdf64ccdaa8c98cce21d32ae7b20e82231f6d5fe Mon Sep 17 00:00:00 2001 From: Steve Repsher Date: Mon, 6 May 2024 09:17:01 -0400 Subject: [PATCH] Refactor translation merges to use native transform stream (#20666) --- build-scripts/gulp/translations.js | 55 ++++++++++++++++++++-------- package.json | 3 +- yarn.lock | 58 +++++++----------------------- 3 files changed, 54 insertions(+), 62 deletions(-) diff --git a/build-scripts/gulp/translations.js b/build-scripts/gulp/translations.js index 6c267a0587..e1abf487ac 100755 --- a/build-scripts/gulp/translations.js +++ b/build-scripts/gulp/translations.js @@ -1,8 +1,10 @@ +/* eslint-disable max-classes-per-file */ + import { deleteAsync } from "del"; import { glob } from "glob"; import gulp from "gulp"; -import merge from "gulp-merge-json"; import rename from "gulp-rename"; +import merge from "lodash.merge"; import { createHash } from "node:crypto"; import { mkdir, readFile } from "node:fs/promises"; import { basename, join } from "node:path"; @@ -54,6 +56,39 @@ class CustomJSON extends Transform { } } +// Transform stream to merge Vinyl JSON files (buffer mode only). +class MergeJSON extends Transform { + _objects = []; + + constructor(stem, startObj = {}, reviver = null) { + super({ objectMode: true, allowHalfOpen: false }); + this._stem = stem; + this._startObj = structuredClone(startObj); + this._reviver = reviver; + } + + async _transform(file, _, callback) { + try { + this._objects.push(JSON.parse(file.contents.toString(), this._reviver)); + if (!this._outFile) this._outFile = file.clone({ contents: false }); + callback(null); + } catch (err) { + callback(err); + } + } + + async _flush(callback) { + try { + const mergedObj = merge(this._startObj, ...this._objects); + this._outFile.contents = Buffer.from(JSON.stringify(mergedObj)); + this._outFile.stem = this._stem; + callback(null, this._outFile); + } catch (err) { + callback(err); + } + } +} + // Utility to flatten object keys to single level using separator const flatten = (data, prefix = "", sep = ".") => { const output = {}; @@ -131,12 +166,7 @@ const createMasterTranslation = () => gulp .src([EN_SRC, ...(mergeBackend ? [`${inBackendDir}/en.json`] : [])]) .pipe(new CustomJSON(lokaliseTransform)) - .pipe( - merge({ - fileName: "en.json", - jsonSpace: undefined, - }) - ) + .pipe(new MergeJSON("en")) .pipe(gulp.dest(workDir)); const FRAGMENTS = ["base"]; @@ -233,14 +263,9 @@ const createTranslations = async () => { } } } - const mergeStream = gulp.src(mergeFiles, { allowEmpty: true }).pipe( - merge({ - fileName: `${locale}.json`, - startObj: enMaster, - jsonReviver: emptyReviver, - jsonSpace: undefined, - }) - ); + const mergeStream = gulp + .src(mergeFiles, { allowEmpty: true }) + .pipe(new MergeJSON(locale, enMaster, emptyReviver)); mergesFinished.push(finished(mergeStream)); mergeStream.pipe(hashStream, { end: false }); } diff --git a/package.json b/package.json index 5900967e2e..cc91d854b6 100644 --- a/package.json +++ b/package.json @@ -177,6 +177,7 @@ "@types/js-yaml": "4.0.9", "@types/leaflet": "1.9.12", "@types/leaflet-draw": "1.0.11", + "@types/lodash.merge": "4.6.9", "@types/luxon": "3.4.2", "@types/mocha": "10.0.6", "@types/qrcode": "1.5.5", @@ -209,7 +210,6 @@ "glob": "10.3.12", "gulp": "5.0.0", "gulp-json-transform": "0.5.0", - "gulp-merge-json": "2.2.1", "gulp-rename": "2.0.0", "gulp-zopfli-green": "6.0.1", "html-minifier-terser": "7.2.0", @@ -218,6 +218,7 @@ "jszip": "3.10.1", "lint-staged": "15.2.2", "lit-analyzer": "2.0.3", + "lodash.merge": "4.6.2", "lodash.template": "4.5.0", "magic-string": "0.30.10", "map-stream": "0.0.7", diff --git a/yarn.lock b/yarn.lock index b4c7e8f769..cd77754e5e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4336,6 +4336,15 @@ __metadata: languageName: node linkType: hard +"@types/lodash.merge@npm:4.6.9": + version: 4.6.9 + resolution: "@types/lodash.merge@npm:4.6.9" + dependencies: + "@types/lodash": "npm:*" + checksum: 10/d0dd6654547c9d8d905184d14aa5c2a37a1ed1c3204f5ab20b7d591a05f34859ef09d3b72c065e94ca1989abf9109eb8230f67c4d64a5768b1d65b9ed8baf8e7 + languageName: node + linkType: hard + "@types/lodash@npm:*": version: 4.17.0 resolution: "@types/lodash@npm:4.17.0" @@ -8779,20 +8788,6 @@ __metadata: languageName: node linkType: hard -"gulp-merge-json@npm:2.2.1": - version: 2.2.1 - resolution: "gulp-merge-json@npm:2.2.1" - dependencies: - json5: "npm:^2.2.3" - lodash.clonedeep: "npm:^4.5.0" - lodash.mergewith: "npm:^4.6.1" - plugin-error: "npm:^2.0.1" - through: "npm:^2.3.8" - vinyl: "npm:^3.0.0" - checksum: 10/fe87fd2101d0a35f2c5d488d546affaa6115d3be376fcf07b56029eed22ccfa8811206ac8bdf13ff79e5f518d0a65d886506204d2bdc3c0f871dc7cfa82c75eb - languageName: node - linkType: hard - "gulp-rename@npm:2.0.0": version: 2.0.0 resolution: "gulp-rename@npm:2.0.0" @@ -9025,6 +9020,7 @@ __metadata: "@types/js-yaml": "npm:4.0.9" "@types/leaflet": "npm:1.9.12" "@types/leaflet-draw": "npm:1.0.11" + "@types/lodash.merge": "npm:4.6.9" "@types/luxon": "npm:3.4.2" "@types/mocha": "npm:10.0.6" "@types/qrcode": "npm:1.5.5" @@ -9078,7 +9074,6 @@ __metadata: google-timezones-json: "npm:1.2.0" gulp: "npm:5.0.0" gulp-json-transform: "npm:0.5.0" - gulp-merge-json: "npm:2.2.1" gulp-rename: "npm:2.0.0" gulp-zopfli-green: "npm:6.0.1" hls.js: "patch:hls.js@npm%3A1.5.7#~/.yarn/patches/hls.js-npm-1.5.7-f5bbd3d060.patch" @@ -9095,6 +9090,7 @@ __metadata: lint-staged: "npm:15.2.2" lit: "npm:2.8.0" lit-analyzer: "npm:2.0.3" + lodash.merge: "npm:4.6.2" lodash.template: "npm:4.5.0" luxon: "npm:3.4.4" magic-string: "npm:0.30.10" @@ -10665,13 +10661,6 @@ __metadata: languageName: node linkType: hard -"lodash.clonedeep@npm:^4.5.0": - version: 4.5.0 - resolution: "lodash.clonedeep@npm:4.5.0" - checksum: 10/957ed243f84ba6791d4992d5c222ffffca339a3b79dbe81d2eaf0c90504160b500641c5a0f56e27630030b18b8e971ea10b44f928a977d5ced3c8948841b555f - languageName: node - linkType: hard - "lodash.debounce@npm:^4.0.8": version: 4.0.8 resolution: "lodash.debounce@npm:4.0.8" @@ -10693,20 +10682,13 @@ __metadata: languageName: node linkType: hard -"lodash.merge@npm:^4.6.2": +"lodash.merge@npm:4.6.2, lodash.merge@npm:^4.6.2": version: 4.6.2 resolution: "lodash.merge@npm:4.6.2" checksum: 10/d0ea2dd0097e6201be083865d50c3fb54fbfbdb247d9cc5950e086c991f448b7ab0cdab0d57eacccb43473d3f2acd21e134db39f22dac2d6c9ba6bf26978e3d6 languageName: node linkType: hard -"lodash.mergewith@npm:^4.6.1": - version: 4.6.2 - resolution: "lodash.mergewith@npm:4.6.2" - checksum: 10/aea75a4492541a4902ac7e551dc6c54b722da0c187f84385d02e8fc33a7ae3454b837822446e5f63fcd5ad1671534ea408740b776670ea4d9c7890b10105fce0 - languageName: node - linkType: hard - "lodash.sortby@npm:^4.7.0": version: 4.7.0 resolution: "lodash.sortby@npm:4.7.0" @@ -12076,15 +12058,6 @@ __metadata: languageName: node linkType: hard -"plugin-error@npm:^2.0.1": - version: 2.0.1 - resolution: "plugin-error@npm:2.0.1" - dependencies: - ansi-colors: "npm:^1.0.1" - checksum: 10/9a4f91461cd24cce401112098969991d7aa6b4c94f78e0381234280c07da779570a8b21ab143292b534ec0117c09705a67e5d756c1c303d4706fdd7f861bf5bc - languageName: node - linkType: hard - "pngjs@npm:^3.0.0, pngjs@npm:^3.3.3": version: 3.4.0 resolution: "pngjs@npm:3.4.0" @@ -13870,13 +13843,6 @@ __metadata: languageName: node linkType: hard -"through@npm:^2.3.8": - version: 2.3.8 - resolution: "through@npm:2.3.8" - checksum: 10/5da78346f70139a7d213b65a0106f3c398d6bc5301f9248b5275f420abc2c4b1e77c2abc72d218dedc28c41efb2e7c312cb76a7730d04f9c2d37d247da3f4198 - languageName: node - linkType: hard - "thunky@npm:^1.0.2": version: 1.1.0 resolution: "thunky@npm:1.1.0"