frontend/build-scripts/gulp/translations.js

import { createHash } from "crypto";
import { deleteSync } from "del";
import {
  mkdirSync,
  readdirSync,
  readFileSync,
  renameSync,
  writeFile,
} from "fs";
import gulp from "gulp";
import flatmap from "gulp-flatmap";
import transform from "gulp-json-transform";
import merge from "gulp-merge-json";
import rename from "gulp-rename";
import path from "path";
import vinylBuffer from "vinyl-buffer";
import source from "vinyl-source-stream";
import env from "../env.cjs";
import paths from "../paths.cjs";
import { mapFiles } from "../util.cjs";
import "./fetch-nightly-translations.js";

const inFrontendDir = "translations/frontend";
const inBackendDir = "translations/backend";
const workDir = "build/translations";
const fullDir = workDir + "/full";
const coreDir = workDir + "/core";
const outDir = workDir + "/output";
let mergeBackend = false;

gulp.task(
  "translations-enable-merge-backend",
  gulp.parallel((done) => {
    mergeBackend = true;
    done();
  }, "allow-setup-fetch-nightly-translations")
);

// Panel translations which should be split from the core translations.
const TRANSLATION_FRAGMENTS = Object.keys(
  JSON.parse(
    readFileSync(
      path.resolve(paths.polymer_dir, "src/translations/en.json"),
      "utf-8"
    )
  ).ui.panel
);

function recursiveFlatten(prefix, data) {
  let output = {};
  Object.keys(data).forEach((key) => {
    if (typeof data[key] === "object") {
      output = {
        ...output,
        ...recursiveFlatten(prefix + key + ".", data[key]),
      };
    } else {
      output[prefix + key] = data[key];
    }
  });
  return output;
}

function flatten(data) {
  return recursiveFlatten("", data);
}

function emptyFilter(data) {
  const newData = {};
  Object.keys(data).forEach((key) => {
    if (data[key]) {
      if (typeof data[key] === "object") {
        newData[key] = emptyFilter(data[key]);
      } else {
        newData[key] = data[key];
      }
    }
  });
  return newData;
}

function recursiveEmpty(data) {
  const newData = {};
  Object.keys(data).forEach((key) => {
    if (data[key]) {
      if (typeof data[key] === "object") {
        newData[key] = recursiveEmpty(data[key]);
      } else {
        newData[key] = "TRANSLATED";
      }
    }
  });
  return newData;
}

/**
 * Replace Lokalise key placeholders with their actual values.
 *
 * We duplicate the behavior of Lokalise here so that placeholders can
 * be included in src/translations/en.json, but still be usable while
 * developing locally.
 *
 * @link https://docs.lokalise.co/article/KO5SZWLLsy-key-referencing
 */
const re_key_reference = /\[%key:([^%]+)%\]/;
function lokaliseTransform(data, original, file) {
  const output = {};
  Object.entries(data).forEach(([key, value]) => {
    if (value instanceof Object) {
      output[key] = lokaliseTransform(value, original, file);
    } else {
      output[key] = value.replace(re_key_reference, (_match, lokalise_key) => {
        const replace = lokalise_key.split("::").reduce((tr, k) => {
          if (!tr) {
            throw Error(
              `Invalid key placeholder ${lokalise_key} in ${file.path}`
            );
          }
          return tr[k];
        }, original);
        if (typeof replace !== "string") {
          throw Error(
            `Invalid key placeholder ${lokalise_key} in ${file.path}`
          );
        }
        return replace;
      });
    }
  });
  return output;
}

gulp.task("clean-translations", async () => deleteSync([workDir]));

gulp.task("ensure-translations-build-dir", async () => {
  mkdirSync(workDir, { recursive: true });
});

gulp.task("create-test-metadata", (cb) => {
  writeFile(
    workDir + "/testMetadata.json",
    JSON.stringify({
      test: {
        nativeName: "Test",
      },
    }),
    cb
  );
});

gulp.task(
  "create-test-translation",
  gulp.series("create-test-metadata", () =>
    gulp
      .src(path.join(paths.translations_src, "en.json"))
      .pipe(transform((data, _file) => recursiveEmpty(data)))
      .pipe(rename("test.json"))
      .pipe(gulp.dest(workDir))
  )
);

/**
 * This task will build a master translation file, to be used as the base for
 * all languages. This starts with src/translations/en.json, and replaces all
 * Lokalise key placeholders with their target values. Under normal circumstances,
 * this will be the same as translations/en.json However, we build it here to
 * facilitate both making changes in development mode, and to ensure that the
 * project is buildable immediately after merging new translation keys, since
 * the Lokalise update to translations/en.json will not happen immediately.
 */
gulp.task("build-master-translation", () => {
  const src = [path.join(paths.translations_src, "en.json")];

  if (mergeBackend) {
    src.push(path.join(inBackendDir, "en.json"));
  }

  return gulp
    .src(src)
    .pipe(transform((data, file) => lokaliseTransform(data, data, file)))
    .pipe(
      merge({
        fileName: "en.json",
      })
    )
    .pipe(gulp.dest(fullDir));
});

gulp.task("build-merged-translations", () =>
  gulp
    .src(
      [
        inFrontendDir + "/*.json",
        "!" + inFrontendDir + "/en.json",
        workDir + "/test.json",
      ],
      {
        allowEmpty: true,
      }
    )
    .pipe(transform((data, file) => lokaliseTransform(data, data, file)))
    .pipe(
      flatmap((stream, file) => {
        // For each language generate a merged json file. It begins with the master
        // translation as a failsafe for untranslated strings, and merges all parent
        // tags into one file for each specific subtag
        //
        // TODO: This is a naive interpretation of BCP47 that should be improved.
        //       Will be OK for now as long as we don't have anything more complicated
        //       than a base translation + region.
        const tr = path.basename(file.history[0], ".json");
        const subtags = tr.split("-");
        const src = [fullDir + "/en.json"];
        for (let i = 1; i <= subtags.length; i++) {
          const lang = subtags.slice(0, i).join("-");
          if (lang === "test") {
            src.push(workDir + "/test.json");
          } else if (lang !== "en") {
            src.push(inFrontendDir + "/" + lang + ".json");
            if (mergeBackend) {
              src.push(inBackendDir + "/" + lang + ".json");
            }
          }
        }
        return gulp
          .src(src, { allowEmpty: true })
          .pipe(transform((data) => emptyFilter(data)))
          .pipe(
            merge({
              fileName: tr + ".json",
            })
          )
          .pipe(gulp.dest(fullDir));
      })
    )
);

let taskName;

const splitTasks = [];
TRANSLATION_FRAGMENTS.forEach((fragment) => {
  taskName = "build-translation-fragment-" + fragment;
  gulp.task(taskName, () =>
    // Return only the translations for this fragment.
    gulp
      .src(fullDir + "/*.json")
      .pipe(
        transform((data) => ({
          ui: {
            panel: {
              [fragment]: data.ui.panel[fragment],
            },
          },
        }))
      )
      .pipe(gulp.dest(workDir + "/" + fragment))
  );
  splitTasks.push(taskName);
});

taskName = "build-translation-core";
gulp.task(taskName, () =>
  // Remove the fragment translations from the core translation.
  gulp
    .src(fullDir + "/*.json")
    .pipe(
      transform((data, _file) => {
        TRANSLATION_FRAGMENTS.forEach((fragment) => {
          delete data.ui.panel[fragment];
        });
        delete data.supervisor;
        return data;
      })
    )
    .pipe(gulp.dest(coreDir))
);

splitTasks.push(taskName);

gulp.task("build-flattened-translations", () =>
  // Flatten the split versions of our translations, and move them into outDir
  gulp
    .src(
      TRANSLATION_FRAGMENTS.map(
        (fragment) => workDir + "/" + fragment + "/*.json"
      ).concat(coreDir + "/*.json"),
      { base: workDir }
    )
    .pipe(
      transform((data) =>
        // Polymer.AppLocalizeBehavior requires flattened json
        flatten(data)
      )
    )
    .pipe(
      rename((filePath) => {
        if (filePath.dirname === "core") {
          filePath.dirname = "";
        }
        // In dev we create the file with the fake hash in the filename
        if (!env.isProdBuild()) {
          filePath.basename += "-dev";
        }
      })
    )
    .pipe(gulp.dest(outDir))
);

const fingerprints = {};

gulp.task("build-translation-fingerprints", () => {
  // Fingerprint full file of each language
  const files = readdirSync(fullDir);

  for (let i = 0; i < files.length; i++) {
    fingerprints[files[i].split(".")[0]] = {
      // In dev we create fake hashes
      hash: env.isProdBuild()
        ? createHash("md5")
            .update(readFileSync(path.join(fullDir, files[i]), "utf-8"))
            .digest("hex")
        : "dev",
    };
  }

  // In dev we create the file with the fake hash in the filename
  if (env.isProdBuild()) {
    mapFiles(outDir, ".json", (filename) => {
      const parsed = path.parse(filename);

      // nl.json -> nl-<hash>.json
      if (!(parsed.name in fingerprints)) {
        throw new Error(`Unable to find hash for ${filename}`);
      }

      renameSync(
        filename,
        `${parsed.dir}/${parsed.name}-${fingerprints[parsed.name].hash}${
          parsed.ext
        }`
      );
    });
  }

  const stream = source("translationFingerprints.json");
  stream.write(JSON.stringify(fingerprints));
  process.nextTick(() => stream.end());
  return stream.pipe(vinylBuffer()).pipe(gulp.dest(workDir));
});

gulp.task("build-translation-fragment-supervisor", () =>
  gulp
    .src(fullDir + "/*.json")
    .pipe(transform((data) => data.supervisor))
    .pipe(
      rename((filePath) => {
        // In dev we create the file with the fake hash in the filename
        if (!env.isProdBuild()) {
          filePath.basename += "-dev";
        }
      })
    )
    .pipe(gulp.dest(workDir + "/supervisor"))
);

gulp.task("build-translation-flatten-supervisor", () =>
  gulp
    .src(workDir + "/supervisor/*.json")
    .pipe(
      transform((data) =>
        // Polymer.AppLocalizeBehavior requires flattened json
        flatten(data)
      )
    )
    .pipe(gulp.dest(outDir))
);

gulp.task("build-translation-write-metadata", () =>
  gulp
    .src(
      [
        path.join(paths.translations_src, "translationMetadata.json"),
        workDir + "/testMetadata.json",
        workDir + "/translationFingerprints.json",
      ],
      { allowEmpty: true }
    )
    .pipe(merge({}))
    .pipe(
      transform((data) => {
        const newData = {};
        Object.entries(data).forEach(([key, value]) => {
          // Filter out translations without native name.
          if (value.nativeName) {
            newData[key] = value;
          } else {
            console.warn(
              `Skipping language ${key}. Native name was not translated.`
            );
          }
        });
        return newData;
      })
    )
    .pipe(
      transform((data) => ({
        fragments: TRANSLATION_FRAGMENTS,
        translations: data,
      }))
    )
    .pipe(rename("translationMetadata.json"))
    .pipe(gulp.dest(workDir))
);

gulp.task(
  "create-translations",
  gulp.series(
    env.isProdBuild() ? (done) => done() : "create-test-translation",
    "build-master-translation",
    "build-merged-translations",
    gulp.parallel(...splitTasks),
    "build-flattened-translations"
  )
);

gulp.task(
  "build-translations",
  gulp.series(
    gulp.parallel(
      "fetch-nightly-translations",
      gulp.series("clean-translations", "ensure-translations-build-dir")
    ),
    "create-translations",
    "build-translation-fingerprints",
    "build-translation-write-metadata"
  )
);

gulp.task(
  "build-supervisor-translations",
  gulp.series(
    gulp.parallel(
      "fetch-nightly-translations",
      gulp.series("clean-translations", "ensure-translations-build-dir")
    ),
    "build-master-translation",
    "build-merged-translations",
    "build-translation-fragment-supervisor",
    "build-translation-flatten-supervisor",
    "build-translation-fingerprints",
    "build-translation-write-metadata"
  )
);