From e6671299fee0676acceaee06f448cd52960f128c Mon Sep 17 00:00:00 2001
From: Bram Kragten <mail@bramkragten.nl>
Date: Tue, 10 Sep 2019 20:18:57 +0200
Subject: [PATCH] Filter html from translations (#3665)

* Filter html from translations

* Error when html found
---
 build-scripts/gulp/download_translations.js | 73 +++++++++++++++++++++
 package.json                                |  1 +
 script/translations_download                |  4 +-
 yarn.lock                                   |  5 ++
 4 files changed, 82 insertions(+), 1 deletion(-)
 create mode 100644 build-scripts/gulp/download_translations.js

diff --git a/build-scripts/gulp/download_translations.js b/build-scripts/gulp/download_translations.js
new file mode 100644
index 0000000000..56a369b87d
--- /dev/null
+++ b/build-scripts/gulp/download_translations.js
@@ -0,0 +1,73 @@
+const del = require("del");
+const gulp = require("gulp");
+const mapStream = require("map-stream");
+
+const inDir = "translations";
+const downloadDir = inDir + "/downloads";
+
+const tasks = [];
+
+function hasHtml(data) {
+  return /<[a-z][\s\S]*>/i.test(data);
+}
+
+function recursiveCheckHasHtml(file, data, errors, recKey) {
+  Object.keys(data).forEach(function(key) {
+    if (typeof data[key] === "object") {
+      nextRecKey = recKey ? `${recKey}.${key}` : key;
+      recursiveCheckHasHtml(file, data[key], errors, nextRecKey);
+    } else if (hasHtml(data[key])) {
+      errors.push(`HTML found in ${file.path} at key ${recKey}.${key}`);
+    }
+  });
+}
+
+function checkHtml() {
+  let errors = [];
+
+  return mapStream(function(file, cb) {
+    const content = file.contents;
+    let error;
+    if (content) {
+      if (hasHtml(String(content))) {
+        const data = JSON.parse(String(content));
+        recursiveCheckHasHtml(file, data, errors);
+        if (errors.length > 0) {
+          error = errors.join("\r\n");
+        }
+      }
+    }
+    cb(error, file);
+  });
+}
+
+let taskName = "clean-downloaded-translations";
+gulp.task(taskName, function() {
+  return del([`${downloadDir}/**`]);
+});
+tasks.push(taskName);
+
+taskName = "check-translations-html";
+gulp.task(taskName, function() {
+  return gulp.src(`${downloadDir}/*.json`).pipe(checkHtml());
+});
+tasks.push(taskName);
+
+taskName = "move-downloaded-translations";
+gulp.task(taskName, function() {
+  return gulp.src(`${downloadDir}/*.json`).pipe(gulp.dest(inDir));
+});
+tasks.push(taskName);
+
+taskName = "check-downloaded-translations";
+gulp.task(
+  taskName,
+  gulp.series(
+    "check-translations-html",
+    "move-downloaded-translations",
+    "clean-downloaded-translations"
+  )
+);
+tasks.push(taskName);
+
+module.exports = tasks;
diff --git a/package.json b/package.json
index 4c4c03e79e..e5a2b241d0 100644
--- a/package.json
+++ b/package.json
@@ -139,6 +139,7 @@
     "husky": "^1.3.1",
     "lint-staged": "^8.1.5",
     "lodash.template": "^4.5.0",
+    "map-stream": "^0.0.7",
     "merge-stream": "^1.0.1",
     "mocha": "^6.0.2",
     "parse5": "^5.1.0",
diff --git a/script/translations_download b/script/translations_download
index 46089668d0..dcf6aa999f 100755
--- a/script/translations_download
+++ b/script/translations_download
@@ -21,7 +21,7 @@ fi
 [ -z "${LOKALISE_TOKEN-}" ] && LOKALISE_TOKEN="$(<.lokalise_token)"
 
 PROJECT_ID="3420425759f6d6d241f598.13594006"
-LOCAL_DIR="$(pwd)/translations"
+LOCAL_DIR="$(pwd)/translations/downloads"
 FILE_FORMAT=json
 
 mkdir -p ${LOCAL_DIR}
@@ -35,3 +35,5 @@ docker run \
      --export_empty skip \
      --type json \
      --unzip_to /opt/dest
+
+./node_modules/.bin/gulp check-downloaded-translations
\ No newline at end of file
diff --git a/yarn.lock b/yarn.lock
index 6481a79e33..e56483761e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -9021,6 +9021,11 @@ map-obj@^1.0.0, map-obj@^1.0.1:
   resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d"
   integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=
 
+map-stream@^0.0.7:
+  version "0.0.7"
+  resolved "https://registry.yarnpkg.com/map-stream/-/map-stream-0.0.7.tgz#8a1f07896d82b10926bd3744a2420009f88974a8"
+  integrity sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=
+
 map-visit@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f"