Gulpify build pipeline (#3145)

* Gulpify build pipeline

* Update build frontend script

* Fixes

* Limit service worker to latest build

* Use shorthand

* Fix hassio build

build-scripts/.eslintrc

@@ -0,0 +1,7 @@
+{
+  "rules": {
+    "import/no-extraneous-dependencies": 0,
+    "no-restricted-syntax": 0,
+    "no-console": 0
+  }
+}

build-scripts/.eslintrc.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../.eslintrc.json",
+  "rules": {
+    "import/no-extraneous-dependencies": 0,
+    "global-require": 0
+  }
+}

build-scripts/babel.js

@@ -0,0 +1,48 @@
+module.exports.babelLoaderConfig = ({ latestBuild }) => {
+  if (latestBuild === undefined) {
+    throw Error("latestBuild not defined for babel loader config");
+  }
+  return {
+    test: /\.m?js$|\.tsx?$/,
+    use: {
+      loader: "babel-loader",
+      options: {
+        presets: [
+          !latestBuild && [
+            require("@babel/preset-env").default,
+            { modules: false },
+          ],
+          [
+            require("@babel/preset-typescript").default,
+            {
+              jsxPragma: "h",
+            },
+          ],
+        ].filter(Boolean),
+        plugins: [
+          // Part of ES2018. Converts {...a, b: 2} to Object.assign({}, a, {b: 2})
+          [
+            "@babel/plugin-proposal-object-rest-spread",
+            { loose: true, useBuiltIns: true },
+          ],
+          // Only support the syntax, Webpack will handle it.
+          "@babel/syntax-dynamic-import",
+          [
+            "@babel/transform-react-jsx",
+            {
+              pragma: "h",
+            },
+          ],
+          [
+            require("@babel/plugin-proposal-decorators").default,
+            { decoratorsBeforeExport: true },
+          ],
+          [
+            require("@babel/plugin-proposal-class-properties").default,
+            { loose: true },
+          ],
+        ],
+      },
+    },
+  };
+};

build-scripts/gulp/clean.js

@@ -0,0 +1,5 @@
+const del = require("del");
+const gulp = require("gulp");
+const config = require("../paths");
+
+gulp.task("clean", () => del([config.root, config.build_dir]));

build-scripts/gulp/develop.js

@@ -0,0 +1,29 @@
+// Run HA develop mode
+const gulp = require("gulp");
+
+require("./clean.js");
+require("./translations.js");
+require("./gen-icons.js");
+require("./gather-static.js");
+require("./webpack.js");
+require("./service-worker.js");
+require("./entry-html.js");
+
+gulp.task(
+  "develop",
+  gulp.series(
+    async function setEnv() {
+      process.env.NODE_ENV = "development";
+    },
+    "clean",
+    gulp.parallel(
+      "copy-static",
+      "gen-service-worker-dev",
+      "gen-icons",
+      "gen-pages-dev",
+      "gen-index-html-dev",
+      gulp.series("build-translations", "copy-translations")
+    ),
+    "webpack-watch"
+  )
+);

build-scripts/gulp/entry-html.js

@@ -0,0 +1,108 @@
+// Tasks to generate entry HTML
+/* eslint-disable import/no-dynamic-require */
+/* eslint-disable global-require */
+const gulp = require("gulp");
+const fs = require("fs-extra");
+const path = require("path");
+const template = require("lodash.template");
+const minify = require("html-minifier").minify;
+const config = require("../paths.js");
+
+const templatePath = (tpl) =>
+  path.resolve(config.polymer_dir, "src/html/", `${tpl}.html.template`);
+
+const readFile = (pth) => fs.readFileSync(pth).toString();
+
+const renderTemplate = (pth, data = {}) => {
+  const compiled = template(readFile(templatePath(pth)));
+  return compiled({ ...data, renderTemplate });
+};
+
+const minifyHtml = (content) =>
+  minify(content, {
+    collapseWhitespace: true,
+    minifyJS: true,
+    minifyCSS: true,
+    removeComments: true,
+  });
+
+const PAGES = ["onboarding", "authorize"];
+
+gulp.task("gen-pages-dev", (done) => {
+  for (const page of PAGES) {
+    const content = renderTemplate(page, {
+      latestPageJS: `/frontend_latest/${page}.js`,
+      latestHassIconsJS: "/frontend_latest/hass-icons.js",
+
+      es5Compatibility: "/frontend_es5/compatibility.js",
+      es5PageJS: `/frontend_es5/${page}.js`,
+      es5HassIconsJS: "/frontend_es5/hass-icons.js",
+    });
+
+    fs.outputFileSync(path.resolve(config.root, `${page}.html`), content);
+  }
+  done();
+});
+
+gulp.task("gen-pages-prod", (done) => {
+  const latestManifest = require(path.resolve(config.output, "manifest.json"));
+  const es5Manifest = require(path.resolve(config.output_es5, "manifest.json"));
+
+  for (const page of PAGES) {
+    const content = renderTemplate(page, {
+      latestPageJS: latestManifest[`${page}.js`],
+      latestHassIconsJS: latestManifest["hass-icons.js"],
+
+      es5Compatibility: es5Manifest["compatibility.js"],
+      es5PageJS: es5Manifest[`${page}.js`],
+      es5HassIconsJS: es5Manifest["hass-icons.js"],
+    });
+
+    fs.outputFileSync(
+      path.resolve(config.root, `${page}.html`),
+      minifyHtml(content)
+    );
+  }
+  done();
+});
+
+gulp.task("gen-index-html-dev", (done) => {
+  // In dev mode we don't mangle names, so we hardcode urls. That way we can
+  // run webpack as last in watch mode, which blocks output.
+  const content = renderTemplate("index", {
+    latestAppJS: "/frontend_latest/app.js",
+    latestCoreJS: "/frontend_latest/core.js",
+    latestCustomPanelJS: "/frontend_latest/custom-panel.js",
+    latestHassIconsJS: "/frontend_latest/hass-icons.js",
+
+    es5Compatibility: "/frontend_es5/compatibility.js",
+    es5AppJS: "/frontend_es5/app.js",
+    es5CoreJS: "/frontend_es5/core.js",
+    es5CustomPanelJS: "/frontend_es5/custom-panel.js",
+    es5HassIconsJS: "/frontend_es5/hass-icons.js",
+  });
+
+  fs.outputFileSync(path.resolve(config.root, "index.html"), content);
+  done();
+});
+
+gulp.task("gen-index-html-prod", (done) => {
+  const latestManifest = require(path.resolve(config.output, "manifest.json"));
+  const es5Manifest = require(path.resolve(config.output_es5, "manifest.json"));
+  const content = renderTemplate("index", {
+    latestAppJS: latestManifest["app.js"],
+    latestCoreJS: latestManifest["core.js"],
+    latestCustomPanelJS: latestManifest["custom-panel.js"],
+    latestHassIconsJS: latestManifest["hass-icons.js"],
+
+    es5Compatibility: es5Manifest["compatibility.js"],
+    es5AppJS: es5Manifest["app.js"],
+    es5CoreJS: es5Manifest["core.js"],
+    es5CustomPanelJS: es5Manifest["custom-panel.js"],
+    es5HassIconsJS: es5Manifest["hass-icons.js"],
+  });
+  const minified = minifyHtml(content).replace(/#THEMEC/g, "{{ theme_color }}");
+
+  fs.outputFileSync(path.resolve(config.root, "index.html"), minified);
+  done();
+});

build-scripts/gulp/gather-static.js

@@ -0,0 +1,87 @@
+// Gulp task to gather all static files.
+
+const gulp = require("gulp");
+const path = require("path");
+const fs = require("fs-extra");
+const zopfli = require("gulp-zopfli-green");
+const merge = require("merge-stream");
+const config = require("../paths");
+
+const npmPath = (...parts) =>
+  path.resolve(config.polymer_dir, "node_modules", ...parts);
+const polyPath = (...parts) => path.resolve(config.polymer_dir, ...parts);
+const staticPath = (...parts) => path.resolve(config.root, "static", ...parts);
+
+const copyFileDir = (fromFile, toDir) =>
+  fs.copySync(fromFile, path.join(toDir, path.basename(fromFile)));
+
+function copyTranslations() {
+  // Translation output
+  fs.copySync(
+    polyPath("build-translations/output"),
+    staticPath("translations")
+  );
+}
+
+function copyStatic() {
+  // Basic static files
+  fs.copySync(polyPath("public"), config.root);
+
+  // Web Component polyfills and adapters
+  copyFileDir(
+    npmPath("@webcomponents/webcomponentsjs/custom-elements-es5-adapter.js"),
+    staticPath("polyfills/")
+  );
+  copyFileDir(
+    npmPath("@webcomponents/webcomponentsjs/webcomponents-bundle.js"),
+    staticPath("polyfills/")
+  );
+  copyFileDir(
+    npmPath("@webcomponents/webcomponentsjs/webcomponents-bundle.js.map"),
+    staticPath("polyfills/")
+  );
+
+  // Local fonts
+  fs.copySync(npmPath("@polymer/font-roboto-local/fonts"), staticPath("fonts"));
+
+  // External dependency assets
+  copyFileDir(
+    npmPath("react-big-calendar/lib/css/react-big-calendar.css"),
+    staticPath("panels/calendar/")
+  );
+  copyFileDir(
+    npmPath("leaflet/dist/leaflet.css"),
+    staticPath("images/leaflet/")
+  );
+  fs.copySync(
+    npmPath("leaflet/dist/images"),
+    staticPath("images/leaflet/images/")
+  );
+}
+
+gulp.task("copy-static", (done) => {
+  copyStatic();
+  done();
+});
+
+gulp.task("compress-static", () => {
+  const fonts = gulp
+    .src(staticPath("fonts/**/*.ttf"))
+    .pipe(zopfli())
+    .pipe(gulp.dest(staticPath("fonts")));
+  const polyfills = gulp
+    .src(staticPath("polyfills/*.js"))
+    .pipe(zopfli())
+    .pipe(gulp.dest(staticPath("polyfills")));
+  const translations = gulp
+    .src(staticPath("translations/*.json"))
+    .pipe(zopfli())
+    .pipe(gulp.dest(staticPath("translations")));
+
+  return merge(fonts, polyfills, translations);
+});
+
+gulp.task("copy-translations", (done) => {
+  copyTranslations();
+  done();
+});

build-scripts/gulp/gen-icons.js

@@ -0,0 +1,125 @@
+const gulp = require("gulp");
+const path = require("path");
+const fs = require("fs");
+
+const ICON_PACKAGE_PATH = path.resolve(
+  __dirname,
+  "../../node_modules/@mdi/svg/"
+);
+const META_PATH = path.resolve(ICON_PACKAGE_PATH, "meta.json");
+const ICON_PATH = path.resolve(ICON_PACKAGE_PATH, "svg");
+const OUTPUT_DIR = path.resolve(__dirname, "../../build");
+const MDI_OUTPUT_PATH = path.resolve(OUTPUT_DIR, "mdi.html");
+const HASS_OUTPUT_PATH = path.resolve(OUTPUT_DIR, "hass-icons.html");
+
+const BUILT_IN_PANEL_ICONS = [
+  "calendar", // Calendar
+  "settings", // Config
+  "home-assistant", // Hass.io
+  "poll-box", // History panel
+  "format-list-bulleted-type", // Logbook
+  "mailbox", // Mailbox
+  "tooltip-account", // Map
+  "cart", // Shopping List
+];
+
+// Given an icon name, load the SVG file
+function loadIcon(name) {
+  const iconPath = path.resolve(ICON_PATH, `${name}.svg`);
+  try {
+    return fs.readFileSync(iconPath, "utf-8");
+  } catch (err) {
+    return null;
+  }
+}
+
+// Given an SVG file, convert it to an iron-iconset-svg definition
+function transformXMLtoPolymer(name, xml) {
+  const start = xml.indexOf("><path") + 1;
+  const end = xml.length - start - 6;
+  const pth = xml.substr(start, end);
+  return `<g id="${name}">${pth}</g>`;
+}
+
+// Given an iconset name and icon names, generate a polymer iconset
+function generateIconset(iconsetName, iconNames) {
+  const iconDefs = Array.from(iconNames)
+    .map((name) => {
+      const iconDef = loadIcon(name);
+      if (!iconDef) {
+        throw new Error(`Unknown icon referenced: ${name}`);
+      }
+      return transformXMLtoPolymer(name, iconDef);
+    })
+    .join("");
+  return `<ha-iconset-svg name="${iconsetName}" size="24"><svg><defs>${iconDefs}</defs></svg></ha-iconset-svg>`;
+}
+
+// Generate the full MDI iconset
+function genMDIIcons() {
+  const meta = JSON.parse(
+    fs.readFileSync(path.resolve(ICON_PACKAGE_PATH, META_PATH), "UTF-8")
+  );
+  const iconNames = meta.map((iconInfo) => iconInfo.name);
+  if (!fs.existsSync(OUTPUT_DIR)) {
+    fs.mkdirSync(OUTPUT_DIR);
+  }
+  fs.writeFileSync(MDI_OUTPUT_PATH, generateIconset("mdi", iconNames));
+}
+
+// Helper function to map recursively over files in a folder and it's subfolders
+function mapFiles(startPath, filter, mapFunc) {
+  const files = fs.readdirSync(startPath);
+  for (let i = 0; i < files.length; i++) {
+    const filename = path.join(startPath, files[i]);
+    const stat = fs.lstatSync(filename);
+    if (stat.isDirectory()) {
+      mapFiles(filename, filter, mapFunc);
+    } else if (filename.indexOf(filter) >= 0) {
+      mapFunc(filename);
+    }
+  }
+}
+
+// Find all icons used by the project.
+function findIcons(searchPath, iconsetName) {
+  const iconRegex = new RegExp(`${iconsetName}:[\\w-]+`, "g");
+  const icons = new Set();
+  function processFile(filename) {
+    const content = fs.readFileSync(filename);
+    let match;
+    // eslint-disable-next-line
+    while ((match = iconRegex.exec(content))) {
+      // strip off "hass:" and add to set
+      icons.add(match[0].substr(iconsetName.length + 1));
+    }
+  }
+  mapFiles(searchPath, ".js", processFile);
+  mapFiles(searchPath, ".ts", processFile);
+  return icons;
+}
+
+function genHassIcons() {
+  const iconNames = findIcons("./src", "hass");
+  BUILT_IN_PANEL_ICONS.forEach((name) => iconNames.add(name));
+  if (!fs.existsSync(OUTPUT_DIR)) {
+    fs.mkdirSync(OUTPUT_DIR);
+  }
+  fs.writeFileSync(HASS_OUTPUT_PATH, generateIconset("hass", iconNames));
+}
+
+gulp.task("gen-icons-mdi", (done) => {
+  genMDIIcons();
+  done();
+});
+gulp.task("gen-icons-hass", (done) => {
+  genHassIcons();
+  done();
+});
+gulp.task("gen-icons", gulp.series("gen-icons-hass", "gen-icons-mdi"));
+
+module.exports = {
+  findIcons,
+  generateIconset,
+  genMDIIcons,
+};

build-scripts/gulp/release.js

@@ -0,0 +1,31 @@
+// Run HA develop mode
+const gulp = require("gulp");
+
+require("./clean.js");
+require("./translations.js");
+require("./gen-icons.js");
+require("./gather-static.js");
+require("./webpack.js");
+require("./service-worker.js");
+require("./entry-html.js");
+
+gulp.task(
+  "build-release",
+  gulp.series(
+    async function setEnv() {
+      process.env.NODE_ENV = "production";
+    },
+    "clean",
+    gulp.parallel(
+      "copy-static",
+      "gen-icons",
+      gulp.series("build-translations", "copy-translations")
+    ),
+    gulp.parallel("webpack-prod", "compress-static"),
+    gulp.parallel(
+      "gen-pages-prod",
+      "gen-index-html-prod",
+      "gen-service-worker-prod"
+    )
+  )
+);

build-scripts/gulp/service-worker.js

@@ -0,0 +1,29 @@
+// Generate service worker.
+// Based on manifest, create a file with the content as service_worker.js
+/* eslint-disable import/no-dynamic-require */
+/* eslint-disable global-require */
+const gulp = require("gulp");
+const path = require("path");
+const fs = require("fs-extra");
+const config = require("../paths.js");
+
+const swPath = path.resolve(config.root, "service_worker.js");
+
+const writeSW = (content) => fs.outputFileSync(swPath, content.trim() + "\n");
+
+gulp.task("gen-service-worker-dev", (done) => {
+  writeSW(
+    `
+console.debug('Service worker disabled in development');
+  `
+  );
+  done();
+});
+
+gulp.task("gen-service-worker-prod", (done) => {
+  fs.copySync(
+    path.resolve(config.output, "service_worker.js"),
+    path.resolve(config.root, "service_worker.js")
+  );
+  done();
+});

build-scripts/gulp/translations.js

@@ -0,0 +1,336 @@
+const del = require("del");
+const path = require("path");
+const gulp = require("gulp");
+const foreach = require("gulp-foreach");
+const hash = require("gulp-hash");
+const hashFilename = require("gulp-hash-filename");
+const merge = require("gulp-merge-json");
+const minify = require("gulp-jsonminify");
+const rename = require("gulp-rename");
+const transform = require("gulp-json-transform");
+
+const inDir = "translations";
+const workDir = "build-translations";
+const fullDir = workDir + "/full";
+const coreDir = workDir + "/core";
+const outDir = workDir + "/output";
+
+String.prototype.rsplit = function(sep, maxsplit) {
+  var split = this.split(sep);
+  return maxsplit
+    ? [split.slice(0, -maxsplit).join(sep)].concat(split.slice(-maxsplit))
+    : split;
+};
+
+// Panel translations which should be split from the core translations. These
+// should mirror the fragment definitions in polymer.json, so that we load
+// additional resources at equivalent points.
+const TRANSLATION_FRAGMENTS = [
+  "config",
+  "history",
+  "logbook",
+  "mailbox",
+  "profile",
+  "shopping-list",
+  "page-authorize",
+  "page-onboarding",
+];
+
+const tasks = [];
+
+function recursiveFlatten(prefix, data) {
+  let output = {};
+  Object.keys(data).forEach(function(key) {
+    if (typeof data[key] === "object") {
+      output = Object.assign(
+        {},
+        output,
+        recursiveFlatten(prefix + key + ".", data[key])
+      );
+    } else {
+      output[prefix + key] = data[key];
+    }
+  });
+  return output;
+}
+
+function flatten(data) {
+  return recursiveFlatten("", data);
+}
+
+function emptyFilter(data) {
+  const newData = {};
+  Object.keys(data).forEach((key) => {
+    if (data[key]) {
+      if (typeof data[key] === "object") {
+        newData[key] = emptyFilter(data[key]);
+      } else {
+        newData[key] = data[key];
+      }
+    }
+  });
+  return newData;
+}
+
+/**
+ * Replace Lokalise key placeholders with their actual values.
+ *
+ * We duplicate the behavior of Lokalise here so that placeholders can
+ * be included in src/translations/en.json, but still be usable while
+ * developing locally.
+ *
+ * @link https://docs.lokalise.co/article/KO5SZWLLsy-key-referencing
+ */
+const re_key_reference = /\[%key:([^%]+)%\]/;
+function lokalise_transform(data, original) {
+  const output = {};
+  Object.entries(data).forEach(([key, value]) => {
+    if (value instanceof Object) {
+      output[key] = lokalise_transform(value, original);
+    } else {
+      output[key] = value.replace(re_key_reference, (match, key) => {
+        const replace = key.split("::").reduce((tr, k) => tr[k], original);
+        if (typeof replace !== "string") {
+          throw Error(
+            `Invalid key placeholder ${key} in src/translations/en.json`
+          );
+        }
+        return replace;
+      });
+    }
+  });
+  return output;
+}
+
+let taskName = "clean-translations";
+gulp.task(taskName, function() {
+  return del([`${outDir}/**/*.json`]);
+});
+tasks.push(taskName);
+
+/**
+ * This task will build a master translation file, to be used as the base for
+ * all languages. This starts with src/translations/en.json, and replaces all
+ * Lokalise key placeholders with their target values. Under normal circumstances,
+ * this will be the same as translations/en.json However, we build it here to
+ * facilitate both making changes in development mode, and to ensure that the
+ * project is buildable immediately after merging new translation keys, since
+ * the Lokalise update to translations/en.json will not happen immediately.
+ */
+taskName = "build-master-translation";
+gulp.task(
+  taskName,
+  gulp.series("clean-translations", function() {
+    return gulp
+      .src("src/translations/en.json")
+      .pipe(
+        transform(function(data, file) {
+          return lokalise_transform(data, data);
+        })
+      )
+      .pipe(rename("translationMaster.json"))
+      .pipe(gulp.dest(workDir));
+  })
+);
+tasks.push(taskName);
+
+taskName = "build-merged-translations";
+gulp.task(
+  taskName,
+  gulp.series("build-master-translation", function() {
+    return gulp.src(inDir + "/*.json").pipe(
+      foreach(function(stream, file) {
+        // For each language generate a merged json file. It begins with the master
+        // translation as a failsafe for untranslated strings, and merges all parent
+        // tags into one file for each specific subtag
+        //
+        // TODO: This is a naive interpretation of BCP47 that should be improved.
+        //       Will be OK for now as long as we don't have anything more complicated
+        //       than a base translation + region.
+        const tr = path.basename(file.history[0], ".json");
+        const subtags = tr.split("-");
+        const src = [workDir + "/translationMaster.json"];
+        for (let i = 1; i <= subtags.length; i++) {
+          const lang = subtags.slice(0, i).join("-");
+          src.push(inDir + "/" + lang + ".json");
+        }
+        return gulp
+          .src(src, { allowEmpty: true })
+          .pipe(transform((data) => emptyFilter(data)))
+          .pipe(
+            merge({
+              fileName: tr + ".json",
+            })
+          )
+          .pipe(gulp.dest(fullDir));
+      })
+    );
+  })
+);
+tasks.push(taskName);
+
+const splitTasks = [];
+TRANSLATION_FRAGMENTS.forEach((fragment) => {
+  taskName = "build-translation-fragment-" + fragment;
+  gulp.task(
+    taskName,
+    gulp.series("build-merged-translations", function() {
+      // Return only the translations for this fragment.
+      return gulp
+        .src(fullDir + "/*.json")
+        .pipe(
+          transform((data) => ({
+            ui: {
+              panel: {
+                [fragment]: data.ui.panel[fragment],
+              },
+            },
+          }))
+        )
+        .pipe(gulp.dest(workDir + "/" + fragment));
+    })
+  );
+  tasks.push(taskName);
+  splitTasks.push(taskName);
+});
+
+taskName = "build-translation-core";
+gulp.task(
+  taskName,
+  gulp.series("build-merged-translations", function() {
+    // Remove the fragment translations from the core translation.
+    return gulp
+      .src(fullDir + "/*.json")
+      .pipe(
+        transform((data) => {
+          TRANSLATION_FRAGMENTS.forEach((fragment) => {
+            delete data.ui.panel[fragment];
+          });
+          return data;
+        })
+      )
+      .pipe(gulp.dest(coreDir));
+  })
+);
+tasks.push(taskName);
+splitTasks.push(taskName);
+
+taskName = "build-flattened-translations";
+gulp.task(
+  taskName,
+  gulp.series(...splitTasks, function() {
+    // Flatten the split versions of our translations, and move them into outDir
+    return gulp
+      .src(
+        TRANSLATION_FRAGMENTS.map(
+          (fragment) => workDir + "/" + fragment + "/*.json"
+        ).concat(coreDir + "/*.json"),
+        { base: workDir }
+      )
+      .pipe(
+        transform(function(data) {
+          // Polymer.AppLocalizeBehavior requires flattened json
+          return flatten(data);
+        })
+      )
+      .pipe(minify())
+      .pipe(hashFilename())
+      .pipe(
+        rename((filePath) => {
+          if (filePath.dirname === "core") {
+            filePath.dirname = "";
+          }
+        })
+      )
+      .pipe(gulp.dest(outDir));
+  })
+);
+tasks.push(taskName);
+
+taskName = "build-translation-fingerprints";
+gulp.task(
+  taskName,
+  gulp.series("build-flattened-translations", function() {
+    return gulp
+      .src(outDir + "/**/*.json")
+      .pipe(
+        rename({
+          extname: "",
+        })
+      )
+      .pipe(
+        hash({
+          algorithm: "md5",
+          hashLength: 32,
+          template: "<%= name %>.json",
+        })
+      )
+      .pipe(hash.manifest("translationFingerprints.json"))
+      .pipe(
+        transform(function(data) {
+          // After generating fingerprints of our translation files, consolidate
+          // all translation fragment fingerprints under the translation name key
+          const newData = {};
+          Object.entries(data).forEach(([key, value]) => {
+            const [path, _md5] = key.rsplit("-", 1);
+            // let translation = key;
+            let translation = path;
+            const parts = translation.split("/");
+            if (parts.length === 2) {
+              translation = parts[1];
+            }
+            if (!(translation in newData)) {
+              newData[translation] = {
+                fingerprints: {},
+              };
+            }
+            newData[translation].fingerprints[path] = value;
+          });
+          return newData;
+        })
+      )
+      .pipe(gulp.dest(workDir));
+  })
+);
+tasks.push(taskName);
+
+taskName = "build-translations";
+gulp.task(
+  taskName,
+  gulp.series("build-translation-fingerprints", function() {
+    return gulp
+      .src([
+        "src/translations/translationMetadata.json",
+        workDir + "/translationFingerprints.json",
+      ])
+      .pipe(merge({}))
+      .pipe(
+        transform(function(data) {
+          const newData = {};
+          Object.entries(data).forEach(([key, value]) => {
+            // Filter out translations without native name.
+            if (data[key].nativeName) {
+              newData[key] = data[key];
+            } else {
+              console.warn(
+                `Skipping language ${key}. Native name was not translated.`
+              );
+            }
+            if (data[key]) newData[key] = value;
+          });
+          return newData;
+        })
+      )
+      .pipe(
+        transform((data) => ({
+          fragments: TRANSLATION_FRAGMENTS,
+          translations: data,
+        }))
+      )
+      .pipe(rename("translationMetadata.json"))
+      .pipe(gulp.dest(workDir));
+  })
+);
+tasks.push(taskName);
+
+module.exports = tasks;

build-scripts/gulp/webpack.js

@@ -0,0 +1,63 @@
+// Tasks to run webpack.
+const gulp = require("gulp");
+const webpack = require("webpack");
+const { createAppConfig } = require("../webpack");
+
+const handler = (done) => (err, stats) => {
+  if (err) {
+    console.log(err.stack || err);
+    if (err.details) {
+      console.log(err.details);
+    }
+    return;
+  }
+
+  console.log(`Build done @ ${new Date().toLocaleTimeString()}`);
+
+  if (stats.hasErrors() || stats.hasWarnings()) {
+    console.log(stats.toString("minimal"));
+  }
+
+  if (done) {
+    done();
+  }
+};
+
+gulp.task("webpack-watch", () => {
+  const compiler = webpack([
+    createAppConfig({
+      isProdBuild: false,
+      latestBuild: true,
+      isStatsBuild: false,
+    }),
+    createAppConfig({
+      isProdBuild: false,
+      latestBuild: false,
+      isStatsBuild: false,
+    }),
+  ]);
+  compiler.watch({}, handler());
+  // we are not calling done, so this command will run forever
+});
+
+gulp.task(
+  "webpack-prod",
+  () =>
+    new Promise((resolve) =>
+      webpack(
+        [
+          createAppConfig({
+            isProdBuild: true,
+            latestBuild: true,
+            isStatsBuild: false,
+          }),
+          createAppConfig({
+            isProdBuild: true,
+            latestBuild: false,
+            isStatsBuild: false,
+          }),
+        ],
+        handler(resolve)
+      )
+    )
+);

build-scripts/paths.js

@@ -0,0 +1,10 @@
+var path = require("path");
+
+module.exports = {
+  polymer_dir: path.resolve(__dirname, ".."),
+  build_dir: path.resolve(__dirname, "../build"),
+  root: path.resolve(__dirname, "../hass_frontend"),
+  static: path.resolve(__dirname, "../hass_frontend/static"),
+  output: path.resolve(__dirname, "../hass_frontend/frontend_latest"),
+  output_es5: path.resolve(__dirname, "../hass_frontend/frontend_es5"),
+};

build-scripts/webpack.js

@@ -0,0 +1,184 @@
+const webpack = require("webpack");
+const fs = require("fs");
+const path = require("path");
+const TerserPlugin = require("terser-webpack-plugin");
+const WorkboxPlugin = require("workbox-webpack-plugin");
+const CompressionPlugin = require("compression-webpack-plugin");
+const zopfli = require("@gfx/zopfli");
+const ManifestPlugin = require("webpack-manifest-plugin");
+const paths = require("./paths.js");
+const { babelLoaderConfig } = require("./babel.js");
+
+let version = fs
+  .readFileSync(path.resolve(paths.polymer_dir, "setup.py"), "utf8")
+  .match(/\d{8}\.\d+/);
+if (!version) {
+  throw Error("Version not found");
+}
+version = version[0];
+
+const resolve = {
+  extensions: [".ts", ".js", ".json", ".tsx"],
+  alias: {
+    react: "preact-compat",
+    "react-dom": "preact-compat",
+    // Not necessary unless you consume a module using `createClass`
+    "create-react-class": "preact-compat/lib/create-react-class",
+    // Not necessary unless you consume a module requiring `react-dom-factories`
+    "react-dom-factories": "preact-compat/lib/react-dom-factories",
+  },
+};
+
+const plugins = [
+  // Ignore moment.js locales
+  new webpack.IgnorePlugin(/^\.\/locale$/, /moment$/),
+  // Color.js is bloated, it contains all color definitions for all material color sets.
+  new webpack.NormalModuleReplacementPlugin(
+    /@polymer\/paper-styles\/color\.js$/,
+    path.resolve(paths.polymer_dir, "src/util/empty.js")
+  ),
+  // Ignore roboto pointing at CDN. We use local font-roboto-local.
+  new webpack.NormalModuleReplacementPlugin(
+    /@polymer\/font-roboto\/roboto\.js$/,
+    path.resolve(paths.polymer_dir, "src/util/empty.js")
+  ),
+  // Ignore mwc icons pointing at CDN.
+  new webpack.NormalModuleReplacementPlugin(
+    /@material\/mwc-icon\/mwc-icon-font\.js$/,
+    path.resolve(paths.polymer_dir, "src/util/empty.js")
+  ),
+];
+
+const optimization = (latestBuild) => ({
+  minimizer: [
+    new TerserPlugin({
+      cache: true,
+      parallel: true,
+      extractComments: true,
+      terserOptions: {
+        ecma: latestBuild ? undefined : 5,
+      },
+    }),
+  ],
+});
+
+const createAppConfig = ({ isProdBuild, latestBuild, isStatsBuild }) => {
+  const isCI = process.env.CI === "true";
+
+  // Create an object mapping browser urls to their paths during build
+  const translationMetadata = require("../build-translations/translationMetadata.json");
+  const workBoxTranslationsTemplatedURLs = {};
+  const englishFP = translationMetadata["translations"]["en"]["fingerprints"];
+  Object.keys(englishFP).forEach((key) => {
+    workBoxTranslationsTemplatedURLs[
+      `/static/translations/${englishFP[key]}`
+    ] = `build-translations/output/${key}.json`;
+  });
+
+  const publicPath = latestBuild ? "/frontend_latest/" : "/frontend_es5/";
+
+  const entry = {
+    app: "./src/entrypoints/app.ts",
+    authorize: "./src/entrypoints/authorize.ts",
+    onboarding: "./src/entrypoints/onboarding.ts",
+    core: "./src/entrypoints/core.ts",
+    compatibility: "./src/entrypoints/compatibility.ts",
+    "custom-panel": "./src/entrypoints/custom-panel.ts",
+    "hass-icons": "./src/entrypoints/hass-icons.ts",
+  };
+
+  return {
+    mode: isProdBuild ? "production" : "development",
+    devtool: isProdBuild
+      ? "cheap-source-map "
+      : "inline-cheap-module-source-map",
+    entry,
+    module: {
+      rules: [
+        babelLoaderConfig({ latestBuild }),
+        {
+          test: /\.css$/,
+          use: "raw-loader",
+        },
+        {
+          test: /\.(html)$/,
+          use: {
+            loader: "html-loader",
+            options: {
+              exportAsEs6Default: true,
+            },
+          },
+        },
+      ],
+    },
+    optimization: optimization(latestBuild),
+    plugins: [
+      new ManifestPlugin(),
+      new webpack.DefinePlugin({
+        __DEV__: JSON.stringify(!isProdBuild),
+        __DEMO__: false,
+        __BUILD__: JSON.stringify(latestBuild ? "latest" : "es5"),
+        __VERSION__: JSON.stringify(version),
+        __STATIC_PATH__: "/static/",
+        "process.env.NODE_ENV": JSON.stringify(
+          isProdBuild ? "production" : "development"
+        ),
+      }),
+      ...plugins,
+      isProdBuild &&
+        !isCI &&
+        !isStatsBuild &&
+        new CompressionPlugin({
+          cache: true,
+          exclude: [/\.js\.map$/, /\.LICENSE$/, /\.py$/, /\.txt$/],
+          algorithm(input, compressionOptions, callback) {
+            return zopfli.gzip(input, compressionOptions, callback);
+          },
+        }),
+      latestBuild &&
+        new WorkboxPlugin.InjectManifest({
+          swSrc: "./src/entrypoints/service-worker-hass.js",
+          swDest: "service_worker.js",
+          importWorkboxFrom: "local",
+          include: [/\.js$/],
+          templatedURLs: {
+            ...workBoxTranslationsTemplatedURLs,
+            "/static/icons/favicon-192x192.png":
+              "public/icons/favicon-192x192.png",
+            "/static/fonts/roboto/Roboto-Light.ttf":
+              "node_modules/@polymer/font-roboto-local/fonts/roboto/Roboto-Light.ttf",
+            "/static/fonts/roboto/Roboto-Medium.ttf":
+              "node_modules/@polymer/font-roboto-local/fonts/roboto/Roboto-Medium.ttf",
+            "/static/fonts/roboto/Roboto-Regular.ttf":
+              "node_modules/@polymer/font-roboto-local/fonts/roboto/Roboto-Regular.ttf",
+            "/static/fonts/roboto/Roboto-Bold.ttf":
+              "node_modules/@polymer/font-roboto-local/fonts/roboto/Roboto-Bold.ttf",
+          },
+        }),
+    ].filter(Boolean),
+    output: {
+      filename: ({ chunk }) => {
+        const dontHash = new Set([
+          // Files who'se names should not be hashed.
+          // We currently have none.
+        ]);
+        if (!isProdBuild || dontHash.has(chunk.name)) return `${chunk.name}.js`;
+        return `${chunk.name}.${chunk.hash.substr(0, 8)}.js`;
+      },
+      chunkFilename:
+        isProdBuild && !isStatsBuild
+          ? "chunk.[chunkhash].js"
+          : "[name].chunk.js",
+      path: latestBuild ? paths.output : paths.output_es5,
+      publicPath,
+    },
+    resolve,
+  };
+};
+
+module.exports = {
+  resolve,
+  plugins,
+  optimization,
+  createAppConfig,
+};