Implement own locale loader

Why?
* Because i18n sucks multi-browser, especially on Chrome.
* What's more, this will allow overlaying "incomplete" locales with
  missing strings over the base locale and get a semi-translated one,
  which probably is better than nothing.
* Additionally we kinda need that implementation anyway for node-based
  tests.
* Also, while not currently implemented, it could allow (hot) reloading
  of locales, or loading external ones, which would help translators,
  or providing an option to the user to choose a locale.
* And finally, not calling i18n will avoid the "context switch" into
  browserland.

Some implementation details:
* Before code can use a locale, it has to be loaded. Sadly sync loading
  is not really supported. So `await locale` or `await localize`.
* Background force reloads locales right now, and caches them in
  localStorage. Windows will look into localStorage for that cache.
* The locale loader will not verify locales other than some rudimentary
  checks. It is assumed that shipped locales where verified before
  check-in.

lib/background.ts

@@ -5,7 +5,7 @@ import { ALLOWED_SCHEMES, TRANSFERABLE_PROPERTIES } from "./constants";
 import { API } from "./api";
 import { Finisher, makeUniqueItems } from "./item";
 import { Prefs } from "./prefs";
-import { _ } from "./i18n";
+import { _, locale } from "./i18n";
 import { openPrefs, openManager } from "./windowutils";
 import { filters } from "./filters";
 import { getManager } from "./manager/man";
@@ -106,6 +106,7 @@ class Handler {
   }
 }
 
+locale.then(() => {
   new class Action extends Handler {
     constructor() {
       super();
@@ -512,3 +513,4 @@ function adjustAction(globalTurbo: boolean) {
   })().catch(ex => {
     console.error("Failed to init components", ex.toString(), ex.stack, ex);
   });
+});

lib/browser.ts

@@ -3,7 +3,6 @@
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const polyfill = require("webextension-polyfill");
 
-export const {i18n} = polyfill;
 export const {extension} = polyfill;
 export const {notifications} = polyfill;
 export const {browserAction} = polyfill;

lib/filters.ts

@@ -4,13 +4,14 @@
 import uuid from "./uuid";
 
 import "./objectoverlay";
-import { storage, i18n } from "./browser";
+import { storage } from "./browser";
 import { EventEmitter } from "./events";
 import { TYPE_LINK, TYPE_MEDIA, TYPE_ALL } from "./constants";
 // eslint-disable-next-line no-unused-vars
 import { Overlayable } from "./objectoverlay";
 import * as DEFAULT_FILTERS from "../data/filters.json";
 import { FASTFILTER } from "./recentlist";
+import { _, locale } from "./i18n";
 
 const REG_ESCAPE = /[{}()[\]\\^$.]/g;
 const REG_FNMATCH = /[*?]/;
@@ -205,7 +206,7 @@ export class Filter {
     this._label = this.raw.label;
     if (this.id !== FAST && this.id.startsWith("deffilter-") &&
       !this.raw.isOverridden("label")) {
-      this._label = i18n.getMessage(this.id) || this._label;
+      this._label = _(this.id) || this._label;
     }
     this._reg = Matcher.fromExpression(this.expr);
     Object.seal(this);
@@ -475,6 +476,7 @@ class Filters extends EventEmitter {
   }
 
   async load() {
+    await locale;
     const defaultFilters = DEFAULT_FILTERS as any;
     let savedFilters = (await storage.local.get("userFilters"));
     if (savedFilters && "userFilters" in savedFilters) {

lib/i18n.ts

@@ -3,57 +3,170 @@
 
 import {memoize} from "./memoize";
 
-
 declare let browser: any;
 declare let chrome: any;
 
-function load() {
+interface JSONEntry {
+  message: string;
+  placeholders: any;
+}
+
+class Entry {
+  private message: string;
+
+  constructor(entry: JSONEntry) {
+    if (!entry.message.includes("$")) {
+      throw new Error("Not entry-able");
+    }
+    let hit = false;
+    this.message = entry.message.replace(/\$[A-Z0-9]+\$/g, (r: string) => {
+      hit = true;
+      const id = r.substr(1, r.length - 2).toLocaleLowerCase();
+      const pholder = entry.placeholders[id];
+      if (!pholder || !pholder.content) {
+        throw new Error(`Invalid placeholder: ${id}`);
+      }
+      return `${pholder.content}$`;
+    });
+    if (!hit) {
+      throw new Error("Not entry-able");
+    }
+  }
+
+  localize(args: any[]) {
+    return this.message.replace(/\$\d+\$/g, (r: string) => {
+      const idx = parseInt(r.substr(1, r.length - 2), 10) - 1;
+      return args[idx] || "";
+    });
+  }
+}
+
+class Localization {
+  private strings: Map<string, Entry | string>;
+
+  constructor(baseLanguage: any, ...overlayLanguages: any) {
+    this.strings = new Map();
+    const mapLanguage = (lang: any) => {
+      for (const [id, entry] of Object.entries<JSONEntry>(lang)) {
+        if (!entry.message) {
+          continue;
+        }
         try {
+          this.strings.set(id, new Entry(entry));
+        }
+        catch (ex) {
+          this.strings.set(id, entry.message);
+        }
+      }
+    };
+    mapLanguage(baseLanguage);
+    overlayLanguages.forEach(() => {});
+  }
+
+  localize(id: string, ...args: any[]) {
+    const entry = this.strings.get(id);
+    if (!entry) {
+      return "";
+    }
+    if (typeof entry === "string") {
+      return entry;
+    }
+    if (args.length === 1 && Array.isArray(args)) {
+      [args] = args;
+    }
+    return entry.localize(args);
+  }
+}
+
+function checkBrowser() {
   // eslint-disable-next-line @typescript-eslint/no-var-requires
   if (typeof browser !== "undefined" && browser.i18n) {
-      return browser.i18n;
+    return;
   }
   if (typeof chrome !== "undefined" && chrome.i18n) {
-      return chrome.i18n;
+    return;
   }
   throw new Error("not in a webext");
 }
+
+async function fetchLanguage(code: string) {
+  try {
+    const resp = await fetch(`/_locales/${code}/messages.json`);
+    return await resp.json();
+  }
+  catch {
+    return null;
+  }
+}
+
+const CACHE_KEY = "_cached_locales";
+
+function loadCached() {
+  if (document.location.pathname.includes("/windows/")) {
+    const cached = localStorage.getItem(CACHE_KEY);
+    if (cached) {
+      return JSON.parse(cached) as any[];
+    }
+  }
+  return null;
+}
+
+async function loadRawLocales() {
+  // en is the base locale
+  const langs = new Set<string>(["en"]);
+  const ui = (browser.i18n || chrome.i18n).getUILanguage();
+  langs.add(ui);
+  if (ui.includes("-")) {
+    // Try the base too
+    langs.add(ui.split(/[_-]+/)[0]);
+  }
+
+  const fetched = await Promise.all(Array.from(langs, fetchLanguage));
+  return fetched.filter(e => !!e);
+}
+
+async function load(): Promise<Localization> {
+  try {
+    checkBrowser();
+    try {
+      // en is the base locale
+      let valid = loadCached();
+      if (!valid) {
+        valid = await loadRawLocales();
+        localStorage.setItem(CACHE_KEY, JSON.stringify(valid));
+      }
+      if (!valid.length) {
+        throw new Error("Could not lood ANY of these locales");
+      }
+
+      const base = valid.shift();
+      const rv = new Localization(base, ...valid);
+      return rv;
+    }
     catch (ex) {
+      console.error("Failed to load locale", ex.toString(), ex.stack, ex);
+      return new Localization({});
+    }
+  }
+  catch {
     // We might be running under node for tests
 
     // eslint-disable-next-line @typescript-eslint/no-var-requires
     const messages = require("../_locales/en/messages.json");
 
-    const map = new Map();
+    return new Localization(messages);
-    for (const [k, v] of Object.entries<any>(messages)) {
-      const {placeholders = {}} = v;
-      let {message = ""} = v;
-      for (const [pname, pval] of Object.entries<any>(placeholders)) {
-        message = message.replace(`$${pname.toUpperCase()}$`, `${pval.content}$`);
   }
-      map.set(k, message);
 }
 
-    return {
+type MemoLocalize = (id: string, ...args: any[]) => string;
-      getMessage(id: string, subst: string[]) {
+
-        const m = map.get(id);
+export const locale = load();
-        if (typeof subst === undefined) {
+let loc: Localization | null;
-          return m;
+let memoLocalize: MemoLocalize | null = null;
-        }
+locale.then(l => {
-        if (!Array.isArray(subst)) {
+  loc = l;
-          subst = [subst];
+  memoLocalize = memoize(loc.localize.bind(loc), 10 * 1000, 10);
-        }
-        return m.replace(/\$\d+\$/g, (r: string) => {
-          const idx = parseInt(r.substr(1, r.length - 2), 10) - 1;
-          return subst[idx] || "";
 });
-      }
-    };
-  }
-}
-
-const i18n = load();
-const memoGetMessage = memoize(i18n.getMessage, 10 * 1000, 10);
 
 /**
  * Localize a message
@@ -62,21 +175,21 @@ const memoGetMessage = memoize(i18n.getMessage, 10 * 1000, 10);
  * @returns {string} Localized message
  */
 export function _(id: string, ...subst: any[]) {
+  if (!loc || !memoLocalize) {
+    console.trace("TOO SOON");
+    throw new Error("Called too soon");
+  }
   if (!subst.length) {
-    return memoGetMessage(id);
+    return memoLocalize(id);
   }
-  if (subst.length === 1 && Array.isArray(subst[0])) {
+  return loc.localize(id, subst);
-    subst = subst.pop();
+}
-  }
+
-  return i18n.getMessage(id, subst);
+function localize_<T extends HTMLElement | DocumentFragment>(elem: T): T {
+  for (const tmpl of elem.querySelectorAll<HTMLTemplateElement>("template")) {
+    localize_(tmpl.content);
   }
 
-/**
- * Localize a DOM
- * @param {Element} elem DOM to localize
- * @returns {Element} Passed in element (fluent)
- */
-export function localize<T extends HTMLElement | DocumentFragment>(elem: T): T {
   for (const el of elem.querySelectorAll<HTMLElement>("*[data-i18n]")) {
     const {i18n: i} = el.dataset;
     if (!i) {
@@ -109,3 +222,14 @@ export function localize<T extends HTMLElement | DocumentFragment>(elem: T): T {
   }
   return elem as T;
 }
+
+/**
+ * Localize a DOM
+ * @param {Element} elem DOM to localize
+ * @returns {Element} Passed in element (fluent)
+ */
+export async function localize<T extends HTMLElement | DocumentFragment>(
+    elem: T): Promise<T> {
+  await locale;
+  return localize_(elem);
+}

uikit/lib/modal.ts

@@ -23,7 +23,7 @@ export default class ModalDialog {
     this._default = null;
   }
 
-  _makeEl() {
+  async _makeEl() {
     this._dismiss = null;
     this._default = null;
 
@@ -35,7 +35,7 @@ export default class ModalDialog {
 
     const body = document.createElement("article");
     body.classList.add("modal-body");
-    body.appendChild(this.content);
+    body.appendChild(await this.getContent());
     cont.appendChild(body);
 
     const footer = document.createElement("footer");
@@ -87,7 +87,7 @@ export default class ModalDialog {
     return el;
   }
 
-  get content(): DocumentFragment | HTMLElement {
+  getContent(): Promise<DocumentFragment | HTMLElement> {
     throw new Error("Not implemented");
   }
 
@@ -160,7 +160,7 @@ export default class ModalDialog {
       return;
     };
 
-    document.body.appendChild(this.el = this._makeEl());
+    document.body.appendChild(this.el = await this._makeEl());
     this.shown();
     addEventListener("keydown", escapeHandler);
     addEventListener("keydown", enterHandler);

windows/manager.ts

@@ -20,7 +20,29 @@ const LOADED = new Promise(resolve => {
   });
 });
 
-LOADED.then(async () => {
+addEventListener("DOMContentLoaded", function dom() {
+  removeEventListener("DOMContentLoaded", dom);
+
+  const platformed = (async () => {
+    try {
+      const platform = (await runtime.getPlatformInfo()).os;
+      document.documentElement.dataset.platform = platform;
+      if (platform === "mac") {
+        const ctx = $("#table-context").content;
+        ctx.querySelector("#ctx-open-file").dataset.key = "ACCEL-KeyO";
+        ctx.querySelector("#ctx-open-directory").dataset.key = "ALT-ACCEL-KeyO";
+      }
+    }
+    catch (ex) {
+      console.error("failed to setup platform", ex.toString(), ex.stack, ex);
+    }
+  })();
+
+  const tabled = new Promised();
+  const localized = localize(document.documentElement);
+  const loaded = Promise.all([LOADED, platformed, localized]);
+  const fullyloaded = Promise.all([LOADED, platformed, tabled, localized]);
+  fullyloaded.then(async () => {
     const nag = await Prefs.get("nagging", 0);
     const nagnext = await Prefs.get("nagging-next", 6);
     const next = Math.ceil(Math.log2(Math.max(1, nag)));
@@ -51,29 +73,6 @@ LOADED.then(async () => {
     }, 2 * 1000);
   });
 
-addEventListener("DOMContentLoaded", function dom() {
-  removeEventListener("DOMContentLoaded", dom);
-
-  const platformed = (async () => {
-    try {
-      const platform = (await runtime.getPlatformInfo()).os;
-      document.documentElement.dataset.platform = platform;
-      if (platform === "mac") {
-        const ctx = $("#table-context").content;
-        ctx.querySelector("#ctx-open-file").dataset.key = "ACCEL-KeyO";
-        ctx.querySelector("#ctx-open-directory").dataset.key = "ALT-ACCEL-KeyO";
-      }
-    }
-    catch (ex) {
-      console.error("failed to setup platform", ex.toString(), ex.stack, ex);
-    }
-  })();
-
-  const tabled = new Promised();
-  const loaded = Promise.all([LOADED, platformed]);
-  const fullyloaded = Promise.all([LOADED, platformed, tabled]);
-
-  localize(document.documentElement);
   $("#donate").addEventListener("click", () => {
     PORT.post("donate");
   });
@@ -110,7 +109,8 @@ addEventListener("DOMContentLoaded", function dom() {
   statusNetwork.addEventListener("click", () => {
     PORT.post("toggle-active");
   });
-  PORT.on("active", active => {
+  PORT.on("active", async (active: boolean) => {
+    await loaded;
     if (active) {
       statusNetwork.className = "icon-network-on";
       statusNetwork.setAttribute("title", _("statusNetwork-active.title"));

windows/manager/itemfilters.ts

@@ -20,7 +20,6 @@ import {DownloadItem, DownloadTable} from "./table";
 import {formatSize} from "../../lib/formatters";
 import {_} from "../../lib/i18n";
 import {$} from "../winutil";
-import {StateTexts} from "./state";
 
 const TIMEOUT_SEARCH = 750;
 
@@ -260,7 +259,9 @@ class FixedMenuFilter extends MenuFilter {
 }
 
 export class StateMenuFilter extends FixedMenuFilter {
-  constructor(collection: FilteredCollection) {
+  constructor(
+      collection: FilteredCollection,
+      StateTexts: Readonly<Map<number, string>>) {
     const items = Array.from(StateTexts.entries()).map(([state, text]) => {
       return {
         state,

windows/manager/removaldlg.ts

@@ -21,10 +21,10 @@ export default class RemovalModalDialog extends ModalDialog {
     this.check = null;
   }
 
-  get content() {
+  async getContent() {
     const content = $<HTMLTemplateElement>("#removal-template").
       content.cloneNode(true) as DocumentFragment;
-    localize(content);
+    await localize(content);
     this.check = content.querySelector(".removal-remember");
     $(".removal-text", content).textContent = this.text;
     return content;

windows/manager/state.ts

@@ -2,11 +2,11 @@
 // License: MIT
 
 import * as _DownloadState from "../../lib/manager/state";
-import { _ } from "../../lib/i18n";
+import { _, locale } from "../../lib/i18n";
 
 export const DownloadState = _DownloadState;
 
-export const StateTexts = Object.freeze(new Map([
+export const StateTexts = locale.then(() => Object.freeze(new Map([
   [DownloadState.QUEUED, _("queued")],
   [DownloadState.RUNNING, _("running")],
   [DownloadState.FINISHING, _("finishing")],
@@ -14,7 +14,7 @@ export const StateTexts = Object.freeze(new Map([
   [DownloadState.DONE, _("done")],
   [DownloadState.CANCELED, _("canceled")],
   [DownloadState.MISSING, _("missing")],
-]));
+])));
 
 export const StateClasses = Object.freeze(new Map([
   [DownloadState.QUEUED, "queued"],

windows/manager/table.ts

@@ -11,7 +11,7 @@ import {
 import { iconForPath } from "../../lib/windowutils";
 import { formatSpeed, formatSize, formatTimeDelta } from "../../lib/formatters";
 import { filters } from "../../lib/filters";
-import { _, localize } from "../../lib/i18n";
+import { _ } from "../../lib/i18n";
 import { EventEmitter } from "../../lib/events";
 import { Prefs, PrefWatcher } from "../../lib/prefs";
 // eslint-disable-next-line no-unused-vars
@@ -52,7 +52,11 @@ const COL_SEGS = 8;
 
 const ICON_BASE_SIZE = 16;
 
-const TEXT_SIZE_UNKNOWM = _("size-unknown");
+let TEXT_SIZE_UNKNOWM = "unknown";
+let REAL_STATE_TEXTS = Object.freeze(new Map<number, string>());
+StateTexts.then(v => {
+  REAL_STATE_TEXTS = v;
+});
 
 const prettyNumber = (function() {
 const rv = new Intl.NumberFormat(undefined, {
@@ -190,7 +194,7 @@ export class DownloadItem extends EventEmitter {
     if (this.error) {
       return _(this.error) || this.error;
     }
-    return StateTexts.get(this.state);
+    return REAL_STATE_TEXTS.get(this.state) || "";
   }
 
   get fmtSpeed() {
@@ -342,6 +346,8 @@ export class DownloadTable extends VirtualTable {
   constructor(treeConfig: any) {
     super("#items", treeConfig, TREE_CONFIG_VERSION);
 
+    TEXT_SIZE_UNKNOWM = _("size-unknown");
+
     this.finished = 0;
     this.running = new Set();
     this.runningTimer = null;
@@ -362,7 +368,7 @@ export class DownloadTable extends VirtualTable {
     new TextFilter(this.downloads);
     const menufilters = new Map<string, MenuFilter>([
       ["colURL", new UrlMenuFilter(this.downloads)],
-      ["colETA", new StateMenuFilter(this.downloads)],
+      ["colETA", new StateMenuFilter(this.downloads, REAL_STATE_TEXTS)],
       ["colSize", new SizeMenuFilter(this.downloads)],
     ]);
     this.on("column-clicked", (id, evt, col) => {
@@ -402,7 +408,6 @@ export class DownloadTable extends VirtualTable {
     this.sids = new Map<number, DownloadItem>();
     this.icons = new Icons($("#icons"));
 
-    localize($<HTMLTemplateElement>("#table-context").content);
     const ctx = this.contextMenu = new ContextMenu("#table-context");
     Keys.adoptContext(ctx);
     Keys.adoptButtons($("#toolbar"));

windows/manager/tooltip.ts

@@ -2,7 +2,7 @@
 "use strict";
 // License: MIT
 
-import { _, localize } from "../../lib/i18n";
+import { _ } from "../../lib/i18n";
 import { formatSpeed } from "../../lib/formatters";
 import { DownloadState } from "./state";
 import { Rect } from "../../uikit/lib/rect";
@@ -155,7 +155,7 @@ export class Tooltip {
     if (!el) {
       throw new Error("invalid template");
     }
-    this.elem = localize(el.cloneNode(true) as HTMLElement);
+    this.elem = el.cloneNode(true) as HTMLElement;
     this.adjust(pos);
 
     // eslint-disable-next-line @typescript-eslint/no-this-alias

windows/prefs.ts

@@ -109,15 +109,14 @@ class CreateFilterDialog extends ModalDialog {
 
   media: HTMLInputElement;
 
-  get content() {
+  getContent() {
-    const tmpl = $<HTMLTemplateElement>("#create-filter-template").
+    const rv = $<HTMLTemplateElement>("#create-filter-template").
       content.cloneNode(true) as DocumentFragment;
-    const rv = localize(tmpl);
     this.label = $("#filter-create-label", rv);
     this.expr = $("#filter-create-expr", rv);
     this.link = $("#filter-create-type-link", rv);
     this.media = $("#filter-create-type-media", rv);
-    return rv;
+    return Promise.resolve(rv);
   }
 
   get buttons() {

windows/single.ts

@@ -198,15 +198,15 @@ function cancel() {
 }
 
 async function init() {
+  await localize(document.documentElement);
   await Promise.all([MASK.init()]);
   Mask = new Dropdown("#mask", MASK.values);
 }
 
-addEventListener("DOMContentLoaded", function dom() {
+addEventListener("DOMContentLoaded", async function dom() {
   removeEventListener("DOMContentLoaded", dom);
-  init().catch(console.error);
+  await init();
 
-  localize(document.documentElement);
   $("#btnDownload").addEventListener("click", () => download(false));
   $("#btnPaused").addEventListener("click", () => download(true));
   $("#btnCancel").addEventListener(