From f99de4cd866b044328ea798aee273110408ba147 Mon Sep 17 00:00:00 2001
From: 18alantom <2.alan.tom@gmail.com>
Date: Mon, 14 Feb 2022 16:44:21 +0530
Subject: [PATCH] feat: add script to generate translation file

---
 package.json                    |   3 +-
 scripts/generateTranslations.js | 247 ++++++++++++++++++++++++++++++++
 scripts/helpers.js              |  93 ++++++++++++
 3 files changed, 342 insertions(+), 1 deletion(-)
 create mode 100644 scripts/generateTranslations.js
 create mode 100644 scripts/helpers.js

diff --git a/package.json b/package.json
index 14859340..ab0ee181 100644
--- a/package.json
+++ b/package.json
@@ -13,7 +13,8 @@
     "electron:build": "vue-cli-service electron:build",
     "electron:serve": "vue-cli-service electron:serve",
     "postinstall": "electron-builder install-app-deps",
-    "postuninstall": "electron-builder install-app-deps"
+    "postuninstall": "electron-builder install-app-deps",
+    "script:translate": "node scripts/generateTranslations.js"
   },
   "main": "background.js",
   "dependencies": {
diff --git a/scripts/generateTranslations.js b/scripts/generateTranslations.js
new file mode 100644
index 00000000..6a884f94
--- /dev/null
+++ b/scripts/generateTranslations.js
@@ -0,0 +1,247 @@
+const fs = require('fs/promises');
+const path = require('path');
+const {
+  getIndexFormat,
+  getWhitespaceSanitized,
+  wrap,
+  splitCsvLine,
+} = require('./helpers');
+
+const translationsFolder = path.resolve(__dirname, '..', 'translations');
+const PATTERN = /(?<!\w)t`((?:[^`]|(?<=\\)`)+)`/g;
+
+function shouldIgnore(p, ignoreList) {
+  return ignoreList.includes(p.split(path.sep).at(-1));
+}
+
+async function getFileList(root, ignoreList) {
+  const contents = await fs.readdir(root);
+  const files = [];
+  const promises = [];
+
+  for (let c of contents) {
+    const absPath = path.resolve(root, c);
+    const isDir = (await fs.stat(absPath)).isDirectory();
+
+    if (isDir && !shouldIgnore(absPath, ignoreList)) {
+      const pr = getFileList(absPath, ignoreList).then((fl) => {
+        files.push(...fl);
+      });
+      promises.push(pr);
+    } else if (absPath.match(/\.(js|ts|vue)$/) !== null) {
+      files.push(absPath);
+    }
+  }
+
+  await Promise.all(promises);
+  return files;
+}
+
+async function getFileContents(fileList) {
+  const contents = [];
+  const promises = [];
+  for (let file of fileList) {
+    const pr = fs.readFile(file, { encoding: 'utf-8' }).then((c) => {
+      contents.push([file, c]);
+    });
+    promises.push(pr);
+  }
+  await Promise.all(promises);
+  return contents;
+}
+
+async function getAllTStringsMap(contents) {
+  const strings = new Map();
+  const promises = [];
+
+  contents.forEach(([f, c]) => {
+    const pr = getTStrings(c).then((ts) => {
+      if (ts.length === 0) {
+        return;
+      }
+      strings.set(f, ts);
+    });
+    promises.push(pr);
+  });
+
+  await Promise.all(promises);
+  return strings;
+}
+
+function getTStrings(content) {
+  return new Promise((resolve) => {
+    const tStrings = tStringFinder(content);
+    resolve(tStrings);
+  });
+}
+
+function tStringFinder(content) {
+  return [...content.matchAll(PATTERN)].map(([_, t]) => {
+    t = getIndexFormat(t);
+    return getWhitespaceSanitized(t);
+  });
+}
+
+function mapToTStringArray(tMap) {
+  const tSet = new Set();
+  for (let k of tMap.keys()) {
+    tMap.get(k).forEach((s) => tSet.add(s));
+  }
+  const tArray = [...tSet];
+  return tArray.sort();
+}
+
+function printHelp() {
+  const shouldPrint = process.argv.findIndex((i) => i === '-h') !== -1;
+  if (shouldPrint) {
+    console.log(
+      `Usage: ` +
+        `\tyarn script:translate\n` +
+        `\tyarn script:translate -h\n` +
+        `\tyarn script:translate -l [language_code]\n` +
+        `\n` +
+        `Example: $ yarn script:translate -l de\n` +
+        `\n` +
+        `Description:\n` +
+        `\tPassing a language code will create a '.csv' file in\n` +
+        `\tthe 'translations' subdirectory. Translated strings are to\n` +
+        `\tbe added to this file.\n\n` +
+        `\tCalling the script without args will update the translation csv\n` +
+        `\tfile with new strings if any. Existing translations won't\n` +
+        `\tbe removed.\n` +
+        `\n` +
+        `Parameters:\n` +
+        `\tlanguage_code : An ISO 693-1 code which has 2 characters eg: en\n` +
+        `\n` +
+        `Reference:\n` +
+        `\tISO 693-1 codes: https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes`
+    );
+  }
+  return shouldPrint;
+}
+
+function getLanguageCode() {
+  const i = process.argv.findIndex((i) => i === '-l');
+  if (i === -1) {
+    return '';
+  }
+  return process.argv[i + 1]?.toLowerCase() ?? '';
+}
+
+function getTranslationFilePath(languageCode) {
+  return path.resolve(translationsFolder, `${languageCode}.csv`);
+}
+
+async function regenerateTranslation(tArray, path) {
+  // Removes old strings, adds new strings
+  const contents = await fs.readFile(path, { encoding: 'utf-8' });
+  const map = new Map();
+
+  // Populate map
+  contents
+    .split('\n')
+    .filter((l) => l.length)
+    .map(splitCsvLine)
+    .forEach((l) => {
+      if (l[1] === '' || !l[1]) {
+        return;
+      }
+
+      map.set(l[0].trim(), l.slice(1));
+    });
+
+  const regenContent = tArray
+    .map((l) => {
+      const source = wrap(l);
+      const translations = map.get(source);
+      return [source, ...(translations ?? [])].join(',');
+    })
+    .join('\n');
+  await fs.writeFile(path, regenContent, { encoding: 'utf-8' });
+  console.log(`\tregenerated: ${path}`);
+}
+
+async function regenerateTranslations(languageCode, tArray) {
+  // regenerate one file
+  if (languageCode) {
+    const path = getTranslationFilePath(languageCode);
+    regenerateTranslation(tArray, path);
+    return;
+  }
+
+  // regenerate all translation files
+  console.log(`Language code not passed, regenerating all translations.`);
+  const contents = (await fs.readdir(translationsFolder)).filter((f) =>
+    f.endsWith('.csv')
+  );
+  contents.forEach((f) =>
+    regenerateTranslation(tArray, path.resolve(translationsFolder, f))
+  );
+}
+
+async function writeTranslations(languageCode, tArray) {
+  const path = getTranslationFilePath(languageCode);
+  try {
+    const stat = await fs.stat(path);
+    if (!stat.isFile()) {
+      throw new Error(`${path} is not a translation file`);
+    }
+
+    console.log(
+      `Existing file found for '${languageCode}': ${path}\n` +
+        `regenerating it's translations.`
+    );
+    regenerateTranslations(languageCode, tArray);
+  } catch (err) {
+    if (err.errno !== -2) {
+      throw err;
+    }
+
+    const content = tArray.map(wrap).join(',\n') + ',';
+    await fs.writeFile(path, content, { encoding: 'utf-8' });
+    console.log(`Generated translation file for '${languageCode}': ${path}`);
+  }
+}
+
+async function run() {
+  if (printHelp()) {
+    return;
+  }
+
+  const root = path.resolve(__dirname, '..');
+  const ignoreList = ['node_modules', 'dist_electron', 'scripts'];
+  const languageCode = getLanguageCode();
+
+  console.log();
+  if (languageCode.length !== 0 && languageCode.length !== 2) {
+    console.error(
+      `Invalid language code passed: '${languageCode}'.\n` +
+        `Please use an ISO 639-1 language code: ${'https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes'}`
+    );
+    return;
+  }
+
+  const fileList = await getFileList(root, ignoreList);
+  const contents = await getFileContents(fileList);
+  const tMap = await getAllTStringsMap(contents);
+  const tArray = mapToTStringArray(tMap);
+
+  try {
+    await fs.stat(translationsFolder);
+  } catch (err) {
+    if (err.errno !== -2) {
+      throw err;
+    }
+
+    await fs.mkdir(translationsFolder);
+  }
+
+  if (languageCode === '') {
+    regenerateTranslations('', tArray);
+    return;
+  }
+
+  writeTranslations(languageCode, tArray);
+}
+
+run();
diff --git a/scripts/helpers.js b/scripts/helpers.js
new file mode 100644
index 00000000..9b0fe05b
--- /dev/null
+++ b/scripts/helpers.js
@@ -0,0 +1,93 @@
+function getIndexFormat(inp) {
+  // converts:
+  // ['This is an ', ,' interpolated ',' string.'] and
+  // 'This is an ${variableA} interpolated ${variableB} string.'
+  // to 'This is an ${0} interpolated ${1} string.'
+  let string, snippets;
+  if (typeof inp === 'string') {
+    string = inp;
+  } else if (inp instanceof Array) {
+    snippets = inp;
+  } else {
+    throw new Error(`invalid input ${inp} of type ${typeof inp}`);
+  }
+
+  if (snippets === undefined) {
+    snippets = getSnippets(string);
+  }
+
+  if (snippets.length === 1) {
+    return snippets[0];
+  }
+
+  let str = '';
+  snippets.forEach((s, i) => {
+    if (i === snippets.length - 1) {
+      str += s;
+      return;
+    }
+    str += s + '${' + i + '}';
+  });
+  return str;
+}
+
+function getSnippets(string) {
+  let start = 0;
+  snippets = [...string.matchAll(/\${[^}]+}/g)].map((m) => {
+    let end = m.index;
+    let snip = string.slice(start, end);
+    start = end + m[0].length;
+    return snip;
+  });
+
+  snippets.push(string.slice(start));
+  return snippets;
+}
+
+function getWhitespaceSanitized(s) {
+  return s.replace(/\s+/g, ' ').trim();
+}
+
+function getIndexList(s) {
+  return [...s.matchAll(/\${([^}]+)}/g)].map(([_, i]) => parseInt(i));
+}
+
+function wrap(s) {
+  return '`' + s + '`';
+}
+
+function splitCsvLine(line) {
+  let t = true;
+  const chars = [...line];
+  const indices = chars
+    .map((c, i) => {
+      if (c === '`') {
+        t = !t;
+      }
+
+      if (c === ',' && t) {
+        return i;
+      }
+
+      return -1;
+    })
+    .filter((i) => i !== -1);
+
+  let s = 0;
+  const splits = indices.map((i) => {
+    const split = line.slice(s, i);
+    s = i + 1;
+    return split.trim();
+  });
+  splits.push(line.slice(s).trim());
+  return splits.filter((s) => s !== ',' && s !== '');
+}
+
+module.exports = {
+  getIndexFormat,
+  getWhitespaceSanitized,
+  getSnippets,
+  getIndexList,
+  wrap,
+  splitCsvLine,
+};