mirror of
https://github.com/frappe/books.git
synced 2024-12-31 05:51:49 +00:00
incr: fetch translateables from schema.json
- use legit csv
This commit is contained in:
18alantom
parent
56e5a7fc74
commit
03ee043696
@ -14,7 +14,7 @@
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { splitCsvLine } from 'utils/translationHelpers';
|
||||
import { parseCSV } from 'utils/csvParser';
|
||||
import { LanguageMap } from 'utils/types';
|
||||
|
||||
const fetch = require('node-fetch').default;
|
||||
@ -23,26 +23,32 @@ const VALENTINES_DAY = 1644796800000;
|
||||
|
||||
export async function getLanguageMap(code: string): Promise<LanguageMap> {
|
||||
const contents = await getContents(code);
|
||||
return getMapFromContents(contents);
|
||||
return getMapFromCsv(contents);
|
||||
}
|
||||
|
||||
function getMapFromContents(contents: string): LanguageMap {
|
||||
const lines: string[] = contents.split('\n').slice(1);
|
||||
return lines
|
||||
.map((l) => splitCsvLine(l) as string[])
|
||||
.filter((l) => l.length >= 2)
|
||||
.reduce((acc, l) => {
|
||||
const key = l[0].slice(1, -1);
|
||||
const translation = l[1].slice(1, -1);
|
||||
acc[key] = { translation };
|
||||
function getMapFromCsv(csv: string): LanguageMap {
|
||||
const matrix = parseCSV(csv);
|
||||
const languageMap: LanguageMap = {};
|
||||
|
||||
const context = l.slice(2)[0];
|
||||
if (context?.length) {
|
||||
acc[key].context = context;
|
||||
}
|
||||
for (const row of matrix) {
|
||||
/**
|
||||
* Ignore lines that have no translations
|
||||
*/
|
||||
if (!row[0] || !row[1]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return acc;
|
||||
}, {} as LanguageMap);
|
||||
const source = row[0];
|
||||
const translation = row[1];
|
||||
const context = row[3];
|
||||
|
||||
languageMap[source] = { translation };
|
||||
if (context?.length) {
|
||||
languageMap[source].context = context;
|
||||
}
|
||||
}
|
||||
|
||||
return languageMap;
|
||||
}
|
||||
|
||||
async function getContents(code: string) {
|
||||
|
||||
@ -344,7 +344,7 @@ export class GeneralLedger extends LedgerReport {
|
||||
getColumns(): ColumnField[] {
|
||||
let columns = [
|
||||
{
|
||||
label: t`#`,
|
||||
label: '#',
|
||||
fieldtype: 'Int',
|
||||
fieldname: 'index',
|
||||
align: 'right',
|
||||
|
||||
@ -1,15 +1,17 @@
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { generateCSV, parseCSV } from '../utils/csvParser';
|
||||
import {
|
||||
getIndexFormat,
|
||||
getWhitespaceSanitized,
|
||||
splitCsvLine,
|
||||
wrap,
|
||||
} from '../utils/translationHelpers';
|
||||
|
||||
const translationsFolder = path.resolve(__dirname, '..', 'translations');
|
||||
const PATTERN = /(?<!\w)t`([^`]+)`/g;
|
||||
|
||||
type Content = { fileName: string; content: string };
|
||||
type UnknownMap = Record<string, unknown>;
|
||||
|
||||
function shouldIgnore(p: string, ignoreList: string[]): boolean {
|
||||
const name = p.split(path.sep).at(-1) ?? '';
|
||||
return ignoreList.includes(name);
|
||||
@ -17,7 +19,8 @@ function shouldIgnore(p: string, ignoreList: string[]): boolean {
|
||||
|
||||
async function getFileList(
|
||||
root: string,
|
||||
ignoreList: string[]
|
||||
ignoreList: string[],
|
||||
extPattern: RegExp = /\.(js|ts|vue)$/
|
||||
): Promise<string[]> {
|
||||
const contents: string[] = await fs.readdir(root);
|
||||
const files: string[] = [];
|
||||
@ -28,11 +31,11 @@ async function getFileList(
|
||||
const isDir = (await fs.stat(absPath)).isDirectory();
|
||||
|
||||
if (isDir && !shouldIgnore(absPath, ignoreList)) {
|
||||
const pr = getFileList(absPath, ignoreList).then((fl) => {
|
||||
const pr = getFileList(absPath, ignoreList, extPattern).then((fl) => {
|
||||
files.push(...fl);
|
||||
});
|
||||
promises.push(pr);
|
||||
} else if (absPath.match(/\.(js|ts|vue)$/) !== null) {
|
||||
} else if (absPath.match(extPattern) !== null) {
|
||||
files.push(absPath);
|
||||
}
|
||||
}
|
||||
@ -88,13 +91,20 @@ function tStringFinder(content: string): string[] {
|
||||
});
|
||||
}
|
||||
|
||||
function mapToTStringArray(tMap: Map<string, string[]>): string[] {
|
||||
function tStringsToArray(
|
||||
tMap: Map<string, string[]>,
|
||||
tStrings: string[]
|
||||
): string[] {
|
||||
const tSet: Set<string> = new Set();
|
||||
for (const k of tMap.keys()) {
|
||||
tMap.get(k)!.forEach((s) => tSet.add(s));
|
||||
}
|
||||
const tArray = [...tSet];
|
||||
return tArray.sort();
|
||||
|
||||
for (const ts of tStrings) {
|
||||
tSet.add(ts);
|
||||
}
|
||||
|
||||
return Array.from(tSet).sort();
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
@ -141,36 +151,31 @@ function getTranslationFilePath(languageCode: string) {
|
||||
|
||||
async function regenerateTranslation(tArray: string[], path: string) {
|
||||
// Removes old strings, adds new strings
|
||||
const contents = await fs.readFile(path, { encoding: 'utf-8' });
|
||||
const storedCSV = await fs.readFile(path, { encoding: 'utf-8' });
|
||||
const storedMatrix = parseCSV(storedCSV);
|
||||
|
||||
const map: Map<string, string[]> = new Map();
|
||||
for (const row of storedMatrix) {
|
||||
const tstring = row[0];
|
||||
map.set(tstring, row.slice(1));
|
||||
}
|
||||
|
||||
// Populate map
|
||||
contents
|
||||
.split('\n')
|
||||
.filter((l) => l.length)
|
||||
.map(splitCsvLine)
|
||||
.forEach((l) => {
|
||||
if (l[1] === '' || !l[1]) {
|
||||
return;
|
||||
}
|
||||
const matrix = tArray.map((source) => {
|
||||
const stored = map.get(source) ?? [];
|
||||
const translation = stored[0] ?? '';
|
||||
const context = stored[1] ?? '';
|
||||
|
||||
map.set(l[0].trim(), l.slice(1));
|
||||
});
|
||||
return [source, translation, context];
|
||||
});
|
||||
const csv = generateCSV(matrix);
|
||||
|
||||
const regenContent = tArray
|
||||
.map((l) => {
|
||||
const source = wrap(l);
|
||||
const translations = map.get(source);
|
||||
return [source, ...(translations ?? [])].join(',');
|
||||
})
|
||||
.join('\n');
|
||||
await fs.writeFile(path, regenContent, { encoding: 'utf-8' });
|
||||
await fs.writeFile(path, csv, { encoding: 'utf-8' });
|
||||
console.log(`\tregenerated: ${path}`);
|
||||
}
|
||||
|
||||
async function regenerateTranslations(languageCode: string, tArray: string[]) {
|
||||
// regenerate one file
|
||||
if (languageCode.length === 0) {
|
||||
if (languageCode.length !== 0) {
|
||||
const path = getTranslationFilePath(languageCode);
|
||||
regenerateTranslation(tArray, path);
|
||||
return;
|
||||
@ -178,12 +183,13 @@ async function regenerateTranslations(languageCode: string, tArray: string[]) {
|
||||
|
||||
// regenerate all translation files
|
||||
console.log(`Language code not passed, regenerating all translations.`);
|
||||
const contents = (await fs.readdir(translationsFolder)).filter((f) =>
|
||||
f.endsWith('.csv')
|
||||
);
|
||||
contents.forEach((f) =>
|
||||
regenerateTranslation(tArray, path.resolve(translationsFolder, f))
|
||||
);
|
||||
for (const filePath of await fs.readdir(translationsFolder)) {
|
||||
if (!filePath.endsWith('.csv')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
regenerateTranslation(tArray, path.resolve(translationsFolder, filePath));
|
||||
}
|
||||
}
|
||||
|
||||
async function writeTranslations(languageCode: string, tArray: string[]) {
|
||||
@ -204,13 +210,82 @@ async function writeTranslations(languageCode: string, tArray: string[]) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
const content = tArray.map(wrap).join(',\n') + ',';
|
||||
await fs.writeFile(path, content, { encoding: 'utf-8' });
|
||||
const matrix = tArray.map((s) => [s, '', '']);
|
||||
const csv = generateCSV(matrix);
|
||||
await fs.writeFile(path, csv, { encoding: 'utf-8' });
|
||||
console.log(`Generated translation file for '${languageCode}': ${path}`);
|
||||
}
|
||||
}
|
||||
|
||||
type Content = { fileName: string; content: string };
|
||||
async function getTStringsFromJsonFileList(
|
||||
fileList: string[]
|
||||
): Promise<string[]> {
|
||||
const promises: Promise<void>[] = [];
|
||||
const schemaTStrings: string[][] = [];
|
||||
|
||||
for (const filePath of fileList) {
|
||||
const promise = fs
|
||||
.readFile(filePath, { encoding: 'utf8' })
|
||||
.then((content) => {
|
||||
const schema = JSON.parse(content) as Record<string, unknown>;
|
||||
const tStrings: string[] = [];
|
||||
pushTStringsFromSchema(schema, tStrings, [
|
||||
'label',
|
||||
'description',
|
||||
'placeholder',
|
||||
]);
|
||||
return tStrings;
|
||||
})
|
||||
.then((ts) => {
|
||||
schemaTStrings.push(ts);
|
||||
});
|
||||
|
||||
promises.push(promise);
|
||||
}
|
||||
|
||||
await Promise.all(promises);
|
||||
return schemaTStrings.flat();
|
||||
}
|
||||
|
||||
function pushTStringsFromSchema(
|
||||
map: UnknownMap | UnknownMap[],
|
||||
array: string[],
|
||||
translateable: string[]
|
||||
) {
|
||||
if (Array.isArray(map)) {
|
||||
for (const item of map) {
|
||||
pushTStringsFromSchema(item, array, translateable);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof map !== 'object') {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const key of Object.keys(map)) {
|
||||
const value = map[key];
|
||||
if (translateable.includes(key) && typeof value === 'string') {
|
||||
array.push(value);
|
||||
}
|
||||
|
||||
if (typeof value !== 'object') {
|
||||
continue;
|
||||
}
|
||||
|
||||
pushTStringsFromSchema(
|
||||
value as UnknownMap | UnknownMap[],
|
||||
array,
|
||||
translateable
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function getSchemaTStrings() {
|
||||
const root = path.resolve(__dirname, '../schemas');
|
||||
const fileList = await getFileList(root, ['tests', 'regional'], /\.json$/);
|
||||
return await getTStringsFromJsonFileList(fileList);
|
||||
}
|
||||
|
||||
async function run() {
|
||||
if (printHelp()) {
|
||||
@ -225,7 +300,8 @@ async function run() {
|
||||
const fileList: string[] = await getFileList(root, ignoreList);
|
||||
const contents: Content[] = await getFileContents(fileList);
|
||||
const tMap: Map<string, string[]> = await getAllTStringsMap(contents);
|
||||
const tArray: string[] = mapToTStringArray(tMap);
|
||||
const schemaTStrings: string[] = await getSchemaTStrings();
|
||||
const tArray: string[] = tStringsToArray(tMap, schemaTStrings);
|
||||
|
||||
try {
|
||||
await fs.stat(translationsFolder);
|
||||
|
||||
@ -49,11 +49,8 @@ function splitCsvBlock(text: string, splitter: string = '\r\n'): string[] {
|
||||
return lines;
|
||||
}
|
||||
|
||||
function splitCsvLine(line: string): string[] {
|
||||
if (line.at(-1) !== ',') {
|
||||
// if conforming to spec, it should not end with ','
|
||||
line += ',';
|
||||
}
|
||||
export function splitCsvLine(line: string): string[] {
|
||||
line += ',';
|
||||
|
||||
const items = [];
|
||||
let item = '';
|
||||
|
||||
@ -57,34 +57,3 @@ export function getWhitespaceSanitized(str: string) {
|
||||
export function getIndexList(str: string) {
|
||||
return [...str.matchAll(/\${([^}]+)}/g)].map(([_, i]) => parseInt(i));
|
||||
}
|
||||
|
||||
export function wrap(str: string) {
|
||||
return '`' + str + '`';
|
||||
}
|
||||
|
||||
export function splitCsvLine(line: string) {
|
||||
let t = true;
|
||||
const chars = [...line];
|
||||
const indices = chars
|
||||
.map((c, i) => {
|
||||
if (c === '`') {
|
||||
t = !t;
|
||||
}
|
||||
|
||||
if (c === ',' && t) {
|
||||
return i;
|
||||
}
|
||||
|
||||
return -1;
|
||||
})
|
||||
.filter((i) => i !== -1);
|
||||
|
||||
let s = 0;
|
||||
const splits = indices.map((i) => {
|
||||
const split = line.slice(s, i);
|
||||
s = i + 1;
|
||||
return split.trim();
|
||||
});
|
||||
splits.push(line.slice(s).trim());
|
||||
return splits.filter((s) => s !== ',' && s !== '');
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user