incr: add patch for v0.4 to v0.5

backend/database/manager.ts

@@ -1,5 +1,7 @@
 import { constants } from 'fs';
 import fs from 'fs/promises';
+import os from 'os';
+import path from 'path';
 import { DatabaseDemuxBase, DatabaseMethod } from 'utils/db/types';
 import { getSchemas } from '../../schemas';
 import { databaseMethodSet } from '../helpers';
@@ -26,18 +28,75 @@ export class DatabaseManager extends DatabaseDemuxBase {
   }
 
   async connectToDatabase(dbPath: string, countryCode?: string) {
-    countryCode ??= await DatabaseCore.getCountryCode(dbPath);
+    countryCode = await this._connect(dbPath, countryCode);
-
-    this.db = new DatabaseCore(dbPath);
-    await this.db.connect();
-
-    const schemaMap = getSchemas(countryCode);
-    this.db.setSchemaMap(schemaMap);
-
     await this.#migrate();
     return countryCode;
   }
 
+  async _connect(dbPath: string, countryCode?: string) {
+    countryCode ??= await DatabaseCore.getCountryCode(dbPath);
+    this.db = new DatabaseCore(dbPath);
+    await this.db.connect();
+    const schemaMap = getSchemas(countryCode);
+    this.db.setSchemaMap(schemaMap);
+    return countryCode;
+  }
+
+  async #migrate(): Promise<void> {
+    if (!this.#isInitialized) {
+      return;
+    }
+
+    const isFirstRun = await this.#getIsFirstRun();
+    if (isFirstRun) {
+      await this.db!.migrate();
+    }
+
+    /**
+     * This needs to be replaced with transactions
+     * TODO: Add transactions in core.ts
+     */
+    const dbPath = this.db!.dbPath;
+    const copyPath = await this.#makeTempCopy();
+    try {
+      await this.#runPatchesAndMigrate();
+    } catch (err) {
+      await this.db!.close();
+      await fs.copyFile(copyPath, dbPath);
+      await this._connect(dbPath);
+
+      throw err;
+    }
+
+    await fs.unlink(copyPath);
+  }
+
+  async #runPatchesAndMigrate() {
+    const patchesToExecute = await this.#getPatchesToExecute();
+    const preMigrationPatches = patchesToExecute.filter(
+      (p) => p.patch.beforeMigrate
+    );
+    const postMigrationPatches = patchesToExecute.filter(
+      (p) => !p.patch.beforeMigrate
+    );
+
+    await runPatches(preMigrationPatches, this);
+    await this.db!.migrate();
+    await runPatches(postMigrationPatches, this);
+  }
+
+  async #getPatchesToExecute(): Promise<Patch[]> {
+    if (this.db === undefined) {
+      return [];
+    }
+
+    const query: { name: string }[] = await this.db.knex!('PatchRun').select(
+      'name'
+    );
+    const executedPatches = query.map((q) => q.name);
+    return patches.filter((p) => !executedPatches.includes(p.name));
+  }
+
   async call(method: DatabaseMethod, ...args: unknown[]) {
     if (!this.#isInitialized) {
       return;
@@ -70,41 +129,6 @@ export class DatabaseManager extends DatabaseDemuxBase {
     return await queryFunction(this.db!, ...args);
   }
 
-  async #migrate(): Promise<void> {
-    if (!this.#isInitialized) {
-      return;
-    }
-
-    const isFirstRun = await this.#getIsFirstRun();
-    if (isFirstRun) {
-      await this.db!.migrate();
-    }
-
-    const patchesToExecute = await this.#getPatchesToExecute();
-    const preMigrationPatches = patchesToExecute.filter(
-      (p) => p.patch.beforeMigrate
-    );
-    const postMigrationPatches = patchesToExecute.filter(
-      (p) => !p.patch.beforeMigrate
-    );
-
-    await runPatches(preMigrationPatches, this);
-    await this.db!.migrate();
-    await runPatches(postMigrationPatches, this);
-  }
-
-  async #getPatchesToExecute(): Promise<Patch[]> {
-    if (this.db === undefined) {
-      return [];
-    }
-
-    const query: { name: string }[] = await this.db.knex!('PatchRun').select(
-      'name'
-    );
-    const executedPatches = query.map((q) => q.name);
-    return patches.filter((p) => !executedPatches.includes(p.name));
-  }
-
   async #unlinkIfExists(dbPath: string) {
     const exists = await fs
       .access(dbPath, constants.W_OK)
@@ -126,6 +150,13 @@ export class DatabaseManager extends DatabaseDemuxBase {
     );
     return tableList.length === 0;
   }
+
+  async #makeTempCopy() {
+    const src = this.db!.dbPath;
+    const dest = path.join(os.tmpdir(), 'temp.db');
+    await fs.copyFile(src, dest);
+    return dest;
+  }
 }
 
 export default new DatabaseManager();

backend/patches/index.ts

@@ -1,6 +1,8 @@
 import { Patch } from '../database/types';
 import testPatch from './testPatch';
+import updateSchemas from './updateSchemas';
 
 export default [
   { name: 'testPatch', version: '0.5.0-beta.0', patch: testPatch },
+  { name: 'updateSchemas', version: '0.5.0-beta.0', patch: updateSchemas },
 ] as Patch[];

backend/patches/updateSchemas.ts

@@ -0,0 +1,285 @@
+import fs from 'fs/promises';
+import { RawValueMap } from 'fyo/core/types';
+import { Knex } from 'knex';
+import { ModelNameEnum } from 'models/types';
+import os from 'os';
+import path from 'path';
+import { SchemaMap } from 'schemas/types';
+import { changeKeys, deleteKeys, invertMap } from 'utils';
+import { getCountryCodeFromCountry } from 'utils/misc';
+import { Version } from 'utils/version';
+import { DatabaseManager } from '../database/manager';
+
+const ignoreColumns = ['keywords'];
+const columnMap = { creation: 'created', owner: 'createdBy' };
+const childTableColumnMap = {
+  parenttype: 'parentSchemaName',
+  parentfield: 'parentFieldname',
+};
+
+const defaultNumberSeriesMap = {
+  [ModelNameEnum.Payment]: 'PAY-',
+  [ModelNameEnum.JournalEntry]: 'JE-',
+  [ModelNameEnum.SalesInvoice]: 'SINV-',
+  [ModelNameEnum.PurchaseInvoice]: 'PINV-',
+} as Record<ModelNameEnum, string>;
+
+async function execute(dm: DatabaseManager) {
+  const sourceKnex = dm.db!.knex!;
+  const version = (
+    await sourceKnex('SingleValue')
+      .select('value')
+      .where({ fieldname: 'version' })
+  )?.[0]?.value;
+
+  /**
+   * Versions after this should have the new schemas
+   */
+
+  if (version && Version.gt(version, '0.4.3-beta.0')) {
+    return;
+  }
+
+  /**
+   * Initialize a different db to copy all the updated
+   * data into.
+   */
+  const countryCode = await getCountryCode(sourceKnex);
+  const destDm = await getDestinationDM(sourceKnex, countryCode);
+
+  /**
+   * Copy data from all the relevant tables
+   * the other tables will be empty cause unused.
+   */
+  await copyData(sourceKnex, destDm);
+
+  /**
+   * Version will update when migration completes, this
+   * is set to prevent this patch from running again.
+   */
+  await destDm.db!.update(ModelNameEnum.SystemSettings, {
+    version: '0.5.0-beta.0',
+  });
+
+  /**
+   * Replace the database with the new one.
+   */
+  await replaceDatabaseCore(dm, destDm);
+}
+
+async function replaceDatabaseCore(
+  dm: DatabaseManager,
+  destDm: DatabaseManager
+) {
+  const sourceDbPath = destDm.db!.dbPath; // new db with new schema
+  const destDbPath = dm.db!.dbPath; // old db to be replaced
+
+  await dm.db!.close();
+  await destDm.db!.close();
+
+  await fs.copyFile(sourceDbPath, destDbPath);
+  await dm._connect(destDbPath);
+}
+
+async function copyData(sourceKnex: Knex, destDm: DatabaseManager) {
+  const destKnex = destDm.db!.knex!;
+  const schemaMap = destDm.getSchemaMap();
+  await destKnex!.raw('PRAGMA foreign_keys=OFF');
+  await copySingleValues(sourceKnex, destKnex, schemaMap);
+  await copyParty(sourceKnex, destKnex);
+  await copyItem(sourceKnex, destKnex);
+  await copyChildTables(sourceKnex, destKnex, schemaMap);
+  await copyOtherTables(sourceKnex, destKnex);
+  await copyTransactionalTables(sourceKnex, destKnex);
+  await copyLedgerEntries(sourceKnex, destKnex);
+  await copyNumberSeries(sourceKnex, destKnex);
+  await destKnex!.raw('PRAGMA foreign_keys=ON');
+}
+
+async function copyNumberSeries(sourceKnex: Knex, destKnex: Knex) {
+  const values = (await sourceKnex(
+    ModelNameEnum.NumberSeries
+  )) as RawValueMap[];
+
+  const refMap = invertMap(defaultNumberSeriesMap);
+
+  for (const value of values) {
+    if (value.referenceType) {
+      continue;
+    }
+
+    const name = value.name as string;
+    const referenceType = refMap[name];
+    const indices = await sourceKnex.raw(
+      `
+      select cast(substr(name, ??) as int) as idx
+      from ?? 
+      order by idx desc 
+      limit 1`,
+      [name.length + 1, referenceType]
+    );
+
+    value.start = 1001;
+    value.current = indices[0]?.idx ?? value.current ?? value.start;
+    value.referenceType = referenceType;
+  }
+
+  await copyValues(destKnex, ModelNameEnum.NumberSeries, values);
+}
+
+async function copyLedgerEntries(sourceKnex: Knex, destKnex: Knex) {
+  const values = (await sourceKnex(
+    ModelNameEnum.AccountingLedgerEntry
+  )) as RawValueMap[];
+  await copyValues(destKnex, ModelNameEnum.AccountingLedgerEntry, values, [
+    'description',
+    'againstAccount',
+    'balance',
+  ]);
+}
+
+async function copyOtherTables(sourceKnex: Knex, destKnex: Knex) {
+  const schemaNames = [
+    ModelNameEnum.Account,
+    ModelNameEnum.Currency,
+    ModelNameEnum.Address,
+    ModelNameEnum.Color,
+    ModelNameEnum.Tax,
+  ];
+
+  for (const sn of schemaNames) {
+    const values = (await sourceKnex(sn)) as RawValueMap[];
+    await copyValues(destKnex, sn, values);
+  }
+}
+
+async function copyTransactionalTables(sourceKnex: Knex, destKnex: Knex) {
+  const schemaNames = [
+    ModelNameEnum.JournalEntry,
+    ModelNameEnum.Payment,
+    ModelNameEnum.SalesInvoice,
+    ModelNameEnum.PurchaseInvoice,
+  ];
+
+  for (const sn of schemaNames) {
+    const values = (await sourceKnex(sn)) as RawValueMap[];
+    values.forEach((v) => {
+      if (!v.submitted) {
+        v.submitted = 0;
+      }
+
+      if (!v.cancelled) {
+        v.cancelled = 0;
+      }
+
+      if (!v.numberSeries) {
+        v.numberSeries = defaultNumberSeriesMap[sn];
+      }
+    });
+    await copyValues(destKnex, sn, values, [], childTableColumnMap);
+  }
+}
+
+async function copyChildTables(
+  sourceKnex: Knex,
+  destKnex: Knex,
+  schemaMap: SchemaMap
+) {
+  const childSchemaNames = Object.keys(schemaMap).filter(
+    (sn) => schemaMap[sn]?.isChild
+  );
+
+  for (const sn of childSchemaNames) {
+    const values = (await sourceKnex(sn)) as RawValueMap[];
+    await copyValues(destKnex, sn, values, [], childTableColumnMap);
+  }
+}
+
+async function copyItem(sourceKnex: Knex, destKnex: Knex) {
+  const values = (await sourceKnex(ModelNameEnum.Item)) as RawValueMap[];
+  values.forEach((value) => {
+    value.for = 'Both';
+  });
+
+  await copyValues(destKnex, ModelNameEnum.Item, values);
+}
+
+async function copyParty(sourceKnex: Knex, destKnex: Knex) {
+  const values = (await sourceKnex(ModelNameEnum.Party)) as RawValueMap[];
+  values.forEach((value) => {
+    // customer will be mapped onto role
+    if (Number(value.supplier) === 1) {
+      value.customer = 'Supplier';
+    } else {
+      value.customer = 'Customer';
+    }
+  });
+
+  await copyValues(
+    destKnex,
+    ModelNameEnum.Party,
+    values,
+    ['supplier', 'addressDisplay'],
+    { customer: 'role' }
+  );
+}
+
+async function copySingleValues(
+  sourceKnex: Knex,
+  destKnex: Knex,
+  schemaMap: SchemaMap
+) {
+  const singleSchemaNames = Object.keys(schemaMap).filter(
+    (k) => schemaMap[k]?.isSingle
+  );
+  const singleValues = (await sourceKnex(ModelNameEnum.SingleValue).whereIn(
+    'parent',
+    singleSchemaNames
+  )) as RawValueMap[];
+
+  await copyValues(destKnex, ModelNameEnum.SingleValue, singleValues);
+}
+
+async function copyValues(
+  destKnex: Knex,
+  destTableName: string,
+  values: RawValueMap[],
+  keysToDelete: string[] = [],
+  keyMap: Record<string, string> = {}
+) {
+  keysToDelete = [...keysToDelete, ...ignoreColumns];
+  keyMap = { ...keyMap, ...columnMap };
+
+  values = values.map((sv) => deleteKeys(sv, keysToDelete));
+  values = values.map((sv) => changeKeys(sv, keyMap));
+
+  await destKnex.batchInsert(destTableName, values, 100);
+}
+
+async function getDestinationDM(knex: Knex, countryCode: string) {
+  /**
+   * This is where all the stuff from the old db will be copied.
+   * That won't be altered cause schema update will cause data loss.
+   */
+  const dbPath = path.join(os.tmpdir(), '__patch_db.db');
+  const dm = new DatabaseManager();
+  await dm.createNewDatabase(dbPath, countryCode);
+  return dm;
+}
+
+async function getCountryCode(knex: Knex) {
+  /**
+   * Need to account for schema changes, in 0.4.3-beta.0
+   */
+  const country = (
+    await knex('SingleValue').select('value').where({ fieldname: 'country' })
+  )?.[0]?.value;
+
+  if (!country) {
+    return '';
+  }
+
+  return getCountryCodeFromCountry(country);
+}
+
+export default { execute, beforeMigrate: true };

utils/index.ts

@@ -125,3 +125,31 @@ export async function timeAsync<K, T>(
   console.timeEnd(name);
   return stuff;
 }
+
+export function changeKeys<T>(
+  source: Record<string, T>,
+  keyMap: Record<string, string | undefined>
+) {
+  const dest: Record<string, T> = {};
+  for (const key of Object.keys(source)) {
+    const newKey = keyMap[key] ?? key;
+    dest[newKey] = source[key];
+  }
+
+  return dest;
+}
+
+export function deleteKeys<T>(
+  source: Record<string, T>,
+  keysToDelete: string[]
+) {
+  const dest: Record<string, T> = {};
+  for (const key of Object.keys(source)) {
+    if (keysToDelete.includes(key)) {
+      continue;
+    }
+    dest[key] = source[key];
+  }
+
+  return dest;
+}