diff --git a/backend/database/manager.ts b/backend/database/manager.ts index 6ba6487d..27ba1eb8 100644 --- a/backend/database/manager.ts +++ b/backend/database/manager.ts @@ -1,5 +1,7 @@ import { constants } from 'fs'; import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; import { DatabaseDemuxBase, DatabaseMethod } from 'utils/db/types'; import { getSchemas } from '../../schemas'; import { databaseMethodSet } from '../helpers'; @@ -26,18 +28,75 @@ export class DatabaseManager extends DatabaseDemuxBase { } async connectToDatabase(dbPath: string, countryCode?: string) { - countryCode ??= await DatabaseCore.getCountryCode(dbPath); - - this.db = new DatabaseCore(dbPath); - await this.db.connect(); - - const schemaMap = getSchemas(countryCode); - this.db.setSchemaMap(schemaMap); - + countryCode = await this._connect(dbPath, countryCode); await this.#migrate(); return countryCode; } + async _connect(dbPath: string, countryCode?: string) { + countryCode ??= await DatabaseCore.getCountryCode(dbPath); + this.db = new DatabaseCore(dbPath); + await this.db.connect(); + const schemaMap = getSchemas(countryCode); + this.db.setSchemaMap(schemaMap); + return countryCode; + } + + async #migrate(): Promise { + if (!this.#isInitialized) { + return; + } + + const isFirstRun = await this.#getIsFirstRun(); + if (isFirstRun) { + await this.db!.migrate(); + } + + /** + * This needs to be replaced with transactions + * TODO: Add transactions in core.ts + */ + const dbPath = this.db!.dbPath; + const copyPath = await this.#makeTempCopy(); + try { + await this.#runPatchesAndMigrate(); + } catch (err) { + await this.db!.close(); + await fs.copyFile(copyPath, dbPath); + await this._connect(dbPath); + + throw err; + } + + await fs.unlink(copyPath); + } + + async #runPatchesAndMigrate() { + const patchesToExecute = await this.#getPatchesToExecute(); + const preMigrationPatches = patchesToExecute.filter( + (p) => p.patch.beforeMigrate + ); + const postMigrationPatches = patchesToExecute.filter( + (p) => !p.patch.beforeMigrate + ); + + await runPatches(preMigrationPatches, this); + await this.db!.migrate(); + await runPatches(postMigrationPatches, this); + } + + async #getPatchesToExecute(): Promise { + if (this.db === undefined) { + return []; + } + + const query: { name: string }[] = await this.db.knex!('PatchRun').select( + 'name' + ); + const executedPatches = query.map((q) => q.name); + return patches.filter((p) => !executedPatches.includes(p.name)); + } + async call(method: DatabaseMethod, ...args: unknown[]) { if (!this.#isInitialized) { return; @@ -70,41 +129,6 @@ export class DatabaseManager extends DatabaseDemuxBase { return await queryFunction(this.db!, ...args); } - async #migrate(): Promise { - if (!this.#isInitialized) { - return; - } - - const isFirstRun = await this.#getIsFirstRun(); - if (isFirstRun) { - await this.db!.migrate(); - } - - const patchesToExecute = await this.#getPatchesToExecute(); - const preMigrationPatches = patchesToExecute.filter( - (p) => p.patch.beforeMigrate - ); - const postMigrationPatches = patchesToExecute.filter( - (p) => !p.patch.beforeMigrate - ); - - await runPatches(preMigrationPatches, this); - await this.db!.migrate(); - await runPatches(postMigrationPatches, this); - } - - async #getPatchesToExecute(): Promise { - if (this.db === undefined) { - return []; - } - - const query: { name: string }[] = await this.db.knex!('PatchRun').select( - 'name' - ); - const executedPatches = query.map((q) => q.name); - return patches.filter((p) => !executedPatches.includes(p.name)); - } - async #unlinkIfExists(dbPath: string) { const exists = await fs .access(dbPath, constants.W_OK) @@ -126,6 +150,13 @@ export class DatabaseManager extends DatabaseDemuxBase { ); return tableList.length === 0; } + + async #makeTempCopy() { + const src = this.db!.dbPath; + const dest = path.join(os.tmpdir(), 'temp.db'); + await fs.copyFile(src, dest); + return dest; + } } export default new DatabaseManager(); diff --git a/backend/patches/index.ts b/backend/patches/index.ts index 3630b598..5f917c47 100644 --- a/backend/patches/index.ts +++ b/backend/patches/index.ts @@ -1,6 +1,8 @@ import { Patch } from '../database/types'; import testPatch from './testPatch'; +import updateSchemas from './updateSchemas'; export default [ { name: 'testPatch', version: '0.5.0-beta.0', patch: testPatch }, + { name: 'updateSchemas', version: '0.5.0-beta.0', patch: updateSchemas }, ] as Patch[]; diff --git a/backend/patches/updateSchemas.ts b/backend/patches/updateSchemas.ts new file mode 100644 index 00000000..516b5d03 --- /dev/null +++ b/backend/patches/updateSchemas.ts @@ -0,0 +1,285 @@ +import fs from 'fs/promises'; +import { RawValueMap } from 'fyo/core/types'; +import { Knex } from 'knex'; +import { ModelNameEnum } from 'models/types'; +import os from 'os'; +import path from 'path'; +import { SchemaMap } from 'schemas/types'; +import { changeKeys, deleteKeys, invertMap } from 'utils'; +import { getCountryCodeFromCountry } from 'utils/misc'; +import { Version } from 'utils/version'; +import { DatabaseManager } from '../database/manager'; + +const ignoreColumns = ['keywords']; +const columnMap = { creation: 'created', owner: 'createdBy' }; +const childTableColumnMap = { + parenttype: 'parentSchemaName', + parentfield: 'parentFieldname', +}; + +const defaultNumberSeriesMap = { + [ModelNameEnum.Payment]: 'PAY-', + [ModelNameEnum.JournalEntry]: 'JE-', + [ModelNameEnum.SalesInvoice]: 'SINV-', + [ModelNameEnum.PurchaseInvoice]: 'PINV-', +} as Record; + +async function execute(dm: DatabaseManager) { + const sourceKnex = dm.db!.knex!; + const version = ( + await sourceKnex('SingleValue') + .select('value') + .where({ fieldname: 'version' }) + )?.[0]?.value; + + /** + * Versions after this should have the new schemas + */ + + if (version && Version.gt(version, '0.4.3-beta.0')) { + return; + } + + /** + * Initialize a different db to copy all the updated + * data into. + */ + const countryCode = await getCountryCode(sourceKnex); + const destDm = await getDestinationDM(sourceKnex, countryCode); + + /** + * Copy data from all the relevant tables + * the other tables will be empty cause unused. + */ + await copyData(sourceKnex, destDm); + + /** + * Version will update when migration completes, this + * is set to prevent this patch from running again. + */ + await destDm.db!.update(ModelNameEnum.SystemSettings, { + version: '0.5.0-beta.0', + }); + + /** + * Replace the database with the new one. + */ + await replaceDatabaseCore(dm, destDm); +} + +async function replaceDatabaseCore( + dm: DatabaseManager, + destDm: DatabaseManager +) { + const sourceDbPath = destDm.db!.dbPath; // new db with new schema + const destDbPath = dm.db!.dbPath; // old db to be replaced + + await dm.db!.close(); + await destDm.db!.close(); + + await fs.copyFile(sourceDbPath, destDbPath); + await dm._connect(destDbPath); +} + +async function copyData(sourceKnex: Knex, destDm: DatabaseManager) { + const destKnex = destDm.db!.knex!; + const schemaMap = destDm.getSchemaMap(); + await destKnex!.raw('PRAGMA foreign_keys=OFF'); + await copySingleValues(sourceKnex, destKnex, schemaMap); + await copyParty(sourceKnex, destKnex); + await copyItem(sourceKnex, destKnex); + await copyChildTables(sourceKnex, destKnex, schemaMap); + await copyOtherTables(sourceKnex, destKnex); + await copyTransactionalTables(sourceKnex, destKnex); + await copyLedgerEntries(sourceKnex, destKnex); + await copyNumberSeries(sourceKnex, destKnex); + await destKnex!.raw('PRAGMA foreign_keys=ON'); +} + +async function copyNumberSeries(sourceKnex: Knex, destKnex: Knex) { + const values = (await sourceKnex( + ModelNameEnum.NumberSeries + )) as RawValueMap[]; + + const refMap = invertMap(defaultNumberSeriesMap); + + for (const value of values) { + if (value.referenceType) { + continue; + } + + const name = value.name as string; + const referenceType = refMap[name]; + const indices = await sourceKnex.raw( + ` + select cast(substr(name, ??) as int) as idx + from ?? + order by idx desc + limit 1`, + [name.length + 1, referenceType] + ); + + value.start = 1001; + value.current = indices[0]?.idx ?? value.current ?? value.start; + value.referenceType = referenceType; + } + + await copyValues(destKnex, ModelNameEnum.NumberSeries, values); +} + +async function copyLedgerEntries(sourceKnex: Knex, destKnex: Knex) { + const values = (await sourceKnex( + ModelNameEnum.AccountingLedgerEntry + )) as RawValueMap[]; + await copyValues(destKnex, ModelNameEnum.AccountingLedgerEntry, values, [ + 'description', + 'againstAccount', + 'balance', + ]); +} + +async function copyOtherTables(sourceKnex: Knex, destKnex: Knex) { + const schemaNames = [ + ModelNameEnum.Account, + ModelNameEnum.Currency, + ModelNameEnum.Address, + ModelNameEnum.Color, + ModelNameEnum.Tax, + ]; + + for (const sn of schemaNames) { + const values = (await sourceKnex(sn)) as RawValueMap[]; + await copyValues(destKnex, sn, values); + } +} + +async function copyTransactionalTables(sourceKnex: Knex, destKnex: Knex) { + const schemaNames = [ + ModelNameEnum.JournalEntry, + ModelNameEnum.Payment, + ModelNameEnum.SalesInvoice, + ModelNameEnum.PurchaseInvoice, + ]; + + for (const sn of schemaNames) { + const values = (await sourceKnex(sn)) as RawValueMap[]; + values.forEach((v) => { + if (!v.submitted) { + v.submitted = 0; + } + + if (!v.cancelled) { + v.cancelled = 0; + } + + if (!v.numberSeries) { + v.numberSeries = defaultNumberSeriesMap[sn]; + } + }); + await copyValues(destKnex, sn, values, [], childTableColumnMap); + } +} + +async function copyChildTables( + sourceKnex: Knex, + destKnex: Knex, + schemaMap: SchemaMap +) { + const childSchemaNames = Object.keys(schemaMap).filter( + (sn) => schemaMap[sn]?.isChild + ); + + for (const sn of childSchemaNames) { + const values = (await sourceKnex(sn)) as RawValueMap[]; + await copyValues(destKnex, sn, values, [], childTableColumnMap); + } +} + +async function copyItem(sourceKnex: Knex, destKnex: Knex) { + const values = (await sourceKnex(ModelNameEnum.Item)) as RawValueMap[]; + values.forEach((value) => { + value.for = 'Both'; + }); + + await copyValues(destKnex, ModelNameEnum.Item, values); +} + +async function copyParty(sourceKnex: Knex, destKnex: Knex) { + const values = (await sourceKnex(ModelNameEnum.Party)) as RawValueMap[]; + values.forEach((value) => { + // customer will be mapped onto role + if (Number(value.supplier) === 1) { + value.customer = 'Supplier'; + } else { + value.customer = 'Customer'; + } + }); + + await copyValues( + destKnex, + ModelNameEnum.Party, + values, + ['supplier', 'addressDisplay'], + { customer: 'role' } + ); +} + +async function copySingleValues( + sourceKnex: Knex, + destKnex: Knex, + schemaMap: SchemaMap +) { + const singleSchemaNames = Object.keys(schemaMap).filter( + (k) => schemaMap[k]?.isSingle + ); + const singleValues = (await sourceKnex(ModelNameEnum.SingleValue).whereIn( + 'parent', + singleSchemaNames + )) as RawValueMap[]; + + await copyValues(destKnex, ModelNameEnum.SingleValue, singleValues); +} + +async function copyValues( + destKnex: Knex, + destTableName: string, + values: RawValueMap[], + keysToDelete: string[] = [], + keyMap: Record = {} +) { + keysToDelete = [...keysToDelete, ...ignoreColumns]; + keyMap = { ...keyMap, ...columnMap }; + + values = values.map((sv) => deleteKeys(sv, keysToDelete)); + values = values.map((sv) => changeKeys(sv, keyMap)); + + await destKnex.batchInsert(destTableName, values, 100); +} + +async function getDestinationDM(knex: Knex, countryCode: string) { + /** + * This is where all the stuff from the old db will be copied. + * That won't be altered cause schema update will cause data loss. + */ + const dbPath = path.join(os.tmpdir(), '__patch_db.db'); + const dm = new DatabaseManager(); + await dm.createNewDatabase(dbPath, countryCode); + return dm; +} + +async function getCountryCode(knex: Knex) { + /** + * Need to account for schema changes, in 0.4.3-beta.0 + */ + const country = ( + await knex('SingleValue').select('value').where({ fieldname: 'country' }) + )?.[0]?.value; + + if (!country) { + return ''; + } + + return getCountryCodeFromCountry(country); +} + +export default { execute, beforeMigrate: true }; diff --git a/utils/index.ts b/utils/index.ts index 029a38f7..c6f55b4d 100644 --- a/utils/index.ts +++ b/utils/index.ts @@ -125,3 +125,31 @@ export async function timeAsync( console.timeEnd(name); return stuff; } + +export function changeKeys( + source: Record, + keyMap: Record +) { + const dest: Record = {}; + for (const key of Object.keys(source)) { + const newKey = keyMap[key] ?? key; + dest[newKey] = source[key]; + } + + return dest; +} + +export function deleteKeys( + source: Record, + keysToDelete: string[] +) { + const dest: Record = {}; + for (const key of Object.keys(source)) { + if (keysToDelete.includes(key)) { + continue; + } + dest[key] = source[key]; + } + + return dest; +}