From 823739f5641c83f947c8ba10aa7de9ba19f88a65 Mon Sep 17 00:00:00 2001 From: 18alantom <2.alan.tom@gmail.com> Date: Wed, 12 Jul 2023 14:58:14 +0530 Subject: [PATCH] fix: create backup on migrate or patch --- backend/database/manager.ts | 186 ++++++++++++++++++++++--------- backend/database/runPatch.ts | 38 +++++-- package.json | 1 + schemas/core/PatchRun.json | 12 ++ schemas/core/SystemSettings.json | 1 + src/pages/GetStarted.vue | 3 - yarn.lock | 7 ++ 7 files changed, 180 insertions(+), 68 deletions(-) diff --git a/backend/database/manager.ts b/backend/database/manager.ts index fe78f3ca..07235c3c 100644 --- a/backend/database/manager.ts +++ b/backend/database/manager.ts @@ -1,4 +1,4 @@ -import fs from 'fs/promises'; +import fs from 'fs-extra'; import { DatabaseError } from 'fyo/utils/errors'; import path from 'path'; import { DatabaseDemuxBase, DatabaseMethod } from 'utils/db/types'; @@ -9,6 +9,9 @@ import { BespokeQueries } from './bespoke'; import DatabaseCore from './core'; import { runPatches } from './runPatch'; import { BespokeFunction, Patch } from './types'; +import BetterSQLite3 from 'better-sqlite3'; +import { getMapFromList } from 'utils/index'; +import { Version } from 'utils/version'; export class DatabaseManager extends DatabaseDemuxBase { db?: DatabaseCore; @@ -50,25 +53,12 @@ export class DatabaseManager extends DatabaseDemuxBase { return; } - const isFirstRun = await this.#getIsFirstRun(); + const isFirstRun = this.#getIsFirstRun(); if (isFirstRun) { await this.db!.migrate(); } - /** - * This needs to be supplimented with transactions - * TODO: Add transactions in core.ts - */ - const dbPath = this.db!.dbPath; - const copyPath = await this.#makeTempCopy(); - - try { - await this.#runPatchesAndMigrate(); - } catch (error) { - await this.#handleFailedMigration(error, dbPath, copyPath); - } finally { - await unlinkIfExists(copyPath); - } + await this.#executeMigration(); } async #handleFailedMigration( @@ -89,32 +79,67 @@ export class DatabaseManager extends DatabaseDemuxBase { throw error; } - async #runPatchesAndMigrate() { - const patchesToExecute = await this.#getPatchesToExecute(); + async #executeMigration() { + const version = this.#getAppVersion(); + const patches = await this.#getPatchesToExecute(version); - patchesToExecute.sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0)); - const preMigrationPatches = patchesToExecute.filter( - (p) => p.patch.beforeMigrate - ); - const postMigrationPatches = patchesToExecute.filter( - (p) => !p.patch.beforeMigrate - ); - - await runPatches(preMigrationPatches, this); - await this.db!.migrate(); - await runPatches(postMigrationPatches, this); - } - - async #getPatchesToExecute(): Promise { - if (this.db === undefined) { - return []; + const hasPatches = !!patches.pre.length || !!patches.post.length; + if (hasPatches) { + await this.#createBackup(); } - const query: { name: string }[] = await this.db.knex!('PatchRun').select( - 'name' - ); - const executedPatches = query.map((q) => q.name); - return patches.filter((p) => !executedPatches.includes(p.name)); + await runPatches(patches.pre, this, version); + await this.db!.migrate({ + pre: async () => { + if (hasPatches) { + return; + } + + await this.#createBackup(); + }, + }); + await runPatches(patches.post, this, version); + } + + async #getPatchesToExecute( + version: string + ): Promise<{ pre: Patch[]; post: Patch[] }> { + if (this.db === undefined) { + return { pre: [], post: [] }; + } + + const query = (await this.db.knex!('PatchRun').select()) as { + name: string; + version?: string; + failed?: boolean; + }[]; + + const runPatchesMap = getMapFromList(query, 'name'); + /** + * A patch is run only if: + * - it hasn't run and was added in a future version + * i.e. app version is before patch added version + * - it ran but failed in some other version (i.e fixed) + */ + const filtered = patches + .filter((p) => { + const exec = runPatchesMap[p.name]; + if (!exec && Version.lte(version, p.version)) { + return true; + } + + if (exec?.failed && exec?.version !== version) { + return true; + } + + return false; + }) + .sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0)); + + return { + pre: filtered.filter((p) => p.patch.beforeMigrate), + post: filtered.filter((p) => !p.patch.beforeMigrate), + }; } async call(method: DatabaseMethod, ...args: unknown[]) { @@ -149,33 +174,86 @@ export class DatabaseManager extends DatabaseDemuxBase { return await queryFunction(this.db!, ...args); } - async #getIsFirstRun(): Promise { - if (!this.#isInitialized) { + #getIsFirstRun(): boolean { + const db = this.getDriver(); + if (!db) { return true; } - const tableList: unknown[] = await this.db!.knex!.raw( - "SELECT name FROM sqlite_master WHERE type='table'" - ); - return tableList.length === 0; + const noPatchRun = + db + .prepare( + `select name from sqlite_master + where + type = 'table' and + name = 'PatchRun'` + ) + .all().length === 0; + + db.close(); + return noPatchRun; } - async #makeTempCopy() { - const src = this.db!.dbPath; - if (src === ':memory:') { + async #createBackup() { + const { dbPath } = this.db ?? {}; + if (!dbPath) { + return; + } + + const backupPath = this.#getBackupFilePath(); + if (!backupPath) { + return; + } + + const db = this.getDriver(); + await db?.backup(backupPath); + db?.close(); + } + + #getBackupFilePath() { + const { dbPath } = this.db ?? {}; + if (dbPath === ':memory:' || !dbPath) { return null; } - const dir = path.parse(src).dir; - const dest = path.join(dir, '__premigratory_temp.db'); + let fileName = path.parse(dbPath).name; + if (fileName.endsWith('.books')) { + fileName = fileName.slice(0, -6); + } - try { - await fs.copyFile(src, dest); - } catch (err) { + const backupFolder = path.join(path.dirname(dbPath), 'backups'); + const date = new Date().toISOString().split('.')[0]; + const version = this.#getAppVersion(); + const backupFile = `${fileName}-${version}-${date}.books.db`; + fs.ensureDirSync(backupFolder); + return path.join(backupFolder, backupFile); + } + + #getAppVersion() { + const db = this.getDriver(); + if (!db) { + return '0.0.0'; + } + + const query = db + .prepare( + `select value from SingleValue + where + fieldname = 'version' and + parent = 'SystemSettings'` + ) + .get() as undefined | { value: string }; + db.close(); + return query?.value || '0.0.0'; + } + + getDriver() { + const { dbPath } = this.db ?? {}; + if (!dbPath) { return null; } - return dest; + return BetterSQLite3(dbPath, { readonly: true }); } } diff --git a/backend/database/runPatch.ts b/backend/database/runPatch.ts index 22fae13a..ecc6a85a 100644 --- a/backend/database/runPatch.ts +++ b/backend/database/runPatch.ts @@ -2,34 +2,50 @@ import { emitMainProcessError, getDefaultMetaFieldValueMap } from '../helpers'; import { DatabaseManager } from './manager'; import { FieldValueMap, Patch } from './types'; -export async function runPatches(patches: Patch[], dm: DatabaseManager) { +export async function runPatches( + patches: Patch[], + dm: DatabaseManager, + version: string +) { const list: { name: string; success: boolean }[] = []; for (const patch of patches) { - const success = await runPatch(patch, dm); + const success = await runPatch(patch, dm, version); list.push({ name: patch.name, success }); } return list; } -async function runPatch(patch: Patch, dm: DatabaseManager): Promise { +async function runPatch( + patch: Patch, + dm: DatabaseManager, + version: string +): Promise { + let failed = false; try { await patch.patch.execute(dm); } catch (error) { - if (!(error instanceof Error)) { - return false; + failed = true; + if (error instanceof Error) { + error.message = `Patch Failed: ${patch.name}\n${error.message}`; + emitMainProcessError(error, { patchName: patch.name, notifyUser: false }); } - - error.message = `Patch Failed: ${patch.name}\n${error.message}`; - emitMainProcessError(error, { patchName: patch.name, notifyUser: false }); - return false; } - await makeEntry(patch.name, dm); + await makeEntry(patch.name, version, failed, dm); return true; } -async function makeEntry(patchName: string, dm: DatabaseManager) { +async function makeEntry( + patchName: string, + version: string, + failed: boolean, + dm: DatabaseManager +) { const defaultFieldValueMap = getDefaultMetaFieldValueMap() as FieldValueMap; + defaultFieldValueMap.name = patchName; + defaultFieldValueMap.failed = failed; + defaultFieldValueMap.version = version; + await dm.db!.insert('PatchRun', defaultFieldValueMap); } diff --git a/package.json b/package.json index 389de7ef..4172dbaf 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "@codemirror/view": "^6.0.0", "@lezer/common": "^1.0.0", "@types/assert": "^1.5.6", + "@types/better-sqlite3": "^7.6.4", "@types/electron-devtools-installer": "^2.2.0", "@types/lodash": "^4.14.179", "@types/luxon": "^2.3.1", diff --git a/schemas/core/PatchRun.json b/schemas/core/PatchRun.json index b92a39db..f638cd62 100644 --- a/schemas/core/PatchRun.json +++ b/schemas/core/PatchRun.json @@ -8,6 +8,18 @@ "fieldtype": "Data", "label": "Name", "required": true + }, + { + "fieldname": "failed", + "fieldtype": "Check", + "label": "Failed", + "default": false + }, + { + "fieldname": "version", + "fieldtype": "Data", + "label": "Version", + "default": "0.0.0" } ] } diff --git a/schemas/core/SystemSettings.json b/schemas/core/SystemSettings.json index e597c40f..e5570c90 100644 --- a/schemas/core/SystemSettings.json +++ b/schemas/core/SystemSettings.json @@ -60,6 +60,7 @@ "fieldname": "version", "label": "Version", "fieldtype": "Data", + "default": "0.0.0", "readOnly": true, "section": "Default" }, diff --git a/src/pages/GetStarted.vue b/src/pages/GetStarted.vue index a06d3d56..63f35559 100644 --- a/src/pages/GetStarted.vue +++ b/src/pages/GetStarted.vue @@ -94,9 +94,6 @@ export default defineComponent({ sections: getGetStartedConfig(), }; }, - mounted() { - // this.sections = getGetStartedConfig(); - }, async activated() { await fyo.doc.getDoc('GetStarted'); await this.checkForCompletedTasks(); diff --git a/yarn.lock b/yarn.lock index dcd2ff87..789a62c7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -585,6 +585,13 @@ resolved "https://registry.yarnpkg.com/@types/assert/-/assert-1.5.6.tgz#a8b5a94ce5fb8f4ba65fdc37fc9507609114189e" integrity sha512-Y7gDJiIqb9qKUHfBQYOWGngUpLORtirAVPuj/CWJrU2C6ZM4/y3XLwuwfGMF8s7QzW746LQZx23m0+1FSgjfug== +"@types/better-sqlite3@^7.6.4": + version "7.6.4" + resolved "https://registry.yarnpkg.com/@types/better-sqlite3/-/better-sqlite3-7.6.4.tgz#102462611e67aadf950d3ccca10292de91e6f35b" + integrity sha512-dzrRZCYPXIXfSR1/surNbJ/grU3scTaygS0OMzjlGf71i9sc2fGyHPXXiXmEvNIoE0cGwsanEFMVJxPXmco9Eg== + dependencies: + "@types/node" "*" + "@types/cacheable-request@^6.0.1": version "6.0.2" resolved "https://registry.yarnpkg.com/@types/cacheable-request/-/cacheable-request-6.0.2.tgz#c324da0197de0a98a2312156536ae262429ff6b9"