fix: create backup on migrate or patch

backend/database/manager.ts

@@ -1,4 +1,4 @@
-import fs from 'fs/promises';
+import fs from 'fs-extra';
 import { DatabaseError } from 'fyo/utils/errors';
 import path from 'path';
 import { DatabaseDemuxBase, DatabaseMethod } from 'utils/db/types';
@@ -9,6 +9,9 @@ import { BespokeQueries } from './bespoke';
 import DatabaseCore from './core';
 import { runPatches } from './runPatch';
 import { BespokeFunction, Patch } from './types';
+import BetterSQLite3 from 'better-sqlite3';
+import { getMapFromList } from 'utils/index';
+import { Version } from 'utils/version';
 
 export class DatabaseManager extends DatabaseDemuxBase {
   db?: DatabaseCore;
@@ -50,25 +53,12 @@ export class DatabaseManager extends DatabaseDemuxBase {
       return;
     }
 
-    const isFirstRun = await this.#getIsFirstRun();
+    const isFirstRun = this.#getIsFirstRun();
     if (isFirstRun) {
       await this.db!.migrate();
     }
 
-    /**
-     * This needs to be supplimented with transactions
-     * TODO: Add transactions in core.ts
-     */
-    const dbPath = this.db!.dbPath;
-    const copyPath = await this.#makeTempCopy();
-
-    try {
-      await this.#runPatchesAndMigrate();
-    } catch (error) {
-      await this.#handleFailedMigration(error, dbPath, copyPath);
-    } finally {
-      await unlinkIfExists(copyPath);
-    }
+    await this.#executeMigration();
   }
 
   async #handleFailedMigration(
@@ -89,32 +79,67 @@ export class DatabaseManager extends DatabaseDemuxBase {
     throw error;
   }
 
-  async #runPatchesAndMigrate() {
-    const patchesToExecute = await this.#getPatchesToExecute();
+  async #executeMigration() {
+    const version = this.#getAppVersion();
+    const patches = await this.#getPatchesToExecute(version);
 
-    patchesToExecute.sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
-    const preMigrationPatches = patchesToExecute.filter(
-      (p) => p.patch.beforeMigrate
-    );
-    const postMigrationPatches = patchesToExecute.filter(
-      (p) => !p.patch.beforeMigrate
-    );
-
-    await runPatches(preMigrationPatches, this);
-    await this.db!.migrate();
-    await runPatches(postMigrationPatches, this);
-  }
-
-  async #getPatchesToExecute(): Promise<Patch[]> {
-    if (this.db === undefined) {
-      return [];
+    const hasPatches = !!patches.pre.length || !!patches.post.length;
+    if (hasPatches) {
+      await this.#createBackup();
     }
 
-    const query: { name: string }[] = await this.db.knex!('PatchRun').select(
-      'name'
-    );
-    const executedPatches = query.map((q) => q.name);
-    return patches.filter((p) => !executedPatches.includes(p.name));
+    await runPatches(patches.pre, this, version);
+    await this.db!.migrate({
+      pre: async () => {
+        if (hasPatches) {
+          return;
+        }
+
+        await this.#createBackup();
+      },
+    });
+    await runPatches(patches.post, this, version);
+  }
+
+  async #getPatchesToExecute(
+    version: string
+  ): Promise<{ pre: Patch[]; post: Patch[] }> {
+    if (this.db === undefined) {
+      return { pre: [], post: [] };
+    }
+
+    const query = (await this.db.knex!('PatchRun').select()) as {
+      name: string;
+      version?: string;
+      failed?: boolean;
+    }[];
+
+    const runPatchesMap = getMapFromList(query, 'name');
+    /**
+     * A patch is run only if:
+     * - it hasn't run and was added in a future version
+     *    i.e. app version is before patch added version
+     * - it ran but failed in some other version (i.e fixed)
+     */
+    const filtered = patches
+      .filter((p) => {
+        const exec = runPatchesMap[p.name];
+        if (!exec && Version.lte(version, p.version)) {
+          return true;
+        }
+
+        if (exec?.failed && exec?.version !== version) {
+          return true;
+        }
+
+        return false;
+      })
+      .sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0));
+
+    return {
+      pre: filtered.filter((p) => p.patch.beforeMigrate),
+      post: filtered.filter((p) => !p.patch.beforeMigrate),
+    };
   }
 
   async call(method: DatabaseMethod, ...args: unknown[]) {
@@ -149,33 +174,86 @@ export class DatabaseManager extends DatabaseDemuxBase {
     return await queryFunction(this.db!, ...args);
   }
 
-  async #getIsFirstRun(): Promise<boolean> {
-    if (!this.#isInitialized) {
+  #getIsFirstRun(): boolean {
+    const db = this.getDriver();
+    if (!db) {
       return true;
     }
 
-    const tableList: unknown[] = await this.db!.knex!.raw(
-      "SELECT name FROM sqlite_master WHERE type='table'"
-    );
-    return tableList.length === 0;
+    const noPatchRun =
+      db
+        .prepare(
+          `select name from sqlite_master
+           where
+            type = 'table' and
+            name = 'PatchRun'`
+        )
+        .all().length === 0;
+
+    db.close();
+    return noPatchRun;
   }
 
-  async #makeTempCopy() {
-    const src = this.db!.dbPath;
-    if (src === ':memory:') {
+  async #createBackup() {
+    const { dbPath } = this.db ?? {};
+    if (!dbPath) {
+      return;
+    }
+
+    const backupPath = this.#getBackupFilePath();
+    if (!backupPath) {
+      return;
+    }
+
+    const db = this.getDriver();
+    await db?.backup(backupPath);
+    db?.close();
+  }
+
+  #getBackupFilePath() {
+    const { dbPath } = this.db ?? {};
+    if (dbPath === ':memory:' || !dbPath) {
       return null;
     }
 
-    const dir = path.parse(src).dir;
-    const dest = path.join(dir, '__premigratory_temp.db');
+    let fileName = path.parse(dbPath).name;
+    if (fileName.endsWith('.books')) {
+      fileName = fileName.slice(0, -6);
+    }
 
-    try {
-      await fs.copyFile(src, dest);
-    } catch (err) {
+    const backupFolder = path.join(path.dirname(dbPath), 'backups');
+    const date = new Date().toISOString().split('.')[0];
+    const version = this.#getAppVersion();
+    const backupFile = `${fileName}-${version}-${date}.books.db`;
+    fs.ensureDirSync(backupFolder);
+    return path.join(backupFolder, backupFile);
+  }
+
+  #getAppVersion() {
+    const db = this.getDriver();
+    if (!db) {
+      return '0.0.0';
+    }
+
+    const query = db
+      .prepare(
+        `select value from SingleValue
+         where
+          fieldname = 'version' and
+          parent = 'SystemSettings'`
+      )
+      .get() as undefined | { value: string };
+    db.close();
+    return query?.value || '0.0.0';
+  }
+
+  getDriver() {
+    const { dbPath } = this.db ?? {};
+    if (!dbPath) {
       return null;
     }
 
-    return dest;
+    return BetterSQLite3(dbPath, { readonly: true });
   }
 }
 

backend/database/runPatch.ts

@@ -2,34 +2,50 @@ import { emitMainProcessError, getDefaultMetaFieldValueMap } from '../helpers';
 import { DatabaseManager } from './manager';
 import { FieldValueMap, Patch } from './types';
 
-export async function runPatches(patches: Patch[], dm: DatabaseManager) {
+export async function runPatches(
+  patches: Patch[],
+  dm: DatabaseManager,
+  version: string
+) {
   const list: { name: string; success: boolean }[] = [];
   for (const patch of patches) {
-    const success = await runPatch(patch, dm);
+    const success = await runPatch(patch, dm, version);
     list.push({ name: patch.name, success });
   }
   return list;
 }
 
-async function runPatch(patch: Patch, dm: DatabaseManager): Promise<boolean> {
+async function runPatch(
+  patch: Patch,
+  dm: DatabaseManager,
+  version: string
+): Promise<boolean> {
+  let failed = false;
   try {
     await patch.patch.execute(dm);
   } catch (error) {
-    if (!(error instanceof Error)) {
-      return false;
+    failed = true;
+    if (error instanceof Error) {
+      error.message = `Patch Failed: ${patch.name}\n${error.message}`;
+      emitMainProcessError(error, { patchName: patch.name, notifyUser: false });
     }
-
-    error.message = `Patch Failed: ${patch.name}\n${error.message}`;
-    emitMainProcessError(error, { patchName: patch.name, notifyUser: false });
-    return false;
   }
 
-  await makeEntry(patch.name, dm);
+  await makeEntry(patch.name, version, failed, dm);
   return true;
 }
 
-async function makeEntry(patchName: string, dm: DatabaseManager) {
+async function makeEntry(
+  patchName: string,
+  version: string,
+  failed: boolean,
+  dm: DatabaseManager
+) {
   const defaultFieldValueMap = getDefaultMetaFieldValueMap() as FieldValueMap;
+
   defaultFieldValueMap.name = patchName;
+  defaultFieldValueMap.failed = failed;
+  defaultFieldValueMap.version = version;
+
   await dm.db!.insert('PatchRun', defaultFieldValueMap);
 }

package.json

@@ -43,6 +43,7 @@
     "@codemirror/view": "^6.0.0",
     "@lezer/common": "^1.0.0",
     "@types/assert": "^1.5.6",
+    "@types/better-sqlite3": "^7.6.4",
     "@types/electron-devtools-installer": "^2.2.0",
     "@types/lodash": "^4.14.179",
     "@types/luxon": "^2.3.1",

schemas/core/PatchRun.json

@@ -8,6 +8,18 @@
       "fieldtype": "Data",
       "label": "Name",
       "required": true
+    },
+    {
+      "fieldname": "failed",
+      "fieldtype": "Check",
+      "label": "Failed",
+      "default": false
+    },
+    {
+      "fieldname": "version",
+      "fieldtype": "Data",
+      "label": "Version",
+      "default": "0.0.0"
     }
   ]
 }

schemas/core/SystemSettings.json

@@ -60,6 +60,7 @@
       "fieldname": "version",
       "label": "Version",
       "fieldtype": "Data",
+      "default": "0.0.0",
       "readOnly": true,
       "section": "Default"
     },

src/pages/GetStarted.vue

@@ -94,9 +94,6 @@ export default defineComponent({
       sections: getGetStartedConfig(),
     };
   },
-  mounted() {
-    // this.sections = getGetStartedConfig();
-  },
   async activated() {
     await fyo.doc.getDoc('GetStarted');
     await this.checkForCompletedTasks();

yarn.lock

@@ -585,6 +585,13 @@
   resolved "https://registry.yarnpkg.com/@types/assert/-/assert-1.5.6.tgz#a8b5a94ce5fb8f4ba65fdc37fc9507609114189e"
   integrity sha512-Y7gDJiIqb9qKUHfBQYOWGngUpLORtirAVPuj/CWJrU2C6ZM4/y3XLwuwfGMF8s7QzW746LQZx23m0+1FSgjfug==
 
+"@types/better-sqlite3@^7.6.4":
+  version "7.6.4"
+  resolved "https://registry.yarnpkg.com/@types/better-sqlite3/-/better-sqlite3-7.6.4.tgz#102462611e67aadf950d3ccca10292de91e6f35b"
+  integrity sha512-dzrRZCYPXIXfSR1/surNbJ/grU3scTaygS0OMzjlGf71i9sc2fGyHPXXiXmEvNIoE0cGwsanEFMVJxPXmco9Eg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/cacheable-request@^6.0.1":
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/@types/cacheable-request/-/cacheable-request-6.0.2.tgz#c324da0197de0a98a2312156536ae262429ff6b9"