incr: add databaseManager, privitize methods

backend/common.ts

@@ -1,5 +1,3 @@
-import { Field, FieldTypeEnum, SchemaMap } from '../schemas/types';
-
 export const sqliteTypeMap = {
   AutoComplete: 'text',
   Currency: 'text',
@@ -24,12 +22,11 @@ export const sqliteTypeMap = {
 };
 
 export const validTypes = Object.keys(sqliteTypeMap);
-
+export function getDefaultMetaFieldValueMap() {
-export function getFieldsByType(
+  return {
-  schemaName: string,
+    createdBy: '__SYSTEM__',
-  schemaMap: SchemaMap,
+    modifiedBy: '__SYSTEM__',
-  type: FieldTypeEnum
+    created: new Date().toISOString(),
-): Field[] {
+    modified: new Date().toISOString(),
-  const fields = schemaMap[schemaName].fields ?? [];
+  };
-  return fields.filter((f) => f.fieldtype === type);
 }

backend/database/core.ts

@@ -1,6 +1,5 @@
 import { knex, Knex } from 'knex';
 import { getRandomString } from '../../frappe/utils';
-import CacheManager from '../../frappe/utils/cacheManager';
 import {
   CannotCommitError,
   DatabaseError,
@@ -16,7 +15,7 @@ import {
   SchemaMap,
   TargetField,
 } from '../../schemas/types';
-import { getFieldsByType, sqliteTypeMap } from '../common';
+import { sqliteTypeMap } from '../common';
 import {
   ColumnDiff,
   FieldValueMap,
@@ -25,18 +24,15 @@ import {
   QueryFilter,
 } from './types';
 
-export default class Database {
+export default class DatabaseCore {
   knex: Knex;
-  cache: CacheManager;
   typeMap = sqliteTypeMap;
   dbPath: string;
-  schemaMap: SchemaMap;
+  schemaMap: SchemaMap = {};
   connectionParams: Knex.Config;
 
-  constructor(dbPath: string, schemaMap: SchemaMap) {
+  constructor(dbPath?: string) {
-    this.schemaMap = schemaMap;
+    this.dbPath = dbPath ?? ':memory:';
-    this.cache = new CacheManager();
-    this.dbPath = dbPath;
     this.connectionParams = {
       client: 'sqlite3',
       connection: {
@@ -53,10 +49,14 @@ export default class Database {
     };
   }
 
+  setSchemaMap(schemaMap: SchemaMap) {
+    this.schemaMap = schemaMap;
+  }
+
   connect() {
     this.knex = knex(this.connectionParams);
     this.knex.on('query-error', (error) => {
-      error.type = this.getError(error);
+      error.type = this.#getError(error);
     });
   }
 
@@ -64,11 +64,228 @@ export default class Database {
     this.knex.destroy();
   }
 
-  async tableExists(schemaName: string) {
+  async commit() {
+    try {
+      await this.raw('commit');
+    } catch (err) {
+      if (err.type !== CannotCommitError) {
+        throw err;
+      }
+    }
+  }
+
+  async raw(query: string, params: Knex.RawBinding[] = []) {
+    return await this.knex.raw(query, params);
+  }
+
+  async migrate() {
+    for (const schemaName in this.schemaMap) {
+      const schema = this.schemaMap[schemaName];
+      if (schema.isSingle) {
+        continue;
+      }
+
+      if (await this.#tableExists(schemaName)) {
+        await this.#alterTable(schemaName);
+      } else {
+        await this.#createTable(schemaName);
+      }
+    }
+
+    await this.commit();
+    await this.#initializeSingles();
+  }
+
+  async exists(schemaName: string, name?: string): Promise<boolean> {
+    const schema = this.schemaMap[schemaName];
+    if (schema.isSingle) {
+      return this.#singleExists(schemaName);
+    }
+
+    let row = [];
+    try {
+      const qb = this.knex(schemaName);
+      if (name !== undefined) {
+        qb.where({ name });
+      }
+      row = await qb.limit(1);
+    } catch (err) {
+      if (this.#getError(err as Error) !== NotFoundError) {
+        throw err;
+      }
+    }
+    return row.length > 0;
+  }
+
+  async insert(schemaName: string, fieldValueMap: FieldValueMap) {
+    // insert parent
+    if (this.schemaMap[schemaName].isSingle) {
+      await this.#updateSingleValues(schemaName, fieldValueMap);
+    } else {
+      await this.#insertOne(schemaName, fieldValueMap);
+    }
+
+    // insert children
+    await this.#insertOrUpdateChildren(schemaName, fieldValueMap, false);
+    return fieldValueMap;
+  }
+
+  async get(
+    schemaName: string,
+    name: string = '',
+    fields: string | string[] = '*'
+  ): Promise<FieldValueMap> {
+    const isSingle = this.schemaMap[schemaName].isSingle;
+    if (!isSingle && !name) {
+      throw new ValueError('name is mandatory');
+    }
+
+    /**
+     * If schema is single return all the values
+     * of the single type schema, in this case field
+     * is ignored.
+     */
+    let fieldValueMap: FieldValueMap = {};
+    if (isSingle) {
+      fieldValueMap = await this.#getSingle(schemaName);
+      fieldValueMap.name = schemaName;
+      return fieldValueMap;
+    }
+
+    if (fields !== '*' && typeof fields === 'string') {
+      fields = [fields];
+    }
+
+    /**
+     * Separate table fields and non table fields
+     */
+    const allTableFields = this.#getTableFields(schemaName);
+    const allTableFieldNames = allTableFields.map((f) => f.fieldname);
+
+    let tableFields: TargetField[] = [];
+    let nonTableFieldNames: string[] = [];
+
+    if (Array.isArray(fields)) {
+      tableFields = tableFields.filter((f) => fields.includes(f.fieldname));
+      nonTableFieldNames = fields.filter(
+        (f) => !allTableFieldNames.includes(f)
+      );
+    } else if (fields === '*') {
+      tableFields = allTableFields;
+    }
+
+    /**
+     * If schema is not single then return specific fields
+     * if child fields are selected, all child fields are returned.
+     */
+    if (nonTableFieldNames.length) {
+      fieldValueMap = (await this.#getOne(schemaName, name, fields)) ?? {};
+    }
+
+    if (tableFields.length) {
+      await this.#loadChildren(fieldValueMap, tableFields);
+    }
+    return fieldValueMap;
+  }
+
+  async getAll({
+    schemaName,
+    fields,
+    filters,
+    start,
+    limit,
+    groupBy,
+    orderBy = 'creation',
+    order = 'desc',
+  }: GetAllOptions = {}): Promise<FieldValueMap[]> {
+    const schema = this.schemaMap[schemaName];
+    if (!fields) {
+      fields = ['name', ...(schema.keywordFields ?? [])];
+    }
+
+    if (typeof fields === 'string') {
+      fields = [fields];
+    }
+
+    return (await this.#getQueryBuilder(schemaName, fields, filters, {
+      offset: start,
+      limit,
+      groupBy,
+      orderBy,
+      order,
+    })) as FieldValueMap[];
+  }
+
+  async getSingleValues(
+    ...fieldnames: { fieldname: string; parent?: string }[]
+  ): Promise<{ fieldname: string; parent: string; value: RawValue }[]> {
+    fieldnames = fieldnames.map((fieldname) => {
+      if (typeof fieldname === 'string') {
+        return { fieldname };
+      }
+      return fieldname;
+    });
+
+    let builder = this.knex('SingleValue');
+    builder = builder.where(fieldnames[0]);
+
+    fieldnames.slice(1).forEach(({ fieldname, parent }) => {
+      if (typeof parent === 'undefined') {
+        builder = builder.orWhere({ fieldname });
+      } else {
+        builder = builder.orWhere({ fieldname, parent });
+      }
+    });
+
+    let values: { fieldname: string; parent: string; value: RawValue }[] = [];
+    try {
+      values = await builder.select('fieldname', 'value', 'parent');
+    } catch (err) {
+      if (this.#getError(err as Error) === NotFoundError) {
+        return [];
+      }
+
+      throw err;
+    }
+
+    return values;
+  }
+
+  async rename(schemaName: string, oldName: string, newName: string) {
+    await this.knex(schemaName)
+      .update({ name: newName })
+      .where('name', oldName);
+    await this.commit();
+  }
+
+  async update(schemaName: string, fieldValueMap: FieldValueMap) {
+    // update parent
+    if (this.schemaMap[schemaName].isSingle) {
+      await this.#updateSingleValues(schemaName, fieldValueMap);
+    } else {
+      await this.#updateOne(schemaName, fieldValueMap);
+    }
+
+    // insert or update children
+    await this.#insertOrUpdateChildren(schemaName, fieldValueMap, true);
+  }
+
+  async delete(schemaName: string, name: string) {
+    await this.#deleteOne(schemaName, name);
+
+    // delete children
+    const tableFields = this.#getTableFields(schemaName);
+
+    for (const field of tableFields) {
+      await this.#deleteChildren(field.target, name);
+    }
+  }
+
+  async #tableExists(schemaName: string) {
     return await this.knex.schema.hasTable(schemaName);
   }
 
-  async singleExists(singleSchemaName: string) {
+  async #singleExists(singleSchemaName: string) {
     const res = await this.knex('SingleValue')
       .count('parent as count')
       .where('parent', singleSchemaName)
@@ -76,31 +293,17 @@ export default class Database {
     return res.count > 0;
   }
 
-  async removeColumns(schemaName: string, targetColumns: string[]) {
+  async #removeColumns(schemaName: string, targetColumns: string[]) {
     // TODO: Implement this for sqlite
   }
 
-  async deleteSingleValues(singleSchemaName: string) {
+  async #deleteSingleValues(singleSchemaName: string) {
     return await this.knex('SingleValue')
       .where('parent', singleSchemaName)
       .delete();
   }
 
-  async sql(query: string, params: Knex.RawBinding[] = []) {
+  #getError(err: Error) {
-    return await this.knex.raw(query, params);
-  }
-
-  async commit() {
-    try {
-      await this.sql('commit');
-    } catch (e) {
-      if (e.type !== CannotCommitError) {
-        throw e;
-      }
-    }
-  }
-
-  getError(err: Error) {
     let errorType = DatabaseError;
     if (err.message.includes('SQLITE_ERROR: no such table')) {
       errorType = NotFoundError;
@@ -125,7 +328,7 @@ export default class Database {
     await this.knex.schema.dropTableIfExists(tempName);
 
     await this.knex.raw('PRAGMA foreign_keys=OFF');
-    await this.createTable(schemaName, tempName);
+    await this.#createTable(schemaName, tempName);
 
     if (tableRows.length > 200) {
       const fi = Math.floor(tableRows.length / max);
@@ -145,19 +348,19 @@ export default class Database {
     await this.knex.raw('PRAGMA foreign_keys=ON');
   }
 
-  async getTableColumns(schemaName: string): Promise<string[]> {
+  async #getTableColumns(schemaName: string): Promise<string[]> {
-    const info = await this.sql(`PRAGMA table_info(${schemaName})`);
+    const info = await this.raw(`PRAGMA table_info(${schemaName})`);
     return info.map((d) => d.name);
   }
 
-  async getForeignKeys(schemaName: string): Promise<string[]> {
+  async #getForeignKeys(schemaName: string): Promise<string[]> {
-    const foreignKeyList = await this.sql(
+    const foreignKeyList = await this.raw(
       `PRAGMA foreign_key_list(${schemaName})`
     );
     return foreignKeyList.map((d) => d.from);
   }
 
-  getQueryBuilder(
+  #getQueryBuilder(
     schemaName: string,
     fields: string[],
     filters: QueryFilter,
@@ -165,7 +368,7 @@ export default class Database {
   ): Knex.QueryBuilder {
     const builder = this.knex.select(fields).from(schemaName);
 
-    this.applyFiltersToBuilder(builder, filters);
+    this.#applyFiltersToBuilder(builder, filters);
 
     if (options.orderBy) {
       builder.orderBy(options.orderBy, options.order);
@@ -186,7 +389,7 @@ export default class Database {
     return builder;
   }
 
-  applyFiltersToBuilder(builder: Knex.QueryBuilder, filters: QueryFilter) {
+  #applyFiltersToBuilder(builder: Knex.QueryBuilder, filters: QueryFilter) {
     // {"status": "Open"} => `status = "Open"`
 
     // {"status": "Open", "name": ["like", "apple%"]}
@@ -236,8 +439,8 @@ export default class Database {
     });
   }
 
-  async getColumnDiff(schemaName: string): Promise<ColumnDiff> {
+  async #getColumnDiff(schemaName: string): Promise<ColumnDiff> {
-    const tableColumns = await this.getTableColumns(schemaName);
+    const tableColumns = await this.#getTableColumns(schemaName);
     const validFields = this.schemaMap[schemaName].fields;
     const diff: ColumnDiff = { added: [], removed: [] };
 
@@ -260,8 +463,8 @@ export default class Database {
     return diff;
   }
 
-  async getNewForeignKeys(schemaName): Promise<Field[]> {
+  async #getNewForeignKeys(schemaName): Promise<Field[]> {
-    const foreignKeys = await this.getForeignKeys(schemaName);
+    const foreignKeys = await this.#getForeignKeys(schemaName);
     const newForeignKeys: Field[] = [];
     const schema = this.schemaMap[schemaName];
     for (const field of schema.fields) {
@@ -275,7 +478,7 @@ export default class Database {
     return newForeignKeys;
   }
 
-  buildColumnForTable(table: Knex.AlterTableBuilder, field: Field) {
+  #buildColumnForTable(table: Knex.AlterTableBuilder, field: Field) {
     if (field.fieldtype === FieldTypeEnum.Table) {
       // In case columnType is "Table"
       // childTable links are handled using the childTable's "parent" field
@@ -317,45 +520,45 @@ export default class Database {
     }
   }
 
-  async alterTable(schemaName: string) {
+  async #alterTable(schemaName: string) {
     // get columns
-    const diff: ColumnDiff = await this.getColumnDiff(schemaName);
+    const diff: ColumnDiff = await this.#getColumnDiff(schemaName);
-    const newForeignKeys: Field[] = await this.getNewForeignKeys(schemaName);
+    const newForeignKeys: Field[] = await this.#getNewForeignKeys(schemaName);
 
     return this.knex.schema
       .table(schemaName, (table) => {
         if (diff.added.length) {
           for (const field of diff.added) {
-            this.buildColumnForTable(table, field);
+            this.#buildColumnForTable(table, field);
           }
         }
 
         if (diff.removed.length) {
-          this.removeColumns(schemaName, diff.removed);
+          this.#removeColumns(schemaName, diff.removed);
         }
       })
       .then(() => {
         if (newForeignKeys.length) {
-          return this.addForeignKeys(schemaName, newForeignKeys);
+          return this.#addForeignKeys(schemaName, newForeignKeys);
         }
       });
   }
 
-  async createTable(schemaName: string, tableName?: string) {
+  async #createTable(schemaName: string, tableName?: string) {
     tableName ??= schemaName;
     const fields = this.schemaMap[schemaName].fields;
-    return await this.runCreateTableQuery(tableName, fields);
+    return await this.#runCreateTableQuery(tableName, fields);
   }
 
-  runCreateTableQuery(schemaName: string, fields: Field[]) {
+  #runCreateTableQuery(schemaName: string, fields: Field[]) {
     return this.knex.schema.createTable(schemaName, (table) => {
       for (const field of fields) {
-        this.buildColumnForTable(table, field);
+        this.#buildColumnForTable(table, field);
       }
     });
   }
 
-  async getNonExtantSingleValues(singleSchemaName: string) {
+  async #getNonExtantSingleValues(singleSchemaName: string) {
     const existingFields = (
       await this.knex('SingleValue')
         .where({ parent: singleSchemaName })
@@ -373,41 +576,15 @@ export default class Database {
       );
   }
 
-  async delete(schemaName: string, name: string) {
+  async #deleteOne(schemaName: string, name: string) {
-    await this.deleteOne(schemaName, name);
+    return this.knex(schemaName).where('name', name).delete();
-
-    // delete children
-    const tableFields = getFieldsByType(
-      schemaName,
-      this.schemaMap,
-      FieldTypeEnum.Table
-    ) as TargetField[];
-
-    for (const field of tableFields) {
-      await this.deleteChildren(field.target, name);
-    }
   }
 
-  async deleteMany(schemaName: string, names: string[]) {
+  #deleteChildren(schemaName: string, parentName: string) {
-    for (const name of names) {
-      await this.delete(schemaName, name);
-    }
-  }
-
-  async deleteOne(schemaName: string, name: string) {
-    return this.knex(schemaName)
-      .where('name', name)
-      .delete()
-      .then(() => {
-        this.clearValueCache(schemaName, name);
-      });
-  }
-
-  deleteChildren(schemaName: string, parentName: string) {
     return this.knex(schemaName).where('parent', parentName).delete();
   }
 
-  runDeleteOtherChildren(
+  #runDeleteOtherChildren(
     field: TargetField,
     parentName: string,
     added: string[]
@@ -419,17 +596,7 @@ export default class Database {
       .delete();
   }
 
-  async rename(schemaName: string, oldName: string, newName: string) {
+  #prepareChild(
-    await this.knex(schemaName)
-      .update({ name: newName })
-      .where('name', oldName)
-      .then(() => {
-        this.clearValueCache(schemaName, oldName);
-      });
-    await this.commit();
-  }
-
-  prepareChild(
     parentSchemaName: string,
     parentName: string,
     child: FieldValueMap,
@@ -445,26 +612,21 @@ export default class Database {
     child.idx = idx;
   }
 
-  clearValueCache(schemaName: string, name: string) {
+  async #addForeignKeys(schemaName: string, newForeignKeys: Field[]) {
-    const cacheKey = `${schemaName}:${name}`;
+    await this.raw('PRAGMA foreign_keys=OFF');
-    this.cache.hclear(cacheKey);
+    await this.raw('BEGIN TRANSACTION');
-  }
-
-  async addForeignKeys(schemaName: string, newForeignKeys: Field[]) {
-    await this.sql('PRAGMA foreign_keys=OFF');
-    await this.sql('BEGIN TRANSACTION');
 
     const tempName = 'TEMP' + schemaName;
 
     // create temp table
-    await this.createTable(schemaName, tempName);
+    await this.#createTable(schemaName, tempName);
 
     try {
       // copy from old to new table
       await this.knex(tempName).insert(this.knex.select().from(schemaName));
     } catch (err) {
-      await this.sql('ROLLBACK');
+      await this.raw('ROLLBACK');
-      await this.sql('PRAGMA foreign_keys=ON');
+      await this.raw('PRAGMA foreign_keys=ON');
 
       const rows = await this.knex.select().from(schemaName);
       await this.prestigeTheTable(schemaName, rows);
@@ -477,62 +639,26 @@ export default class Database {
     // rename new table
     await this.knex.schema.renameTable(tempName, schemaName);
 
-    await this.sql('COMMIT');
+    await this.raw('COMMIT');
-    await this.sql('PRAGMA foreign_keys=ON');
+    await this.raw('PRAGMA foreign_keys=ON');
   }
 
-  async getAll({
+  async #loadChildren(
-    schemaName,
+    fieldValueMap: FieldValueMap,
-    fields,
+    tableFields: TargetField[]
-    filters,
-    start,
-    limit,
-    groupBy,
-    orderBy = 'creation',
-    order = 'desc',
-  }: GetAllOptions = {}): Promise<FieldValueMap[]> {
-    const schema = this.schemaMap[schemaName];
-    if (!fields) {
-      fields = ['name', ...(schema.keywordFields ?? [])];
-    }
-
-    if (typeof fields === 'string') {
-      fields = [fields];
-    }
-
-    return (await this.getQueryBuilder(schemaName, fields, filters, {
-      offset: start,
-      limit,
-      groupBy,
-      orderBy,
-      order,
-    })) as FieldValueMap[];
-  }
-
-  async get(
-    schemaName: string,
-    name: string = '',
-    fields: string | string[] = '*'
   ) {
-    const schema = this.schemaMap[schemaName];
+    for (const field of tableFields) {
-    let fieldValueMap: FieldValueMap;
+      fieldValueMap[field.fieldname] = await this.getAll({
-    if (schema.isSingle) {
+        schemaName: field.target,
-      fieldValueMap = await this.getSingle(schemaName);
+        fields: ['*'],
-      fieldValueMap.name = schemaName;
+        filters: { parent: fieldValueMap.name as string },
-    } else {
+        orderBy: 'idx',
-      if (!name) {
+        order: 'asc',
-        throw new ValueError('name is mandatory');
+      });
-      }
-      fieldValueMap = await this.getOne(schemaName, name, fields);
     }
-    if (!fieldValueMap) {
-      return;
-    }
-    await this.loadChildren(fieldValueMap, schemaName);
-    return fieldValueMap;
   }
 
-  async getOne(
+  async #getOne(
     schemaName: string,
     name: string,
     fields: string | string[] = '*'
@@ -545,7 +671,7 @@ export default class Database {
     return fieldValueMap;
   }
 
-  async getSingle(schemaName: string): Promise<FieldValueMap> {
+  async #getSingle(schemaName: string): Promise<FieldValueMap> {
     const values = await this.getAll({
       schemaName: 'SingleValue',
       fields: ['fieldname', 'value'],
@@ -562,39 +688,7 @@ export default class Database {
     return fieldValueMap;
   }
 
-  async loadChildren(fieldValueMap: FieldValueMap, schemaName: string) {
+  #insertOne(schemaName: string, fieldValueMap: FieldValueMap) {
-    // Sets children on a field
-    const tableFields = getFieldsByType(
-      schemaName,
-      this.schemaMap,
-      FieldTypeEnum.Table
-    ) as TargetField[];
-
-    for (const field of tableFields) {
-      fieldValueMap[field.fieldname] = await this.getAll({
-        schemaName: field.target,
-        fields: ['*'],
-        filters: { parent: fieldValueMap.name as string },
-        orderBy: 'idx',
-        order: 'asc',
-      });
-    }
-  }
-
-  async insert(schemaName: string, fieldValueMap: FieldValueMap) {
-    // insert parent
-    if (this.schemaMap[schemaName].isSingle) {
-      await this.updateSingleValues(schemaName, fieldValueMap);
-    } else {
-      await this.insertOne(schemaName, fieldValueMap);
-    }
-
-    // insert children
-    await this.insertOrUpdateChildren(schemaName, fieldValueMap, false);
-    return fieldValueMap;
-  }
-
-  insertOne(schemaName: string, fieldValueMap: FieldValueMap) {
     const fields = this.schemaMap[schemaName].fields;
     if (!fieldValueMap.name) {
       fieldValueMap.name = getRandomString();
@@ -612,12 +706,12 @@ export default class Database {
     return this.knex(schemaName).insert(fieldValueMap);
   }
 
-  async updateSingleValues(
+  async #updateSingleValues(
     singleSchemaName: string,
     fieldValueMap: FieldValueMap
   ) {
     const fields = this.schemaMap[singleSchemaName].fields;
-    await this.deleteSingleValues(singleSchemaName);
+    await this.#deleteSingleValues(singleSchemaName);
 
     for (const field of fields) {
       const value = fieldValueMap[field.fieldname] as RawValue | undefined;
@@ -625,11 +719,11 @@ export default class Database {
         continue;
       }
 
-      await this.updateSingleValue(singleSchemaName, field.fieldname, value);
+      await this.#updateSingleValue(singleSchemaName, field.fieldname, value);
     }
   }
 
-  async updateSingleValue(
+  async #updateSingleValue(
     singleSchemaName: string,
     fieldname: string,
     value: RawValue
@@ -642,32 +736,14 @@ export default class Database {
       .update({ value });
   }
 
-  async migrate() {
+  async #initializeSingles() {
-    for (const schemaName in this.schemaMap) {
-      const schema = this.schemaMap[schemaName];
-      if (schema.isSingle) {
-        continue;
-      }
-
-      if (await this.tableExists(schemaName)) {
-        await this.alterTable(schemaName);
-      } else {
-        await this.createTable(schemaName);
-      }
-    }
-
-    await this.commit();
-    await this.initializeSingles();
-  }
-
-  async initializeSingles() {
     const singleSchemaNames = Object.keys(this.schemaMap).filter(
       (n) => this.schemaMap[n].isSingle
     );
 
     for (const schemaName of singleSchemaNames) {
-      if (await this.singleExists(schemaName)) {
+      if (await this.#singleExists(schemaName)) {
-        await this.updateNonExtantSingleValues(schemaName);
+        await this.#updateNonExtantSingleValues(schemaName);
         continue;
       }
 
@@ -684,59 +760,32 @@ export default class Database {
         return acc;
       }, {});
 
-      await this.updateSingleValues(schemaName, defaultValues);
+      await this.#updateSingleValues(schemaName, defaultValues);
     }
   }
 
-  async updateNonExtantSingleValues(schemaName: string) {
+  async #updateNonExtantSingleValues(schemaName: string) {
-    const singleValues = await this.getNonExtantSingleValues(schemaName);
+    const singleValues = await this.#getNonExtantSingleValues(schemaName);
     for (const sv of singleValues) {
-      await this.updateSingleValue(schemaName, sv.fieldname, sv.value);
+      await this.#updateSingleValue(schemaName, sv.fieldname, sv.value);
     }
   }
 
-  async updateOne(schemaName: string, fieldValueMap: FieldValueMap) {
+  async #updateOne(schemaName: string, fieldValueMap: FieldValueMap) {
     const updateMap = { ...fieldValueMap };
     delete updateMap.name;
 
     return await this.knex(schemaName)
       .where('name', fieldValueMap.name as string)
-      .update(updateMap)
+      .update(updateMap);
-      .then(() => {
-        const cacheKey = `${schemaName}:${fieldValueMap.name}`;
-        if (!this.cache.hexists(cacheKey)) {
-          return;
-        }
-
-        for (const fieldname in updateMap) {
-          const value = updateMap[fieldname];
-          this.cache.hset(cacheKey, fieldname, value);
-        }
-      });
   }
 
-  async update(schemaName: string, fieldValueMap: FieldValueMap) {
+  async #insertOrUpdateChildren(
-    // update parent
-    if (this.schemaMap[schemaName].isSingle) {
-      await this.updateSingleValues(schemaName, fieldValueMap);
-    } else {
-      await this.updateOne(schemaName, fieldValueMap);
-    }
-
-    // insert or update children
-    await this.insertOrUpdateChildren(schemaName, fieldValueMap, true);
-  }
-
-  async insertOrUpdateChildren(
     schemaName: string,
     fieldValueMap: FieldValueMap,
     isUpdate: boolean
   ) {
-    const tableFields = getFieldsByType(
+    const tableFields = this.#getTableFields(schemaName);
-      schemaName,
-      this.schemaMap,
-      FieldTypeEnum.Table
-    ) as TargetField[];
 
     const parentName = fieldValueMap.name as string;
     for (const field of tableFields) {
@@ -745,146 +794,29 @@ export default class Database {
       const tableFieldValue = (fieldValueMap[field.fieldname] ??
         []) as FieldValueMap[];
       for (const child of tableFieldValue) {
-        this.prepareChild(schemaName, parentName, child, field, added.length);
+        this.#prepareChild(schemaName, parentName, child, field, added.length);
 
         if (
           isUpdate &&
           (await this.exists(field.target, child.name as string))
         ) {
-          await this.updateOne(field.target, child);
+          await this.#updateOne(field.target, child);
         } else {
-          await this.insertOne(field.target, child);
+          await this.#insertOne(field.target, child);
         }
 
         added.push(child.name as string);
       }
 
       if (isUpdate) {
-        await this.runDeleteOtherChildren(field, parentName, added);
+        await this.#runDeleteOtherChildren(field, parentName, added);
       }
     }
   }
 
-  async exists(schemaName: string, name?: string): Promise<boolean> {
+  #getTableFields(schemaName: string): TargetField[] {
-    const schema = this.schemaMap[schemaName];
+    return this.schemaMap[schemaName].fields.filter(
-    if (schema.isSingle) {
+      (f) => f.fieldtype === FieldTypeEnum.Table
-      return this.singleExists(schemaName);
+    ) as TargetField[];
-    }
-
-    let row = [];
-    try {
-      const qb = this.knex(schemaName);
-      if (name !== undefined) {
-        qb.where({ name });
-      }
-      row = await qb.limit(1);
-    } catch (err) {
-      if (this.getError(err as Error) !== NotFoundError) {
-        throw err;
-      }
-    }
-    return row.length > 0;
-  }
-
-  /**
-   * Get list of values from the singles table.
-   * @param  {...string | Object} fieldnames list of fieldnames to get the values of
-   * @returns {Array<Object>} array of {parent, value, fieldname}.
-   * @example
-   * Database.getSingleValues('internalPrecision');
-   * // returns [{ fieldname: 'internalPrecision', parent: 'SystemSettings', value: '12' }]
-   * @example
-   * Database.getSingleValues({fieldname:'internalPrecision', parent: 'SystemSettings'});
-   * // returns [{ fieldname: 'internalPrecision', parent: 'SystemSettings', value: '12' }]
-   */
-  async getSingleValues(
-    ...fieldnames: { fieldname: string; parent?: string }[]
-  ) {
-    fieldnames = fieldnames.map((fieldname) => {
-      if (typeof fieldname === 'string') {
-        return { fieldname };
-      }
-      return fieldname;
-    });
-
-    let builder = this.knex('SingleValue');
-    builder = builder.where(fieldnames[0]);
-
-    fieldnames.slice(1).forEach(({ fieldname, parent }) => {
-      if (typeof parent === 'undefined') {
-        builder = builder.orWhere({ fieldname });
-      } else {
-        builder = builder.orWhere({ fieldname, parent });
-      }
-    });
-
-    let values: { fieldname: string; parent: string; value: RawValue }[] = [];
-    try {
-      values = await builder.select('fieldname', 'value', 'parent');
-    } catch (err) {
-      if (this.getError(err as Error) === NotFoundError) {
-        return [];
-      }
-
-      throw err;
-    }
-
-    return values;
-  }
-
-  async getValue(
-    schemaName: string,
-    filters: string | Record<string, string>,
-    fieldname = 'name'
-  ): Promise<RawValue | undefined> {
-    if (typeof filters === 'string') {
-      filters = { name: filters };
-    }
-
-    const row = await this.getAll({
-      schemaName,
-      fields: [fieldname],
-      filters: filters,
-      start: 0,
-      limit: 1,
-      orderBy: 'name',
-      order: 'asc',
-    });
-
-    if (row.length === 1) {
-      return row[0][fieldname] as RawValue;
-    }
-
-    return undefined;
-  }
-
-  async setValue(
-    schemaName: string,
-    name: string,
-    fieldname: string,
-    value: RawValue
-  ) {
-    return await this.setValues(schemaName, name, {
-      [fieldname]: value,
-    });
-  }
-
-  async setValues(
-    schemaName: string,
-    name: string,
-    fieldValueMap: FieldValueMap
-  ) {
-    return this.updateOne(
-      schemaName,
-      Object.assign({}, fieldValueMap, { name })
-    );
-  }
-
-  async getCachedValue(schemaName: string, name: string, fieldname: string) {
-    let value = this.cache.hget(`${schemaName}:${name}`, fieldname);
-    if (value == null) {
-      value = await this.getValue(schemaName, name, fieldname);
-    }
-    return value;
   }
 }

backend/database/database.ts

@@ -1,925 +0,0 @@
-import { knex, Knex } from 'knex';
-import { pesa } from 'pesa';
-import { getRandomString } from '../../frappe/utils';
-import CacheManager from '../../frappe/utils/cacheManager';
-import {
-  CannotCommitError,
-  DatabaseError,
-  DuplicateEntryError,
-  LinkValidationError,
-  ValueError,
-} from '../../frappe/utils/errors';
-import Observable from '../../frappe/utils/observable';
-import { getMeta, getModels, getNewDoc, sqliteTypeMap } from '../common';
-
-interface GetAllOptions {
-  doctype?: string;
-  fields?: string[];
-  filters?: Record<string, string>;
-  start?: number;
-  limit?: number;
-  groupBy?: string;
-  orderBy?: string;
-  order?: string;
-}
-
-export default class Database extends Observable<never> {
-  knex: Knex;
-  cache: CacheManager;
-
-  constructor(dbPath: string) {
-    super();
-    this.typeMap = sqliteTypeMap;
-    this.cache = new CacheManager();
-    this.dbPath = dbPath;
-    this.connectionParams = {
-      client: 'sqlite3',
-      connection: {
-        filename: this.dbPath,
-      },
-      pool: {
-        afterCreate(conn, done) {
-          conn.run('PRAGMA foreign_keys=ON');
-          done();
-        },
-      },
-      useNullAsDefault: true,
-      asyncStackTraces: process.env.NODE_ENV === 'development',
-    };
-  }
-
-  connect() {
-    this.knex = knex(this.connectionParams);
-    this.knex.on('query-error', (error) => {
-      error.type = this.getError(error);
-    });
-  }
-
-  close() {
-    //
-  }
-
-  /**
-   * TODO: Refactor this
-   *
-   * essentially it's not models that are required by the database
-   * but the schema, all the information that is relevant to building
-   * tables needs to given to this function.
-   */
-  async migrate() {
-    const models = getModels();
-    for (const doctype in models) {
-      // check if controller module
-      const meta = getMeta(doctype);
-      const baseDoctype = meta.getBaseDocType();
-
-      if (!meta.isSingle) {
-        if (await this.tableExists(baseDoctype)) {
-          await this.alterTable(baseDoctype);
-        } else {
-          await this.createTable(baseDoctype);
-        }
-      }
-    }
-
-    await this.commit();
-    await this.initializeSingles();
-  }
-
-  async initializeSingles() {
-    const models = getModels();
-    const singleDoctypes = Object.keys(models)
-      .filter((n) => models[n].isSingle)
-      .map((n) => models[n].name);
-
-    for (const doctype of singleDoctypes) {
-      if (await this.singleExists(doctype)) {
-        const singleValues = await this.getSingleFieldsToInsert(doctype);
-        singleValues.forEach(({ fieldname, value }) => {
-          const singleValue = getNewDoc({
-            doctype: 'SingleValue',
-            parent: doctype,
-            fieldname,
-            value,
-          });
-          singleValue.insert();
-        });
-        continue;
-      }
-      const meta = getMeta(doctype);
-      if (meta.fields.every((df) => df.default == null)) {
-        continue;
-      }
-      const defaultValues = meta.fields.reduce((doc, df) => {
-        if (df.default != null) {
-          doc[df.fieldname] = df.default;
-        }
-        return doc;
-      }, {});
-      await this.updateSingle(doctype, defaultValues);
-    }
-  }
-
-  async singleExists(doctype) {
-    const res = await this.knex('SingleValue')
-      .count('parent as count')
-      .where('parent', doctype)
-      .first();
-    return res.count > 0;
-  }
-
-  async getSingleFieldsToInsert(doctype) {
-    const existingFields = (
-      await this.knex('SingleValue')
-        .where({ parent: doctype })
-        .select('fieldname')
-    ).map(({ fieldname }) => fieldname);
-
-    return getMeta(doctype)
-      .fields.map(({ fieldname, default: value }) => ({
-        fieldname,
-        value,
-      }))
-      .filter(
-        ({ fieldname, value }) =>
-          !existingFields.includes(fieldname) && value !== undefined
-      );
-  }
-
-  tableExists(table) {
-    return this.knex.schema.hasTable(table);
-  }
-
-  async createTable(doctype, tableName = null) {
-    const fields = this.getValidFields(doctype);
-    return await this.runCreateTableQuery(tableName || doctype, fields);
-  }
-
-  runCreateTableQuery(doctype, fields) {
-    return this.knex.schema.createTable(doctype, (table) => {
-      for (const field of fields) {
-        this.buildColumnForTable(table, field);
-      }
-    });
-  }
-
-  async alterTable(doctype) {
-    // get columns
-    const diff = await this.getColumnDiff(doctype);
-    const newForeignKeys = await this.getNewForeignKeys(doctype);
-
-    return this.knex.schema
-      .table(doctype, (table) => {
-        if (diff.added.length) {
-          for (const field of diff.added) {
-            this.buildColumnForTable(table, field);
-          }
-        }
-
-        if (diff.removed.length) {
-          this.removeColumns(doctype, diff.removed);
-        }
-      })
-      .then(() => {
-        if (newForeignKeys.length) {
-          return this.addForeignKeys(doctype, newForeignKeys);
-        }
-      });
-  }
-
-  buildColumnForTable(table, field) {
-    const columnType = this.getColumnType(field);
-    if (!columnType) {
-      // In case columnType is "Table"
-      // childTable links are handled using the childTable's "parent" field
-      return;
-    }
-
-    const column = table[columnType](field.fieldname);
-
-    // primary key
-    if (field.fieldname === 'name') {
-      column.primary();
-    }
-
-    // default value
-    if (!!field.default && !(field.default instanceof Function)) {
-      column.defaultTo(field.default);
-    }
-
-    // required
-    if (
-      (!!field.required && !(field.required instanceof Function)) ||
-      field.fieldtype === 'Currency'
-    ) {
-      column.notNullable();
-    }
-
-    // link
-    if (field.fieldtype === 'Link' && field.target) {
-      const meta = getMeta(field.target);
-      table
-        .foreign(field.fieldname)
-        .references('name')
-        .inTable(meta.getBaseDocType())
-        .onUpdate('CASCADE')
-        .onDelete('RESTRICT');
-    }
-  }
-
-  async getColumnDiff(doctype) {
-    const tableColumns = await this.getTableColumns(doctype);
-    const validFields = this.getValidFields(doctype);
-    const diff = { added: [], removed: [] };
-
-    for (const field of validFields) {
-      if (
-        !tableColumns.includes(field.fieldname) &&
-        this.getColumnType(field)
-      ) {
-        diff.added.push(field);
-      }
-    }
-
-    const validFieldNames = validFields.map((field) => field.fieldname);
-    for (const column of tableColumns) {
-      if (!validFieldNames.includes(column)) {
-        diff.removed.push(column);
-      }
-    }
-
-    return diff;
-  }
-
-  async removeColumns(doctype: string, removed: string[]) {
-    // TODO: Implement this for sqlite
-  }
-
-  async getNewForeignKeys(doctype) {
-    const foreignKeys = await this.getForeignKeys(doctype);
-    const newForeignKeys = [];
-    const meta = getMeta(doctype);
-    for (const field of meta.getValidFields({ withChildren: false })) {
-      if (
-        field.fieldtype === 'Link' &&
-        !foreignKeys.includes(field.fieldname)
-      ) {
-        newForeignKeys.push(field);
-      }
-    }
-    return newForeignKeys;
-  }
-
-  async get(doctype, name = null, fields = '*') {
-    const meta = getMeta(doctype);
-    let doc;
-    if (meta.isSingle) {
-      doc = await this.getSingle(doctype);
-      doc.name = doctype;
-    } else {
-      if (!name) {
-        throw new ValueError('name is mandatory');
-      }
-      doc = await this.getOne(doctype, name, fields);
-    }
-    if (!doc) {
-      return;
-    }
-    await this.loadChildren(doc, meta);
-    return doc;
-  }
-
-  async loadChildren(doc, meta) {
-    // load children
-    const tableFields = meta.getTableFields();
-    for (const field of tableFields) {
-      doc[field.fieldname] = await this.getAll({
-        doctype: field.childtype,
-        fields: ['*'],
-        filters: { parent: doc.name },
-        orderBy: 'idx',
-        order: 'asc',
-      });
-    }
-  }
-
-  async getSingle(doctype) {
-    const values = await this.getAll({
-      doctype: 'SingleValue',
-      fields: ['fieldname', 'value'],
-      filters: { parent: doctype },
-      orderBy: 'fieldname',
-      order: 'asc',
-    });
-    const doc = {};
-    for (const row of values) {
-      doc[row.fieldname] = row.value;
-    }
-    return doc;
-  }
-
-  /**
-   * Get list of values from the singles table.
-   * @param  {...string | Object} fieldnames list of fieldnames to get the values of
-   * @returns {Array<Object>} array of {parent, value, fieldname}.
-   * @example
-   * Database.getSingleValues('internalPrecision');
-   * // returns [{ fieldname: 'internalPrecision', parent: 'SystemSettings', value: '12' }]
-   * @example
-   * Database.getSingleValues({fieldname:'internalPrecision', parent: 'SystemSettings'});
-   * // returns [{ fieldname: 'internalPrecision', parent: 'SystemSettings', value: '12' }]
-   */
-  async getSingleValues(...fieldnames) {
-    fieldnames = fieldnames.map((fieldname) => {
-      if (typeof fieldname === 'string') {
-        return { fieldname };
-      }
-      return fieldname;
-    });
-
-    let builder = this.knex('SingleValue');
-    builder = builder.where(fieldnames[0]);
-
-    fieldnames.slice(1).forEach(({ fieldname, parent }) => {
-      if (typeof parent === 'undefined') {
-        builder = builder.orWhere({ fieldname });
-      } else {
-        builder = builder.orWhere({ fieldname, parent });
-      }
-    });
-
-    let values = [];
-    try {
-      values = await builder.select('fieldname', 'value', 'parent');
-    } catch (error) {
-      if (error.message.includes('no such table')) {
-        return [];
-      }
-      throw error;
-    }
-
-    return values.map((value) => {
-      const fields = getMeta(value.parent).fields;
-      return this.getDocFormattedDoc(fields, values);
-    });
-  }
-
-  async getOne(doctype, name, fields = '*') {
-    const meta = getMeta(doctype);
-    const baseDoctype = meta.getBaseDocType();
-
-    const doc = await this.knex
-      .select(fields)
-      .from(baseDoctype)
-      .where('name', name)
-      .first();
-
-    if (!doc) {
-      return doc;
-    }
-
-    return this.getDocFormattedDoc(meta.fields, doc);
-  }
-
-  getDocFormattedDoc(fields, doc) {
-    // format for usage, not going into the db
-    const docFields = Object.keys(doc);
-    const filteredFields = fields.filter(({ fieldname }) =>
-      docFields.includes(fieldname)
-    );
-
-    const formattedValues = filteredFields.reduce((d, field) => {
-      const { fieldname } = field;
-      d[fieldname] = this.getDocFormattedValues(field, doc[fieldname]);
-      return d;
-    }, {});
-
-    return Object.assign(doc, formattedValues);
-  }
-
-  getDocFormattedValues(field, value) {
-    // format for usage, not going into the db
-    try {
-      if (field.fieldtype === 'Currency') {
-        return pesa(value);
-      }
-    } catch (err) {
-      err.message += ` value: '${value}' of type: ${typeof value}, fieldname: '${
-        field.fieldname
-      }', label: '${field.label}'`;
-      throw err;
-    }
-    return value;
-  }
-
-  triggerChange(doctype, name) {
-    this.trigger(`change:${doctype}`, { name }, 500);
-    this.trigger(`change`, { doctype, name }, 500);
-    // also trigger change for basedOn doctype
-    const meta = getMeta(doctype);
-    if (meta.basedOn) {
-      this.triggerChange(meta.basedOn, name);
-    }
-  }
-
-  async insert(doctype, doc) {
-    const meta = getMeta(doctype);
-    const baseDoctype = meta.getBaseDocType();
-    doc = this.applyBaseDocTypeFilters(doctype, doc);
-
-    // insert parent
-    if (meta.isSingle) {
-      await this.updateSingle(doctype, doc);
-    } else {
-      await this.insertOne(baseDoctype, doc);
-    }
-
-    // insert children
-    await this.insertChildren(meta, doc, baseDoctype);
-
-    this.triggerChange(doctype, doc.name);
-
-    return doc;
-  }
-
-  async insertChildren(meta, doc, doctype) {
-    const tableFields = meta.getTableFields();
-    for (const field of tableFields) {
-      let idx = 0;
-      for (const child of doc[field.fieldname] || []) {
-        this.prepareChild(doctype, doc.name, child, field, idx);
-        await this.insertOne(field.childtype, child);
-        idx++;
-      }
-    }
-  }
-
-  insertOne(doctype, doc) {
-    const fields = this.getValidFields(doctype);
-
-    if (!doc.name) {
-      doc.name = getRandomString();
-    }
-
-    const formattedDoc = this.getFormattedDoc(fields, doc);
-    return this.knex(doctype).insert(formattedDoc);
-  }
-
-  async update(doctype, doc) {
-    const meta = getMeta(doctype);
-    const baseDoctype = meta.getBaseDocType();
-    doc = this.applyBaseDocTypeFilters(doctype, doc);
-
-    // update parent
-    if (meta.isSingle) {
-      await this.updateSingle(doctype, doc);
-    } else {
-      await this.updateOne(baseDoctype, doc);
-    }
-
-    // insert or update children
-    await this.updateChildren(meta, doc, baseDoctype);
-
-    this.triggerChange(doctype, doc.name);
-
-    return doc;
-  }
-
-  async updateChildren(meta, doc, doctype) {
-    const tableFields = meta.getTableFields();
-    for (const field of tableFields) {
-      const added = [];
-      for (const child of doc[field.fieldname] || []) {
-        this.prepareChild(doctype, doc.name, child, field, added.length);
-        if (await this.exists(field.childtype, child.name)) {
-          await this.updateOne(field.childtype, child);
-        } else {
-          await this.insertOne(field.childtype, child);
-        }
-        added.push(child.name);
-      }
-      await this.runDeleteOtherChildren(field, doc.name, added);
-    }
-  }
-
-  updateOne(doctype, doc) {
-    const validFields = this.getValidFields(doctype);
-    const fieldsToUpdate = Object.keys(doc).filter((f) => f !== 'name');
-    const fields = validFields.filter((df) =>
-      fieldsToUpdate.includes(df.fieldname)
-    );
-    const formattedDoc = this.getFormattedDoc(fields, doc);
-
-    return this.knex(doctype)
-      .where('name', doc.name)
-      .update(formattedDoc)
-      .then(() => {
-        const cacheKey = `${doctype}:${doc.name}`;
-        if (this.cache.hexists(cacheKey)) {
-          for (const fieldname in formattedDoc) {
-            const value = formattedDoc[fieldname];
-            this.cache.hset(cacheKey, fieldname, value);
-          }
-        }
-      });
-  }
-
-  runDeleteOtherChildren(field, parent, added) {
-    // delete other children
-    return this.knex(field.childtype)
-      .where('parent', parent)
-      .andWhere('name', 'not in', added)
-      .delete();
-  }
-
-  async updateSingle(doctype, doc) {
-    const meta = getMeta(doctype);
-    await this.deleteSingleValues(doctype);
-    for (const field of meta.getValidFields({ withChildren: false })) {
-      const value = doc[field.fieldname];
-      if (value != null) {
-        const singleValue = getNewDoc({
-          doctype: 'SingleValue',
-          parent: doctype,
-          fieldname: field.fieldname,
-          value: value,
-        });
-        await singleValue.insert();
-      }
-    }
-  }
-
-  deleteSingleValues(name) {
-    return this.knex('SingleValue').where('parent', name).delete();
-  }
-
-  async rename(doctype, oldName, newName) {
-    const meta = getMeta(doctype);
-    const baseDoctype = meta.getBaseDocType();
-    await this.knex(baseDoctype)
-      .update({ name: newName })
-      .where('name', oldName)
-      .then(() => {
-        this.clearValueCache(doctype, oldName);
-      });
-    await this.commit();
-
-    this.triggerChange(doctype, newName);
-  }
-
-  prepareChild(parenttype, parent, child, field, idx) {
-    if (!child.name) {
-      child.name = getRandomString();
-    }
-    child.parent = parent;
-    child.parenttype = parenttype;
-    child.parentfield = field.fieldname;
-    child.idx = idx;
-  }
-
-  getValidFields(doctype) {
-    return getMeta(doctype).getValidFields({ withChildren: false });
-  }
-
-  getFormattedDoc(fields, doc) {
-    // format for storage, going into the db
-    const formattedDoc = {};
-    fields.map((field) => {
-      const value = doc[field.fieldname];
-      formattedDoc[field.fieldname] = this.getFormattedValue(field, value);
-    });
-    return formattedDoc;
-  }
-
-  getFormattedValue(field, value) {
-    // format for storage, going into the db
-    const type = typeof value;
-    if (field.fieldtype === 'Currency') {
-      let currency = value;
-
-      if (type === 'number' || type === 'string') {
-        currency = pesa(value);
-      }
-
-      const currencyValue = currency.store;
-      if (typeof currencyValue !== 'string') {
-        throw new Error(
-          `invalid currencyValue '${currencyValue}' of type '${typeof currencyValue}' on converting from '${value}' of type '${type}'`
-        );
-      }
-
-      return currencyValue;
-    }
-
-    if (value instanceof Date) {
-      if (field.fieldtype === 'Date') {
-        // date
-        return value.toISOString().substr(0, 10);
-      } else {
-        // datetime
-        return value.toISOString();
-      }
-    } else if (field.fieldtype === 'Link' && !value) {
-      // empty value must be null to satisfy
-      // foreign key constraint
-      return null;
-    } else {
-      return value;
-    }
-  }
-
-  applyBaseDocTypeFilters(doctype, doc) {
-    const meta = getMeta(doctype);
-    if (meta.filters) {
-      for (const fieldname in meta.filters) {
-        const value = meta.filters[fieldname];
-        if (typeof value !== 'object') {
-          doc[fieldname] = value;
-        }
-      }
-    }
-    return doc;
-  }
-
-  async deleteMany(doctype, names) {
-    for (const name of names) {
-      await this.delete(doctype, name);
-    }
-  }
-
-  async delete(doctype, name) {
-    const meta = getMeta(doctype);
-    const baseDoctype = meta.getBaseDocType();
-    await this.deleteOne(baseDoctype, name);
-
-    // delete children
-    const tableFields = getMeta(doctype).getTableFields();
-    for (const field of tableFields) {
-      await this.deleteChildren(field.childtype, name);
-    }
-
-    this.triggerChange(doctype, name);
-  }
-
-  async deleteOne(doctype, name) {
-    return this.knex(doctype)
-      .where('name', name)
-      .delete()
-      .then(() => {
-        this.clearValueCache(doctype, name);
-      });
-  }
-
-  deleteChildren(parenttype, parent) {
-    return this.knex(parenttype).where('parent', parent).delete();
-  }
-
-  async exists(doctype, name) {
-    return (await this.getValue(doctype, name)) ? true : false;
-  }
-
-  async getValue(doctype, filters, fieldname = 'name') {
-    const meta = getMeta(doctype);
-    const baseDoctype = meta.getBaseDocType();
-    if (typeof filters === 'string') {
-      filters = { name: filters };
-    }
-    if (meta.filters) {
-      Object.assign(filters, meta.filters);
-    }
-
-    const row = await this.getAll({
-      doctype: baseDoctype,
-      fields: [fieldname],
-      filters: filters,
-      start: 0,
-      limit: 1,
-      orderBy: 'name',
-      order: 'asc',
-    });
-    return row.length ? row[0][fieldname] : null;
-  }
-
-  async setValue(doctype, name, fieldname, value) {
-    return await this.setValues(doctype, name, {
-      [fieldname]: value,
-    });
-  }
-
-  async setValues(doctype, name, fieldValuePair) {
-    const doc = Object.assign({}, fieldValuePair, { name });
-    return this.updateOne(doctype, doc);
-  }
-
-  async getCachedValue(doctype, name, fieldname) {
-    let value = this.cache.hget(`${doctype}:${name}`, fieldname);
-    if (value == null) {
-      value = await this.getValue(doctype, name, fieldname);
-    }
-    return value;
-  }
-
-  async getAll({
-    doctype,
-    fields,
-    filters,
-    start,
-    limit,
-    groupBy,
-    orderBy = 'creation',
-    order = 'desc',
-  }: GetAllOptions = {}) {
-    const meta = getMeta(doctype);
-    const baseDoctype = meta.getBaseDocType();
-    if (!fields) {
-      fields = meta.getKeywordFields();
-      fields.push('name');
-    }
-    if (typeof fields === 'string') {
-      fields = [fields];
-    }
-    if (meta.filters) {
-      filters = Object.assign({}, filters, meta.filters);
-    }
-
-    const builder = this.knex.select(fields).from(baseDoctype);
-
-    this.applyFiltersToBuilder(builder, filters);
-
-    if (orderBy) {
-      builder.orderBy(orderBy, order);
-    }
-
-    if (groupBy) {
-      builder.groupBy(groupBy);
-    }
-
-    if (start) {
-      builder.offset(start);
-    }
-
-    if (limit) {
-      builder.limit(limit);
-    }
-
-    const docs = await builder;
-    return docs.map((doc) => this.getDocFormattedDoc(meta.fields, doc));
-  }
-
-  applyFiltersToBuilder(builder, filters) {
-    // {"status": "Open"} => `status = "Open"`
-
-    // {"status": "Open", "name": ["like", "apple%"]}
-    // => `status="Open" and name like "apple%"
-
-    // {"date": [">=", "2017-09-09", "<=", "2017-11-01"]}
-    // => `date >= 2017-09-09 and date <= 2017-11-01`
-
-    const filtersArray = [];
-
-    for (const field in filters) {
-      const value = filters[field];
-      let operator = '=';
-      let comparisonValue = value;
-
-      if (Array.isArray(value)) {
-        operator = value[0];
-        comparisonValue = value[1];
-        operator = operator.toLowerCase();
-
-        if (operator === 'includes') {
-          operator = 'like';
-        }
-
-        if (operator === 'like' && !comparisonValue.includes('%')) {
-          comparisonValue = `%${comparisonValue}%`;
-        }
-      }
-
-      filtersArray.push([field, operator, comparisonValue]);
-
-      if (Array.isArray(value) && value.length > 2) {
-        // multiple conditions
-        const operator = value[2];
-        const comparisonValue = value[3];
-        filtersArray.push([field, operator, comparisonValue]);
-      }
-    }
-
-    filtersArray.map((filter) => {
-      const [field, operator, comparisonValue] = filter;
-      if (operator === '=') {
-        builder.where(field, comparisonValue);
-      } else {
-        builder.where(field, operator, comparisonValue);
-      }
-    });
-  }
-
-  sql(query: string, params: Knex.RawBinding[] = []) {
-    // run raw query
-    return this.knex.raw(query, params);
-  }
-
-  async commit() {
-    try {
-      await this.sql('commit');
-    } catch (e) {
-      if (e.type !== CannotCommitError) {
-        throw e;
-      }
-    }
-  }
-
-  clearValueCache(doctype, name) {
-    const cacheKey = `${doctype}:${name}`;
-    this.cache.hclear(cacheKey);
-  }
-
-  getColumnType(field) {
-    return this.typeMap[field.fieldtype];
-  }
-
-  async addForeignKeys(doctype, newForeignKeys) {
-    await this.sql('PRAGMA foreign_keys=OFF');
-    await this.sql('BEGIN TRANSACTION');
-
-    const tempName = 'TEMP' + doctype;
-
-    // create temp table
-    await this.createTable(doctype, tempName);
-
-    try {
-      // copy from old to new table
-      await this.knex(tempName).insert(this.knex.select().from(doctype));
-    } catch (err) {
-      await this.sql('ROLLBACK');
-      await this.sql('PRAGMA foreign_keys=ON');
-
-      const rows = await this.knex.select().from(doctype);
-      await this.prestigeTheTable(doctype, rows);
-      return;
-    }
-
-    // drop old table
-    await this.knex.schema.dropTable(doctype);
-
-    // rename new table
-    await this.knex.schema.renameTable(tempName, doctype);
-
-    await this.sql('COMMIT');
-    await this.sql('PRAGMA foreign_keys=ON');
-  }
-
-  async getTableColumns(doctype: string) {
-    return (await this.sql(`PRAGMA table_info(${doctype})`)).map((d) => d.name);
-  }
-
-  async getForeignKeys(doctype: string) {
-    return (await this.sql(`PRAGMA foreign_key_list(${doctype})`)).map(
-      (d) => d.from
-    );
-  }
-
-  getError(err) {
-    let errorType = DatabaseError;
-    if (err.message.includes('FOREIGN KEY')) {
-      errorType = LinkValidationError;
-    }
-    if (err.message.includes('SQLITE_ERROR: cannot commit')) {
-      errorType = CannotCommitError;
-    }
-    if (err.message.includes('SQLITE_CONSTRAINT: UNIQUE constraint failed:')) {
-      errorType = DuplicateEntryError;
-    }
-    return errorType;
-  }
-
-  async prestigeTheTable(tableName, tableRows) {
-    const max = 200;
-
-    // Alter table hacx for sqlite in case of schema change.
-    const tempName = `__${tableName}`;
-    await this.knex.schema.dropTableIfExists(tempName);
-
-    await this.knex.raw('PRAGMA foreign_keys=OFF');
-    await this.createTable(tableName, tempName);
-
-    if (tableRows.length > 200) {
-      const fi = Math.floor(tableRows.length / max);
-      for (let i = 0; i <= fi; i++) {
-        const rowSlice = tableRows.slice(i * max, i + 1 * max);
-        if (rowSlice.length === 0) {
-          break;
-        }
-        await this.knex.batchInsert(tempName, rowSlice);
-      }
-    } else {
-      await this.knex.batchInsert(tempName, tableRows);
-    }
-
-    await this.knex.schema.dropTable(tableName);
-    await this.knex.schema.renameTable(tempName, tableName);
-    await this.knex.raw('PRAGMA foreign_keys=ON');
-  }
-}

backend/database/manager.ts

@@ -0,0 +1,79 @@
+import fs from 'fs/promises';
+import { getSchemas } from '../../schemas';
+import patches from '../patches';
+import DatabaseCore from './core';
+import { runPatches } from './runPatch';
+import { Patch } from './types';
+
+export class DatabaseManager {
+  db: DatabaseCore;
+  constructor() {}
+
+  async createNewDatabase(dbPath: string, countryCode: string) {
+    await this.#unlinkIfExists(dbPath);
+    this.connectToDatabase(dbPath, countryCode);
+  }
+
+  async connectToDatabase(dbPath: string, countryCode?: string) {
+    this.db = new DatabaseCore(dbPath);
+    this.db.connect();
+
+    countryCode ??= await this.#getCountryCode();
+    const schemaMap = getSchemas(countryCode);
+    this.db.setSchemaMap(schemaMap);
+
+    await this.migrate();
+  }
+
+  async migrate() {
+    const patchesToExecute = await this.#getPatchesToExecute();
+    const preMigrationPatches = patchesToExecute.filter(
+      (p) => p.patch.beforeMigrate
+    );
+    const postMigrationPatches = patchesToExecute.filter(
+      (p) => !p.patch.beforeMigrate
+    );
+
+    await runPatches(preMigrationPatches, this);
+    await this.db.migrate();
+    await runPatches(postMigrationPatches, this);
+  }
+
+  async #getPatchesToExecute(): Promise<Patch[]> {
+    const query: { name: string }[] = await this.db
+      .knex('PatchRun')
+      .select('name');
+    const executedPatches = query.map((q) => q.name);
+    return patches.filter((p) => !executedPatches.includes(p.name));
+  }
+
+  async #getCountryCode(): Promise<string | undefined> {
+    if (!this.db) {
+      return undefined;
+    }
+
+    const query = await this.db
+      .knex('SingleValue')
+      .where({ fieldname: 'countryCode', parent: 'SystemSettings' });
+
+    if (query.length > 0) {
+      return query[0].countryCode as string;
+    }
+
+    return undefined;
+  }
+
+  async #unlinkIfExists(dbPath: string) {
+    try {
+      fs.unlink(dbPath);
+    } catch (err) {
+      if (err.code === 'ENOENT') {
+        return;
+      }
+
+      throw err;
+    }
+  }
+}
+
+export default new DatabaseManager();

backend/database/runPatch.ts

@@ -0,0 +1,31 @@
+import { getDefaultMetaFieldValueMap } from '../common';
+import { DatabaseManager } from './manager';
+import { FieldValueMap, Patch } from './types';
+
+export async function runPatches(patches: Patch[], dm: DatabaseManager) {
+  const list: { name: string; success: boolean }[] = [];
+  for (const patch of patches) {
+    const success = await runPatch(patch, dm);
+    list.push({ name: patch.name, success });
+  }
+  return list;
+}
+
+async function runPatch(patch: Patch, dm: DatabaseManager): Promise<boolean> {
+  try {
+    await patch.patch.execute(dm);
+  } catch (err) {
+    console.error('PATCH FAILED: ', patch.name);
+    console.error(err);
+    return false;
+  }
+
+  await makeEntry(patch.name, dm);
+  return true;
+}
+
+async function makeEntry(patchName: string, dm: DatabaseManager) {
+  const defaultFieldValueMap = getDefaultMetaFieldValueMap() as FieldValueMap;
+  defaultFieldValueMap.name = patchName;
+  await dm.db.insert('PatchRun', defaultFieldValueMap);
+}

backend/database/types.ts

@@ -26,3 +26,12 @@ export type FieldValueMap = Record<
   string,
   RawValue | undefined | FieldValueMap[]
 >;
+
+export interface Patch {
+  name: string;
+  version: string;
+  patch: {
+    execute: (DatabaseManager) => Promise<void>;
+    beforeMigrate?: boolean;
+  };
+}

backend/patches/index.ts

@@ -0,0 +1,6 @@
+import { Patch } from '../database/types';
+import testPatch from './testPatch';
+
+export default [
+  { name: 'testPatch', version: '0.4.2-beta.0', patch: testPatch },
+] as Patch[];

backend/patches/testPatch.ts

@@ -0,0 +1,10 @@
+import { DatabaseManager } from '../database/manager';
+
+async function execute(dm: DatabaseManager) {
+  /**
+   * Execute function will receive the DatabaseManager which is to be used
+   * to apply database patches.
+   */
+}
+
+export default { execute, beforeMigrate: true };

models/doctype/Account/AccountDocument.js

@@ -4,11 +4,8 @@ import Document from 'frappe/model/document';
 export default class Account extends Document {
   async validate() {
     if (!this.accountType && this.parentAccount) {
-      this.accountType = await frappe.db.getValue(
+      const account = frappe.db.get('Account', this.parentAccount);
-        'Account',
+      this.accountType = account.accountType;
-        this.parentAccount,
-        'accountType'
-      );
     }
   }
 }

models/doctype/Transaction/TransactionServer.js

@@ -30,12 +30,10 @@ export default {
     await entries.post();
 
     // update outstanding amounts
-    await frappe.db.setValue(
+    await frappe.db.update(this.doctype, {
-      this.doctype,
+      name: this.name,
-      this.name,
+      outstandingAmount: this.baseGrandTotal,
-      'outstandingAmount',
+    });
-      this.baseGrandTotal
-    );
 
     let party = await frappe.getDoc('Party', this.customer || this.supplier);
     await party.updateOutstandingAmount();

schemas/core/SystemSettings.json

@@ -73,6 +73,12 @@
       "fieldtype": "Check",
       "default": false,
       "description": "Hides the Get Started section from the sidebar. Change will be visible on restart or refreshing the app."
+    },
+    {
+      "fieldname": "countryCode",
+      "label": "Country Code",
+      "fieldtype": "Data",
+      "description": "Country code used to initialize regional settings."
     }
   ],
   "quickEditFields": [

src/initialization.js

@@ -27,9 +27,7 @@ async function runRegionalModelUpdates() {
     return;
   }
 
-  const { country, setupComplete } = await frappe.db.getSingle(
+  const { country, setupComplete } = await frappe.db.get('AccountingSettings');
-    'AccountingSettings'
-  );
   if (!parseInt(setupComplete)) return;
   await regionalModelUpdates({ country });
 }

src/utils.js

@@ -124,7 +124,8 @@ export function partyWithAvatar(party) {
       Avatar,
     },
     async mounted() {
-      this.imageURL = await frappe.db.getValue('Party', party, 'image');
+      const p = await frappe.db.get('Party', party);
+      this.imageURL = p.image;
       this.label = party;
     },
     template: `