test: start with dbCore tests

backend/database/core.ts

@@ -24,6 +24,17 @@ import {
   QueryFilter
 } from './types';
 
+/**
+ * Db Core Call Sequence
+ *
+ * 1. Init core: `const db = new DatabaseCore(dbPath)`.
+ * 2. Connect db: `db.connect()`. This will allow for raw queries to be executed.
+ * 3. Set schemas: `bb.setSchemaMap(schemaMap)`. This will allow for ORM functions to be executed.
+ * 4. Migrate: `await db.migrate()`. This will create absent tables and update the tables' shape.
+ * 5. ORM function execution: `db.get(...)`, `db.insert(...)`, etc.
+ * 6. Close connection: `await db.close()`.
+ */
+
 export default class DatabaseCore {
   knex?: Knex;
   typeMap = sqliteTypeMap;
@@ -60,13 +71,13 @@ export default class DatabaseCore {
     });
   }
 
-  close() {
-    this.knex!.destroy();
+  async close() {
+    await this.knex!.destroy();
   }
 
   async commit() {
     try {
-      await this.raw('commit');
+      await this.knex!.raw('commit');
     } catch (err) {
       const type = this.#getError(err as Error);
       if (type !== CannotCommitError) {
@@ -75,10 +86,6 @@ export default class DatabaseCore {
     }
   }
 
-  async raw(query: string, params: Knex.RawBinding[] = []) {
-    return await this.knex!.raw(query, params);
-  }
-
   async migrate() {
     for (const schemaName in this.schemaMap) {
       const schema = this.schemaMap[schemaName];
@@ -350,14 +357,14 @@ export default class DatabaseCore {
   }
 
   async #getTableColumns(schemaName: string): Promise<string[]> {
-    const info: FieldValueMap[] = await this.raw(
+    const info: FieldValueMap[] = await this.knex!.raw(
       `PRAGMA table_info(${schemaName})`
     );
     return info.map((d) => d.name as string);
   }
 
   async #getForeignKeys(schemaName: string): Promise<string[]> {
-    const foreignKeyList: FieldValueMap[] = await this.raw(
+    const foreignKeyList: FieldValueMap[] = await this.knex!.raw(
       `PRAGMA foreign_key_list(${schemaName})`
     );
     return foreignKeyList.map((d) => d.from as string);
@@ -506,14 +513,17 @@ export default class DatabaseCore {
     }
 
     // required
-    if (field.required || field.fieldtype === 'Currency') {
+    if (field.required) {
       column.notNullable();
     }
 
     // link
-    if (field.fieldtype === 'Link' && (field as TargetField).target) {
-      const targetschemaName = (field as TargetField).target as string;
-      const schema = this.schemaMap[targetschemaName];
+    if (
+      field.fieldtype === FieldTypeEnum.Link &&
+      (field as TargetField).target
+    ) {
+      const targetSchemaName = (field as TargetField).target as string;
+      const schema = this.schemaMap[targetSchemaName];
       table
         .foreign(field.fieldname)
         .references('name')
@@ -614,8 +624,8 @@ export default class DatabaseCore {
   }
 
   async #addForeignKeys(schemaName: string, newForeignKeys: Field[]) {
-    await this.raw('PRAGMA foreign_keys=OFF');
-    await this.raw('BEGIN TRANSACTION');
+    await this.knex!.raw('PRAGMA foreign_keys=OFF');
+    await this.knex!.raw('BEGIN TRANSACTION');
 
     const tempName = 'TEMP' + schemaName;
 
@@ -626,8 +636,8 @@ export default class DatabaseCore {
       // copy from old to new table
       await this.knex!(tempName).insert(this.knex!.select().from(schemaName));
     } catch (err) {
-      await this.raw('ROLLBACK');
-      await this.raw('PRAGMA foreign_keys=ON');
+      await this.knex!.raw('ROLLBACK');
+      await this.knex!.raw('PRAGMA foreign_keys=ON');
 
       const rows = await this.knex!.select().from(schemaName);
       await this.prestigeTheTable(schemaName, rows);
@@ -640,8 +650,8 @@ export default class DatabaseCore {
     // rename new table
     await this.knex!.schema.renameTable(tempName, schemaName);
 
-    await this.raw('COMMIT');
-    await this.raw('PRAGMA foreign_keys=ON');
+    await this.knex!.raw('COMMIT');
+    await this.knex!.raw('PRAGMA foreign_keys=ON');
   }
 
   async #loadChildren(

backend/database/tests/helpers.ts

@@ -0,0 +1,164 @@
+import { cloneDeep } from 'lodash';
+import { SchemaMap, SchemaStub, SchemaStubMap } from 'schemas/types';
+import {
+  addMetaFields,
+  cleanSchemas,
+  getAbstractCombinedSchemas,
+} from '../../../schemas';
+import SingleValue from '../../../schemas/core/SingleValue.json';
+
+const Customer = {
+  name: 'Customer',
+  label: 'Customer',
+  fields: [
+    {
+      fieldname: 'name',
+      label: 'Name',
+      fieldtype: 'Data',
+      default: 'John Thoe',
+      required: true,
+    },
+    {
+      fieldname: 'email',
+      label: 'Email',
+      fieldtype: 'Data',
+      placeholder: 'john@thoe.com',
+    },
+  ],
+  quickEditFields: ['email'],
+  keywordFields: ['name'],
+};
+
+const SalesInvoiceItem = {
+  name: 'SalesInvoiceItem',
+  label: 'Sales Invoice Item',
+  isChild: true,
+  fields: [
+    {
+      fieldname: 'item',
+      label: 'Item',
+      fieldtype: 'Data',
+      required: true,
+    },
+    {
+      fieldname: 'quantity',
+      label: 'Quantity',
+      fieldtype: 'Float',
+      required: true,
+      default: 1,
+    },
+    {
+      fieldname: 'rate',
+      label: 'Rate',
+      fieldtype: 'Currency',
+      required: true,
+    },
+    {
+      fieldname: 'amount',
+      label: 'Amount',
+      fieldtype: 'Currency',
+      computed: true,
+      readOnly: true,
+    },
+  ],
+  tableFields: ['item', 'quantity', 'rate', 'amount'],
+};
+
+const SalesInvoice = {
+  name: 'SalesInvoice',
+  label: 'Sales Invoice',
+  isSingle: false,
+  isChild: false,
+  isSubmittable: true,
+  keywordFields: ['name', 'customer'],
+  fields: [
+    {
+      label: 'Invoice No',
+      fieldname: 'name',
+      fieldtype: 'Data',
+      required: true,
+      readOnly: true,
+    },
+    {
+      fieldname: 'date',
+      label: 'Date',
+      fieldtype: 'Date',
+    },
+    {
+      fieldname: 'customer',
+      label: 'Customer',
+      fieldtype: 'Link',
+      target: 'Customer',
+      required: true,
+    },
+    {
+      fieldname: 'account',
+      label: 'Account',
+      fieldtype: 'Data',
+      required: true,
+    },
+    {
+      fieldname: 'items',
+      label: 'Items',
+      fieldtype: 'Table',
+      target: 'SalesInvoiceItem',
+      required: true,
+    },
+    {
+      fieldname: 'grandTotal',
+      label: 'Grand Total',
+      fieldtype: 'Currency',
+      computed: true,
+      readOnly: true,
+    },
+  ],
+};
+
+const SystemSettings = {
+  name: 'SystemSettings',
+  label: 'System Settings',
+  isSingle: true,
+  isChild: false,
+  fields: [
+    {
+      fieldname: 'dateFormat',
+      label: 'Date Format',
+      fieldtype: 'Select',
+      options: [
+        {
+          label: '23/03/2022',
+          value: 'dd/MM/yyyy',
+        },
+        {
+          label: '03/23/2022',
+          value: 'MM/dd/yyyy',
+        },
+      ],
+      default: 'dd/MM/yyyy',
+      required: true,
+    },
+    {
+      fieldname: 'locale',
+      label: 'Locale',
+      fieldtype: 'Data',
+      default: 'en-IN',
+    },
+  ],
+  quickEditFields: ['locale', 'dateFormat'],
+  keywordFields: [],
+};
+
+export function getBuiltTestSchemaMap(): SchemaMap {
+  const testSchemaMap: SchemaStubMap = {
+    SingleValue: SingleValue as SchemaStub,
+    Customer: Customer as SchemaStub,
+    SalesInvoice: SalesInvoice as SchemaStub,
+    SalesInvoiceItem: SalesInvoiceItem as SchemaStub,
+    SystemSettings: SystemSettings as SchemaStub,
+  };
+
+  const schemaMapClone = cloneDeep(testSchemaMap);
+  const abstractCombined = getAbstractCombinedSchemas(schemaMapClone);
+  const cleanedSchemas = cleanSchemas(abstractCombined);
+  return addMetaFields(cleanedSchemas);
+}

backend/database/tests/testCore.spec.ts

@@ -0,0 +1,126 @@
+import * as assert from 'assert';
+import 'mocha';
+import { getMapFromList } from 'schemas/helpers';
+import { FieldTypeEnum } from 'schemas/types';
+import { sqliteTypeMap } from '../../common';
+import DatabaseCore from '../core';
+import { SqliteTableInfo } from '../types';
+import { getBuiltTestSchemaMap } from './helpers';
+
+describe('DatabaseCore: Connect Migrate Close', async function () {
+  const db = new DatabaseCore();
+  specify('dbPath', function () {
+    assert.strictEqual(db.dbPath, ':memory:');
+  });
+
+  const schemaMap = getBuiltTestSchemaMap();
+  db.setSchemaMap(schemaMap);
+  specify('schemaMap', function () {
+    assert.strictEqual(schemaMap, db.schemaMap);
+  });
+
+  specify('connect', function () {
+    assert.doesNotThrow(() => db.connect());
+    assert.notStrictEqual(db.knex, undefined);
+  });
+
+  specify('migrate and close', async function () {
+    // Does not throw
+    await db.migrate();
+    // Does not throw
+    await db.close();
+  });
+});
+
+describe('DatabaseCore: Migrate and Check Db', function () {
+  let db: DatabaseCore;
+  const schemaMap = getBuiltTestSchemaMap();
+
+  this.beforeEach(async function () {
+    db = new DatabaseCore();
+    db.connect();
+    db.setSchemaMap(schemaMap);
+  });
+
+  this.afterEach(async function () {
+    await db.close();
+  });
+
+  specify(`Pre Migrate TableInfo`, async function () {
+    for (const schemaName in schemaMap) {
+      const columns = await db.knex?.raw('pragma table_info(??)', schemaName);
+      assert.strictEqual(columns.length, 0, `column count ${schemaName}`);
+    }
+  });
+
+  specify('Post Migrate TableInfo', async function () {
+    await db.migrate();
+    for (const schemaName in schemaMap) {
+      const schema = schemaMap[schemaName];
+      const fieldMap = getMapFromList(schema.fields, 'fieldname');
+      const columns: SqliteTableInfo[] = await db.knex!.raw(
+        'pragma table_info(??)',
+        schemaName
+      );
+
+      let columnCount = schema.fields.filter(
+        (f) => f.fieldtype !== FieldTypeEnum.Table
+      ).length;
+
+      if (schema.isSingle) {
+        columnCount = 0;
+      }
+
+      assert.strictEqual(
+        columns.length,
+        columnCount,
+        `${schemaName}:: column count: ${columns.length}, ${columnCount}`
+      );
+
+      for (const column of columns) {
+        const field = fieldMap[column.name];
+        const dbColType = sqliteTypeMap[field.fieldtype];
+
+        assert.strictEqual(
+          column.name,
+          field.fieldname,
+          `${schemaName}.${column.name}:: name check: ${column.name}, ${field.fieldname}`
+        );
+
+        assert.strictEqual(
+          column.type,
+          dbColType,
+          `${schemaName}.${column.name}:: type check: ${column.type}, ${dbColType}`
+        );
+
+        if (field.required !== undefined) {
+          assert.strictEqual(
+            !!column.notnull,
+            field.required,
+            `${schemaName}.${column.name}:: notnull check: ${column.notnull}, ${field.required}`
+          );
+        } else {
+          assert.strictEqual(
+            column.notnull,
+            0,
+            `${schemaName}.${column.name}:: notnull check: ${column.notnull}, ${field.required}`
+          );
+        }
+
+        if (column.dflt_value === null) {
+          assert.strictEqual(
+            field.default,
+            undefined,
+            `${schemaName}.${column.name}:: dflt_value check: ${column.dflt_value}, ${field.default}`
+          );
+        } else {
+          assert.strictEqual(
+            column.dflt_value.slice(1, -1),
+            String(field.default),
+            `${schemaName}.${column.name}:: dflt_value check: ${column.type}, ${dbColType}`
+          );
+        }
+      }
+    }
+  });
+});

backend/database/types.ts

@@ -46,3 +46,13 @@ export type KnexColumnType =
   | 'datetime'
   | 'time'
   | 'binary';
+
+// Returned by pragma table_info
+export interface SqliteTableInfo {
+  pk: number;
+  cid: number;
+  name: string;
+  type: string;
+  notnull: number; // 0 | 1
+  dflt_value: string | null;
+}

package.json

@@ -16,7 +16,7 @@
     "postinstall": "electron-builder install-app-deps",
     "postuninstall": "electron-builder install-app-deps",
     "script:translate": "node scripts/generateTranslations.js",
-    "test": "TS_NODE_COMPILER_OPTIONS='{\"module\":\"commonjs\"}' mocha --reporter nyan --require ts-node/register ./**/tests/**/*.spec.ts"
+    "test": "TS_NODE_COMPILER_OPTIONS='{\"module\":\"commonjs\"}' mocha --reporter nyan --require ts-node/register --require tsconfig-paths/register ./**/tests/**/*.spec.ts"
   },
   "dependencies": {
     "@popperjs/core": "^2.10.2",
@@ -66,6 +66,7 @@
     "raw-loader": "^4.0.2",
     "tailwindcss": "npm:@tailwindcss/postcss7-compat",
     "ts-node": "^10.7.0",
+    "tsconfig-paths": "^3.14.1",
     "tslib": "^2.3.1",
     "typescript": "^4.6.2",
     "vue-cli-plugin-electron-builder": "^2.0.0",

schemas/app/Payment.json

@@ -4,7 +4,6 @@
   "isSingle": false,
   "isChild": false,
   "isSubmittable": true,
-  "settings": "PaymentSettings",
   "fields": [
     {
       "label": "Payment No",

schemas/app/SalesInvoice.json

@@ -5,7 +5,6 @@
   "isChild": false,
   "isSubmittable": true,
   "keywordFields": ["name", "customer"],
-  "settings": "SalesInvoiceSettings",
   "fields": [
     {
       "label": "Invoice No",

schemas/index.ts

@@ -2,7 +2,15 @@ import { cloneDeep } from 'lodash';
 import { getListFromMap, getMapFromList } from './helpers';
 import regionalSchemas from './regional';
 import { appSchemas, coreSchemas, metaSchemas } from './schemas';
-import { Schema, SchemaMap, SchemaStub, SchemaStubMap } from './types';
+import { Field, Schema, SchemaMap, SchemaStub, SchemaStubMap } from './types';
+
+const NAME_FIELD = {
+  fieldname: 'name',
+  label: `ID`,
+  fieldtype: 'Data',
+  required: true,
+  readOnly: true,
+};
 
 export function getSchemas(countryCode: string = '-'): SchemaMap {
   const builtCoreSchemas = getCoreSchemas();
@@ -41,9 +49,26 @@ export function addMetaFields(schemaMap: SchemaMap): SchemaMap {
     }
   }
 
+  addNameField(schemaMap);
   return schemaMap;
 }
 
+function addNameField(schemaMap: SchemaMap) {
+  for (const name in schemaMap) {
+    const schema = schemaMap[name];
+    if (schema.isSingle) {
+      continue;
+    }
+
+    const pkField = schema.fields.find((f) => f.fieldname === 'name');
+    if (pkField !== undefined) {
+      continue;
+    }
+
+    schema.fields.push(NAME_FIELD as Field);
+  }
+}
+
 function getCoreSchemas(): SchemaMap {
   const rawSchemaMap = getMapFromList(coreSchemas, 'name');
   const coreSchemaMap = getAbstractCombinedSchemas(rawSchemaMap);

schemas/schemas.ts

@@ -29,9 +29,8 @@ import PatchRun from './core/PatchRun.json';
 import SingleValue from './core/SingleValue.json';
 import SystemSettings from './core/SystemSettings.json';
 import base from './meta/base.json';
-import submittable from './meta/submittable.json';
-//asdf
 import child from './meta/child.json';
+import submittable from './meta/submittable.json';
 import tree from './meta/tree.json';
 import { Schema, SchemaStub } from './types';
 

schemas/tests/helpers.ts

@@ -1,3 +1,4 @@
+import { cloneDeep } from 'lodash';
 import Account from '../app/Account.json';
 import Customer from '../app/Customer.json';
 import JournalEntry from '../app/JournalEntry.json';
@@ -31,10 +32,10 @@ export function getTestSchemaMap(): {
   } as AppSchemaMap;
   const regionalSchemaMap = { Party: PartyRegional } as RegionalSchemaMap;
 
-  return {
+  return cloneDeep({
     appSchemaMap,
     regionalSchemaMap,
-  };
+  });
 }
 
 export function everyFieldExists(fieldList: string[], schema: Schema): boolean {

schemas/types.ts

@@ -9,7 +9,6 @@
  * If any field has to have a dynamic value, it should be added to the controller
  * file by the same name.
  *
- *
  * There are a few types of schemas:
  * - _Regional_: Schemas that are in the '../regional' subdirectories
  *      these can be of any of the below types.
@@ -31,6 +30,11 @@
  *
  * Note: if a Regional schema is not present as a non regional variant it's used
  * as it is.
+ * 
+ * ## Additional Notes
+ * 
+ * In all the schemas, the 'name' field/column is the primary key. If it isn't
+ * explicitly added, the schema builder will add it in.
  */
 
 export enum FieldTypeEnum {
@@ -107,9 +111,9 @@ export type Field =
 
 export type TreeSettings = { parentField: string };
 
-// @formattoer:off
+// @formatter:off
 export interface Schema {
-  name: string;                  // Table PK
+  name: string;                  // Table name
   label: string;                 // Translateable UI facing name
   fields: Field[];               // Maps to database columns
   isTree?: boolean;              // Used for nested set, eg for Chart of Accounts
@@ -117,6 +121,7 @@ export interface Schema {
   isChild?: boolean;             // Indicates a child table, i.e table with "parent" FK column
   isSingle?: boolean;            // Fields will be values in SingleValue, i.e. an Entity Attr. Value
   isAbstract?: boolean;          // Not entered into db, used to extend a Subclass schema
+  tableFields?: string[]         // Used for displaying childTableFields
   isSubmittable?: boolean;       // For transactional types, values considered only after submit
   keywordFields?: string[];      // Used to get fields that are to be used for search.
   quickEditFields?: string[];    // Used to get fields for the quickEditForm

src/main.js

@@ -1,6 +1,7 @@
 import { ipcRenderer } from 'electron';
 import frappe from 'frappe';
 import { createApp } from 'vue';
+import { getBuiltTestSchemaMap } from '../backend/database/tests/helpers';
 import { getSchemas } from '../schemas';
 import App from './App';
 import FeatherIcon from './components/FeatherIcon';
@@ -104,3 +105,4 @@ import { setLanguageMap, stringifyCircular } from './utils';
 })();
 
 window.gs = getSchemas;
+window.gst = getBuiltTestSchemaMap;

tsconfig.json

@@ -15,7 +15,10 @@
     "baseUrl": ".",
     "types": ["webpack-env"],
     "paths": {
-      "@/*": ["src/*"]
+      "@/*": ["src/*"],
+      "schemas/*": ["schemas/*"],
+      "backend/*": ["backend/*"],
+      "common/*": ["common/*"]
     },
     "lib": ["esnext", "dom", "dom.iterable", "scripthost"]
   },

vue.config.js

@@ -40,6 +40,9 @@ module.exports = {
     Object.assign(config.resolve.alias, {
       frappe: path.resolve(__dirname, './frappe'),
       '~': path.resolve('.'),
+      schemas: path.resolve(__dirname, './schemas'),
+      backend: path.resolve(__dirname, './backend'),
+      common: path.resolve(__dirname, './common'),
     });
 
     config.plugins.push(

yarn.lock

@@ -1338,6 +1338,11 @@
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.9.tgz#97edc9037ea0c38585320b28964dde3b39e4660d"
   integrity sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==
 
+"@types/json5@^0.0.29":
+  version "0.0.29"
+  resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
+  integrity sha1-7ihweulOEdK4J7y+UnC86n8+ce4=
+
 "@types/lodash@^4.14.179":
   version "4.14.179"
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.179.tgz#490ec3288088c91295780237d2497a3aa9dfb5c5"
@@ -7887,6 +7892,11 @@ minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5:
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
   integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
 
+minimist@^1.2.6:
+  version "1.2.6"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
+  integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
+
 minipass-collect@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/minipass-collect/-/minipass-collect-1.0.2.tgz#22b813bf745dc6edba2576b940022ad6edc8c617"
@@ -10705,6 +10715,11 @@ strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   dependencies:
     ansi-regex "^5.0.1"
 
+strip-bom@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3"
+  integrity sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=
+
 strip-eof@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf"
@@ -11177,6 +11192,16 @@ ts-pnp@^1.1.6:
   resolved "https://registry.yarnpkg.com/ts-pnp/-/ts-pnp-1.2.0.tgz#a500ad084b0798f1c3071af391e65912c86bca92"
   integrity sha512-csd+vJOb/gkzvcCHgTGSChYpy5f1/XKNsmvBGO4JXS+z1v2HobugDz4s1IeFXM3wZB44uczs+eazB5Q/ccdhQw==
 
+tsconfig-paths@^3.14.1:
+  version "3.14.1"
+  resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz#ba0734599e8ea36c862798e920bcf163277b137a"
+  integrity sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ==
+  dependencies:
+    "@types/json5" "^0.0.29"
+    json5 "^1.0.1"
+    minimist "^1.2.6"
+    strip-bom "^3.0.0"
+
 tslib@^1.8.0, tslib@^1.8.1, tslib@^1.9.0:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"