test: add more dbcore tests

- fix bugs in core.ts

backend/database/core.ts

@@ -6,14 +6,14 @@ import {
   DuplicateEntryError,
   LinkValidationError,
   NotFoundError,
-  ValueError
+  ValueError,
 } from '../../frappe/utils/errors';
 import {
   Field,
   FieldTypeEnum,
   RawValue,
   SchemaMap,
-  TargetField
+  TargetField,
 } from '../../schemas/types';
 import { getDefaultMetaFieldValueMap, sqliteTypeMap, SYSTEM } from '../common';
 import {
@@ -21,11 +21,15 @@ import {
   FieldValueMap,
   GetAllOptions,
   GetQueryBuilderOptions,
-  QueryFilter
+  QueryFilter,
 } from './types';
 
 /**
- * Db Core Call Sequence
+ * # DatabaseCore
+ * This is the ORM, the DatabaseCore interface (function signatures) should be
+ * replicated by the frontend demuxes and all the backend muxes.
+ *
+ * ## Db Core Call Sequence
  *
  * 1. Init core: `const db = new DatabaseCore(dbPath)`.
  * 2. Connect db: `db.connect()`. This will allow for raw queries to be executed.
@@ -33,6 +37,10 @@ import {
  * 4. Migrate: `await db.migrate()`. This will create absent tables and update the tables' shape.
  * 5. ORM function execution: `db.get(...)`, `db.insert(...)`, etc.
  * 6. Close connection: `await db.close()`.
+ *
+ * Note: Meta values: created, modified, createdBy, modifiedBy are set by DatabaseCore
+ * only for schemas that are SingleValue. Else they have to be passed by the caller in
+ * the `fieldValueMap`.
  */
 
 export default class DatabaseCore {
@@ -141,10 +149,10 @@ export default class DatabaseCore {
   async get(
     schemaName: string,
     name: string = '',
-    fields: string | string[] = '*'
+    fields?: string | string[]
   ): Promise<FieldValueMap> {
-    const isSingle = this.schemaMap[schemaName].isSingle;
+    const schema = this.schemaMap[schemaName];
-    if (!isSingle && !name) {
+    if (!schema.isSingle && !name) {
       throw new ValueError('name is mandatory');
     }
 
@@ -154,38 +162,37 @@ export default class DatabaseCore {
      * is ignored.
      */
     let fieldValueMap: FieldValueMap = {};
-    if (isSingle) {
+    if (schema.isSingle) {
       return await this.#getSingle(schemaName);
     }
 
-    if (fields !== '*' && typeof fields === 'string') {
+    if (typeof fields === 'string') {
       fields = [fields];
     }
 
+    if (fields === undefined) {
+      fields = schema.fields.map((f) => f.fieldname);
+    }
+
     /**
      * Separate table fields and non table fields
      */
-    const allTableFields = this.#getTableFields(schemaName);
+    const allTableFields: TargetField[] = this.#getTableFields(schemaName);
-    const allTableFieldNames = allTableFields.map((f) => f.fieldname);
+    const allTableFieldNames: string[] = allTableFields.map((f) => f.fieldname);
-
+    const tableFields: TargetField[] = allTableFields.filter((f) =>
-    let tableFields: TargetField[] = [];
+      fields!.includes(f.fieldname)
-    let nonTableFieldNames: string[] = [];
+    );
-
+    const nonTableFieldNames: string[] = fields.filter(
-    if (Array.isArray(fields)) {
+      (f) => !allTableFieldNames.includes(f)
-      tableFields = tableFields.filter((f) => fields.includes(f.fieldname));
+    );
-      nonTableFieldNames = fields.filter(
-        (f) => !allTableFieldNames.includes(f)
-      );
-    } else if (fields === '*') {
-      tableFields = allTableFields;
-    }
 
     /**
      * If schema is not single then return specific fields
      * if child fields are selected, all child fields are returned.
      */
     if (nonTableFieldNames.length) {
-      fieldValueMap = (await this.#getOne(schemaName, name, fields)) ?? {};
+      fieldValueMap =
+        (await this.#getOne(schemaName, name, nonTableFieldNames)) ?? {};
     }
 
     if (tableFields.length) {
@@ -194,32 +201,34 @@ export default class DatabaseCore {
     return fieldValueMap;
   }
 
-  async getAll({
+  async getAll(
-    schemaName,
+    schemaName: string,
-    fields,
+    options: GetAllOptions = {}
-    filters,
+  ): Promise<FieldValueMap[]> {
-    start,
-    limit,
-    groupBy,
-    orderBy = 'created',
-    order = 'desc',
-  }: GetAllOptions): Promise<FieldValueMap[]> {
     const schema = this.schemaMap[schemaName];
-    if (!fields) {
+    const hasCreated = !!schema.fields.find((f) => f.fieldname === 'created');
-      fields = ['name', ...(schema.keywordFields ?? [])];
+    const {
-    }
+      fields = ['name', ...(schema.keywordFields ?? [])],
-
+      filters,
-    if (typeof fields === 'string') {
+      offset,
-      fields = [fields];
-    }
-
-    return (await this.#getQueryBuilder(schemaName, fields, filters ?? {}, {
-      offset: start,
       limit,
       groupBy,
-      orderBy,
+      orderBy = hasCreated ? 'created' : undefined,
-      order,
+      order = 'desc',
-    })) as FieldValueMap[];
+    } = options;
+
+    return (await this.#getQueryBuilder(
+      schemaName,
+      typeof fields === 'string' ? [fields] : fields,
+      filters ?? {},
+      {
+        offset,
+        limit,
+        groupBy,
+        orderBy,
+        order,
+      }
+    )) as FieldValueMap[];
   }
 
   async getSingleValues(
@@ -258,6 +267,11 @@ export default class DatabaseCore {
   }
 
   async rename(schemaName: string, oldName: string, newName: string) {
+    /**
+     * Rename is expensive mostly won't allow it.
+     * TODO: rename all links
+     * TODO: rename in childtables
+     */
     await this.knex!(schemaName)
       .update({ name: newName })
       .where('name', oldName);
@@ -621,7 +635,7 @@ export default class DatabaseCore {
     if (!child.name) {
       child.name = getRandomString();
     }
-    child.parentName = parentName;
+    child.parent = parentName;
     child.parentSchemaName = parentSchemaName;
     child.parentFieldname = field.fieldname;
     child.idx = idx;
@@ -663,8 +677,7 @@ export default class DatabaseCore {
     tableFields: TargetField[]
   ) {
     for (const field of tableFields) {
-      fieldValueMap[field.fieldname] = await this.getAll({
+      fieldValueMap[field.fieldname] = await this.getAll(field.target, {
-        schemaName: field.target,
         fields: ['*'],
         filters: { parent: fieldValueMap.name as string },
         orderBy: 'idx',
@@ -673,11 +686,7 @@ export default class DatabaseCore {
     }
   }
 
-  async #getOne(
+  async #getOne(schemaName: string, name: string, fields: string[]) {
-    schemaName: string,
-    name: string,
-    fields: string | string[] = '*'
-  ) {
     const fieldValueMap: FieldValueMap = await this.knex!.select(fields)
       .from(schemaName)
       .where('name', name)
@@ -686,8 +695,7 @@ export default class DatabaseCore {
   }
 
   async #getSingle(schemaName: string): Promise<FieldValueMap> {
-    const values = await this.getAll({
+    const values = await this.getAll('SingleValue', {
-      schemaName: 'SingleValue',
       fields: ['fieldname', 'value'],
       filters: { parent: schemaName },
       orderBy: 'fieldname',
@@ -698,21 +706,21 @@ export default class DatabaseCore {
   }
 
   #insertOne(schemaName: string, fieldValueMap: FieldValueMap) {
-    const fields = this.schemaMap[schemaName].fields;
     if (!fieldValueMap.name) {
       fieldValueMap.name = getRandomString();
     }
 
-    const validMap: FieldValueMap = {};
+    // Non Table Fields
-    for (const { fieldname, fieldtype } of fields) {
+    const fields = this.schemaMap[schemaName].fields.filter(
-      if (fieldtype === FieldTypeEnum.Table) {
+      (f) => f.fieldtype !== FieldTypeEnum.Table
-        continue;
+    );
-      }
 
+    const validMap: FieldValueMap = {};
+    for (const { fieldname } of fields) {
       validMap[fieldname] = fieldValueMap[fieldname];
     }
 
-    return this.knex!(schemaName).insert(fieldValueMap);
+    return this.knex!(schemaName).insert(validMap);
   }
 
   async #updateSingleValues(
@@ -808,6 +816,18 @@ export default class DatabaseCore {
   async #updateOne(schemaName: string, fieldValueMap: FieldValueMap) {
     const updateMap = { ...fieldValueMap };
     delete updateMap.name;
+    const schema = this.schemaMap[schemaName];
+    for (const { fieldname, fieldtype } of schema.fields) {
+      if (fieldtype !== FieldTypeEnum.Table) {
+        continue;
+      }
+
+      delete updateMap[fieldname];
+    }
+
+    if (Object.keys(updateMap).length === 0) {
+      return;
+    }
 
     return await this.knex!(schemaName)
       .where('name', fieldValueMap.name as string)

backend/database/tests/helpers.ts

@@ -1,3 +1,4 @@
+import assert from 'assert';
 import { cloneDeep } from 'lodash';
 import { SchemaMap, SchemaStub, SchemaStubMap } from 'schemas/types';
 import {
@@ -15,7 +16,6 @@ const Customer = {
       fieldname: 'name',
       label: 'Name',
       fieldtype: 'Data',
-      default: 'John Thoe',
       required: true,
     },
     {
@@ -50,13 +50,13 @@ const SalesInvoiceItem = {
     {
       fieldname: 'rate',
       label: 'Rate',
-      fieldtype: 'Currency',
+      fieldtype: 'Float',
       required: true,
     },
     {
       fieldname: 'amount',
       label: 'Amount',
-      fieldtype: 'Currency',
+      fieldtype: 'Float',
       computed: true,
       readOnly: true,
     },
@@ -170,4 +170,39 @@ export function getBaseMeta() {
     created: new Date().toISOString(),
     modified: new Date().toISOString(),
   };
-}
+}
+
+export async function assertThrows(
+  func: () => Promise<unknown>,
+  message?: string
+) {
+  let threw = true;
+  try {
+    await func();
+    threw = false;
+  } catch {
+  } finally {
+    if (!threw) {
+      throw new assert.AssertionError({
+        message: `Missing expected exception: ${message}`,
+      });
+    }
+  }
+}
+
+export async function assertDoesNotThrow(
+  func: () => Promise<unknown>,
+  message?: string
+) {
+  try {
+    await func();
+  } catch (err) {
+    throw new assert.AssertionError({
+      message: `Missing expected exception: ${message} Error: ${
+        (err as Error).message
+      }`,
+    });
+  }
+}
+
+export type BaseMetaKey = 'created' | 'modified' | 'createdBy' | 'modifiedBy';

backend/database/tests/testCore.spec.ts

@@ -2,13 +2,29 @@ import * as assert from 'assert';
 import 'mocha';
 import { getMapFromList } from 'schemas/helpers';
 import { FieldTypeEnum, RawValue } from 'schemas/types';
-import { getValueMapFromList } from 'utils';
+import { getValueMapFromList, sleep } from 'utils';
-import { sqliteTypeMap } from '../../common';
+import { getDefaultMetaFieldValueMap, sqliteTypeMap } from '../../common';
 import DatabaseCore from '../core';
-import { SqliteTableInfo } from '../types';
+import { FieldValueMap, SqliteTableInfo } from '../types';
-import { getBuiltTestSchemaMap } from './helpers';
+import {
+  assertDoesNotThrow,
+  assertThrows,
+  BaseMetaKey,
+  getBuiltTestSchemaMap,
+} from './helpers';
 
-describe('DatabaseCore: Connect Migrate Close', async function () {
+/**
+ * Note: these tests have a strange structure where multiple tests are
+ * inside a `specify`, this is cause `describe` doesn't support `async` or waiting
+ * on promises.
+ *
+ * Due to this `async` db operations need to be handled in `specify`. And `specify`
+ * can't be nested in the `describe` can, hence the strange structure.
+ *
+ * This also implies that assert calls should have discriptive
+ */
+
+describe('DatabaseCore: Connect Migrate Close', function () {
   const db = new DatabaseCore();
   specify('dbPath', function () {
     assert.strictEqual(db.dbPath, ':memory:');
@@ -98,7 +114,7 @@ describe('DatabaseCore: Migrate and Check Db', function () {
           assert.strictEqual(
             !!column.notnull,
             field.required,
-            `${schemaName}.${column.name}:: notnull check: ${column.notnull}, ${field.required}`
+            `${schemaName}.${column.name}:: iotnull iheck: ${column.notnull}, ${field.required}`
           );
         } else {
           assert.strictEqual(
@@ -189,7 +205,7 @@ describe('DatabaseCore: CRUD', function () {
     localeRow = rows.find((r) => r.fieldname === 'locale');
 
     assert.notStrictEqual(localeEntryName, undefined, 'localeEntryName');
-    assert.strictEqual(rows.length, 2, 'row length');
+    assert.strictEqual(rows.length, 2, 'rows length insert');
     assert.strictEqual(
       localeRow?.name as string,
       localeEntryName,
@@ -215,7 +231,7 @@ describe('DatabaseCore: CRUD', function () {
     localeRow = rows.find((r) => r.fieldname === 'locale');
 
     assert.notStrictEqual(localeEntryName, undefined, 'localeEntryName');
-    assert.strictEqual(rows.length, 2, 'row length');
+    assert.strictEqual(rows.length, 2, 'rows length update');
     assert.strictEqual(
       localeRow?.name as string,
       localeEntryName,
@@ -276,5 +292,303 @@ describe('DatabaseCore: CRUD', function () {
     }
   });
 
-  specify('CRUD simple nondependent schema', async function () {});
+  specify('CRUD nondependent schema', async function () {
+    const schemaName = 'Customer';
+    let rows = await db.knex!(schemaName);
+    assert.strictEqual(rows.length, 0, 'rows length before insertion');
+
+    /**
+     * Insert
+     */
+    const metaValues = getDefaultMetaFieldValueMap();
+    const name = 'John Thoe';
+
+    await assertThrows(
+      async () => await db.insert(schemaName, { name }),
+      'insert() did not throw without meta values'
+    );
+
+    const updateMap = Object.assign({}, metaValues, { name });
+    await db.insert(schemaName, updateMap);
+    rows = await db.knex!(schemaName);
+    let firstRow = rows?.[0];
+    assert.strictEqual(rows.length, 1, `rows length insert ${rows.length}`);
+    assert.strictEqual(
+      firstRow.name,
+      name,
+      `name check ${firstRow.name}, ${name}`
+    );
+    assert.strictEqual(firstRow.email, null, `email check ${firstRow.email}`);
+
+    for (const key in metaValues) {
+      assert.strictEqual(
+        firstRow[key],
+        metaValues[key as BaseMetaKey],
+        `${key} check`
+      );
+    }
+
+    /**
+     * Update
+     */
+    const email = 'john@thoe.com';
+    await sleep(1); // required for modified to change
+    await db.update(schemaName, {
+      name,
+      email,
+      modified: new Date().toISOString(),
+    });
+    rows = await db.knex!(schemaName);
+    firstRow = rows?.[0];
+    assert.strictEqual(rows.length, 1, `rows length update ${rows.length}`);
+    assert.strictEqual(
+      firstRow.name,
+      name,
+      `name check update ${firstRow.name}, ${name}`
+    );
+    assert.strictEqual(
+      firstRow.email,
+      email,
+      `email check update ${firstRow.email}`
+    );
+
+    for (const key in metaValues) {
+      const val = firstRow[key];
+      const expected = metaValues[key as BaseMetaKey];
+      if (key !== 'modified') {
+        assert.strictEqual(val, expected, `${key} check ${val}, ${expected}`);
+      } else {
+        assert.notStrictEqual(
+          val,
+          expected,
+          `${key} check ${val}, ${expected}`
+        );
+      }
+    }
+
+    /**
+     * Delete
+     */
+    await db.delete(schemaName, name);
+    rows = await db.knex!(schemaName);
+    assert.strictEqual(rows.length, 0, `rows length delete ${rows.length}`);
+
+    /**
+     * Get
+     */
+    let fvMap = await db.get(schemaName, name);
+    assert.strictEqual(
+      Object.keys(fvMap).length,
+      0,
+      `key count get ${JSON.stringify(fvMap)}`
+    );
+
+    /**
+     * > 1 entries
+     */
+
+    const cOne = { name: 'John Whoe', ...getDefaultMetaFieldValueMap() };
+    const cTwo = { name: 'Jane Whoe', ...getDefaultMetaFieldValueMap() };
+
+    // Insert
+    await db.insert(schemaName, cOne);
+    assert.strictEqual(
+      (await db.knex!(schemaName)).length,
+      1,
+      `rows length minsert`
+    );
+    await db.insert(schemaName, cTwo);
+    rows = await db.knex!(schemaName);
+    assert.strictEqual(rows.length, 2, `rows length minsert`);
+
+    const cs = [cOne, cTwo];
+    for (const i in cs) {
+      for (const k in cs[i]) {
+        const val = cs[i][k as BaseMetaKey];
+        assert.strictEqual(
+          rows?.[i]?.[k],
+          val,
+          `equality check ${i} ${k} ${val} ${rows?.[i]?.[k]}`
+        );
+      }
+    }
+
+    // Update
+    await db.update(schemaName, { name: cOne.name, email });
+    const cOneEmail = await db.get(schemaName, cOne.name, 'email');
+    assert.strictEqual(
+      cOneEmail.email,
+      email,
+      `mi update check one ${cOneEmail}`
+    );
+    const cTwoEmail = await db.get(schemaName, cTwo.name, 'email');
+    assert.strictEqual(
+      cOneEmail.email,
+      email,
+      `mi update check two ${cTwoEmail}`
+    );
+
+    // Rename
+    const newName = 'Johnny Whoe';
+    await db.rename(schemaName, cOne.name, newName);
+
+    fvMap = await db.get(schemaName, cOne.name);
+    assert.strictEqual(
+      Object.keys(fvMap).length,
+      0,
+      `mi rename check old ${JSON.stringify(fvMap)}`
+    );
+
+    fvMap = await db.get(schemaName, newName);
+    assert.strictEqual(
+      fvMap.email,
+      email,
+      `mi rename check new ${JSON.stringify(fvMap)}`
+    );
+
+    // Delete
+    await db.delete(schemaName, newName);
+    rows = await db.knex!(schemaName);
+    assert.strictEqual(rows.length, 1, `mi delete length ${rows.length}`);
+    assert.strictEqual(
+      rows[0].name,
+      cTwo.name,
+      `mi delete name ${rows[0].name}`
+    );
+  });
+
+  specify('CRUD dependent schema', async function () {
+    const Customer = 'Customer';
+    const SalesInvoice = 'SalesInvoice';
+    const SalesInvoiceItem = 'SalesInvoiceItem';
+
+    const customer: FieldValueMap = {
+      name: 'John Whoe',
+      email: 'john@whoe.com',
+      ...getDefaultMetaFieldValueMap(),
+    };
+
+    const invoice: FieldValueMap = {
+      name: 'SINV-1001',
+      date: '2022-01-21',
+      customer: customer.name,
+      account: 'Debtors',
+      submitted: false,
+      cancelled: false,
+      ...getDefaultMetaFieldValueMap(),
+    };
+
+    await assertThrows(
+      async () => await db.insert(SalesInvoice, invoice),
+      'foreign key constraint fail failed'
+    );
+
+    await assertDoesNotThrow(async () => {
+      await db.insert(Customer, customer);
+      await db.insert(SalesInvoice, invoice);
+    }, 'insertion failed');
+
+    await assertThrows(
+      async () => await db.delete(Customer, customer.name as string),
+      'foreign key constraint fail failed'
+    );
+
+    await assertDoesNotThrow(async () => {
+      await db.delete(SalesInvoice, invoice.name as string);
+      await db.delete(Customer, customer.name as string);
+    }, 'deletion failed');
+
+    await db.insert(Customer, customer);
+    await db.insert(SalesInvoice, invoice);
+
+    let fvMap = await db.get(SalesInvoice, invoice.name as string);
+    for (const key in invoice) {
+      let expected = invoice[key];
+      if (typeof expected === 'boolean') {
+        expected = +expected;
+      }
+
+      assert.strictEqual(
+        fvMap[key],
+        expected,
+        `equality check ${key}: ${fvMap[key]}, ${invoice[key]}`
+      );
+    }
+
+    assert.strictEqual(
+      (fvMap.items as unknown[])?.length,
+      0,
+      'empty items check'
+    );
+
+    const items: FieldValueMap[] = [
+      {
+        item: 'Bottle Caps',
+        quantity: 2,
+        rate: 100,
+        amount: 200,
+      },
+    ];
+
+    await assertThrows(
+      async () => await db.insert(SalesInvoice, { name: invoice.name, items }),
+      'invoice insertion with ct did not fail'
+    );
+    await assertDoesNotThrow(
+      async () => await db.update(SalesInvoice, { name: invoice.name, items }),
+      'ct insertion failed'
+    );
+
+    fvMap = await db.get(SalesInvoice, invoice.name as string);
+    const ct = fvMap.items as FieldValueMap[];
+    assert.strictEqual(ct.length, 1, `ct length ${ct.length}`);
+    assert.strictEqual(ct[0].parent, invoice.name, `ct parent ${ct[0].parent}`);
+    assert.strictEqual(
+      ct[0].parentFieldname,
+      'items',
+      `ct parentFieldname ${ct[0].parentFieldname}`
+    );
+    assert.strictEqual(
+      ct[0].parentSchemaName,
+      SalesInvoice,
+      `ct parentSchemaName ${ct[0].parentSchemaName}`
+    );
+    for (const key in items[0]) {
+      assert.strictEqual(
+        ct[0][key],
+        items[0][key],
+        `ct values ${key}: ${ct[0][key]}, ${items[0][key]}`
+      );
+    }
+
+    items.push({
+      item: 'Mentats',
+      quantity: 4,
+      rate: 200,
+      amount: 800,
+    });
+    await assertDoesNotThrow(
+      async () => await db.update(SalesInvoice, { name: invoice.name, items }),
+      'ct updation failed'
+    );
+
+    let rows = await db.getAll(SalesInvoiceItem, {
+      fields: ['item', 'quantity', 'rate', 'amount'],
+    });
+    assert.strictEqual(rows.length, 2, `ct length update ${rows.length}`);
+
+    for (const i in rows) {
+      for (const key in rows[i]) {
+        assert.strictEqual(
+          rows[i][key],
+          items[i][key],
+          `ct values ${i},${key}: ${rows[i][key]}`
+        );
+      }
+    }
+
+    await db.delete(SalesInvoice, invoice.name as string);
+    rows = await db.getAll(SalesInvoiceItem);
+    assert.strictEqual(rows.length, 0, `ct length delete ${rows.length}`);
+  });
 });

backend/database/types.ts

@@ -12,10 +12,9 @@ export interface GetQueryBuilderOptions {
 }
 
 export interface GetAllOptions {
-  schemaName: string;
   fields?: string[];
   filters?: QueryFilter;
-  start?: number;
+  offset?: number;
   limit?: number;
   groupBy?: string;
   orderBy?: string;

utils/index.ts

@@ -27,3 +27,7 @@ export function getValueMapFromList<T, K extends keyof T, V extends keyof T>(
 export function getRandomString(): string {
   return Math.random().toString(36).slice(2, 8);
 }
+
+export async function sleep(durationMilliseconds: number = 1000) {
+  return new Promise((r) => setTimeout(() => r(null), durationMilliseconds));
+}