import { knex, Knex } from 'knex'; import { getRandomString } from '../../frappe/utils'; import { CannotCommitError, DatabaseError, DuplicateEntryError, LinkValidationError, NotFoundError, ValueError } from '../../frappe/utils/errors'; import { Field, FieldTypeEnum, RawValue, SchemaMap, TargetField } from '../../schemas/types'; import { sqliteTypeMap } from '../common'; import { ColumnDiff, FieldValueMap, GetAllOptions, GetQueryBuilderOptions, QueryFilter } from './types'; /** * Db Core Call Sequence * * 1. Init core: `const db = new DatabaseCore(dbPath)`. * 2. Connect db: `db.connect()`. This will allow for raw queries to be executed. * 3. Set schemas: `bb.setSchemaMap(schemaMap)`. This will allow for ORM functions to be executed. * 4. Migrate: `await db.migrate()`. This will create absent tables and update the tables' shape. * 5. ORM function execution: `db.get(...)`, `db.insert(...)`, etc. * 6. Close connection: `await db.close()`. */ export default class DatabaseCore { knex?: Knex; typeMap = sqliteTypeMap; dbPath: string; schemaMap: SchemaMap = {}; connectionParams: Knex.Config; constructor(dbPath?: string) { this.dbPath = dbPath ?? ':memory:'; this.connectionParams = { client: 'sqlite3', connection: { filename: this.dbPath, }, pool: { afterCreate(conn: { run: (query: string) => void }, done: () => void) { conn.run('PRAGMA foreign_keys=ON'); done(); }, }, useNullAsDefault: true, asyncStackTraces: process.env.NODE_ENV === 'development', }; } setSchemaMap(schemaMap: SchemaMap) { this.schemaMap = schemaMap; } connect() { this.knex = knex(this.connectionParams); this.knex.on('query-error', (error) => { error.type = this.#getError(error); }); } async close() { await this.knex!.destroy(); } async commit() { try { await this.knex!.raw('commit'); } catch (err) { const type = this.#getError(err as Error); if (type !== CannotCommitError) { throw err; } } } async migrate() { for (const schemaName in this.schemaMap) { const schema = this.schemaMap[schemaName]; if (schema.isSingle) { continue; } if (await this.#tableExists(schemaName)) { await this.#alterTable(schemaName); } else { await this.#createTable(schemaName); } } await this.commit(); await this.#initializeSingles(); } async exists(schemaName: string, name?: string): Promise { const schema = this.schemaMap[schemaName]; if (schema.isSingle) { return this.#singleExists(schemaName); } let row = []; try { const qb = this.knex!(schemaName); if (name !== undefined) { qb.where({ name }); } row = await qb.limit(1); } catch (err) { if (this.#getError(err as Error) !== NotFoundError) { throw err; } } return row.length > 0; } async insert(schemaName: string, fieldValueMap: FieldValueMap) { // insert parent if (this.schemaMap[schemaName].isSingle) { await this.#updateSingleValues(schemaName, fieldValueMap); } else { await this.#insertOne(schemaName, fieldValueMap); } // insert children await this.#insertOrUpdateChildren(schemaName, fieldValueMap, false); return fieldValueMap; } async get( schemaName: string, name: string = '', fields: string | string[] = '*' ): Promise { const isSingle = this.schemaMap[schemaName].isSingle; if (!isSingle && !name) { throw new ValueError('name is mandatory'); } /** * If schema is single return all the values * of the single type schema, in this case field * is ignored. */ let fieldValueMap: FieldValueMap = {}; if (isSingle) { fieldValueMap = await this.#getSingle(schemaName); fieldValueMap.name = schemaName; return fieldValueMap; } if (fields !== '*' && typeof fields === 'string') { fields = [fields]; } /** * Separate table fields and non table fields */ const allTableFields = this.#getTableFields(schemaName); const allTableFieldNames = allTableFields.map((f) => f.fieldname); let tableFields: TargetField[] = []; let nonTableFieldNames: string[] = []; if (Array.isArray(fields)) { tableFields = tableFields.filter((f) => fields.includes(f.fieldname)); nonTableFieldNames = fields.filter( (f) => !allTableFieldNames.includes(f) ); } else if (fields === '*') { tableFields = allTableFields; } /** * If schema is not single then return specific fields * if child fields are selected, all child fields are returned. */ if (nonTableFieldNames.length) { fieldValueMap = (await this.#getOne(schemaName, name, fields)) ?? {}; } if (tableFields.length) { await this.#loadChildren(fieldValueMap, tableFields); } return fieldValueMap; } async getAll({ schemaName, fields, filters, start, limit, groupBy, orderBy = 'creation', order = 'desc', }: GetAllOptions): Promise { const schema = this.schemaMap[schemaName]; if (!fields) { fields = ['name', ...(schema.keywordFields ?? [])]; } if (typeof fields === 'string') { fields = [fields]; } return (await this.#getQueryBuilder(schemaName, fields, filters ?? {}, { offset: start, limit, groupBy, orderBy, order, })) as FieldValueMap[]; } async getSingleValues( ...fieldnames: { fieldname: string; parent?: string }[] ): Promise<{ fieldname: string; parent: string; value: RawValue }[]> { fieldnames = fieldnames.map((fieldname) => { if (typeof fieldname === 'string') { return { fieldname }; } return fieldname; }); let builder = this.knex!('SingleValue'); builder = builder.where(fieldnames[0]); fieldnames.slice(1).forEach(({ fieldname, parent }) => { if (typeof parent === 'undefined') { builder = builder.orWhere({ fieldname }); } else { builder = builder.orWhere({ fieldname, parent }); } }); let values: { fieldname: string; parent: string; value: RawValue }[] = []; try { values = await builder.select('fieldname', 'value', 'parent'); } catch (err) { if (this.#getError(err as Error) === NotFoundError) { return []; } throw err; } return values; } async rename(schemaName: string, oldName: string, newName: string) { await this.knex!(schemaName) .update({ name: newName }) .where('name', oldName); await this.commit(); } async update(schemaName: string, fieldValueMap: FieldValueMap) { // update parent if (this.schemaMap[schemaName].isSingle) { await this.#updateSingleValues(schemaName, fieldValueMap); } else { await this.#updateOne(schemaName, fieldValueMap); } // insert or update children await this.#insertOrUpdateChildren(schemaName, fieldValueMap, true); } async delete(schemaName: string, name: string) { await this.#deleteOne(schemaName, name); // delete children const tableFields = this.#getTableFields(schemaName); for (const field of tableFields) { await this.#deleteChildren(field.target, name); } } async #tableExists(schemaName: string) { return await this.knex!.schema.hasTable(schemaName); } async #singleExists(singleSchemaName: string) { const res = await this.knex!('SingleValue') .count('parent as count') .where('parent', singleSchemaName) .first(); return (res?.count ?? 0) > 0; } async #removeColumns(schemaName: string, targetColumns: string[]) { // TODO: Implement this for sqlite } async #deleteSingleValues(singleSchemaName: string) { return await this.knex!('SingleValue') .where('parent', singleSchemaName) .delete(); } #getError(err: Error) { let errorType = DatabaseError; if (err.message.includes('SQLITE_ERROR: no such table')) { errorType = NotFoundError; } if (err.message.includes('FOREIGN KEY')) { errorType = LinkValidationError; } if (err.message.includes('SQLITE_ERROR: cannot commit')) { errorType = CannotCommitError; } if (err.message.includes('SQLITE_CONSTRAINT: UNIQUE constraint failed:')) { errorType = DuplicateEntryError; } return errorType; } async prestigeTheTable(schemaName: string, tableRows: FieldValueMap[]) { const max = 200; // Alter table hacx for sqlite in case of schema change. const tempName = `__${schemaName}`; await this.knex!.schema.dropTableIfExists(tempName); await this.knex!.raw('PRAGMA foreign_keys=OFF'); await this.#createTable(schemaName, tempName); if (tableRows.length > 200) { const fi = Math.floor(tableRows.length / max); for (let i = 0; i <= fi; i++) { const rowSlice = tableRows.slice(i * max, i + 1 * max); if (rowSlice.length === 0) { break; } await this.knex!.batchInsert(tempName, rowSlice); } } else { await this.knex!.batchInsert(tempName, tableRows); } await this.knex!.schema.dropTable(schemaName); await this.knex!.schema.renameTable(tempName, schemaName); await this.knex!.raw('PRAGMA foreign_keys=ON'); } async #getTableColumns(schemaName: string): Promise { const info: FieldValueMap[] = await this.knex!.raw( `PRAGMA table_info(${schemaName})` ); return info.map((d) => d.name as string); } async #getForeignKeys(schemaName: string): Promise { const foreignKeyList: FieldValueMap[] = await this.knex!.raw( `PRAGMA foreign_key_list(${schemaName})` ); return foreignKeyList.map((d) => d.from as string); } #getQueryBuilder( schemaName: string, fields: string[], filters: QueryFilter, options: GetQueryBuilderOptions ): Knex.QueryBuilder { const builder = this.knex!.select(fields).from(schemaName); this.#applyFiltersToBuilder(builder, filters); if (options.orderBy) { builder.orderBy(options.orderBy, options.order); } if (options.groupBy) { builder.groupBy(options.groupBy); } if (options.offset) { builder.offset(options.offset); } if (options.limit) { builder.limit(options.limit); } return builder; } #applyFiltersToBuilder(builder: Knex.QueryBuilder, filters: QueryFilter) { // {"status": "Open"} => `status = "Open"` // {"status": "Open", "name": ["like", "apple%"]} // => `status="Open" and name like "apple%" // {"date": [">=", "2017-09-09", "<=", "2017-11-01"]} // => `date >= 2017-09-09 and date <= 2017-11-01` const filtersArray = []; for (const field in filters) { const value = filters[field]; let operator = '='; let comparisonValue = value; if (Array.isArray(value)) { operator = value[0]; comparisonValue = value[1]; operator = operator.toLowerCase(); if (operator === 'includes') { operator = 'like'; } if (operator === 'like' && !comparisonValue.includes('%')) { comparisonValue = `%${comparisonValue}%`; } } filtersArray.push([field, operator, comparisonValue]); if (Array.isArray(value) && value.length > 2) { // multiple conditions const operator = value[2]; const comparisonValue = value[3]; filtersArray.push([field, operator, comparisonValue]); } } filtersArray.map((filter) => { const [field, operator, comparisonValue] = filter; if (operator === '=') { builder.where(field as string, comparisonValue); } else { builder.where(field as string, operator as string, comparisonValue); } }); } async #getColumnDiff(schemaName: string): Promise { const tableColumns = await this.#getTableColumns(schemaName); const validFields = this.schemaMap[schemaName].fields; const diff: ColumnDiff = { added: [], removed: [] }; for (const field of validFields) { const hasDbType = this.typeMap.hasOwnProperty(field.fieldtype); if (!tableColumns.includes(field.fieldname) && hasDbType) { diff.added.push(field); } } const validFieldNames = validFields.map((field) => field.fieldname); for (const column of tableColumns) { if (!validFieldNames.includes(column)) { diff.removed.push(column); } } return diff; } async #getNewForeignKeys(schemaName: string): Promise { const foreignKeys = await this.#getForeignKeys(schemaName); const newForeignKeys: Field[] = []; const schema = this.schemaMap[schemaName]; for (const field of schema.fields) { if ( field.fieldtype === 'Link' && !foreignKeys.includes(field.fieldname) ) { newForeignKeys.push(field); } } return newForeignKeys; } #buildColumnForTable(table: Knex.AlterTableBuilder, field: Field) { if (field.fieldtype === FieldTypeEnum.Table) { // In case columnType is "Table" // childTable links are handled using the childTable's "parent" field return; } const columnType = this.typeMap[field.fieldtype]; if (!columnType) { return; } const column = table[columnType]( field.fieldname ) as Knex.SqlLiteColumnBuilder; // primary key if (field.fieldname === 'name') { column.primary(); } // iefault value if (field.default !== undefined) { column.defaultTo(field.default); } // required if (field.required) { column.notNullable(); } // link if ( field.fieldtype === FieldTypeEnum.Link && (field as TargetField).target ) { const targetSchemaName = (field as TargetField).target as string; const schema = this.schemaMap[targetSchemaName]; table .foreign(field.fieldname) .references('name') .inTable(schema.name) .onUpdate('CASCADE') .onDelete('RESTRICT'); } } async #alterTable(schemaName: string) { // get columns const diff: ColumnDiff = await this.#getColumnDiff(schemaName); const newForeignKeys: Field[] = await this.#getNewForeignKeys(schemaName); return this.knex!.schema.table(schemaName, (table) => { if (diff.added.length) { for (const field of diff.added) { this.#buildColumnForTable(table, field); } } if (diff.removed.length) { this.#removeColumns(schemaName, diff.removed); } }).then(() => { if (newForeignKeys.length) { return this.#addForeignKeys(schemaName, newForeignKeys); } }); } async #createTable(schemaName: string, tableName?: string) { tableName ??= schemaName; const fields = this.schemaMap[schemaName].fields; return await this.#runCreateTableQuery(tableName, fields); } #runCreateTableQuery(schemaName: string, fields: Field[]) { return this.knex!.schema.createTable(schemaName, (table) => { for (const field of fields) { this.#buildColumnForTable(table, field); } }); } async #getNonExtantSingleValues(singleSchemaName: string) { const existingFields = ( await this.knex!('SingleValue') .where({ parent: singleSchemaName }) .select('fieldname') ).map(({ fieldname }) => fieldname); return this.schemaMap[singleSchemaName].fields .map(({ fieldname, default: value }) => ({ fieldname, value: value as RawValue | undefined, })) .filter( ({ fieldname, value }) => !existingFields.includes(fieldname) && value !== undefined ); } async #deleteOne(schemaName: string, name: string) { return this.knex!(schemaName).where('name', name).delete(); } #deleteChildren(schemaName: string, parentName: string) { return this.knex!(schemaName).where('parent', parentName).delete(); } #runDeleteOtherChildren( field: TargetField, parentName: string, added: string[] ) { // delete other children return this.knex!(field.target) .where('parent', parentName) .andWhere('name', 'not in', added) .delete(); } #prepareChild( parentSchemaName: string, parentName: string, child: FieldValueMap, field: Field, idx: number ) { if (!child.name) { child.name = getRandomString(); } child.parentName = parentName; child.parentSchemaName = parentSchemaName; child.parentFieldname = field.fieldname; child.idx = idx; } async #addForeignKeys(schemaName: string, newForeignKeys: Field[]) { await this.knex!.raw('PRAGMA foreign_keys=OFF'); await this.knex!.raw('BEGIN TRANSACTION'); const tempName = 'TEMP' + schemaName; // create temp table await this.#createTable(schemaName, tempName); try { // copy from old to new table await this.knex!(tempName).insert(this.knex!.select().from(schemaName)); } catch (err) { await this.knex!.raw('ROLLBACK'); await this.knex!.raw('PRAGMA foreign_keys=ON'); const rows = await this.knex!.select().from(schemaName); await this.prestigeTheTable(schemaName, rows); return; } // drop old table await this.knex!.schema.dropTable(schemaName); // rename new table await this.knex!.schema.renameTable(tempName, schemaName); await this.knex!.raw('COMMIT'); await this.knex!.raw('PRAGMA foreign_keys=ON'); } async #loadChildren( fieldValueMap: FieldValueMap, tableFields: TargetField[] ) { for (const field of tableFields) { fieldValueMap[field.fieldname] = await this.getAll({ schemaName: field.target, fields: ['*'], filters: { parent: fieldValueMap.name as string }, orderBy: 'idx', order: 'asc', }); } } async #getOne( schemaName: string, name: string, fields: string | string[] = '*' ) { const fieldValueMap: FieldValueMap = await this.knex!.select(fields) .from(schemaName) .where('name', name) .first(); return fieldValueMap; } async #getSingle(schemaName: string): Promise { const values = await this.getAll({ schemaName: 'SingleValue', fields: ['fieldname', 'value'], filters: { parent: schemaName }, orderBy: 'fieldname', order: 'asc', }); const fieldValueMap: FieldValueMap = {}; for (const row of values) { fieldValueMap[row.fieldname as string] = row.value as RawValue; } return fieldValueMap; } #insertOne(schemaName: string, fieldValueMap: FieldValueMap) { const fields = this.schemaMap[schemaName].fields; if (!fieldValueMap.name) { fieldValueMap.name = getRandomString(); } const validMap: FieldValueMap = {}; for (const { fieldname, fieldtype } of fields) { if (fieldtype === FieldTypeEnum.Table) { continue; } validMap[fieldname] = fieldValueMap[fieldname]; } return this.knex!(schemaName).insert(fieldValueMap); } async #updateSingleValues( singleSchemaName: string, fieldValueMap: FieldValueMap ) { const fields = this.schemaMap[singleSchemaName].fields; await this.#deleteSingleValues(singleSchemaName); for (const field of fields) { const value = fieldValueMap[field.fieldname] as RawValue | undefined; if (value === undefined) { continue; } await this.#updateSingleValue(singleSchemaName, field.fieldname, value); } } async #updateSingleValue( singleSchemaName: string, fieldname: string, value: RawValue ) { return await this.knex!('SingleValue') .where({ parent: singleSchemaName, fieldname, }) .update({ value }); } async #initializeSingles() { const singleSchemaNames = Object.keys(this.schemaMap).filter( (n) => this.schemaMap[n].isSingle ); for (const schemaName of singleSchemaNames) { if (await this.#singleExists(schemaName)) { await this.#updateNonExtantSingleValues(schemaName); continue; } const fields = this.schemaMap[schemaName].fields; if (fields.every((f) => f.default === undefined)) { continue; } const defaultValues: FieldValueMap = fields.reduce((acc, f) => { if (f.default !== undefined) { acc[f.fieldname] = f.default; } return acc; }, {} as FieldValueMap); await this.#updateSingleValues(schemaName, defaultValues); } } async #updateNonExtantSingleValues(schemaName: string) { const singleValues = await this.#getNonExtantSingleValues(schemaName); for (const sv of singleValues) { await this.#updateSingleValue(schemaName, sv.fieldname, sv.value!); } } async #updateOne(schemaName: string, fieldValueMap: FieldValueMap) { const updateMap = { ...fieldValueMap }; delete updateMap.name; return await this.knex!(schemaName) .where('name', fieldValueMap.name as string) .update(updateMap); } async #insertOrUpdateChildren( schemaName: string, fieldValueMap: FieldValueMap, isUpdate: boolean ) { const tableFields = this.#getTableFields(schemaName); const parentName = fieldValueMap.name as string; for (const field of tableFields) { const added: string[] = []; const tableFieldValue = (fieldValueMap[field.fieldname] ?? []) as FieldValueMap[]; for (const child of tableFieldValue) { this.#prepareChild(schemaName, parentName, child, field, added.length); if ( isUpdate && (await this.exists(field.target, child.name as string)) ) { await this.#updateOne(field.target, child); } else { await this.#insertOne(field.target, child); } added.push(child.name as string); } if (isUpdate) { await this.#runDeleteOtherChildren(field, parentName, added); } } } #getTableFields(schemaName: string): TargetField[] { return this.schemaMap[schemaName].fields.filter( (f) => f.fieldtype === FieldTypeEnum.Table ) as TargetField[]; } }