books/frappe/backends/database.js

const frappe = require('frappe');
const Observable = require('frappe/utils/observable');
const CacheManager = require('frappe/utils/cacheManager');
const Knex = require('knex');

module.exports = class Database extends Observable {
  constructor() {
    super();
    this.initTypeMap();
    this.connectionParams = {};
    this.cache = new CacheManager();
  }

  connect() {
    this.knex = Knex(this.connectionParams);
    this.knex.on('query-error', (error) => {
      error.type = this.getError(error);
    });
    this.executePostDbConnect();
  }

  close() {
    //
  }

  async migrate() {
    for (let doctype in frappe.models) {
      // check if controller module
      let meta = frappe.getMeta(doctype);
      let baseDoctype = meta.getBaseDocType();
      if (!meta.isSingle) {
        if (await this.tableExists(baseDoctype)) {
          await this.alterTable(baseDoctype);
        } else {
          await this.createTable(baseDoctype);
        }
      }
    }
    await this.commit();
    await this.initializeSingles();
  }

  async initializeSingles() {
    let singleDoctypes = frappe
      .getModels((model) => model.isSingle)
      .map((model) => model.name);

    for (let doctype of singleDoctypes) {
      if (await this.singleExists(doctype)) {
        const singleValues = await this.getSingleFieldsToInsert(doctype);
        singleValues.forEach(({ fieldname, value }) => {
          let singleValue = frappe.newDoc({
            doctype: 'SingleValue',
            parent: doctype,
            fieldname,
            value,
          });
          singleValue.insert();
        });
        continue;
      }
      let meta = frappe.getMeta(doctype);
      if (meta.fields.every((df) => df.default == null)) {
        continue;
      }
      let defaultValues = meta.fields.reduce((doc, df) => {
        if (df.default != null) {
          doc[df.fieldname] = df.default;
        }
        return doc;
      }, {});
      await this.updateSingle(doctype, defaultValues);
    }
  }

  async singleExists(doctype) {
    let res = await this.knex('SingleValue')
      .count('parent as count')
      .where('parent', doctype)
      .first();
    return res.count > 0;
  }

  async getSingleFieldsToInsert(doctype) {
    const existingFields = (
      await frappe.db
        .knex('SingleValue')
        .where({ parent: doctype })
        .select('fieldname')
    ).map(({ fieldname }) => fieldname);

    return frappe
      .getMeta(doctype)
      .fields.map(({ fieldname, default: value }) => ({
        fieldname,
        value,
      }))
      .filter(
        ({ fieldname, value }) =>
          !existingFields.includes(fieldname) && value !== undefined
      );
  }

  tableExists(table) {
    return this.knex.schema.hasTable(table);
  }

  async createTable(doctype, tableName = null) {
    let fields = this.getValidFields(doctype);
    return await this.runCreateTableQuery(tableName || doctype, fields);
  }

  runCreateTableQuery(doctype, fields) {
    return this.knex.schema.createTable(doctype, (table) => {
      for (let field of fields) {
        this.buildColumnForTable(table, field);
      }
    });
  }

  async alterTable(doctype) {
    // get columns
    let diff = await this.getColumnDiff(doctype);
    let newForeignKeys = await this.getNewForeignKeys(doctype);

    return this.knex.schema
      .table(doctype, (table) => {
        if (diff.added.length) {
          for (let field of diff.added) {
            this.buildColumnForTable(table, field);
          }
        }

        if (diff.removed.length) {
          this.removeColumns(doctype, diff.removed);
        }
      })
      .then(() => {
        if (newForeignKeys.length) {
          return this.addForeignKeys(doctype, newForeignKeys);
        }
      });
  }

  buildColumnForTable(table, field) {
    let columnType = this.getColumnType(field);
    if (!columnType) {
      // In case columnType is "Table"
      // childTable links are handled using the childTable's "parent" field
      return;
    }

    let column = table[columnType](field.fieldname);

    // primary key
    if (field.fieldname === 'name') {
      column.primary();
    }

    // default value
    if (!!field.default && !(field.default instanceof Function)) {
      column.defaultTo(field.default);
    }

    // required
    if (
      (!!field.required && !(field.required instanceof Function)) ||
      field.fieldtype === 'Currency'
    ) {
      column.notNullable();
    }

    // link
    if (field.fieldtype === 'Link' && field.target) {
      let meta = frappe.getMeta(field.target);
      table
        .foreign(field.fieldname)
        .references('name')
        .inTable(meta.getBaseDocType())
        .onUpdate('CASCADE')
        .onDelete('RESTRICT');
    }
  }

  async getColumnDiff(doctype) {
    const tableColumns = await this.getTableColumns(doctype);
    const validFields = this.getValidFields(doctype);
    const diff = { added: [], removed: [] };

    for (let field of validFields) {
      if (
        !tableColumns.includes(field.fieldname) &&
        this.getColumnType(field)
      ) {
        diff.added.push(field);
      }
    }

    const validFieldNames = validFields.map((field) => field.fieldname);
    for (let column of tableColumns) {
      if (!validFieldNames.includes(column)) {
        diff.removed.push(column);
      }
    }

    return diff;
  }

  async removeColumns(doctype, removed) {
    for (let column of removed) {
      await this.runRemoveColumnQuery(doctype, column);
    }
  }

  async getNewForeignKeys(doctype) {
    let foreignKeys = await this.getForeignKeys(doctype);
    let newForeignKeys = [];
    let meta = frappe.getMeta(doctype);
    for (let field of meta.getValidFields({ withChildren: false })) {
      if (
        field.fieldtype === 'Link' &&
        !foreignKeys.includes(field.fieldname)
      ) {
        newForeignKeys.push(field);
      }
    }
    return newForeignKeys;
  }

  async addForeignKeys(doctype, newForeignKeys) {
    for (let field of newForeignKeys) {
      this.addForeignKey(doctype, field);
    }
  }

  async getForeignKeys(doctype, field) {
    return [];
  }

  async getTableColumns(doctype) {
    return [];
  }

  async get(doctype, name = null, fields = '*') {
    let meta = frappe.getMeta(doctype);
    let doc;
    if (meta.isSingle) {
      doc = await this.getSingle(doctype);
      doc.name = doctype;
    } else {
      if (!name) {
        throw new frappe.errors.ValueError('name is mandatory');
      }
      doc = await this.getOne(doctype, name, fields);
    }
    if (!doc) {
      return;
    }
    await this.loadChildren(doc, meta);
    return doc;
  }

  async loadChildren(doc, meta) {
    // load children
    let tableFields = meta.getTableFields();
    for (let field of tableFields) {
      doc[field.fieldname] = await this.getAll({
        doctype: field.childtype,
        fields: ['*'],
        filters: { parent: doc.name },
        orderBy: 'idx',
        order: 'asc',
      });
    }
  }

  async getSingle(doctype) {
    let values = await this.getAll({
      doctype: 'SingleValue',
      fields: ['fieldname', 'value'],
      filters: { parent: doctype },
      orderBy: 'fieldname',
      order: 'asc',
    });
    let doc = {};
    for (let row of values) {
      doc[row.fieldname] = row.value;
    }
    return doc;
  }

  /**
   * Get list of values from the singles table.
   * @param  {...string | Object} fieldnames list of fieldnames to get the values of
   * @returns {Array<Object>} array of {parent, value, fieldname}.
   * @example
   * Database.getSingleValues('internalPrecision');
   * // returns [{ fieldname: 'internalPrecision', parent: 'SystemSettings', value: '12' }]
   * @example
   * Database.getSingleValues({fieldname:'internalPrecision', parent: 'SystemSettings'});
   * // returns [{ fieldname: 'internalPrecision', parent: 'SystemSettings', value: '12' }]
   */
  async getSingleValues(...fieldnames) {
    fieldnames = fieldnames.map((fieldname) => {
      if (typeof fieldname === 'string') {
        return { fieldname };
      }
      return fieldname;
    });

    let builder = frappe.db.knex('SingleValue');
    builder = builder.where(fieldnames[0]);

    fieldnames.slice(1).forEach(({ fieldname, parent }) => {
      if (typeof parent === 'undefined') {
        builder = builder.orWhere({ fieldname });
      } else {
        builder = builder.orWhere({ fieldname, parent });
      }
    });

    let values = [];
    try {
      values = await builder.select('fieldname', 'value', 'parent');
    } catch (error) {
      if (error.message.includes('no such table')) {
        return [];
      }
      throw error;
    }

    return values.map((value) => {
      const fields = frappe.getMeta(value.parent).fields;
      return this.getDocFormattedDoc(fields, values);
    });
  }

  async getOne(doctype, name, fields = '*') {
    let meta = frappe.getMeta(doctype);
    let baseDoctype = meta.getBaseDocType();

    const doc = await this.knex
      .select(fields)
      .from(baseDoctype)
      .where('name', name)
      .first();

    if (!doc) {
      return doc;
    }

    return this.getDocFormattedDoc(meta.fields, doc);
  }

  getDocFormattedDoc(fields, doc) {
    // format for usage, not going into the db
    const docFields = Object.keys(doc);
    const filteredFields = fields.filter(({ fieldname }) =>
      docFields.includes(fieldname)
    );

    const formattedValues = filteredFields.reduce((d, field) => {
      const { fieldname } = field;
      d[fieldname] = this.getDocFormattedValues(field, doc[fieldname]);
      return d;
    }, {});

    return Object.assign(doc, formattedValues);
  }

  getDocFormattedValues(field, value) {
    // format for usage, not going into the db
    try {
      if (field.fieldtype === 'Currency') {
        return frappe.pesa(value);
      }
    } catch (err) {
      err.message += ` value: '${value}' of type: ${typeof value}, fieldname: '${
        field.fieldname
      }', label: '${field.label}'`;
      throw err;
    }
    return value;
  }

  triggerChange(doctype, name) {
    this.trigger(`change:${doctype}`, { name }, 500);
    this.trigger(`change`, { doctype, name }, 500);
    // also trigger change for basedOn doctype
    let meta = frappe.getMeta(doctype);
    if (meta.basedOn) {
      this.triggerChange(meta.basedOn, name);
    }
  }

  async insert(doctype, doc) {
    let meta = frappe.getMeta(doctype);
    let baseDoctype = meta.getBaseDocType();
    doc = this.applyBaseDocTypeFilters(doctype, doc);

    // insert parent
    if (meta.isSingle) {
      await this.updateSingle(doctype, doc);
    } else {
      await this.insertOne(baseDoctype, doc);
    }

    // insert children
    await this.insertChildren(meta, doc, baseDoctype);

    this.triggerChange(doctype, doc.name);

    return doc;
  }

  async insertChildren(meta, doc, doctype) {
    let tableFields = meta.getTableFields();
    for (let field of tableFields) {
      let idx = 0;
      for (let child of doc[field.fieldname] || []) {
        this.prepareChild(doctype, doc.name, child, field, idx);
        await this.insertOne(field.childtype, child);
        idx++;
      }
    }
  }

  insertOne(doctype, doc) {
    let fields = this.getValidFields(doctype);

    if (!doc.name) {
      doc.name = frappe.getRandomString();
    }

    let formattedDoc = this.getFormattedDoc(fields, doc);
    return this.knex(doctype).insert(formattedDoc);
  }

  async update(doctype, doc) {
    let meta = frappe.getMeta(doctype);
    let baseDoctype = meta.getBaseDocType();
    doc = this.applyBaseDocTypeFilters(doctype, doc);

    // update parent
    if (meta.isSingle) {
      await this.updateSingle(doctype, doc);
    } else {
      await this.updateOne(baseDoctype, doc);
    }

    // insert or update children
    await this.updateChildren(meta, doc, baseDoctype);

    this.triggerChange(doctype, doc.name);

    return doc;
  }

  async updateChildren(meta, doc, doctype) {
    let tableFields = meta.getTableFields();
    for (let field of tableFields) {
      let added = [];
      for (let child of doc[field.fieldname] || []) {
        this.prepareChild(doctype, doc.name, child, field, added.length);
        if (await this.exists(field.childtype, child.name)) {
          await this.updateOne(field.childtype, child);
        } else {
          await this.insertOne(field.childtype, child);
        }
        added.push(child.name);
      }
      await this.runDeleteOtherChildren(field, doc.name, added);
    }
  }

  updateOne(doctype, doc) {
    let validFields = this.getValidFields(doctype);
    let fieldsToUpdate = Object.keys(doc).filter((f) => f !== 'name');
    let fields = validFields.filter((df) =>
      fieldsToUpdate.includes(df.fieldname)
    );
    let formattedDoc = this.getFormattedDoc(fields, doc);

    return this.knex(doctype)
      .where('name', doc.name)
      .update(formattedDoc)
      .then(() => {
        let cacheKey = `${doctype}:${doc.name}`;
        if (this.cache.hexists(cacheKey)) {
          for (let fieldname in formattedDoc) {
            let value = formattedDoc[fieldname];
            this.cache.hset(cacheKey, fieldname, value);
          }
        }
      });
  }

  runDeleteOtherChildren(field, parent, added) {
    // delete other children
    return this.knex(field.childtype)
      .where('parent', parent)
      .andWhere('name', 'not in', added)
      .delete();
  }

  async updateSingle(doctype, doc) {
    let meta = frappe.getMeta(doctype);
    await this.deleteSingleValues(doctype);
    for (let field of meta.getValidFields({ withChildren: false })) {
      let value = doc[field.fieldname];
      if (value != null) {
        let singleValue = frappe.newDoc({
          doctype: 'SingleValue',
          parent: doctype,
          fieldname: field.fieldname,
          value: value,
        });
        await singleValue.insert();
      }
    }
  }

  deleteSingleValues(name) {
    return this.knex('SingleValue').where('parent', name).delete();
  }

  async rename(doctype, oldName, newName) {
    let meta = frappe.getMeta(doctype);
    let baseDoctype = meta.getBaseDocType();
    await this.knex(baseDoctype)
      .update({ name: newName })
      .where('name', oldName)
      .then(() => {
        this.clearValueCache(doctype, oldName);
      });
    await frappe.db.commit();

    this.triggerChange(doctype, newName);
  }

  prepareChild(parenttype, parent, child, field, idx) {
    if (!child.name) {
      child.name = frappe.getRandomString();
    }
    child.parent = parent;
    child.parenttype = parenttype;
    child.parentfield = field.fieldname;
    child.idx = idx;
  }

  getValidFields(doctype) {
    return frappe.getMeta(doctype).getValidFields({ withChildren: false });
  }

  getFormattedDoc(fields, doc) {
    // format for storage, going into the db
    let formattedDoc = {};
    fields.map((field) => {
      let value = doc[field.fieldname];
      formattedDoc[field.fieldname] = this.getFormattedValue(field, value);
    });
    return formattedDoc;
  }

  getFormattedValue(field, value) {
    // format for storage, going into the db
    const type = typeof value;
    if (field.fieldtype === 'Currency') {
      let currency = value;

      if (type === 'number' || type === 'string') {
        currency = frappe.pesa(value);
      }

      const currencyValue = currency.store;
      if (typeof currencyValue !== 'string') {
        throw new Error(
          `invalid currencyValue '${currencyValue}' of type '${typeof currencyValue}' on converting from '${value}' of type '${type}'`
        );
      }

      return currencyValue;
    }

    if (value instanceof Date) {
      if (field.fieldtype === 'Date') {
        // date
        return value.toISOString().substr(0, 10);
      } else {
        // datetime
        return value.toISOString();
      }
    } else if (field.fieldtype === 'Link' && !value) {
      // empty value must be null to satisfy
      // foreign key constraint
      return null;
    } else {
      return value;
    }
  }

  applyBaseDocTypeFilters(doctype, doc) {
    let meta = frappe.getMeta(doctype);
    if (meta.filters) {
      for (let fieldname in meta.filters) {
        let value = meta.filters[fieldname];
        if (typeof value !== 'object') {
          doc[fieldname] = value;
        }
      }
    }
    return doc;
  }

  async deleteMany(doctype, names) {
    for (const name of names) {
      await this.delete(doctype, name);
    }
  }

  async delete(doctype, name) {
    let meta = frappe.getMeta(doctype);
    let baseDoctype = meta.getBaseDocType();
    await this.deleteOne(baseDoctype, name);

    // delete children
    let tableFields = frappe.getMeta(doctype).getTableFields();
    for (let field of tableFields) {
      await this.deleteChildren(field.childtype, name);
    }

    this.triggerChange(doctype, name);
  }

  async deleteOne(doctype, name) {
    return this.knex(doctype)
      .where('name', name)
      .delete()
      .then(() => {
        this.clearValueCache(doctype, name);
      });
  }

  deleteChildren(parenttype, parent) {
    return this.knex(parenttype).where('parent', parent).delete();
  }

  async exists(doctype, name) {
    return (await this.getValue(doctype, name)) ? true : false;
  }

  async getValue(doctype, filters, fieldname = 'name') {
    let meta = frappe.getMeta(doctype);
    let baseDoctype = meta.getBaseDocType();
    if (typeof filters === 'string') {
      filters = { name: filters };
    }
    if (meta.filters) {
      Object.assign(filters, meta.filters);
    }

    let row = await this.getAll({
      doctype: baseDoctype,
      fields: [fieldname],
      filters: filters,
      start: 0,
      limit: 1,
      orderBy: 'name',
      order: 'asc',
    });
    return row.length ? row[0][fieldname] : null;
  }

  async setValue(doctype, name, fieldname, value) {
    return await this.setValues(doctype, name, {
      [fieldname]: value,
    });
  }

  async setValues(doctype, name, fieldValuePair) {
    let doc = Object.assign({}, fieldValuePair, { name });
    return this.updateOne(doctype, doc);
  }

  async getCachedValue(doctype, name, fieldname) {
    let value = this.cache.hget(`${doctype}:${name}`, fieldname);
    if (value == null) {
      value = await this.getValue(doctype, name, fieldname);
    }
    return value;
  }

  async getAll({
    doctype,
    fields,
    filters,
    start,
    limit,
    groupBy,
    orderBy = 'creation',
    order = 'desc',
  } = {}) {
    let meta = frappe.getMeta(doctype);
    let baseDoctype = meta.getBaseDocType();
    if (!fields) {
      fields = meta.getKeywordFields();
      fields.push('name');
    }
    if (typeof fields === 'string') {
      fields = [fields];
    }
    if (meta.filters) {
      filters = Object.assign({}, filters, meta.filters);
    }

    let builder = this.knex.select(fields).from(baseDoctype);

    this.applyFiltersToBuilder(builder, filters);

    if (orderBy) {
      builder.orderBy(orderBy, order);
    }

    if (groupBy) {
      builder.groupBy(groupBy);
    }

    if (start) {
      builder.offset(start);
    }

    if (limit) {
      builder.limit(limit);
    }

    const docs = await builder;
    return docs.map((doc) => this.getDocFormattedDoc(meta.fields, doc));
  }

  applyFiltersToBuilder(builder, filters) {
    // {"status": "Open"} => `status = "Open"`

    // {"status": "Open", "name": ["like", "apple%"]}
    // => `status="Open" and name like "apple%"

    // {"date": [">=", "2017-09-09", "<=", "2017-11-01"]}
    // => `date >= 2017-09-09 and date <= 2017-11-01`

    let filtersArray = [];

    for (let field in filters) {
      let value = filters[field];
      let operator = '=';
      let comparisonValue = value;

      if (Array.isArray(value)) {
        operator = value[0];
        comparisonValue = value[1];
        operator = operator.toLowerCase();

        if (operator === 'includes') {
          operator = 'like';
        }

        if (operator === 'like' && !comparisonValue.includes('%')) {
          comparisonValue = `%${comparisonValue}%`;
        }
      }

      filtersArray.push([field, operator, comparisonValue]);

      if (Array.isArray(value) && value.length > 2) {
        // multiple conditions
        let operator = value[2];
        let comparisonValue = value[3];
        filtersArray.push([field, operator, comparisonValue]);
      }
    }

    filtersArray.map((filter) => {
      const [field, operator, comparisonValue] = filter;
      if (operator === '=') {
        builder.where(field, comparisonValue);
      } else {
        builder.where(field, operator, comparisonValue);
      }
    });
  }

  run(query, params) {
    // run query
    return this.sql(query, params);
  }

  sql(query, params) {
    // run sql
    return this.knex.raw(query, params);
  }

  async commit() {
    try {
      await this.sql('commit');
    } catch (e) {
      if (e.type !== frappe.errors.CannotCommitError) {
        throw e;
      }
    }
  }

  clearValueCache(doctype, name) {
    let cacheKey = `${doctype}:${name}`;
    this.cache.hclear(cacheKey);
  }

  getColumnType(field) {
    return this.typeMap[field.fieldtype];
  }

  getError(err) {
    return frappe.errors.DatabaseError;
  }

  initTypeMap() {
    this.typeMap = {};
  }

  executePostDbConnect() {
    frappe.initializeMoneyMaker();
  }
};