feat: enable custom fields

- fix date issue in setup test

backend/database/manager.ts

@@ -11,10 +11,11 @@ import patches from '../patches';
 import { BespokeQueries } from './bespoke';
 import DatabaseCore from './core';
 import { runPatches } from './runPatch';
-import { BespokeFunction, Patch } from './types';
+import { BespokeFunction, Patch, RawCustomField } from './types';
 
 export class DatabaseManager extends DatabaseDemuxBase {
   db?: DatabaseCore;
+  rawCustomFields: RawCustomField[] = [];
 
   get #isInitialized(): boolean {
     return this.db !== undefined && this.db.knex !== undefined;
@@ -22,10 +23,10 @@ export class DatabaseManager extends DatabaseDemuxBase {
 
   getSchemaMap() {
     if (this.#isInitialized) {
-      return this.db?.schemaMap ?? getSchemas();
+      return this.db?.schemaMap ?? getSchemas('-', this.rawCustomFields);
     }
 
-    return getSchemas();
+    return getSchemas('-', this.rawCustomFields);
   }
 
   async createNewDatabase(dbPath: string, countryCode: string) {
@@ -43,11 +44,20 @@ export class DatabaseManager extends DatabaseDemuxBase {
     countryCode ??= await DatabaseCore.getCountryCode(dbPath);
     this.db = new DatabaseCore(dbPath);
     await this.db.connect();
-    const schemaMap = getSchemas(countryCode);
+    await this.setRawCustomFields();
+    const schemaMap = getSchemas(countryCode, this.rawCustomFields);
     this.db.setSchemaMap(schemaMap);
     return countryCode;
   }
 
+  async setRawCustomFields() {
+    try {
+      this.rawCustomFields = (await this.db?.knex?.(
+        'CustomField'
+      )) as RawCustomField[];
+    } catch {}
+  }
+
   async #migrate(): Promise<void> {
     if (!this.#isInitialized) {
       return;

backend/database/types.ts

@@ -1,6 +1,6 @@
-import { Field, RawValue } from '../../schemas/types';
-import DatabaseCore from './core';
-import { DatabaseManager } from './manager';
+import type { Field, FieldType, RawValue } from '../../schemas/types';
+import type DatabaseCore from './core';
+import type { DatabaseManager } from './manager';
 
 export interface GetQueryBuilderOptions {
   offset?: number;
@@ -80,3 +80,16 @@ export type SingleValue<T> = {
   parent: string;
   value: T;
 }[];
+
+export type RawCustomField = {
+  parent: string;
+  label: string;
+  fieldname: string;
+  fieldtype: FieldType;
+  isRequired?: boolean;
+  section?: string;
+  tab?: string;
+  options?: string;
+  target?: string;
+  references?: string;
+};

fyo/model/doc.ts

@@ -751,7 +751,7 @@ export class Doc extends Observable<DocValue | Doc[]> {
     if (dbValues && docModified !== dbModified) {
       throw new ConflictError(
         this.fyo
-          .t`${this.schema.label} ${this.name} has been modified after loading` +
+          .t`${this.schema.label} ${this.name} has been modified after loading please reload entry.` +
           ` ${dbModified}, ${docModified}`
       );
     }

fyo/models/CustomField.ts

@@ -1,14 +1,18 @@
+import { DocValue } from 'fyo/core/types';
 import { Doc } from 'fyo/model/doc';
 import type {
   FormulaMap,
   HiddenMap,
   ListsMap,
+  RequiredMap,
   ValidationMap,
 } from 'fyo/model/types';
-import type { FieldType, SelectOption } from 'schemas/types';
-import type { CustomForm } from './CustomForm';
 import { ValueError } from 'fyo/utils/errors';
+import { camelCase } from 'lodash';
 import { ModelNameEnum } from 'models/types';
+import type { FieldType } from 'schemas/types';
+import { FieldTypeEnum } from 'schemas/types';
+import type { CustomForm } from './CustomForm';
 
 export class CustomField extends Doc {
   parentdoc?: CustomForm;
@@ -31,6 +35,19 @@ export class CustomField extends Doc {
     return this.parentdoc?.parentFields;
   }
 
+  formulas: FormulaMap = {
+    fieldname: {
+      formula: () => {
+        if (!this.label?.length) {
+          return;
+        }
+
+        return camelCase(this.label);
+      },
+      dependsOn: ['label'],
+    },
+  };
+
   hidden: HiddenMap = {
     options: () =>
       this.fieldtype !== 'Select' &&
@@ -40,9 +57,15 @@ export class CustomField extends Doc {
     references: () => this.fieldtype !== 'DynamicLink',
   };
 
-  formulas: FormulaMap = {};
-
   validations: ValidationMap = {
+    label: (value) => {
+      if (typeof value !== 'string') {
+        return;
+      }
+
+      const fieldname = camelCase(value);
+      (this.validations.fieldname as (value: DocValue) => void)(fieldname);
+    },
     fieldname: (value) => {
       if (typeof value !== 'string') {
         return;
@@ -94,9 +117,38 @@ export class CustomField extends Doc {
         return [];
       }
 
-      return (doc.parentdoc?.customFields
-        ?.map((cf) => ({ value: cf.fieldname, label: cf.label }))
-        .filter((cf) => cf.label && cf.value) ?? []) as SelectOption[];
+      const referenceType: string[] = [
+        FieldTypeEnum.AutoComplete,
+        FieldTypeEnum.Data,
+        FieldTypeEnum.Text,
+        FieldTypeEnum.Select,
+      ];
+
+      const customFields =
+        doc.parentdoc?.customFields
+          ?.filter(
+            (cf) =>
+              cf.fieldname &&
+              cf.label &&
+              referenceType.includes(cf.fieldtype ?? '')
+          )
+          ?.map((cf) => ({ value: cf.fieldname!, label: cf.label! })) ?? [];
+
+      const schemaFields =
+        doc.parentSchema?.fields
+          .filter(
+            (f) => f.fieldname && f.label && referenceType.includes(f.fieldtype)
+          )
+          .map((f) => ({ value: f.fieldname, label: f.label })) ?? [];
+
+      return [customFields, schemaFields].flat();
     },
   };
+
+  required: RequiredMap = {
+    options: () =>
+      this.fieldtype === 'Select' || this.fieldtype === 'AutoComplete',
+    target: () => this.fieldtype === 'Link' || this.fieldtype === 'Table',
+    references: () => this.fieldtype === 'DynamicLink',
+  };
 }

fyo/models/CustomForm.ts

@@ -1,9 +1,10 @@
 import { Doc } from 'fyo/model/doc';
 import { HiddenMap, ListsMap } from 'fyo/model/types';
+import { ValidationError } from 'fyo/utils/errors';
 import { ModelNameEnum } from 'models/types';
-import type { CustomField } from './CustomField';
-import { getMapFromList } from 'utils/index';
 import { Field } from 'schemas/types';
+import { getMapFromList } from 'utils/index';
+import { CustomField } from './CustomField';
 
 export class CustomForm extends Doc {
   name?: string;
@@ -49,4 +50,29 @@ export class CustomForm extends Doc {
   };
 
   hidden: HiddenMap = { customFields: () => !this.name };
+
+  // eslint-disable-next-line @typescript-eslint/require-await
+  override async validate(): Promise<void> {
+    for (const row of this.customFields ?? []) {
+      if (row.fieldtype === 'Select' || row.fieldtype === 'AutoComplete') {
+        this.validateOptions(row);
+      }
+    }
+  }
+
+  validateOptions(row: CustomField) {
+    const optionString = row.options ?? '';
+    const options = optionString
+      .split('\n')
+      .map((s) => s.trim())
+      .filter(Boolean);
+
+    if (options.length > 1) {
+      return;
+    }
+
+    throw new ValidationError(
+      `At least two options need to be set for the selected fieldtype`
+    );
+  }
 }

fyo/tests/testFyo.spec.ts

@@ -17,7 +17,7 @@ test('Fyo Init', async (t) => {
 test('Fyo Docs', async (t) => {
   const countryCode = 'in';
   const fyo = getTestFyo();
-  const schemaMap = getSchemas(countryCode);
+  const schemaMap = getSchemas(countryCode, []);
   const regionalModels = await getRegionalModels(countryCode);
   await fyo.db.createNewDatabase(':memory:', countryCode);
   await fyo.initializeAndRegister(models, regionalModels);

schemas/index.ts

@@ -1,8 +1,16 @@
+import { RawCustomField } from 'backend/database/types';
 import { cloneDeep } from 'lodash';
 import { getListFromMap, getMapFromList } from 'utils';
 import regionalSchemas from './regional';
 import { appSchemas, coreSchemas, metaSchemas } from './schemas';
-import { Field, Schema, SchemaMap, SchemaStub, SchemaStubMap } from './types';
+import type {
+  Field,
+  Schema,
+  SchemaMap,
+  SchemaStub,
+  SchemaStubMap,
+  SelectOption,
+} from './types';
 
 const NAME_FIELD = {
   fieldname: 'name',
@@ -12,7 +20,10 @@ const NAME_FIELD = {
   readOnly: true,
 };
 
-export function getSchemas(countryCode = '-'): Readonly<SchemaMap> {
+export function getSchemas(
+  countryCode = '-',
+  rawCustomFields: RawCustomField[]
+): Readonly<SchemaMap> {
   const builtCoreSchemas = getCoreSchemas();
   const builtAppSchemas = getAppSchemas(countryCode);
 
@@ -21,6 +32,7 @@ export function getSchemas(countryCode = '-'): Readonly<SchemaMap> {
   schemaMap = removeFields(schemaMap);
   schemaMap = setSchemaNameOnFields(schemaMap);
 
+  addCustomFields(schemaMap, rawCustomFields);
   deepFreeze(schemaMap);
   return schemaMap;
 }
@@ -251,3 +263,74 @@ function getRegionalSchemaMap(countryCode: string): SchemaStubMap {
 
   return getMapFromList(countrySchemas, 'name');
 }
+
+function addCustomFields(
+  schemaMap: SchemaMap,
+  rawCustomFields: RawCustomField[]
+): void {
+  const fieldMap = getFieldMapFromRawCustomFields(rawCustomFields, schemaMap);
+  for (const schemaName in fieldMap) {
+    const fields = fieldMap[schemaName];
+    schemaMap[schemaName]?.fields.push(...fields);
+  }
+}
+
+function getFieldMapFromRawCustomFields(
+  rawCustomFields: RawCustomField[],
+  schemaMap: SchemaMap
+) {
+  const schemaFieldMap: Record<string, Record<string, Field>> = {};
+
+  return rawCustomFields.reduce(
+    (
+      map,
+      {
+        parent,
+        label,
+        fieldname,
+        fieldtype,
+        isRequired,
+        section,
+        tab,
+        options: rawOptions,
+        target,
+        references,
+      }
+    ) => {
+      schemaFieldMap[parent] ??= getMapFromList(
+        schemaMap[parent]?.fields ?? [],
+        'fieldname'
+      );
+
+      if (!schemaFieldMap[parent] || schemaFieldMap[parent][fieldname]) {
+        return map;
+      }
+
+      map[parent] ??= [];
+      const options = rawOptions
+        ?.split('\n')
+        .map((o) => {
+          const value = o.trim();
+          return { value, label: value } as SelectOption;
+        })
+        .filter((o) => o.label && o.value);
+
+      const field = {
+        label,
+        fieldname,
+        fieldtype,
+        required: isRequired,
+        section,
+        tab,
+        target,
+        options,
+        references,
+        isCustom: true,
+      };
+
+      map[parent].push(field as Field);
+      return map;
+    },
+    {} as Record<string, Field[]>
+  );
+}

tests/helpers.ts

@@ -2,6 +2,7 @@ import { DatabaseManager } from 'backend/database/manager';
 import { config } from 'dotenv';
 import { Fyo } from 'fyo';
 import { DummyAuthDemux } from 'fyo/tests/helpers';
+import { DateTime } from 'luxon';
 import path from 'path';
 import setupInstance from 'src/setup/setupInstance';
 import { SetupWizardOptions } from 'src/setup/types';
@@ -17,8 +18,12 @@ export function getTestSetupWizardOptions(): SetupWizardOptions {
     email: 'test@testmyfantasy.com',
     bankName: 'Test Bank of Scriptia',
     currency: 'INR',
-    fiscalYearStart: getFiscalYear('04-01', true)!.toISOString().split('T')[0],
-    fiscalYearEnd: getFiscalYear('04-01', false)!.toISOString().split('T')[0],
+    fiscalYearStart: DateTime.fromJSDate(
+      getFiscalYear('04-01', true)!
+    ).toISODate(),
+    fiscalYearEnd: DateTime.fromJSDate(
+      getFiscalYear('04-01', false)!
+    ).toISODate(),
     chartOfAccounts: 'India - Chart of Accounts',
   };
 }

tests/testSetupInstance.spec.ts

@@ -1,4 +1,5 @@
 import { assertDoesNotThrow } from 'backend/database/tests/helpers';
+import { DateTime } from 'luxon';
 import setupInstance from 'src/setup/setupInstance';
 import { SetupWizardOptions } from 'src/setup/types';
 import test from 'tape';
@@ -39,7 +40,7 @@ test('check setup Singles', async (t) => {
     const optionsValue = setupOptions[field as keyof SetupWizardOptions];
 
     if (dbValue instanceof Date) {
-      dbValue = dbValue.toISOString().split('T')[0];
+      dbValue = DateTime.fromJSDate(dbValue).toISODate();
     }
 
     t.equal(