test: import from csv

- add check links

src/importer.ts

@@ -106,11 +106,73 @@ export class Importer {
     });
   }
 
-  pushFromValueMatrixToDocs() {
+  selectFile(data: string): boolean {
+    try {
+      const parsed = parseCSV(data);
+      this.selectParsed(parsed);
+    } catch {
+      return false;
+    }
+
+    return true;
+  }
+
+  async checkLinks() {
+    const tfKeys = this.assignedTemplateFields
+      .map((key, index) => ({
+        key,
+        index,
+        tf: this.templateFieldsMap.get(key ?? ''),
+      }))
+      .filter(({ key, tf }) => {
+        if (!key || !tf) {
+          return false;
+        }
+
+        return tf.fieldtype === FieldTypeEnum.Link;
+      }) as { key: string; index: number; tf: TemplateField }[];
+
+    const linksNames: Map<string, Set<string>> = new Map();
+    for (const row of this.valueMatrix) {
+      for (const { tf, index } of tfKeys) {
+        const target = (tf as TargetField).target;
+        const value = row[index]?.value;
+        if (typeof value !== 'string' || !value) {
+          continue;
+        }
+
+        if (!linksNames.has(target)) {
+          linksNames.set(target, new Set());
+        }
+
+        linksNames.get(target)?.add(value);
+      }
+    }
+
+    const doesNotExist = [];
+    for (const [target, values] of linksNames.entries()) {
+      for (const value of values) {
+        const exists = await this.fyo.db.exists(target, value);
+        if (exists) {
+          continue;
+        }
+
+        doesNotExist.push({
+          schemaName: target,
+          schemaLabel: this.fyo.schemaMap[this.schemaName]?.label,
+          name: value,
+        });
+      }
+    }
+
+    return doesNotExist;
+  }
+
+  populateDocs() {
     const { dataMap, childTableMap } =
       this.getDataAndChildTableMapFromValueMatrix();
 
-    const schema = this.fyo.db.schemaMap[this.schemaName];
+    const schema = this.fyo.schemaMap[this.schemaName];
     const targetFieldnameMap = schema?.fields
       .filter((f) => f.fieldtype === FieldTypeEnum.Table)
       .reduce((acc, f) => {
@@ -171,7 +233,7 @@ export class Importer {
 
     for (const i in this.valueMatrix) {
       const row = this.valueMatrix[i];
-      const name = row[nameIndex].value;
+      const name = row[nameIndex]?.value;
       if (typeof name !== 'string') {
         continue;
       }
@@ -199,7 +261,7 @@ export class Importer {
         }
 
         const childNameIndex = nameIndices[tf.schemaName];
-        let childName = row[childNameIndex].value;
+        let childName = row[childNameIndex]?.value;
         if (typeof childName !== 'string') {
           childName = `${tf.schemaName}-${i}`;
         }
@@ -224,17 +286,6 @@ export class Importer {
     return { dataMap, childTableMap };
   }
 
-  selectFile(data: string): boolean {
-    try {
-      const parsed = parseCSV(data);
-      this.selectParsed(parsed);
-    } catch {
-      return false;
-    }
-
-    return true;
-  }
-
   selectParsed(parsed: string[][]): void {
     if (!parsed?.length) {
       return;
@@ -337,6 +388,11 @@ export class Importer {
       return vmi;
     }
 
+    if (vmi.rawValue === '') {
+      vmi.value = null;
+      return vmi;
+    }
+
     try {
       vmi.value = Converter.toDocValue(rawValue, tf, this.fyo);
     } catch {
@@ -437,7 +493,12 @@ function getTemplateFields(
     }
 
     for (const field of schema.fields) {
-      if (field.computed || field.meta || field.hidden) {
+      if (
+        field.computed ||
+        field.meta ||
+        field.hidden ||
+        (field.readOnly && !field.required)
+      ) {
         continue;
       }
 
@@ -453,12 +514,10 @@ function getTemplateFields(
         continue;
       }
 
-      if (field.readOnly && field.required) {
+      const tf = { ...field };
-        field.readOnly = false;
-      }
 
-      if (field.readOnly) {
+      if (tf.readOnly) {
-        continue;
+        tf.readOnly = false;
       }
 
       const schemaName = schema.name;
@@ -466,7 +525,7 @@ function getTemplateFields(
       const fieldKey = `${schema.name}.${field.fieldname}`;
 
       fields.push({
-        ...field,
+        ...tf,
         schemaName,
         schemaLabel,
         fieldKey,

src/pages/ImportWizard.vue

@@ -676,7 +676,7 @@ export default defineComponent({
         return;
       }
 
-      this.importer.pushFromValueMatrixToDocs();
+      this.importer.populateDocs();
 
       let doneCount = 0;
       for (const doc of this.importer.docs) {

tests/items.csv

@@ -0,0 +1,13 @@
+"Item Name",Description,"Unit Type",Type,For,"Sales Acc.","Purchase Acc.",Tax,Rate,HSN/SAC,Barcode,"Track Item","Created By","Modified By",Created,Modified
+Item.name,Item.description,Item.unit,Item.itemType,Item.for,Item.incomeAccount,Item.expenseAccount,Item.tax,Item.rate,Item.hsnCode,Item.barcode,Item.trackItem,Item.createdBy,Item.modifiedBy,Item.created,Item.modified
+"Final Item","A final item made from raw items",Unit,Product,Both,Sales,"Stock Received But Not Billed",,500,0,,1,lin@to.co,lin@to.co,2023-01-31T06:46:00.200Z,2023-01-31T06:46:00.200Z
+"Raw Two","Another Raw item used to make a final item",Unit,Product,Both,Sales,"Stock Received But Not Billed",,200,0,,1,lin@to.co,lin@to.co,2023-01-31T06:45:32.449Z,2023-01-31T06:45:32.449Z
+"Raw One","A raw item used to make a final item.",Unit,Product,Both,Sales,"Stock Received But Not Billed",,100,0,,1,lin@to.co,lin@to.co,2023-01-31T06:44:58.047Z,2023-01-31T06:44:58.047Z
+"Test One",,Unit,Product,Both,Sales,"Stock Received But Not Billed",GST-18,200,0,,1,lin@to.co,lin@to.co,2023-01-09T10:46:02.217Z,2023-01-09T10:46:02.217Z
+Stuff,"Some stuff.",Unit,Product,Both,Sales,"Stock Received But Not Billed",GST-18,200,101192,,1,lin@to.co,lin@to.co,2023-01-09T07:14:12.208Z,2023-01-09T07:14:12.208Z
+"Something Sellable",,Unit,Product,Sales,Sales,"Cost of Goods Sold",,300,0,,0,lin@to.co,lin@to.co,2022-10-11T09:15:15.724Z,2023-01-16T08:49:49.267Z
+Ball,"Just a ball..",Unit,Product,Both,Sales,"Cost of Goods Sold",,30,0,,0,Administrator,Administrator,2022-02-24T04:38:09.181Z,2022-02-24T04:38:09.181Z
+Bat,"Regular old bat...",Unit,Product,Both,Sales,"Cost of Goods Sold",,129,0,,0,Administrator,Administrator,2022-02-24T04:38:09.174Z,2022-02-24T04:38:09.174Z
+"Holy Icon","The holiest of icons.",Unit,Product,Both,Sales,"Cost of Goods Sold",GST-3,330,0,,0,Administrator,Administrator,2022-02-11T11:32:33.342Z,2022-02-11T11:32:33.342Z
+"Flower Pot","Just a flower pot.",Unit,Product,Both,Sales,"Cost of Goods Sold",GST-12,200,,,0,Administrator,Administrator,2021-12-16T07:04:08.233Z,2021-12-16T07:04:08.233Z
+Flow,"Used to test the flow of operations.",Unit,Product,Both,Sales,"Cost of Goods Sold",GST-12,100,,,0,Administrator,Administrator,2021-12-16T05:42:02.081Z,2021-12-16T05:48:48.203Z

tests/parties.csv

@@ -0,0 +1,8 @@
+Name,Role,"Default Account","Outstanding Amount",Currency,Email,Phone,Address,"GSTIN No.","GST Registration","Created By","Modified By",Created,Modified
+Party.name,Party.role,Party.defaultAccount,Party.outstandingAmount,Party.currency,Party.email,Party.phone,Party.address,Party.gstin,Party.gstType,Party.createdBy,Party.modifiedBy,Party.created,Party.modified
+Randoe,Both,,259.6,INR,,,,,Unregistered,lin@to.co,lin@to.co,2023-01-09T04:58:16.050Z,2023-01-09T10:46:46.128Z
+Saipan,Customer,Debtors,299.99999999921,USD,sai@pan.co,,,,Unregistered,lin@to.co,lin@to.co,2022-07-18T17:07:35.103Z,2023-01-30T14:36:50.058Z
+Lordham,Customer,Debtors,851.8,INR,lo@gamil.com,8989004444,,,Unregistered,Administrator,lin@to.co,2022-02-04T06:35:19.404Z,2022-07-18T17:05:42.976Z
+Lyn,Customer,Debtors,100,INR,lyn@to.co,,,,Consumer,Administrator,Administrator,2022-02-04T06:21:19.069Z,2022-02-28T05:18:32.743Z
+Bølèn,Customer,Debtors,0,INR,bo@len.co,,,22ABCIK123401Z5,"Registered Regular",Administrator,Administrator,2022-01-12T08:44:58.879Z,2022-01-12T08:45:26.714Z
+Bé,Customer,Debtors,46,INR,bey@more.tips,6969969600,,,Consumer,Administrator,Administrator,2021-12-16T11:32:11.595Z,2021-12-16T12:00:28.558Z

tests/sales_invoices.csv

@@ -0,0 +1,30 @@
+"Invoice No",Date,Party,Account,"Customer Currency","Exchange Rate","Net Total","Grand Total","Base Grand Total","Outstanding Amount","Set Discount Amount","Discount Amount","Discount Percent","Discount After Tax","Entry Currency",Notes,"Stock Not Transferred","Number Series","Created By","Modified By",Created,Modified,Submitted,Cancelled,"Tax Account",Rate,Amount,Item,Description,Quantity,Rate,Account,Tax,Amount,"Set Discount Amount","Discount Amount","Discount Percent",HSN/SAC,"Stock Not Transferred"
+SalesInvoice.name,SalesInvoice.date,SalesInvoice.party,SalesInvoice.account,SalesInvoice.currency,SalesInvoice.exchangeRate,SalesInvoice.netTotal,SalesInvoice.grandTotal,SalesInvoice.baseGrandTotal,SalesInvoice.outstandingAmount,SalesInvoice.setDiscountAmount,SalesInvoice.discountAmount,SalesInvoice.discountPercent,SalesInvoice.discountAfterTax,SalesInvoice.entryCurrency,SalesInvoice.terms,SalesInvoice.stockNotTransferred,SalesInvoice.numberSeries,SalesInvoice.createdBy,SalesInvoice.modifiedBy,SalesInvoice.created,SalesInvoice.modified,SalesInvoice.submitted,SalesInvoice.cancelled,TaxSummary.account,TaxSummary.rate,TaxSummary.amount,SalesInvoiceItem.item,SalesInvoiceItem.description,SalesInvoiceItem.quantity,SalesInvoiceItem.rate,SalesInvoiceItem.account,SalesInvoiceItem.tax,SalesInvoiceItem.amount,SalesInvoiceItem.setItemDiscountAmount,SalesInvoiceItem.itemDiscountAmount,SalesInvoiceItem.itemDiscountPercent,SalesInvoiceItem.hsnCode,SalesInvoiceItem.stockNotTransferred
+1020,2023-01-31,Lordham,Debtors,INR,1,500,500,500,500,0,0,0,0,Party,,1,SINV-,lin@to.co,lin@to.co,2023-01-31T09:00:45.858Z,2023-01-31T09:00:45.858Z,0,0,,,,"Final Item","A final item made from raw items",1,500,Sales,,500,0,0,0,0,1
+1019,2023-01-09,Randoe,Debtors,INR,1,220,259.6,259.6,259.6,0,0,0,0,Party,,1,SINV-,lin@to.co,lin@to.co,2023-01-09T10:46:40.923Z,2023-01-09T10:46:46.085Z,1,0,CGST,9,19.8,,,,,,,,,,,,
+1019,2023-01-09,Randoe,Debtors,INR,1,220,259.6,259.6,259.6,0,0,0,0,Party,,1,SINV-,lin@to.co,lin@to.co,2023-01-09T10:46:40.923Z,2023-01-09T10:46:46.085Z,1,0,SGST,9,19.8,,,,,,,,,,,,
+1019,2023-01-09,Randoe,Debtors,INR,1,220,259.6,259.6,259.6,0,0,0,0,Party,,1,SINV-,lin@to.co,lin@to.co,2023-01-09T10:46:40.923Z,2023-01-09T10:46:46.085Z,1,0,,,,"Test One",,1,220,Sales,GST-18,220,0,0,0,0,1
+1018,2022-10-11,Saipan,Debtors,USD,82.47,3.63768643142,3.63768643142,299.99999999921,299.99999999921,0,0,0,0,Party,,0,SINV-,lin@to.co,lin@to.co,2022-10-11T09:16:50.373Z,2023-01-30T14:36:50.021Z,1,0,,,,"Something Sellable",,1,3.63768643142,Sales,,3.63768643142,0,0,0,0,0
+1014,2022-07-11,Lordham,Debtors,INR,1,660,679.8,679.8,679.8,0,0,,0,Party,,,SINV-,lin@to.co,lin@to.co,2022-07-11T11:06:18.788Z,2022-07-18T17:05:42.922Z,1,0,CGST,1.5,9.9,,,,,,,,,,,,
+1014,2022-07-11,Lordham,Debtors,INR,1,660,679.8,679.8,679.8,0,0,,0,Party,,,SINV-,lin@to.co,lin@to.co,2022-07-11T11:06:18.788Z,2022-07-18T17:05:42.922Z,1,0,SGST,1.5,9.9,,,,,,,,,,,,
+1014,2022-07-11,Lordham,Debtors,INR,1,660,679.8,679.8,679.8,0,0,,0,Party,,,SINV-,lin@to.co,lin@to.co,2022-07-11T11:06:18.788Z,2022-07-18T17:05:42.922Z,1,0,,,,"Holy Icon","The holiest of icons.",2,330,Sales,GST-3,660,0,0,,0,
+1011,2022-02-22,Lyn,Debtors,INR,1,1410,1471.2,1471.2,100,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.656Z,2022-02-28T05:18:32.731Z,1,0,SGST,1.5,30.6,,,,,,,,,,,,
+1011,2022-02-22,Lyn,Debtors,INR,1,1410,1471.2,1471.2,100,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.656Z,2022-02-28T05:18:32.731Z,1,0,CGST,1.5,30.6,,,,,,,,,,,,
+1011,2022-02-22,Lyn,Debtors,INR,1,1410,1471.2,1471.2,100,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.656Z,2022-02-28T05:18:32.731Z,1,0,,,,"Flower Pot","Just a flower pot.",1,210,Sales,GST-12,210,0,0,,,
+1011,2022-02-22,Lyn,Debtors,INR,1,1410,1471.2,1471.2,100,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.656Z,2022-02-28T05:18:32.731Z,1,0,,,,"Holy Icon","The holiest of icons.",3,400,Sales,GST-3,1200,0,0,,0,
+1012,2022-02-20,Lordham,Debtors,INR,1,2230,2353.6,2353.6,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.685Z,2022-02-28T05:18:11.657Z,1,0,SGST,1.5,61.8,,,,,,,,,,,,
+1012,2022-02-20,Lordham,Debtors,INR,1,2230,2353.6,2353.6,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.685Z,2022-02-28T05:18:11.657Z,1,0,CGST,1.5,61.8,,,,,,,,,,,,
+1012,2022-02-20,Lordham,Debtors,INR,1,2230,2353.6,2353.6,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.685Z,2022-02-28T05:18:11.657Z,1,0,,,,"Flower Pot","Just a flower pot.",3,210,Sales,GST-12,630,0,0,,,
+1012,2022-02-20,Lordham,Debtors,INR,1,2230,2353.6,2353.6,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.685Z,2022-02-28T05:18:11.657Z,1,0,,,,"Holy Icon","The holiest of icons.",4,400,Sales,GST-3,1600,0,0,,0,
+1008,2022-02-11,Lordham,Debtors,INR,1,600,672,672,172,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-11T08:40:28.193Z,2022-02-11T09:48:18.274Z,1,0,SGST,6,36,,,,,,,,,,,,
+1008,2022-02-11,Lordham,Debtors,INR,1,600,672,672,172,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-11T08:40:28.193Z,2022-02-11T09:48:18.274Z,1,0,CGST,6,36,,,,,,,,,,,,
+1008,2022-02-11,Lordham,Debtors,INR,1,600,672,672,172,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-11T08:40:28.193Z,2022-02-11T09:48:18.274Z,1,0,,,,"Flower Pot","Just a flower pot.",3,200,Sales,GST-12,600,0,0,,,
+1007,2022-02-04,Lyn,Debtors,INR,1,200,224,224,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-04T06:21:24.058Z,2022-02-04T06:21:46.308Z,1,0,SGST,6,12,,,,,,,,,,,,
+1007,2022-02-04,Lyn,Debtors,INR,1,200,224,224,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-04T06:21:24.058Z,2022-02-04T06:21:46.308Z,1,0,CGST,6,12,,,,,,,,,,,,
+1007,2022-02-04,Lyn,Debtors,INR,1,200,224,224,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-04T06:21:24.058Z,2022-02-04T06:21:46.308Z,1,0,,,,"Flower Pot","Just a flower pot.",1,200,Sales,GST-12,200,0,0,,,
+1004,2022-01-12,Bølèn,Debtors,INR,1,1800,2016,2016,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-01-12T08:45:19.774Z,2022-01-12T08:45:26.702Z,1,0,SGST,6,108,,,,,,,,,,,,
+1004,2022-01-12,Bølèn,Debtors,INR,1,1800,2016,2016,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-01-12T08:45:19.774Z,2022-01-12T08:45:26.702Z,1,0,CGST,6,108,,,,,,,,,,,,
+1004,2022-01-12,Bølèn,Debtors,INR,1,1800,2016,2016,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-01-12T08:45:19.774Z,2022-01-12T08:45:26.702Z,1,0,,,,"Flower Pot","Just a flower pot.",9,200,Sales,GST-12,1800,0,0,,,
+1001,2021-12-16,Bé,Debtors,INR,1,100,112,112,46,0,0,,0,Party,,,SINV-,Administrator,Administrator,2021-12-16T11:34:06.174Z,2021-12-16T12:00:28.526Z,1,0,SGST,6,6,,,,,,,,,,,,
+1001,2021-12-16,Bé,Debtors,INR,1,100,112,112,46,0,0,,0,Party,,,SINV-,Administrator,Administrator,2021-12-16T11:34:06.174Z,2021-12-16T12:00:28.526Z,1,0,CGST,6,6,,,,,,,,,,,,
+1001,2021-12-16,Bé,Debtors,INR,1,100,112,112,46,0,0,,0,Party,,,SINV-,Administrator,Administrator,2021-12-16T11:34:06.174Z,2021-12-16T12:00:28.526Z,1,0,,,,Flow,"Used to test the flow of operations.",1,100,Sales,GST-12,100,0,0,,,

tests/testImporter.spec.ts

@@ -1,4 +1,7 @@
+import { assertDoesNotThrow } from 'backend/database/tests/helpers';
+import { readFileSync } from 'fs';
 import { ModelNameEnum } from 'models/types';
+import { join } from 'path';
 import { Importer } from 'src/importer';
 import test from 'tape';
 import { closeTestFyo, getTestFyo, setupTestFyo } from './helpers';
@@ -6,9 +9,59 @@ import { closeTestFyo, getTestFyo, setupTestFyo } from './helpers';
 const fyo = getTestFyo();
 setupTestFyo(fyo, __filename);
 
-test('importer', async (t) => {
+test('importer init', (t) => {
   const importer = new Importer(ModelNameEnum.SalesInvoice, fyo);
-  t.ok(importer.getCSVTemplate());
+  t.equal(
+    typeof importer.getCSVTemplate(),
+    'string',
+    'csv template is a string'
+  );
+  t.end();
+});
+
+test('import Items', async (t) => {
+  const importer = new Importer(ModelNameEnum.Item, fyo);
+  const csvPath = join(__dirname, 'items.csv');
+  const data = readFileSync(csvPath, { encoding: 'utf-8' });
+  t.equal(importer.selectFile(data), true, 'file selection');
+  t.equal((await importer.checkLinks()).length, 0, 'all links exist');
+  t.doesNotThrow(() => importer.populateDocs(), 'populating docs');
+  for (const doc of importer.docs) {
+    await assertDoesNotThrow(async () => await doc.sync());
+  }
+});
+
+test('import Party', async (t) => {
+  const importer = new Importer(ModelNameEnum.Party, fyo);
+  const csvPath = join(__dirname, 'parties.csv');
+  const data = readFileSync(csvPath, { encoding: 'utf-8' });
+  t.equal(importer.selectFile(data), true, 'file selection');
+  t.equal((await importer.checkLinks()).length, 0, 'all links exist');
+  t.doesNotThrow(() => importer.populateDocs(), 'populating docs');
+  for (const doc of importer.docs) {
+    await assertDoesNotThrow(async () => await doc.sync());
+  }
+});
+
+test('import SalesInvoices', async (t) => {
+  const importer = new Importer(ModelNameEnum.SalesInvoice, fyo);
+  const csvPath = join(__dirname, 'sales_invoices.csv');
+  const data = readFileSync(csvPath, { encoding: 'utf-8' });
+
+  t.equal(importer.selectFile(data), true, 'file selection');
+  t.equal((await importer.checkLinks()).length, 0, 'all links exist');
+  t.doesNotThrow(() => importer.populateDocs(), 'populating docs');
+
+  const names = [];
+  for (const doc of importer.docs.slice(0, 2)) {
+    await assertDoesNotThrow(async () => await doc.sync());
+    names.push(doc.name);
+  }
+
+  t.ok(
+    names.every((n) => n?.startsWith('SINV-')),
+    'numberSeries assigned'
+  );
 });
 
 closeTestFyo(fyo, __filename);