diff --git a/src/importer.ts b/src/importer.ts index fbf67b60..5a2d7013 100644 --- a/src/importer.ts +++ b/src/importer.ts @@ -106,11 +106,73 @@ export class Importer { }); } - pushFromValueMatrixToDocs() { + selectFile(data: string): boolean { + try { + const parsed = parseCSV(data); + this.selectParsed(parsed); + } catch { + return false; + } + + return true; + } + + async checkLinks() { + const tfKeys = this.assignedTemplateFields + .map((key, index) => ({ + key, + index, + tf: this.templateFieldsMap.get(key ?? ''), + })) + .filter(({ key, tf }) => { + if (!key || !tf) { + return false; + } + + return tf.fieldtype === FieldTypeEnum.Link; + }) as { key: string; index: number; tf: TemplateField }[]; + + const linksNames: Map> = new Map(); + for (const row of this.valueMatrix) { + for (const { tf, index } of tfKeys) { + const target = (tf as TargetField).target; + const value = row[index]?.value; + if (typeof value !== 'string' || !value) { + continue; + } + + if (!linksNames.has(target)) { + linksNames.set(target, new Set()); + } + + linksNames.get(target)?.add(value); + } + } + + const doesNotExist = []; + for (const [target, values] of linksNames.entries()) { + for (const value of values) { + const exists = await this.fyo.db.exists(target, value); + if (exists) { + continue; + } + + doesNotExist.push({ + schemaName: target, + schemaLabel: this.fyo.schemaMap[this.schemaName]?.label, + name: value, + }); + } + } + + return doesNotExist; + } + + populateDocs() { const { dataMap, childTableMap } = this.getDataAndChildTableMapFromValueMatrix(); - const schema = this.fyo.db.schemaMap[this.schemaName]; + const schema = this.fyo.schemaMap[this.schemaName]; const targetFieldnameMap = schema?.fields .filter((f) => f.fieldtype === FieldTypeEnum.Table) .reduce((acc, f) => { @@ -171,7 +233,7 @@ export class Importer { for (const i in this.valueMatrix) { const row = this.valueMatrix[i]; - const name = row[nameIndex].value; + const name = row[nameIndex]?.value; if (typeof name !== 'string') { continue; } @@ -199,7 +261,7 @@ export class Importer { } const childNameIndex = nameIndices[tf.schemaName]; - let childName = row[childNameIndex].value; + let childName = row[childNameIndex]?.value; if (typeof childName !== 'string') { childName = `${tf.schemaName}-${i}`; } @@ -224,17 +286,6 @@ export class Importer { return { dataMap, childTableMap }; } - selectFile(data: string): boolean { - try { - const parsed = parseCSV(data); - this.selectParsed(parsed); - } catch { - return false; - } - - return true; - } - selectParsed(parsed: string[][]): void { if (!parsed?.length) { return; @@ -337,6 +388,11 @@ export class Importer { return vmi; } + if (vmi.rawValue === '') { + vmi.value = null; + return vmi; + } + try { vmi.value = Converter.toDocValue(rawValue, tf, this.fyo); } catch { @@ -437,7 +493,12 @@ function getTemplateFields( } for (const field of schema.fields) { - if (field.computed || field.meta || field.hidden) { + if ( + field.computed || + field.meta || + field.hidden || + (field.readOnly && !field.required) + ) { continue; } @@ -453,12 +514,10 @@ function getTemplateFields( continue; } - if (field.readOnly && field.required) { - field.readOnly = false; - } + const tf = { ...field }; - if (field.readOnly) { - continue; + if (tf.readOnly) { + tf.readOnly = false; } const schemaName = schema.name; @@ -466,7 +525,7 @@ function getTemplateFields( const fieldKey = `${schema.name}.${field.fieldname}`; fields.push({ - ...field, + ...tf, schemaName, schemaLabel, fieldKey, diff --git a/src/pages/ImportWizard.vue b/src/pages/ImportWizard.vue index 5ace8853..0ea74aae 100644 --- a/src/pages/ImportWizard.vue +++ b/src/pages/ImportWizard.vue @@ -676,7 +676,7 @@ export default defineComponent({ return; } - this.importer.pushFromValueMatrixToDocs(); + this.importer.populateDocs(); let doneCount = 0; for (const doc of this.importer.docs) { diff --git a/tests/items.csv b/tests/items.csv new file mode 100644 index 00000000..bd5ff56d --- /dev/null +++ b/tests/items.csv @@ -0,0 +1,13 @@ +"Item Name",Description,"Unit Type",Type,For,"Sales Acc.","Purchase Acc.",Tax,Rate,HSN/SAC,Barcode,"Track Item","Created By","Modified By",Created,Modified +Item.name,Item.description,Item.unit,Item.itemType,Item.for,Item.incomeAccount,Item.expenseAccount,Item.tax,Item.rate,Item.hsnCode,Item.barcode,Item.trackItem,Item.createdBy,Item.modifiedBy,Item.created,Item.modified +"Final Item","A final item made from raw items",Unit,Product,Both,Sales,"Stock Received But Not Billed",,500,0,,1,lin@to.co,lin@to.co,2023-01-31T06:46:00.200Z,2023-01-31T06:46:00.200Z +"Raw Two","Another Raw item used to make a final item",Unit,Product,Both,Sales,"Stock Received But Not Billed",,200,0,,1,lin@to.co,lin@to.co,2023-01-31T06:45:32.449Z,2023-01-31T06:45:32.449Z +"Raw One","A raw item used to make a final item.",Unit,Product,Both,Sales,"Stock Received But Not Billed",,100,0,,1,lin@to.co,lin@to.co,2023-01-31T06:44:58.047Z,2023-01-31T06:44:58.047Z +"Test One",,Unit,Product,Both,Sales,"Stock Received But Not Billed",GST-18,200,0,,1,lin@to.co,lin@to.co,2023-01-09T10:46:02.217Z,2023-01-09T10:46:02.217Z +Stuff,"Some stuff.",Unit,Product,Both,Sales,"Stock Received But Not Billed",GST-18,200,101192,,1,lin@to.co,lin@to.co,2023-01-09T07:14:12.208Z,2023-01-09T07:14:12.208Z +"Something Sellable",,Unit,Product,Sales,Sales,"Cost of Goods Sold",,300,0,,0,lin@to.co,lin@to.co,2022-10-11T09:15:15.724Z,2023-01-16T08:49:49.267Z +Ball,"Just a ball..",Unit,Product,Both,Sales,"Cost of Goods Sold",,30,0,,0,Administrator,Administrator,2022-02-24T04:38:09.181Z,2022-02-24T04:38:09.181Z +Bat,"Regular old bat...",Unit,Product,Both,Sales,"Cost of Goods Sold",,129,0,,0,Administrator,Administrator,2022-02-24T04:38:09.174Z,2022-02-24T04:38:09.174Z +"Holy Icon","The holiest of icons.",Unit,Product,Both,Sales,"Cost of Goods Sold",GST-3,330,0,,0,Administrator,Administrator,2022-02-11T11:32:33.342Z,2022-02-11T11:32:33.342Z +"Flower Pot","Just a flower pot.",Unit,Product,Both,Sales,"Cost of Goods Sold",GST-12,200,,,0,Administrator,Administrator,2021-12-16T07:04:08.233Z,2021-12-16T07:04:08.233Z +Flow,"Used to test the flow of operations.",Unit,Product,Both,Sales,"Cost of Goods Sold",GST-12,100,,,0,Administrator,Administrator,2021-12-16T05:42:02.081Z,2021-12-16T05:48:48.203Z \ No newline at end of file diff --git a/tests/parties.csv b/tests/parties.csv new file mode 100644 index 00000000..b3cae66a --- /dev/null +++ b/tests/parties.csv @@ -0,0 +1,8 @@ +Name,Role,"Default Account","Outstanding Amount",Currency,Email,Phone,Address,"GSTIN No.","GST Registration","Created By","Modified By",Created,Modified +Party.name,Party.role,Party.defaultAccount,Party.outstandingAmount,Party.currency,Party.email,Party.phone,Party.address,Party.gstin,Party.gstType,Party.createdBy,Party.modifiedBy,Party.created,Party.modified +Randoe,Both,,259.6,INR,,,,,Unregistered,lin@to.co,lin@to.co,2023-01-09T04:58:16.050Z,2023-01-09T10:46:46.128Z +Saipan,Customer,Debtors,299.99999999921,USD,sai@pan.co,,,,Unregistered,lin@to.co,lin@to.co,2022-07-18T17:07:35.103Z,2023-01-30T14:36:50.058Z +Lordham,Customer,Debtors,851.8,INR,lo@gamil.com,8989004444,,,Unregistered,Administrator,lin@to.co,2022-02-04T06:35:19.404Z,2022-07-18T17:05:42.976Z +Lyn,Customer,Debtors,100,INR,lyn@to.co,,,,Consumer,Administrator,Administrator,2022-02-04T06:21:19.069Z,2022-02-28T05:18:32.743Z +Bølèn,Customer,Debtors,0,INR,bo@len.co,,,22ABCIK123401Z5,"Registered Regular",Administrator,Administrator,2022-01-12T08:44:58.879Z,2022-01-12T08:45:26.714Z +Bé,Customer,Debtors,46,INR,bey@more.tips,6969969600,,,Consumer,Administrator,Administrator,2021-12-16T11:32:11.595Z,2021-12-16T12:00:28.558Z \ No newline at end of file diff --git a/tests/sales_invoices.csv b/tests/sales_invoices.csv new file mode 100644 index 00000000..fb53678e --- /dev/null +++ b/tests/sales_invoices.csv @@ -0,0 +1,30 @@ +"Invoice No",Date,Party,Account,"Customer Currency","Exchange Rate","Net Total","Grand Total","Base Grand Total","Outstanding Amount","Set Discount Amount","Discount Amount","Discount Percent","Discount After Tax","Entry Currency",Notes,"Stock Not Transferred","Number Series","Created By","Modified By",Created,Modified,Submitted,Cancelled,"Tax Account",Rate,Amount,Item,Description,Quantity,Rate,Account,Tax,Amount,"Set Discount Amount","Discount Amount","Discount Percent",HSN/SAC,"Stock Not Transferred" +SalesInvoice.name,SalesInvoice.date,SalesInvoice.party,SalesInvoice.account,SalesInvoice.currency,SalesInvoice.exchangeRate,SalesInvoice.netTotal,SalesInvoice.grandTotal,SalesInvoice.baseGrandTotal,SalesInvoice.outstandingAmount,SalesInvoice.setDiscountAmount,SalesInvoice.discountAmount,SalesInvoice.discountPercent,SalesInvoice.discountAfterTax,SalesInvoice.entryCurrency,SalesInvoice.terms,SalesInvoice.stockNotTransferred,SalesInvoice.numberSeries,SalesInvoice.createdBy,SalesInvoice.modifiedBy,SalesInvoice.created,SalesInvoice.modified,SalesInvoice.submitted,SalesInvoice.cancelled,TaxSummary.account,TaxSummary.rate,TaxSummary.amount,SalesInvoiceItem.item,SalesInvoiceItem.description,SalesInvoiceItem.quantity,SalesInvoiceItem.rate,SalesInvoiceItem.account,SalesInvoiceItem.tax,SalesInvoiceItem.amount,SalesInvoiceItem.setItemDiscountAmount,SalesInvoiceItem.itemDiscountAmount,SalesInvoiceItem.itemDiscountPercent,SalesInvoiceItem.hsnCode,SalesInvoiceItem.stockNotTransferred +1020,2023-01-31,Lordham,Debtors,INR,1,500,500,500,500,0,0,0,0,Party,,1,SINV-,lin@to.co,lin@to.co,2023-01-31T09:00:45.858Z,2023-01-31T09:00:45.858Z,0,0,,,,"Final Item","A final item made from raw items",1,500,Sales,,500,0,0,0,0,1 +1019,2023-01-09,Randoe,Debtors,INR,1,220,259.6,259.6,259.6,0,0,0,0,Party,,1,SINV-,lin@to.co,lin@to.co,2023-01-09T10:46:40.923Z,2023-01-09T10:46:46.085Z,1,0,CGST,9,19.8,,,,,,,,,,,, +1019,2023-01-09,Randoe,Debtors,INR,1,220,259.6,259.6,259.6,0,0,0,0,Party,,1,SINV-,lin@to.co,lin@to.co,2023-01-09T10:46:40.923Z,2023-01-09T10:46:46.085Z,1,0,SGST,9,19.8,,,,,,,,,,,, +1019,2023-01-09,Randoe,Debtors,INR,1,220,259.6,259.6,259.6,0,0,0,0,Party,,1,SINV-,lin@to.co,lin@to.co,2023-01-09T10:46:40.923Z,2023-01-09T10:46:46.085Z,1,0,,,,"Test One",,1,220,Sales,GST-18,220,0,0,0,0,1 +1018,2022-10-11,Saipan,Debtors,USD,82.47,3.63768643142,3.63768643142,299.99999999921,299.99999999921,0,0,0,0,Party,,0,SINV-,lin@to.co,lin@to.co,2022-10-11T09:16:50.373Z,2023-01-30T14:36:50.021Z,1,0,,,,"Something Sellable",,1,3.63768643142,Sales,,3.63768643142,0,0,0,0,0 +1014,2022-07-11,Lordham,Debtors,INR,1,660,679.8,679.8,679.8,0,0,,0,Party,,,SINV-,lin@to.co,lin@to.co,2022-07-11T11:06:18.788Z,2022-07-18T17:05:42.922Z,1,0,CGST,1.5,9.9,,,,,,,,,,,, +1014,2022-07-11,Lordham,Debtors,INR,1,660,679.8,679.8,679.8,0,0,,0,Party,,,SINV-,lin@to.co,lin@to.co,2022-07-11T11:06:18.788Z,2022-07-18T17:05:42.922Z,1,0,SGST,1.5,9.9,,,,,,,,,,,, +1014,2022-07-11,Lordham,Debtors,INR,1,660,679.8,679.8,679.8,0,0,,0,Party,,,SINV-,lin@to.co,lin@to.co,2022-07-11T11:06:18.788Z,2022-07-18T17:05:42.922Z,1,0,,,,"Holy Icon","The holiest of icons.",2,330,Sales,GST-3,660,0,0,,0, +1011,2022-02-22,Lyn,Debtors,INR,1,1410,1471.2,1471.2,100,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.656Z,2022-02-28T05:18:32.731Z,1,0,SGST,1.5,30.6,,,,,,,,,,,, +1011,2022-02-22,Lyn,Debtors,INR,1,1410,1471.2,1471.2,100,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.656Z,2022-02-28T05:18:32.731Z,1,0,CGST,1.5,30.6,,,,,,,,,,,, +1011,2022-02-22,Lyn,Debtors,INR,1,1410,1471.2,1471.2,100,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.656Z,2022-02-28T05:18:32.731Z,1,0,,,,"Flower Pot","Just a flower pot.",1,210,Sales,GST-12,210,0,0,,, +1011,2022-02-22,Lyn,Debtors,INR,1,1410,1471.2,1471.2,100,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.656Z,2022-02-28T05:18:32.731Z,1,0,,,,"Holy Icon","The holiest of icons.",3,400,Sales,GST-3,1200,0,0,,0, +1012,2022-02-20,Lordham,Debtors,INR,1,2230,2353.6,2353.6,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.685Z,2022-02-28T05:18:11.657Z,1,0,SGST,1.5,61.8,,,,,,,,,,,, +1012,2022-02-20,Lordham,Debtors,INR,1,2230,2353.6,2353.6,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.685Z,2022-02-28T05:18:11.657Z,1,0,CGST,1.5,61.8,,,,,,,,,,,, +1012,2022-02-20,Lordham,Debtors,INR,1,2230,2353.6,2353.6,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.685Z,2022-02-28T05:18:11.657Z,1,0,,,,"Flower Pot","Just a flower pot.",3,210,Sales,GST-12,630,0,0,,, +1012,2022-02-20,Lordham,Debtors,INR,1,2230,2353.6,2353.6,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-24T05:07:23.685Z,2022-02-28T05:18:11.657Z,1,0,,,,"Holy Icon","The holiest of icons.",4,400,Sales,GST-3,1600,0,0,,0, +1008,2022-02-11,Lordham,Debtors,INR,1,600,672,672,172,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-11T08:40:28.193Z,2022-02-11T09:48:18.274Z,1,0,SGST,6,36,,,,,,,,,,,, +1008,2022-02-11,Lordham,Debtors,INR,1,600,672,672,172,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-11T08:40:28.193Z,2022-02-11T09:48:18.274Z,1,0,CGST,6,36,,,,,,,,,,,, +1008,2022-02-11,Lordham,Debtors,INR,1,600,672,672,172,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-11T08:40:28.193Z,2022-02-11T09:48:18.274Z,1,0,,,,"Flower Pot","Just a flower pot.",3,200,Sales,GST-12,600,0,0,,, +1007,2022-02-04,Lyn,Debtors,INR,1,200,224,224,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-04T06:21:24.058Z,2022-02-04T06:21:46.308Z,1,0,SGST,6,12,,,,,,,,,,,, +1007,2022-02-04,Lyn,Debtors,INR,1,200,224,224,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-04T06:21:24.058Z,2022-02-04T06:21:46.308Z,1,0,CGST,6,12,,,,,,,,,,,, +1007,2022-02-04,Lyn,Debtors,INR,1,200,224,224,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-02-04T06:21:24.058Z,2022-02-04T06:21:46.308Z,1,0,,,,"Flower Pot","Just a flower pot.",1,200,Sales,GST-12,200,0,0,,, +1004,2022-01-12,Bølèn,Debtors,INR,1,1800,2016,2016,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-01-12T08:45:19.774Z,2022-01-12T08:45:26.702Z,1,0,SGST,6,108,,,,,,,,,,,, +1004,2022-01-12,Bølèn,Debtors,INR,1,1800,2016,2016,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-01-12T08:45:19.774Z,2022-01-12T08:45:26.702Z,1,0,CGST,6,108,,,,,,,,,,,, +1004,2022-01-12,Bølèn,Debtors,INR,1,1800,2016,2016,0,0,0,,0,Party,,,SINV-,Administrator,Administrator,2022-01-12T08:45:19.774Z,2022-01-12T08:45:26.702Z,1,0,,,,"Flower Pot","Just a flower pot.",9,200,Sales,GST-12,1800,0,0,,, +1001,2021-12-16,Bé,Debtors,INR,1,100,112,112,46,0,0,,0,Party,,,SINV-,Administrator,Administrator,2021-12-16T11:34:06.174Z,2021-12-16T12:00:28.526Z,1,0,SGST,6,6,,,,,,,,,,,, +1001,2021-12-16,Bé,Debtors,INR,1,100,112,112,46,0,0,,0,Party,,,SINV-,Administrator,Administrator,2021-12-16T11:34:06.174Z,2021-12-16T12:00:28.526Z,1,0,CGST,6,6,,,,,,,,,,,, +1001,2021-12-16,Bé,Debtors,INR,1,100,112,112,46,0,0,,0,Party,,,SINV-,Administrator,Administrator,2021-12-16T11:34:06.174Z,2021-12-16T12:00:28.526Z,1,0,,,,Flow,"Used to test the flow of operations.",1,100,Sales,GST-12,100,0,0,,, \ No newline at end of file diff --git a/tests/testImporter.spec.ts b/tests/testImporter.spec.ts index 6facce58..fc26a734 100644 --- a/tests/testImporter.spec.ts +++ b/tests/testImporter.spec.ts @@ -1,4 +1,7 @@ +import { assertDoesNotThrow } from 'backend/database/tests/helpers'; +import { readFileSync } from 'fs'; import { ModelNameEnum } from 'models/types'; +import { join } from 'path'; import { Importer } from 'src/importer'; import test from 'tape'; import { closeTestFyo, getTestFyo, setupTestFyo } from './helpers'; @@ -6,9 +9,59 @@ import { closeTestFyo, getTestFyo, setupTestFyo } from './helpers'; const fyo = getTestFyo(); setupTestFyo(fyo, __filename); -test('importer', async (t) => { +test('importer init', (t) => { const importer = new Importer(ModelNameEnum.SalesInvoice, fyo); - t.ok(importer.getCSVTemplate()); + t.equal( + typeof importer.getCSVTemplate(), + 'string', + 'csv template is a string' + ); + t.end(); +}); + +test('import Items', async (t) => { + const importer = new Importer(ModelNameEnum.Item, fyo); + const csvPath = join(__dirname, 'items.csv'); + const data = readFileSync(csvPath, { encoding: 'utf-8' }); + t.equal(importer.selectFile(data), true, 'file selection'); + t.equal((await importer.checkLinks()).length, 0, 'all links exist'); + t.doesNotThrow(() => importer.populateDocs(), 'populating docs'); + for (const doc of importer.docs) { + await assertDoesNotThrow(async () => await doc.sync()); + } +}); + +test('import Party', async (t) => { + const importer = new Importer(ModelNameEnum.Party, fyo); + const csvPath = join(__dirname, 'parties.csv'); + const data = readFileSync(csvPath, { encoding: 'utf-8' }); + t.equal(importer.selectFile(data), true, 'file selection'); + t.equal((await importer.checkLinks()).length, 0, 'all links exist'); + t.doesNotThrow(() => importer.populateDocs(), 'populating docs'); + for (const doc of importer.docs) { + await assertDoesNotThrow(async () => await doc.sync()); + } +}); + +test('import SalesInvoices', async (t) => { + const importer = new Importer(ModelNameEnum.SalesInvoice, fyo); + const csvPath = join(__dirname, 'sales_invoices.csv'); + const data = readFileSync(csvPath, { encoding: 'utf-8' }); + + t.equal(importer.selectFile(data), true, 'file selection'); + t.equal((await importer.checkLinks()).length, 0, 'all links exist'); + t.doesNotThrow(() => importer.populateDocs(), 'populating docs'); + + const names = []; + for (const doc of importer.docs.slice(0, 2)) { + await assertDoesNotThrow(async () => await doc.sync()); + names.push(doc.name); + } + + t.ok( + names.every((n) => n?.startsWith('SINV-')), + 'numberSeries assigned' + ); }); closeTestFyo(fyo, __filename);