feat: exports

src/components/ExportWizard.vue

@@ -7,7 +7,7 @@
     <!-- Export Config -->
     <div class="grid grid-cols-3 p-4 gap-4">
       <Check
-        v-if="configFields.useListFilters"
+        v-if="configFields.useListFilters && Object.keys(listFilters).length"
         :df="configFields.useListFilters"
         :space-between="true"
         :show-label="true"
@@ -23,72 +23,54 @@
         :border="true"
         @change="(value: ExportFormat) => (exportFormat = value)"
       />
+      <Int
+        v-if="configFields.limit"
+        :df="configFields.limit"
+        :value="limit"
+        :border="true"
+        @change="(value: number) => (limit = value)"
+      />
     </div>
     <hr />
 
     <!-- Fields Selection -->
-    <div>
+    <div class="max-h-80 overflow-auto custom-scroll">
-      <!-- Field Selection Header -->
+      <!-- Main Fields -->
-      <button
+      <div class="p-4">
-        class="flex justify-between items-center text-gray-600 p-4 w-full"
+        <h2 class="text-sm font-semibold text-gray-800">
-        @click="showFieldSelection = !showFieldSelection"
+          {{ fyo.schemaMap[schemaName]?.label ?? schemaName }}
-      >
+        </h2>
-        <p class="text-sm">
+        <div class="grid grid-cols-3 border rounded mt-1">
-          {{ t`${numSelected} fields selected` }}
+          <Check
-        </p>
+            v-for="ef of fields"
-        <feather-icon
+            :label-class="
-          :name="showFieldSelection ? 'chevron-down' : 'chevron-up'"
+              ef.fieldtype === 'Table'
-          class="w-4 h-4"
+                ? 'text-sm text-gray-600 font-semibold'
-        />
+                : 'text-sm text-gray-600'
-      </button>
+            "
-
+            :key="ef.fieldname"
-      <!-- Field Selection Body -->
+            :df="getField(ef)"
-      <hr v-if="showFieldSelection" />
+            :show-label="true"
-      <div
+            :value="ef.export"
-        v-if="showFieldSelection"
+            @change="(value: boolean) => setExportFieldValue(ef, value)"
-        class="max-h-96 overflow-auto custom-scroll"
+          />
-      >
-        <!-- Main Fields -->
-        <div class="p-4">
-          <h2 class="text-sm font-semibold text-gray-800">
-            {{ fyo.schemaMap[schemaName]?.label ?? schemaName }}
-          </h2>
-          <div class="grid grid-cols-3 border rounded-md mt-1">
-            <Check
-              v-for="ef of fields"
-              :label-class="
-                ef.fieldtype === 'Table'
-                  ? 'text-sm text-gray-600 font-semibold'
-                  : 'text-sm text-gray-600'
-              "
-              :key="ef.fieldname"
-              :df="getField(ef)"
-              :show-label="true"
-              :value="ef.export"
-              @change="(value: boolean) => setExportFieldValue(ef, value)"
-            />
-          </div>
         </div>
+      </div>
 
-        <!-- Table Fields -->
+      <!-- Table Fields -->
-        <div
+      <div class="p-4" v-for="efs of filteredTableFields" :key="efs.fieldname">
-          class="p-4"
+        <h2 class="text-sm font-semibold text-gray-800">
-          v-for="efs of filteredTableFields"
+          {{ fyo.schemaMap[efs.target]?.label ?? schemaName }}
-          :key="efs.fieldname"
+        </h2>
-        >
+        <div class="grid grid-cols-3 border rounded mt-1">
-          <h2 class="text-sm font-semibold text-gray-800">
+          <Check
-            {{ fyo.schemaMap[efs.target]?.label ?? schemaName }}
+            v-for="ef of efs.fields"
-          </h2>
+            :key="ef.fieldname"
-          <div class="grid grid-cols-3 border rounded-md mt-1">
+            :df="getField(ef)"
-            <Check
+            :show-label="true"
-              v-for="ef of efs.fields"
+            :value="ef.export"
-              :key="ef.fieldname"
+            @change="(value: boolean) => setExportFieldValue(ef, value, efs.target)"
-              :df="getField(ef)"
+          />
-              :show-label="true"
-              :value="ef.export"
-              @change="(value: boolean) => setExportFieldValue(ef, value, efs.target)"
-            />
-          </div>
         </div>
       </div>
     </div>
@@ -96,68 +78,59 @@
     <!-- Export Button -->
     <hr />
     <div class="p-4 flex justify-between items-center">
-      <p class="text-gray-600 text-sm">{{ t`${numEntries} entries` }}</p>
+      <p class="text-sm text-gray-600">
+        {{ t`${numSelected} fields selected` }}
+      </p>
       <Button type="primary" @click="exportData">{{ t`Export` }}</Button>
     </div>
   </div>
 </template>
-
 <script lang="ts">
 import { t } from 'fyo';
-import { Field, FieldTypeEnum, TargetField } from 'schemas/types';
+import { Field, FieldTypeEnum } from 'schemas/types';
 import { fyo } from 'src/initFyo';
-import { defineComponent } from 'vue';
+import {
+getCsvExportData,
+getExportFields,
+getExportTableFields,
+getJsonExportData
+} from 'src/utils/export';
+import { getSavePath, saveData, showExportInFolder } from 'src/utils/ipcCalls';
+import { ExportField, ExportFormat, ExportTableField } from 'src/utils/types';
+import { QueryFilter } from 'utils/db/types';
+import { defineComponent, PropType } from 'vue';
 import Button from './Button.vue';
 import Check from './Controls/Check.vue';
+import Int from './Controls/Int.vue';
 import Select from './Controls/Select.vue';
 import FormHeader from './FormHeader.vue';
 
-interface ExportField {
-  fieldname: string;
-  fieldtype: FieldTypeEnum;
-  label: string;
-  export: boolean;
-}
-
-interface ExportTableField {
-  fieldname: string;
-  label: string;
-  target: string;
-  fields: ExportField[];
-}
-
-type ExportFormat = 'csv' | 'json';
 interface ExportWizardData {
-  numEntries: number;
   useListFilters: boolean;
   exportFormat: ExportFormat;
-  showFieldSelection: boolean;
   fields: ExportField[];
+  limit: number | null;
   tableFields: ExportTableField[];
+  numUnfilteredEntries: number;
 }
 
-const excludedFieldTypes = [
-  FieldTypeEnum.AttachImage,
-  FieldTypeEnum.Attachment,
-];
-
 export default defineComponent({
   props: {
     schemaName: { type: String, required: true },
+    listFilters: { type: Object as PropType<QueryFilter>, default: () => {} },
     pageTitle: String,
   },
   data() {
     const fields = fyo.schemaMap[this.schemaName]?.fields ?? [];
     const exportFields = getExportFields(fields);
-    const exportTableFields = getExportTableFields(fields);
+    const exportTableFields = getExportTableFields(fields, fyo);
 
     return {
-      numEntries: 0,
+      limit: null,
       useListFilters: true,
       exportFormat: 'csv',
       fields: exportFields,
       tableFields: exportTableFields,
-      showFieldSelection: !false,
     } as ExportWizardData;
   },
   methods: {
@@ -194,8 +167,51 @@ export default defineComponent({
 
       field.export = value;
     },
-    exportData() {
+    async exportData() {
-      console.log('export clicked');
+      const filters = JSON.parse(
+        JSON.stringify(this.useListFilters ? this.listFilters : {})
+      );
+
+      let data: string;
+      if (this.exportFormat === 'json') {
+        data = await getJsonExportData(
+          this.schemaName,
+          this.fields,
+          this.tableFields,
+          this.limit,
+          filters,
+          fyo
+        );
+      } else {
+        data = await getCsvExportData(
+          this.schemaName,
+          this.fields,
+          this.tableFields,
+          this.limit,
+          filters,
+          fyo
+        );
+      }
+
+      await this.saveExportData(data);
+    },
+    async saveExportData(data: string) {
+      const fileName = this.getFileName();
+      const { canceled, filePath } = await getSavePath(
+        fileName,
+        this.exportFormat
+      );
+      if (canceled || !filePath) {
+        return;
+      }
+
+      await saveData(data, filePath);
+      showExportInFolder(fyo.t`Export Successful`, filePath);
+    },
+    getFileName() {
+      const fileName = this.label.toLowerCase().replace(/\s/g, '-');
+      const dateString = new Date().toISOString().split('T')[0];
+      return `${fileName}_${dateString}`;
     },
   },
   computed: {
@@ -230,6 +246,12 @@ export default defineComponent({
           label: t`Use List Filters`,
           fieldname: 'useListFilters',
         },
+        limit: {
+          placeholder: 'Limit number of rows',
+          fieldtype: 'Int',
+          label: t`Limit`,
+          fieldname: 'limit',
+        },
         exportFormat: {
           fieldtype: 'Select',
           label: t`Export Format`,
@@ -242,39 +264,6 @@ export default defineComponent({
       };
     },
   },
-  components: { FormHeader, Check, Select, Button },
+  components: { FormHeader, Check, Select, Button, Int },
 });
-
-function getExportFields(fields: Field[]): ExportField[] {
-  return fields
-    .filter((f) => !f.computed && f.label)
-    .map((field) => {
-      const { fieldname, label } = field;
-      const fieldtype = field.fieldtype as FieldTypeEnum;
-      return {
-        fieldname,
-        fieldtype,
-        label,
-        export: !excludedFieldTypes.includes(fieldtype),
-      };
-    });
-}
-
-function getExportTableFields(fields: Field[]): ExportTableField[] {
-  return fields
-    .filter((f) => f.fieldtype === FieldTypeEnum.Table)
-    .map((f) => {
-      const target = (f as TargetField).target;
-      const tableFields = fyo.schemaMap[target]?.fields ?? [];
-      const exportTableFields = getExportFields(tableFields);
-
-      return {
-        fieldname: f.fieldname,
-        label: f.label,
-        target,
-        fields: exportTableFields,
-      };
-    })
-    .filter((f) => !!f.fields.length);
-}
 </script>

src/components/Toast.vue

@@ -9,7 +9,7 @@
       items-center
       mb-3
       w-96
-      z-10
+      z-30
       bg-white
       rounded-lg
     "

src/pages/ListView/List.vue

@@ -101,7 +101,7 @@ import ListCell from './ListCell';
 export default defineComponent({
   name: 'List',
   props: { listConfig: Object, filters: Object, schemaName: String },
-  emits: ['makeNewDoc'],
+  emits: ['makeNewDoc', 'updatedData'],
   components: {
     Row,
     ListCell,
@@ -205,6 +205,7 @@ export default defineComponent({
           orderBy,
         })
       ).map((d) => ({ ...d, schema: fyo.schemaMap[this.schemaName] }));
+      this.$emit('updatedData', filters);
     },
   },
 });

src/pages/ListView/ListView.vue

@@ -25,10 +25,15 @@
       :listConfig="listConfig"
       :filters="filters"
       class="flex-1 flex h-full"
+      @updatedData="updatedData"
       @makeNewDoc="makeNewDoc"
     />
     <Modal :open-modal="openExportModal" @closemodal="openExportModal = false">
-      <ExportWizard :schema-name="schemaName" :title="pageTitle" />
+      <ExportWizard
+        :schema-name="schemaName"
+        :title="pageTitle"
+        :list-filters="listFilters"
+      />
     </Modal>
   </div>
 </template>
@@ -59,7 +64,11 @@ export default {
     ExportWizard,
   },
   data() {
-    return { listConfig: undefined, openExportModal: !false };
+    return {
+      listConfig: undefined,
+      openExportModal: false,
+      listFilters: {},
+    };
   },
   async activated() {
     if (typeof this.filters === 'object') {
@@ -73,6 +82,9 @@ export default {
     docsPath.value = '';
   },
   methods: {
+    updatedData(listFilters) {
+      this.listFilters = listFilters;
+    },
     async makeNewDoc() {
       const doc = await fyo.doc.getNewDoc(this.schemaName, this.filters ?? {});
       const path = this.getFormPath(doc.name);

src/utils/export.ts

@@ -0,0 +1,334 @@
+import { Fyo } from 'fyo';
+import { RawValueMap } from 'fyo/core/types';
+import { Field, FieldTypeEnum, RawValue, TargetField } from 'schemas/types';
+import { generateCSV } from 'utils/csvParser';
+import { GetAllOptions, QueryFilter } from 'utils/db/types';
+import { getMapFromList } from 'utils/index';
+import { ExportField, ExportTableField } from './types';
+
+const excludedFieldTypes = [
+  FieldTypeEnum.AttachImage,
+  FieldTypeEnum.Attachment,
+];
+
+interface CsvHeader {
+  label: string;
+  schemaName: string;
+  fieldname: string;
+  parentFieldname?: string;
+}
+
+export function getExportFields(
+  fields: Field[],
+  exclude: string[] = []
+): ExportField[] {
+  return fields
+    .filter((f) => !f.computed && f.label && !exclude.includes(f.fieldname))
+    .map((field) => {
+      const { fieldname, label } = field;
+      const fieldtype = field.fieldtype as FieldTypeEnum;
+      return {
+        fieldname,
+        fieldtype,
+        label,
+        export: !excludedFieldTypes.includes(fieldtype),
+      };
+    });
+}
+
+export function getExportTableFields(
+  fields: Field[],
+  fyo: Fyo
+): ExportTableField[] {
+  return fields
+    .filter((f) => f.fieldtype === FieldTypeEnum.Table)
+    .map((f) => {
+      const target = (f as TargetField).target;
+      const tableFields = fyo.schemaMap[target]?.fields ?? [];
+      const exportTableFields = getExportFields(tableFields, ['name']);
+
+      return {
+        fieldname: f.fieldname,
+        label: f.label,
+        target,
+        fields: exportTableFields,
+      };
+    })
+    .filter((f) => !!f.fields.length);
+}
+
+export async function getJsonExportData(
+  schemaName: string,
+  fields: ExportField[],
+  tableFields: ExportTableField[],
+  limit: number | null,
+  filters: QueryFilter,
+  fyo: Fyo
+): Promise<string> {
+  const data = await getExportData(
+    schemaName,
+    fields,
+    tableFields,
+    limit,
+    filters,
+    fyo
+  );
+  convertParentDataToJsonExport(data.parentData, data.childTableData);
+  return JSON.stringify(data.parentData);
+}
+
+export async function getCsvExportData(
+  schemaName: string,
+  fields: ExportField[],
+  tableFields: ExportTableField[],
+  limit: number | null,
+  filters: QueryFilter,
+  fyo: Fyo
+): Promise<string> {
+  const { childTableData, parentData } = await getExportData(
+    schemaName,
+    fields,
+    tableFields,
+    limit,
+    filters,
+    fyo
+  );
+  /**
+   * parentNameMap: Record<ParentName, Record<ParentFieldName, Rows[]>>
+   */
+  const parentNameMap = getParentNameMap(childTableData);
+  const headers = getCsvHeaders(schemaName, fields, tableFields);
+
+  const rows: RawValue[][] = [];
+  for (const parentRow of parentData) {
+    const parentName = parentRow.name as string;
+    if (!parentName) {
+      continue;
+    }
+
+    const baseRowData = headers.parent.map(
+      (f) => (parentRow[f.fieldname] as RawValue) ?? ''
+    );
+
+    const tableFieldRowMap = parentNameMap[parentName];
+    if (!tableFieldRowMap || !Object.keys(tableFieldRowMap ?? {}).length) {
+      rows.push([baseRowData, headers.child.map((_) => '')].flat());
+      continue;
+    }
+
+    for (const tableFieldName in tableFieldRowMap) {
+      const tableRows = tableFieldRowMap[tableFieldName] ?? [];
+
+      for (const tableRow of tableRows) {
+        const tableRowData = headers.child.map((f) => {
+          if (f.parentFieldname !== tableFieldName) {
+            return '';
+          }
+
+          return (tableRow[f.fieldname] as RawValue) ?? '';
+        });
+
+        rows.push([baseRowData, tableRowData].flat());
+      }
+    }
+  }
+
+  const flatHeaders = [headers.parent, headers.child].flat();
+  const labels = flatHeaders.map((f) => f.label);
+  const keys = flatHeaders.map((f) => `${f.schemaName}.${f.fieldname}`);
+
+  rows.unshift(keys);
+  rows.unshift(labels);
+
+  return generateCSV(rows);
+}
+
+function getCsvHeaders(
+  schemaName: string,
+  fields: ExportField[],
+  tableFields: ExportTableField[]
+) {
+  const headers = {
+    parent: [] as CsvHeader[],
+    child: [] as CsvHeader[],
+  };
+  for (const { label, fieldname, fieldtype, export: shouldExport } of fields) {
+    if (!shouldExport || fieldtype === FieldTypeEnum.Table) {
+      continue;
+    }
+
+    headers.parent.push({ schemaName, label, fieldname });
+  }
+
+  for (const tf of tableFields) {
+    if (!fields.find((f) => f.fieldname === tf.fieldname)?.export) {
+      continue;
+    }
+
+    for (const field of tf.fields) {
+      if (!field.export) {
+        continue;
+      }
+
+      headers.child.push({
+        schemaName: tf.target,
+        label: field.label,
+        fieldname: field.fieldname,
+        parentFieldname: tf.fieldname,
+      });
+    }
+  }
+
+  return headers;
+}
+
+function getParentNameMap(childTableData: Record<string, RawValueMap[]>) {
+  const parentNameMap: Record<string, Record<string, RawValueMap[]>> = {};
+  for (const key in childTableData) {
+    for (const row of childTableData[key]) {
+      const parent = row.parent as string;
+      if (!parent) {
+        continue;
+      }
+
+      parentNameMap[parent] ??= {};
+      parentNameMap[parent][key] ??= [];
+      parentNameMap[parent][key].push(row);
+    }
+  }
+  return parentNameMap;
+}
+
+async function getExportData(
+  schemaName: string,
+  fields: ExportField[],
+  tableFields: ExportTableField[],
+  limit: number | null,
+  filters: QueryFilter,
+  fyo: Fyo
+) {
+  const parentData = await getParentData(
+    schemaName,
+    filters,
+    fields,
+    limit,
+    fyo
+  );
+  const parentNames = parentData.map((f) => f.name as string).filter(Boolean);
+  const childTableData = await getAllChildTableData(
+    tableFields,
+    fields,
+    parentNames,
+    fyo
+  );
+  return { parentData, childTableData };
+}
+
+function convertParentDataToJsonExport(
+  parentData: RawValueMap[],
+  childTableData: Record<string, RawValueMap[]>
+) {
+  /**
+   * Map from List does not create copies. Map is a
+   * map of references, hence parentData is altered.
+   */
+
+  const nameMap = getMapFromList(parentData, 'name');
+  for (const fieldname in childTableData) {
+    const data = childTableData[fieldname];
+
+    for (const row of data) {
+      const parent = row.parent as string | undefined;
+      if (!parent || !nameMap?.[parent]) {
+        continue;
+      }
+
+      nameMap[parent][fieldname] ??= [];
+
+      delete row.parent;
+      delete row.name;
+
+      (nameMap[parent][fieldname] as RawValueMap[]).push(row);
+    }
+  }
+}
+
+async function getParentData(
+  schemaName: string,
+  filters: QueryFilter,
+  fields: ExportField[],
+  limit: number | null,
+  fyo: Fyo
+) {
+  const orderBy = !!fields.find((f) => f.fieldname === 'date')
+    ? 'date'
+    : 'created';
+  const options: GetAllOptions = { filters, orderBy, order: 'desc' };
+  if (limit) {
+    options.limit = limit;
+  }
+
+  options.fields = fields
+    .filter((f) => f.export && f.fieldtype !== FieldTypeEnum.Table)
+    .map((f) => f.fieldname);
+  if (!options.fields.includes('name')) {
+    options.fields.unshift('name');
+  }
+  const data = await fyo.db.getAllRaw(schemaName, options);
+  convertRawPesaToFloat(data, fields);
+  return data;
+}
+
+async function getAllChildTableData(
+  tableFields: ExportTableField[],
+  parentFields: ExportField[],
+  parentNames: string[],
+  fyo: Fyo
+) {
+  const childTables: Record<string, RawValueMap[]> = {};
+
+  // Getting Child Row data
+  for (const tf of tableFields) {
+    const f = parentFields.find((f) => f.fieldname === tf.fieldname);
+    if (!f?.export) {
+      continue;
+    }
+
+    childTables[tf.fieldname] = await getChildTableData(tf, parentNames, fyo);
+  }
+
+  return childTables;
+}
+
+async function getChildTableData(
+  exportTableField: ExportTableField,
+  parentNames: string[],
+  fyo: Fyo
+) {
+  const exportTableFields = exportTableField.fields
+    .filter((f) => f.export && f.fieldtype !== FieldTypeEnum.Table)
+    .map((f) => f.fieldname);
+  if (!exportTableFields.includes('parent')) {
+    exportTableFields.unshift('parent');
+  }
+
+  const data = await fyo.db.getAllRaw(exportTableField.target, {
+    orderBy: 'idx',
+    fields: exportTableFields,
+    filters: { parent: ['in', parentNames] },
+  });
+  convertRawPesaToFloat(data, exportTableField.fields);
+  return data;
+}
+
+function convertRawPesaToFloat(data: RawValueMap[], fields: Field[]) {
+  const currencyFields = fields.filter(
+    (f) => f.fieldtype === FieldTypeEnum.Currency
+  );
+
+  for (const row of data) {
+    for (const { fieldname } of currencyFields) {
+      row[fieldname] = parseFloat((row[fieldname] ?? '0') as string);
+    }
+  }
+}

src/utils/types.ts

@@ -1,3 +1,5 @@
+import { FieldTypeEnum } from "schemas/types";
+
 export interface MessageDialogButton {
   label: string;
   action: () => Promise<unknown> | unknown;
@@ -47,3 +49,20 @@ export interface SidebarItem {
   schemaName?: string;
   hidden?: () => boolean;
 }
+
+
+export interface ExportField {
+  fieldname: string;
+  fieldtype: FieldTypeEnum;
+  label: string;
+  export: boolean;
+}
+
+export interface ExportTableField {
+  fieldname: string;
+  label: string;
+  target: string;
+  fields: ExportField[];
+}
+
+export type ExportFormat = 'csv' | 'json';

utils/index.ts

@@ -37,6 +37,10 @@ export function getMapFromList<T, K extends keyof T>(
   list: T[],
   name: K
 ): Record<string, T> {
+  /**
+   * Do not convert function to use copies of T
+   * instead of references.
+   */
   const acc: Record<string, T> = {};
   for (const t of list) {
     const key = t[name];