2
0
mirror of https://github.com/frappe/books.git synced 2024-11-09 23:30:56 +00:00

feat: exports

This commit is contained in:
18alantom 2022-10-17 22:24:33 +05:30
parent fb2b026e96
commit 90c8516e62
7 changed files with 489 additions and 130 deletions

View File

@ -7,7 +7,7 @@
<!-- Export Config -->
<div class="grid grid-cols-3 p-4 gap-4">
<Check
v-if="configFields.useListFilters"
v-if="configFields.useListFilters && Object.keys(listFilters).length"
:df="configFields.useListFilters"
:space-between="true"
:show-label="true"
@ -23,72 +23,54 @@
:border="true"
@change="(value: ExportFormat) => (exportFormat = value)"
/>
<Int
v-if="configFields.limit"
:df="configFields.limit"
:value="limit"
:border="true"
@change="(value: number) => (limit = value)"
/>
</div>
<hr />
<!-- Fields Selection -->
<div>
<!-- Field Selection Header -->
<button
class="flex justify-between items-center text-gray-600 p-4 w-full"
@click="showFieldSelection = !showFieldSelection"
>
<p class="text-sm">
{{ t`${numSelected} fields selected` }}
</p>
<feather-icon
:name="showFieldSelection ? 'chevron-down' : 'chevron-up'"
class="w-4 h-4"
/>
</button>
<!-- Field Selection Body -->
<hr v-if="showFieldSelection" />
<div
v-if="showFieldSelection"
class="max-h-96 overflow-auto custom-scroll"
>
<!-- Main Fields -->
<div class="p-4">
<h2 class="text-sm font-semibold text-gray-800">
{{ fyo.schemaMap[schemaName]?.label ?? schemaName }}
</h2>
<div class="grid grid-cols-3 border rounded-md mt-1">
<Check
v-for="ef of fields"
:label-class="
ef.fieldtype === 'Table'
? 'text-sm text-gray-600 font-semibold'
: 'text-sm text-gray-600'
"
:key="ef.fieldname"
:df="getField(ef)"
:show-label="true"
:value="ef.export"
@change="(value: boolean) => setExportFieldValue(ef, value)"
/>
</div>
<div class="max-h-80 overflow-auto custom-scroll">
<!-- Main Fields -->
<div class="p-4">
<h2 class="text-sm font-semibold text-gray-800">
{{ fyo.schemaMap[schemaName]?.label ?? schemaName }}
</h2>
<div class="grid grid-cols-3 border rounded mt-1">
<Check
v-for="ef of fields"
:label-class="
ef.fieldtype === 'Table'
? 'text-sm text-gray-600 font-semibold'
: 'text-sm text-gray-600'
"
:key="ef.fieldname"
:df="getField(ef)"
:show-label="true"
:value="ef.export"
@change="(value: boolean) => setExportFieldValue(ef, value)"
/>
</div>
</div>
<!-- Table Fields -->
<div
class="p-4"
v-for="efs of filteredTableFields"
:key="efs.fieldname"
>
<h2 class="text-sm font-semibold text-gray-800">
{{ fyo.schemaMap[efs.target]?.label ?? schemaName }}
</h2>
<div class="grid grid-cols-3 border rounded-md mt-1">
<Check
v-for="ef of efs.fields"
:key="ef.fieldname"
:df="getField(ef)"
:show-label="true"
:value="ef.export"
@change="(value: boolean) => setExportFieldValue(ef, value, efs.target)"
/>
</div>
<!-- Table Fields -->
<div class="p-4" v-for="efs of filteredTableFields" :key="efs.fieldname">
<h2 class="text-sm font-semibold text-gray-800">
{{ fyo.schemaMap[efs.target]?.label ?? schemaName }}
</h2>
<div class="grid grid-cols-3 border rounded mt-1">
<Check
v-for="ef of efs.fields"
:key="ef.fieldname"
:df="getField(ef)"
:show-label="true"
:value="ef.export"
@change="(value: boolean) => setExportFieldValue(ef, value, efs.target)"
/>
</div>
</div>
</div>
@ -96,68 +78,59 @@
<!-- Export Button -->
<hr />
<div class="p-4 flex justify-between items-center">
<p class="text-gray-600 text-sm">{{ t`${numEntries} entries` }}</p>
<p class="text-sm text-gray-600">
{{ t`${numSelected} fields selected` }}
</p>
<Button type="primary" @click="exportData">{{ t`Export` }}</Button>
</div>
</div>
</template>
<script lang="ts">
import { t } from 'fyo';
import { Field, FieldTypeEnum, TargetField } from 'schemas/types';
import { Field, FieldTypeEnum } from 'schemas/types';
import { fyo } from 'src/initFyo';
import { defineComponent } from 'vue';
import {
getCsvExportData,
getExportFields,
getExportTableFields,
getJsonExportData
} from 'src/utils/export';
import { getSavePath, saveData, showExportInFolder } from 'src/utils/ipcCalls';
import { ExportField, ExportFormat, ExportTableField } from 'src/utils/types';
import { QueryFilter } from 'utils/db/types';
import { defineComponent, PropType } from 'vue';
import Button from './Button.vue';
import Check from './Controls/Check.vue';
import Int from './Controls/Int.vue';
import Select from './Controls/Select.vue';
import FormHeader from './FormHeader.vue';
interface ExportField {
fieldname: string;
fieldtype: FieldTypeEnum;
label: string;
export: boolean;
}
interface ExportTableField {
fieldname: string;
label: string;
target: string;
fields: ExportField[];
}
type ExportFormat = 'csv' | 'json';
interface ExportWizardData {
numEntries: number;
useListFilters: boolean;
exportFormat: ExportFormat;
showFieldSelection: boolean;
fields: ExportField[];
limit: number | null;
tableFields: ExportTableField[];
numUnfilteredEntries: number;
}
const excludedFieldTypes = [
FieldTypeEnum.AttachImage,
FieldTypeEnum.Attachment,
];
export default defineComponent({
props: {
schemaName: { type: String, required: true },
listFilters: { type: Object as PropType<QueryFilter>, default: () => {} },
pageTitle: String,
},
data() {
const fields = fyo.schemaMap[this.schemaName]?.fields ?? [];
const exportFields = getExportFields(fields);
const exportTableFields = getExportTableFields(fields);
const exportTableFields = getExportTableFields(fields, fyo);
return {
numEntries: 0,
limit: null,
useListFilters: true,
exportFormat: 'csv',
fields: exportFields,
tableFields: exportTableFields,
showFieldSelection: !false,
} as ExportWizardData;
},
methods: {
@ -194,8 +167,51 @@ export default defineComponent({
field.export = value;
},
exportData() {
console.log('export clicked');
async exportData() {
const filters = JSON.parse(
JSON.stringify(this.useListFilters ? this.listFilters : {})
);
let data: string;
if (this.exportFormat === 'json') {
data = await getJsonExportData(
this.schemaName,
this.fields,
this.tableFields,
this.limit,
filters,
fyo
);
} else {
data = await getCsvExportData(
this.schemaName,
this.fields,
this.tableFields,
this.limit,
filters,
fyo
);
}
await this.saveExportData(data);
},
async saveExportData(data: string) {
const fileName = this.getFileName();
const { canceled, filePath } = await getSavePath(
fileName,
this.exportFormat
);
if (canceled || !filePath) {
return;
}
await saveData(data, filePath);
showExportInFolder(fyo.t`Export Successful`, filePath);
},
getFileName() {
const fileName = this.label.toLowerCase().replace(/\s/g, '-');
const dateString = new Date().toISOString().split('T')[0];
return `${fileName}_${dateString}`;
},
},
computed: {
@ -230,6 +246,12 @@ export default defineComponent({
label: t`Use List Filters`,
fieldname: 'useListFilters',
},
limit: {
placeholder: 'Limit number of rows',
fieldtype: 'Int',
label: t`Limit`,
fieldname: 'limit',
},
exportFormat: {
fieldtype: 'Select',
label: t`Export Format`,
@ -242,39 +264,6 @@ export default defineComponent({
};
},
},
components: { FormHeader, Check, Select, Button },
components: { FormHeader, Check, Select, Button, Int },
});
function getExportFields(fields: Field[]): ExportField[] {
return fields
.filter((f) => !f.computed && f.label)
.map((field) => {
const { fieldname, label } = field;
const fieldtype = field.fieldtype as FieldTypeEnum;
return {
fieldname,
fieldtype,
label,
export: !excludedFieldTypes.includes(fieldtype),
};
});
}
function getExportTableFields(fields: Field[]): ExportTableField[] {
return fields
.filter((f) => f.fieldtype === FieldTypeEnum.Table)
.map((f) => {
const target = (f as TargetField).target;
const tableFields = fyo.schemaMap[target]?.fields ?? [];
const exportTableFields = getExportFields(tableFields);
return {
fieldname: f.fieldname,
label: f.label,
target,
fields: exportTableFields,
};
})
.filter((f) => !!f.fields.length);
}
</script>

View File

@ -9,7 +9,7 @@
items-center
mb-3
w-96
z-10
z-30
bg-white
rounded-lg
"

View File

@ -101,7 +101,7 @@ import ListCell from './ListCell';
export default defineComponent({
name: 'List',
props: { listConfig: Object, filters: Object, schemaName: String },
emits: ['makeNewDoc'],
emits: ['makeNewDoc', 'updatedData'],
components: {
Row,
ListCell,
@ -205,6 +205,7 @@ export default defineComponent({
orderBy,
})
).map((d) => ({ ...d, schema: fyo.schemaMap[this.schemaName] }));
this.$emit('updatedData', filters);
},
},
});

View File

@ -25,10 +25,15 @@
:listConfig="listConfig"
:filters="filters"
class="flex-1 flex h-full"
@updatedData="updatedData"
@makeNewDoc="makeNewDoc"
/>
<Modal :open-modal="openExportModal" @closemodal="openExportModal = false">
<ExportWizard :schema-name="schemaName" :title="pageTitle" />
<ExportWizard
:schema-name="schemaName"
:title="pageTitle"
:list-filters="listFilters"
/>
</Modal>
</div>
</template>
@ -59,7 +64,11 @@ export default {
ExportWizard,
},
data() {
return { listConfig: undefined, openExportModal: !false };
return {
listConfig: undefined,
openExportModal: false,
listFilters: {},
};
},
async activated() {
if (typeof this.filters === 'object') {
@ -73,6 +82,9 @@ export default {
docsPath.value = '';
},
methods: {
updatedData(listFilters) {
this.listFilters = listFilters;
},
async makeNewDoc() {
const doc = await fyo.doc.getNewDoc(this.schemaName, this.filters ?? {});
const path = this.getFormPath(doc.name);

334
src/utils/export.ts Normal file
View File

@ -0,0 +1,334 @@
import { Fyo } from 'fyo';
import { RawValueMap } from 'fyo/core/types';
import { Field, FieldTypeEnum, RawValue, TargetField } from 'schemas/types';
import { generateCSV } from 'utils/csvParser';
import { GetAllOptions, QueryFilter } from 'utils/db/types';
import { getMapFromList } from 'utils/index';
import { ExportField, ExportTableField } from './types';
const excludedFieldTypes = [
FieldTypeEnum.AttachImage,
FieldTypeEnum.Attachment,
];
interface CsvHeader {
label: string;
schemaName: string;
fieldname: string;
parentFieldname?: string;
}
export function getExportFields(
fields: Field[],
exclude: string[] = []
): ExportField[] {
return fields
.filter((f) => !f.computed && f.label && !exclude.includes(f.fieldname))
.map((field) => {
const { fieldname, label } = field;
const fieldtype = field.fieldtype as FieldTypeEnum;
return {
fieldname,
fieldtype,
label,
export: !excludedFieldTypes.includes(fieldtype),
};
});
}
export function getExportTableFields(
fields: Field[],
fyo: Fyo
): ExportTableField[] {
return fields
.filter((f) => f.fieldtype === FieldTypeEnum.Table)
.map((f) => {
const target = (f as TargetField).target;
const tableFields = fyo.schemaMap[target]?.fields ?? [];
const exportTableFields = getExportFields(tableFields, ['name']);
return {
fieldname: f.fieldname,
label: f.label,
target,
fields: exportTableFields,
};
})
.filter((f) => !!f.fields.length);
}
export async function getJsonExportData(
schemaName: string,
fields: ExportField[],
tableFields: ExportTableField[],
limit: number | null,
filters: QueryFilter,
fyo: Fyo
): Promise<string> {
const data = await getExportData(
schemaName,
fields,
tableFields,
limit,
filters,
fyo
);
convertParentDataToJsonExport(data.parentData, data.childTableData);
return JSON.stringify(data.parentData);
}
export async function getCsvExportData(
schemaName: string,
fields: ExportField[],
tableFields: ExportTableField[],
limit: number | null,
filters: QueryFilter,
fyo: Fyo
): Promise<string> {
const { childTableData, parentData } = await getExportData(
schemaName,
fields,
tableFields,
limit,
filters,
fyo
);
/**
* parentNameMap: Record<ParentName, Record<ParentFieldName, Rows[]>>
*/
const parentNameMap = getParentNameMap(childTableData);
const headers = getCsvHeaders(schemaName, fields, tableFields);
const rows: RawValue[][] = [];
for (const parentRow of parentData) {
const parentName = parentRow.name as string;
if (!parentName) {
continue;
}
const baseRowData = headers.parent.map(
(f) => (parentRow[f.fieldname] as RawValue) ?? ''
);
const tableFieldRowMap = parentNameMap[parentName];
if (!tableFieldRowMap || !Object.keys(tableFieldRowMap ?? {}).length) {
rows.push([baseRowData, headers.child.map((_) => '')].flat());
continue;
}
for (const tableFieldName in tableFieldRowMap) {
const tableRows = tableFieldRowMap[tableFieldName] ?? [];
for (const tableRow of tableRows) {
const tableRowData = headers.child.map((f) => {
if (f.parentFieldname !== tableFieldName) {
return '';
}
return (tableRow[f.fieldname] as RawValue) ?? '';
});
rows.push([baseRowData, tableRowData].flat());
}
}
}
const flatHeaders = [headers.parent, headers.child].flat();
const labels = flatHeaders.map((f) => f.label);
const keys = flatHeaders.map((f) => `${f.schemaName}.${f.fieldname}`);
rows.unshift(keys);
rows.unshift(labels);
return generateCSV(rows);
}
function getCsvHeaders(
schemaName: string,
fields: ExportField[],
tableFields: ExportTableField[]
) {
const headers = {
parent: [] as CsvHeader[],
child: [] as CsvHeader[],
};
for (const { label, fieldname, fieldtype, export: shouldExport } of fields) {
if (!shouldExport || fieldtype === FieldTypeEnum.Table) {
continue;
}
headers.parent.push({ schemaName, label, fieldname });
}
for (const tf of tableFields) {
if (!fields.find((f) => f.fieldname === tf.fieldname)?.export) {
continue;
}
for (const field of tf.fields) {
if (!field.export) {
continue;
}
headers.child.push({
schemaName: tf.target,
label: field.label,
fieldname: field.fieldname,
parentFieldname: tf.fieldname,
});
}
}
return headers;
}
function getParentNameMap(childTableData: Record<string, RawValueMap[]>) {
const parentNameMap: Record<string, Record<string, RawValueMap[]>> = {};
for (const key in childTableData) {
for (const row of childTableData[key]) {
const parent = row.parent as string;
if (!parent) {
continue;
}
parentNameMap[parent] ??= {};
parentNameMap[parent][key] ??= [];
parentNameMap[parent][key].push(row);
}
}
return parentNameMap;
}
async function getExportData(
schemaName: string,
fields: ExportField[],
tableFields: ExportTableField[],
limit: number | null,
filters: QueryFilter,
fyo: Fyo
) {
const parentData = await getParentData(
schemaName,
filters,
fields,
limit,
fyo
);
const parentNames = parentData.map((f) => f.name as string).filter(Boolean);
const childTableData = await getAllChildTableData(
tableFields,
fields,
parentNames,
fyo
);
return { parentData, childTableData };
}
function convertParentDataToJsonExport(
parentData: RawValueMap[],
childTableData: Record<string, RawValueMap[]>
) {
/**
* Map from List does not create copies. Map is a
* map of references, hence parentData is altered.
*/
const nameMap = getMapFromList(parentData, 'name');
for (const fieldname in childTableData) {
const data = childTableData[fieldname];
for (const row of data) {
const parent = row.parent as string | undefined;
if (!parent || !nameMap?.[parent]) {
continue;
}
nameMap[parent][fieldname] ??= [];
delete row.parent;
delete row.name;
(nameMap[parent][fieldname] as RawValueMap[]).push(row);
}
}
}
async function getParentData(
schemaName: string,
filters: QueryFilter,
fields: ExportField[],
limit: number | null,
fyo: Fyo
) {
const orderBy = !!fields.find((f) => f.fieldname === 'date')
? 'date'
: 'created';
const options: GetAllOptions = { filters, orderBy, order: 'desc' };
if (limit) {
options.limit = limit;
}
options.fields = fields
.filter((f) => f.export && f.fieldtype !== FieldTypeEnum.Table)
.map((f) => f.fieldname);
if (!options.fields.includes('name')) {
options.fields.unshift('name');
}
const data = await fyo.db.getAllRaw(schemaName, options);
convertRawPesaToFloat(data, fields);
return data;
}
async function getAllChildTableData(
tableFields: ExportTableField[],
parentFields: ExportField[],
parentNames: string[],
fyo: Fyo
) {
const childTables: Record<string, RawValueMap[]> = {};
// Getting Child Row data
for (const tf of tableFields) {
const f = parentFields.find((f) => f.fieldname === tf.fieldname);
if (!f?.export) {
continue;
}
childTables[tf.fieldname] = await getChildTableData(tf, parentNames, fyo);
}
return childTables;
}
async function getChildTableData(
exportTableField: ExportTableField,
parentNames: string[],
fyo: Fyo
) {
const exportTableFields = exportTableField.fields
.filter((f) => f.export && f.fieldtype !== FieldTypeEnum.Table)
.map((f) => f.fieldname);
if (!exportTableFields.includes('parent')) {
exportTableFields.unshift('parent');
}
const data = await fyo.db.getAllRaw(exportTableField.target, {
orderBy: 'idx',
fields: exportTableFields,
filters: { parent: ['in', parentNames] },
});
convertRawPesaToFloat(data, exportTableField.fields);
return data;
}
function convertRawPesaToFloat(data: RawValueMap[], fields: Field[]) {
const currencyFields = fields.filter(
(f) => f.fieldtype === FieldTypeEnum.Currency
);
for (const row of data) {
for (const { fieldname } of currencyFields) {
row[fieldname] = parseFloat((row[fieldname] ?? '0') as string);
}
}
}

View File

@ -1,3 +1,5 @@
import { FieldTypeEnum } from "schemas/types";
export interface MessageDialogButton {
label: string;
action: () => Promise<unknown> | unknown;
@ -47,3 +49,20 @@ export interface SidebarItem {
schemaName?: string;
hidden?: () => boolean;
}
export interface ExportField {
fieldname: string;
fieldtype: FieldTypeEnum;
label: string;
export: boolean;
}
export interface ExportTableField {
fieldname: string;
label: string;
target: string;
fields: ExportField[];
}
export type ExportFormat = 'csv' | 'json';

View File

@ -37,6 +37,10 @@ export function getMapFromList<T, K extends keyof T>(
list: T[],
name: K
): Record<string, T> {
/**
* Do not convert function to use copies of T
* instead of references.
*/
const acc: Record<string, T> = {};
for (const t of list) {
const key = t[name];