feat(CSV to JSON): typed json + separator autodetection

This commit is contained in:
ShareVB committed 2025-04-21 22:07:59 +02:00
1 parent c7ab527ccc
commit 795f713f5c
7 files changed
+68 -14

No files matched your search

+3
View File
@@ -117,6 +117,7 @@ declare module '@vue/runtime-core' {
DockerMemo: typeof import('./src/tools/docker-memo/docker-memo.vue')['default']
DockerRunToDockerComposeConverter: typeof import('./src/tools/docker-run-to-docker-compose-converter/docker-run-to-docker-compose-converter.vue')['default']
DockerRunToKubernetes: typeof import('./src/tools/docker-run-to-kubernetes/docker-run-to-kubernetes.vue')['default']
DuplicateLineFinder: typeof import('./src/tools/duplicate-line-finder/duplicate-line-finder.vue')['default']
DurationCalculator: typeof import('./src/tools/duration-calculator/duration-calculator.vue')['default']
DynamicValues: typeof import('./src/tools/benchmark-builder/dynamic-values.vue')['default']
EcdsaKeyPairGenerator: typeof import('./src/tools/ecdsa-key-pair-generator/ecdsa-key-pair-generator.vue')['default']
@@ -276,6 +277,7 @@ declare module '@vue/runtime-core' {
NButton: typeof import('naive-ui')['NButton']
NCard: typeof import('naive-ui')['NCard']
NCheckbox: typeof import('naive-ui')['NCheckbox']
NCode: typeof import('naive-ui')['NCode']
NCollapseTransition: typeof import('naive-ui')['NCollapseTransition']
NColorPicker: typeof import('naive-ui')['NColorPicker']
NConfigProvider: typeof import('naive-ui')['NConfigProvider']
@@ -294,6 +296,7 @@ declare module '@vue/runtime-core' {
NModal: typeof import('naive-ui')['NModal']
NP: typeof import('naive-ui')['NP']
NRate: typeof import('naive-ui')['NRate']
NScrollbar: typeof import('naive-ui')['NScrollbar']
NSlider: typeof import('naive-ui')['NSlider']
NSpace: typeof import('naive-ui')['NSpace']
NSwitch: typeof import('naive-ui')['NSwitch']
+1
View File
@@ -115,6 +115,7 @@
"date-fns": "^2.30.0",
"date-holidays": "^3.23.21",
"decomposerize": "^1.4.4",
"detect-csv": "^1.1.0",
"dmarc-report-parser": "^0.1.5",
"dns-query": "^0.11.2",
"docker2kube": "^0.0.14",
+7
View File
@@ -242,6 +242,9 @@ dependencies:
decomposerize:
specifier: ^1.4.4
version: 1.4.4
detect-csv:
specifier: ^1.1.0
version: 1.1.0
dmarc-report-parser:
specifier: ^0.1.5
version: 0.1.5
@@ -9612,6 +9615,10 @@ packages:
resolution: {integrity: sha512-2N3BOUU4gYMpTP24s5rF5iP7BDr7uNTCs4ozw3kf/eKfvWSIu93GEBi5m427YoyJoeOzQ5smuu4nNAPGb8idSQ==}
dev: true
/detect-csv@1.1.0:
resolution: {integrity: sha512-msOkPMiNTtr73yh5G5doGASChThzQiRF3Es+wPKD2+0W+j14lRIfH4P3QDmweMRFfuRoauh531X4vtaaI/oeCw==}
dev: false
/detect-europe-js@0.1.2:
resolution: {integrity: sha512-lgdERlL3u0aUdHocoouzT10d9I89VVhk0qNRmll7mXdGfJT1/wqZ2ZLA4oJAjeACPY5fT1wsbq2AT+GkuInsow==}
dev: false
@@ -4,15 +4,27 @@ import { convertCsvToArray, getHeaders } from './csv-to-json.service';
describe('csv-to-json service', () => {
describe('getHeaders', () => {
it('extracts all the keys from the first line of the CSV', () => {
expect(getHeaders('a,b,c\n1,2,3\n4,5,6')).toEqual(['a', 'b', 'c']);
expect(getHeaders('a,b,c\n1,2,3\n4,5,6', ',')).toEqual(['a', 'b', 'c']);
});
it('returns an empty array if the CSV is empty', () => {
expect(getHeaders('')).toEqual([]);
expect(getHeaders('', ',')).toEqual([]);
});
});
describe('convertCsvToArray', () => {
it('detects and converts with different separators', () => {
const res = [
{ a: '1', b: '2' },
{ a: '3', b: '4' },
];
expect(convertCsvToArray('a,b\n1,2\n3,4')).toEqual(res);
expect(convertCsvToArray('a;b\n1;2\n3;4')).toEqual(res);
expect(convertCsvToArray('a\tb\n1\t2\n3\t4')).toEqual(res);
expect(convertCsvToArray('a|b\n1|2\n3|4')).toEqual(res);
});
it('converts a CSV string to an array of objects', () => {
const csv = 'a,b\n1,2\n3,4';
@@ -22,6 +34,16 @@ describe('csv-to-json service', () => {
]);
});
it('converts a CSV string to an array of typed objects', () => {
const csv = 'a,b\n1,2\ntrue, false\ntruc, bidule';
expect(convertCsvToArray(csv, true)).toEqual([
{ a: 1, b: 2 },
{ a: true, b: false },
{ a: 'truc', b: 'bidule' },
]);
});
it('converts a CSV string with different keys to an array of objects', () => {
const csv = 'a,b,c\n1,2,\n3,,4';
+20 -10
View File
@@ -1,14 +1,16 @@
import detectCSV from 'detect-csv';
export { getHeaders, convertCsvToArray };
function getHeaders(csv: string): string[] {
function getHeaders(csv: string, delimiter: string): string[] {
if (csv.trim() === '') {
return [];
}
const firstLine = csv.split('\n')[0];
return firstLine.split(/[,;]/).map(header => header.trim());
return firstLine.split(new RegExp(`[${delimiter}]`)).map(header => header.trim());
}
function deserializeValue(value: string): unknown {
function deserializeValue(value: string, tryParseValues: boolean): unknown {
if (value === 'null') {
return null;
}
@@ -23,18 +25,26 @@ function deserializeValue(value: string): unknown {
return valueAsString.slice(1, -1);
}
return valueAsString;
if (!tryParseValues) {
return valueAsString;
}
try {
return JSON.parse(valueAsString);
}
catch (_) {
return valueAsString;
}
}
function convertCsvToArray(csv: string): Record<string, unknown>[] {
const lines = csv.split('\n');
const headers = getHeaders(csv);
function convertCsvToArray(csv: string, tryParseValues: boolean = false): Record<string, unknown>[] {
const delimiter = detectCSV(csv)?.delimiter || ',';
const headers = getHeaders(csv, delimiter);
return lines.slice(1).map((line) => {
return csv.split('\n').slice(1).map((line) => {
// Split on comma or semicolon not within quotes
const data = line.split(/[,;](?=(?:(?:[^"]*"){2})*[^"]*$)/).map(value => value.trim());
const data = line.split(new RegExp(`[${delimiter}](?=(?:(?:[^"]*"){2})*[^"]*$)`)).map(value => value.trim());
return headers.reduce((obj, header, index) => {
obj[header] = deserializeValue(data[index]);
obj[header] = deserializeValue(data[index], tryParseValues);
return obj;
}, {} as Record<string, unknown>);
});
+7 -2
View File
@@ -4,24 +4,29 @@ import FormatTransformer from '@/components/FormatTransformer.vue';
import type { UseValidationRule } from '@/composable/validation';
import { withDefaultOnError } from '@/utils/defaults';
const typedValues = ref(false);
function transformer(value: string) {
return withDefaultOnError(() => {
if (value === '') {
return '';
}
return JSON.stringify(convertCsvToArray(value), null, 2);
return JSON.stringify(convertCsvToArray(value, typedValues.value), null, 2);
}, '');
}
const rules: UseValidationRule<string>[] = [
{
validator: (v: string) => v === '' || ((v.includes(',') || v.includes(';')) && v.includes('\n')),
validator: (v: string) => v === '' || ((v.includes(',') || v.includes(';') || v.includes('\t') || v.includes('|')) && v.includes('\n')),
message: 'Provided CSV is not valid.',
},
];
</script>
<template>
<n-form-item label="Typed Values" label-placement="left">
<n-checkbox v-model:value="typedValues" />
</n-form-item>
<FormatTransformer
input-label="Your raw CSV"
input-placeholder="Paste your raw CSV here..."
+6
View File
@@ -0,0 +1,6 @@
declare module "detect-csv" {
export default function DetectCSV(content: string): Nullable<{
delimiter: string
newline: string
}>
}