From 795f713f5cb6362b1dcf56cedd6d08c0cb33dba5 Mon Sep 17 00:00:00 2001 From: ShareVB Date: Mon, 21 Apr 2025 22:07:59 +0200 Subject: [PATCH] feat(CSV to JSON): typed json + separator autodetection --- components.d.ts | 3 ++ package.json | 1 + pnpm-lock.yaml | 7 +++++ .../csv-to-json/csv-to-json.service.test.ts | 26 ++++++++++++++-- src/tools/csv-to-json/csv-to-json.service.ts | 30 ++++++++++++------- src/tools/csv-to-json/csv-to-json.vue | 9 ++++-- src/tools/csv-to-json/detect-csv.d.ts | 6 ++++ 7 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 src/tools/csv-to-json/detect-csv.d.ts diff --git a/components.d.ts b/components.d.ts index c0c638e4..4157a6ea 100644 --- a/components.d.ts +++ b/components.d.ts @@ -117,6 +117,7 @@ declare module '@vue/runtime-core' { DockerMemo: typeof import('./src/tools/docker-memo/docker-memo.vue')['default'] DockerRunToDockerComposeConverter: typeof import('./src/tools/docker-run-to-docker-compose-converter/docker-run-to-docker-compose-converter.vue')['default'] DockerRunToKubernetes: typeof import('./src/tools/docker-run-to-kubernetes/docker-run-to-kubernetes.vue')['default'] + DuplicateLineFinder: typeof import('./src/tools/duplicate-line-finder/duplicate-line-finder.vue')['default'] DurationCalculator: typeof import('./src/tools/duration-calculator/duration-calculator.vue')['default'] DynamicValues: typeof import('./src/tools/benchmark-builder/dynamic-values.vue')['default'] EcdsaKeyPairGenerator: typeof import('./src/tools/ecdsa-key-pair-generator/ecdsa-key-pair-generator.vue')['default'] @@ -276,6 +277,7 @@ declare module '@vue/runtime-core' { NButton: typeof import('naive-ui')['NButton'] NCard: typeof import('naive-ui')['NCard'] NCheckbox: typeof import('naive-ui')['NCheckbox'] + NCode: typeof import('naive-ui')['NCode'] NCollapseTransition: typeof import('naive-ui')['NCollapseTransition'] NColorPicker: typeof import('naive-ui')['NColorPicker'] NConfigProvider: typeof import('naive-ui')['NConfigProvider'] @@ -294,6 +296,7 @@ declare module '@vue/runtime-core' { NModal: typeof import('naive-ui')['NModal'] NP: typeof import('naive-ui')['NP'] NRate: typeof import('naive-ui')['NRate'] + NScrollbar: typeof import('naive-ui')['NScrollbar'] NSlider: typeof import('naive-ui')['NSlider'] NSpace: typeof import('naive-ui')['NSpace'] NSwitch: typeof import('naive-ui')['NSwitch'] diff --git a/package.json b/package.json index 172d2b1e..4b98bf40 100644 --- a/package.json +++ b/package.json @@ -115,6 +115,7 @@ "date-fns": "^2.30.0", "date-holidays": "^3.23.21", "decomposerize": "^1.4.4", + "detect-csv": "^1.1.0", "dmarc-report-parser": "^0.1.5", "dns-query": "^0.11.2", "docker2kube": "^0.0.14", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 35802697..4e14924b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -242,6 +242,9 @@ dependencies: decomposerize: specifier: ^1.4.4 version: 1.4.4 + detect-csv: + specifier: ^1.1.0 + version: 1.1.0 dmarc-report-parser: specifier: ^0.1.5 version: 0.1.5 @@ -9612,6 +9615,10 @@ packages: resolution: {integrity: sha512-2N3BOUU4gYMpTP24s5rF5iP7BDr7uNTCs4ozw3kf/eKfvWSIu93GEBi5m427YoyJoeOzQ5smuu4nNAPGb8idSQ==} dev: true + /detect-csv@1.1.0: + resolution: {integrity: sha512-msOkPMiNTtr73yh5G5doGASChThzQiRF3Es+wPKD2+0W+j14lRIfH4P3QDmweMRFfuRoauh531X4vtaaI/oeCw==} + dev: false + /detect-europe-js@0.1.2: resolution: {integrity: sha512-lgdERlL3u0aUdHocoouzT10d9I89VVhk0qNRmll7mXdGfJT1/wqZ2ZLA4oJAjeACPY5fT1wsbq2AT+GkuInsow==} dev: false diff --git a/src/tools/csv-to-json/csv-to-json.service.test.ts b/src/tools/csv-to-json/csv-to-json.service.test.ts index 932b334a..6ef7a0be 100644 --- a/src/tools/csv-to-json/csv-to-json.service.test.ts +++ b/src/tools/csv-to-json/csv-to-json.service.test.ts @@ -4,15 +4,27 @@ import { convertCsvToArray, getHeaders } from './csv-to-json.service'; describe('csv-to-json service', () => { describe('getHeaders', () => { it('extracts all the keys from the first line of the CSV', () => { - expect(getHeaders('a,b,c\n1,2,3\n4,5,6')).toEqual(['a', 'b', 'c']); + expect(getHeaders('a,b,c\n1,2,3\n4,5,6', ',')).toEqual(['a', 'b', 'c']); }); it('returns an empty array if the CSV is empty', () => { - expect(getHeaders('')).toEqual([]); + expect(getHeaders('', ',')).toEqual([]); }); }); describe('convertCsvToArray', () => { + it('detects and converts with different separators', () => { + const res = [ + { a: '1', b: '2' }, + { a: '3', b: '4' }, + ]; + + expect(convertCsvToArray('a,b\n1,2\n3,4')).toEqual(res); + expect(convertCsvToArray('a;b\n1;2\n3;4')).toEqual(res); + expect(convertCsvToArray('a\tb\n1\t2\n3\t4')).toEqual(res); + expect(convertCsvToArray('a|b\n1|2\n3|4')).toEqual(res); + }); + it('converts a CSV string to an array of objects', () => { const csv = 'a,b\n1,2\n3,4'; @@ -22,6 +34,16 @@ describe('csv-to-json service', () => { ]); }); + it('converts a CSV string to an array of typed objects', () => { + const csv = 'a,b\n1,2\ntrue, false\ntruc, bidule'; + + expect(convertCsvToArray(csv, true)).toEqual([ + { a: 1, b: 2 }, + { a: true, b: false }, + { a: 'truc', b: 'bidule' }, + ]); + }); + it('converts a CSV string with different keys to an array of objects', () => { const csv = 'a,b,c\n1,2,\n3,,4'; diff --git a/src/tools/csv-to-json/csv-to-json.service.ts b/src/tools/csv-to-json/csv-to-json.service.ts index 8de997a3..99c493c9 100644 --- a/src/tools/csv-to-json/csv-to-json.service.ts +++ b/src/tools/csv-to-json/csv-to-json.service.ts @@ -1,14 +1,16 @@ +import detectCSV from 'detect-csv'; + export { getHeaders, convertCsvToArray }; -function getHeaders(csv: string): string[] { +function getHeaders(csv: string, delimiter: string): string[] { if (csv.trim() === '') { return []; } const firstLine = csv.split('\n')[0]; - return firstLine.split(/[,;]/).map(header => header.trim()); + return firstLine.split(new RegExp(`[${delimiter}]`)).map(header => header.trim()); } -function deserializeValue(value: string): unknown { +function deserializeValue(value: string, tryParseValues: boolean): unknown { if (value === 'null') { return null; } @@ -23,18 +25,26 @@ function deserializeValue(value: string): unknown { return valueAsString.slice(1, -1); } - return valueAsString; + if (!tryParseValues) { + return valueAsString; + } + try { + return JSON.parse(valueAsString); + } + catch (_) { + return valueAsString; + } } -function convertCsvToArray(csv: string): Record[] { - const lines = csv.split('\n'); - const headers = getHeaders(csv); +function convertCsvToArray(csv: string, tryParseValues: boolean = false): Record[] { + const delimiter = detectCSV(csv)?.delimiter || ','; + const headers = getHeaders(csv, delimiter); - return lines.slice(1).map((line) => { + return csv.split('\n').slice(1).map((line) => { // Split on comma or semicolon not within quotes - const data = line.split(/[,;](?=(?:(?:[^"]*"){2})*[^"]*$)/).map(value => value.trim()); + const data = line.split(new RegExp(`[${delimiter}](?=(?:(?:[^"]*"){2})*[^"]*$)`)).map(value => value.trim()); return headers.reduce((obj, header, index) => { - obj[header] = deserializeValue(data[index]); + obj[header] = deserializeValue(data[index], tryParseValues); return obj; }, {} as Record); }); diff --git a/src/tools/csv-to-json/csv-to-json.vue b/src/tools/csv-to-json/csv-to-json.vue index 4a55b06d..89303aae 100644 --- a/src/tools/csv-to-json/csv-to-json.vue +++ b/src/tools/csv-to-json/csv-to-json.vue @@ -4,24 +4,29 @@ import FormatTransformer from '@/components/FormatTransformer.vue'; import type { UseValidationRule } from '@/composable/validation'; import { withDefaultOnError } from '@/utils/defaults'; +const typedValues = ref(false); + function transformer(value: string) { return withDefaultOnError(() => { if (value === '') { return ''; } - return JSON.stringify(convertCsvToArray(value), null, 2); + return JSON.stringify(convertCsvToArray(value, typedValues.value), null, 2); }, ''); } const rules: UseValidationRule[] = [ { - validator: (v: string) => v === '' || ((v.includes(',') || v.includes(';')) && v.includes('\n')), + validator: (v: string) => v === '' || ((v.includes(',') || v.includes(';') || v.includes('\t') || v.includes('|')) && v.includes('\n')), message: 'Provided CSV is not valid.', }, ];