feat(new tool): File Splitter (json, txt, xml)

Fix #628, #795
This commit is contained in:
ShareVB committed 2025-11-12 21:37:45 +01:00
1 parent 5dd16d4b49
commit 29c14837b6
4 files changed
+425

No files matched your search

@@ -0,0 +1,94 @@
import { describe, expect, it } from 'vitest';
import { splitContent } from './file-splitter.service';
describe('splitContent - lines mode', () => {
const content = 'a\nb\nc\nd\ne';
it('splits by maxSize', () => {
const result = splitContent(content, 'lines', 6, 'maxSize');
expect(result).toEqual(['a\nb\nc', 'd\ne']);
});
it('splits by fixedCount', () => {
const result = splitContent(content, 'lines', 2, 'fixedCount');
expect(result).toEqual(['a\nb', 'c\nd', 'e']);
});
it('splits by chunkCount', () => {
const result = splitContent(content, 'lines', 2, 'chunkCount');
expect(result).toEqual(['a\nb\nc', 'd\ne']);
});
});
describe('splitContent - nodes mode with JSON array', () => {
const jsonArray = JSON.stringify([1, 2, 3, 4, 5]);
it('splits by maxSize', () => {
const result = splitContent(jsonArray, 'nodes', 6, 'maxSize');
expect(result).toEqual(['[1,2]', '[3,4]', '[5]']);
});
it('splits by fixedCount', () => {
const result = splitContent(jsonArray, 'nodes', 3, 'fixedCount');
expect(result).toEqual(['[1,2,3]', '[4,5]']);
});
it('splits by chunkCount', () => {
const result = splitContent(jsonArray, 'nodes', 4, 'chunkCount');
expect(result).toEqual(['[1,2]', '[3,4]', '[5]']);
});
});
describe('splitContent - nodes mode with JSON object', () => {
const jsonObject = JSON.stringify({ a: 1, b: 2, c: 3, d: 4 });
it('splits by maxSize', () => {
const result = splitContent(jsonObject, 'nodes', 14, 'maxSize');
expect(result).toEqual([
'{"a":1}',
'{"b":2}',
'{"c":3}',
'{"d":4}',
]);
});
it('splits by fixedCount', () => {
const result = splitContent(jsonObject, 'nodes', 3, 'fixedCount');
expect(result).toEqual(['{"a":1,"b":2,"c":3}', '{"d":4}']);
});
it('splits by chunkCount', () => {
const result = splitContent(jsonObject, 'nodes', 2, 'chunkCount');
expect(result).toEqual(['{"a":1,"b":2}', '{"c":3,"d":4}']);
});
});
describe('splitContent - nodes mode with XML', () => {
const xml = '<root><a>1</a><b>2</b><c>3</c><d>4</d></root>';
it('splits by maxSize', () => {
const result = splitContent(xml, 'nodes', 30, 'maxSize');
expect(result).toEqual([
'<root><a>1</a><b>2</b></root>',
'<root><c>3</c><d>4</d></root>',
]);
});
it('splits by fixedCount', () => {
const result = splitContent(xml, 'nodes', 3, 'fixedCount');
expect(result).toEqual([
'<root><a>1</a><b>2</b><c>3</c></root>',
'<root><d>4</d></root>',
]);
});
it('splits by chunkCount', () => {
const result = splitContent(xml, 'nodes', 4, 'chunkCount');
expect(result).toEqual([
'<root><a>1</a></root>',
'<root><b>2</b></root>',
'<root><c>3</c></root>',
'<root><d>4</d></root>',
]);
});
});
@@ -0,0 +1,167 @@
function chunkByFixedCount<T>(arr: T[], count: number): T[][] {
const result: T[][] = [];
for (let i = 0; i < arr.length; i += count) {
result.push(arr.slice(i, i + count));
}
return result;
}
function chunkByChunkCount<T>(arr: T[], chunks: number): T[][] {
const result: T[][] = [];
const chunkSize = Math.ceil(arr.length / chunks);
for (let i = 0; i < arr.length; i += chunkSize) {
result.push(arr.slice(i, i + chunkSize));
}
return result;
}
export function splitContent(
content: string,
mode: 'lines' | 'nodes',
count: number,
strategy: 'maxSize' | 'fixedCount' | 'chunkCount' = 'maxSize',
): string[] {
const getChunks = <T>(arr: T[]): T[][] => {
if (strategy === 'fixedCount') {
return chunkByFixedCount(arr, count);
}
if (strategy === 'chunkCount') {
return chunkByChunkCount(arr, count);
}
return []; // handled separately for maxSize
};
if (mode === 'lines') {
const lines = content.split(/\r?\n/);
if (strategy === 'maxSize') {
const result: string[] = [];
let chunk: string[] = [];
let size = 0;
for (const line of lines) {
const lineSize = line.length + 1;
if (size + lineSize > count && chunk.length > 0) {
result.push(chunk.join('\n'));
chunk = [];
size = 0;
}
chunk.push(line);
size += lineSize;
}
if (chunk.length > 0) {
result.push(chunk.join('\n'));
}
return result;
}
return getChunks(lines).map(group => group.join('\n'));
}
try {
const parsed = JSON.parse(content);
if (Array.isArray(parsed)) {
if (strategy === 'maxSize') {
const result: string[] = [];
let chunk: any[] = [];
let size = 0;
for (const item of parsed) {
const itemStr = JSON.stringify(item);
const itemSize = itemStr.length + 2;
if (size + itemSize > count && chunk.length > 0) {
result.push(JSON.stringify(chunk));
chunk = [];
size = 0;
}
chunk.push(item);
size += itemSize;
}
if (chunk.length > 0) {
result.push(JSON.stringify(chunk));
}
return result;
}
return getChunks(parsed).map(chunk => JSON.stringify(chunk));
}
if (typeof parsed === 'object') {
const entries = Object.entries(parsed);
if (strategy === 'maxSize') {
const result: string[] = [];
let chunk: [string, any][] = [];
let size = 0;
for (const [key, value] of entries) {
const entryStr = JSON.stringify({ [key]: value });
const entrySize = entryStr.length + 2;
if (size + entrySize > count && chunk.length > 0) {
result.push(JSON.stringify(Object.fromEntries(chunk)));
chunk = [];
size = 0;
}
chunk.push([key, value]);
size += entrySize;
}
if (chunk.length > 0) {
result.push(JSON.stringify(Object.fromEntries(chunk)));
}
return result;
}
return getChunks(entries).map(chunk =>
JSON.stringify(Object.fromEntries(chunk)),
);
}
}
catch {
const parser = new DOMParser();
const xmlDoc = parser.parseFromString(content, 'application/xml');
const root = xmlDoc.documentElement;
const children = Array.from(root.children);
if (strategy === 'maxSize') {
const result: string[] = [];
let chunk: Element[] = [];
for (const child of children) {
const tempDoc = document.implementation.createDocument('', root.tagName, null);
chunk.forEach(c => tempDoc.documentElement.appendChild(c.cloneNode(true)));
tempDoc.documentElement.appendChild(child.cloneNode(true));
const serialized = new XMLSerializer().serializeToString(tempDoc);
const chunkSize = serialized.length;
if (chunkSize > count && chunk.length > 0) {
const frag = document.implementation.createDocument('', root.tagName, null);
chunk.forEach(c => frag.documentElement.appendChild(c.cloneNode(true)));
result.push(new XMLSerializer().serializeToString(frag));
chunk = [];
}
chunk.push(child);
}
if (chunk.length > 0) {
const frag = document.implementation.createDocument('', root.tagName, null);
chunk.forEach(c => frag.documentElement.appendChild(c.cloneNode(true)));
result.push(new XMLSerializer().serializeToString(frag));
}
return result;
}
return getChunks(children).map((group) => {
const frag = document.implementation.createDocument('', root.tagName, null);
group.forEach(child => frag.documentElement.appendChild(child.cloneNode(true)));
return new XMLSerializer().serializeToString(frag);
});
}
throw new Error('Unsupported format or invalid content');
}
+151
View File
@@ -0,0 +1,151 @@
<script setup lang="ts">
import { splitContent } from './file-splitter.service';
import { useQueryParamOrStorage } from '@/composable/queryParams';
const splitMode = useQueryParamOrStorage<'lines' | 'nodes'>({ name: 'mode', storageName: 'file-split:m', defaultValue: 'lines' });
const strategy = useQueryParamOrStorage<'maxSize' | 'fixedCount' | 'chunkCount'>({ name: 'strategy', storageName: 'file-split:s', defaultValue: 'fixedCount' });
const count = useQueryParamOrStorage({ name: 'count', storageName: 'file-split:c', defaultValue: 100 });
const extension = ref('');
const preview = ref(false);
const error = ref('');
const fileContent = ref('');
const chunks = ref<string[]>([]);
async function prepareChunks() {
error.value = '';
const splitModeValue = splitMode.value;
const countValue = count.value;
const strategyValue = strategy.value;
const fileContentValue = fileContent.value;
if (!fileContentValue) {
return [];
}
try {
chunks.value = await new Promise((resolve, reject) => {
try {
resolve(splitContent(
fileContentValue,
splitModeValue,
(strategyValue === 'maxSize' ? 1024 : 1) * countValue,
strategyValue));
}
catch (e: any) {
reject(e);
}
});
}
catch (err: any) {
error.value = err.toString();
chunks.value = [];
}
}
function readFileAsString(file: File) {
return new Promise<string>((resolve, reject) => {
const fr = new FileReader();
fr.onload = () => {
resolve(fr.result as string || '');
};
fr.onerror = reject;
fr.readAsText(file);
});
}
async function onFileUploaded(uploadedFile: File) {
extension.value = uploadedFile.name?.split('.').pop() || 'txt';
fileContent.value = await readFileAsString(uploadedFile);
}
function downloadChunk(index: number) {
const blob = new Blob([chunks.value[index]], { type: 'text/plain' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = `chunk-${index + 1}.${extension.value}`;
a.click();
URL.revokeObjectURL(url);
}
function downloadAllChunks() {
for (let i = 0; i < chunks.value.length; i++) {
downloadChunk(i);
}
}
</script>
<template>
<div>
<n-space justify="center" mb-2>
<NRadioGroup v-model:value="splitMode">
<NRadio value="lines">
Split by Lines
</NRadio>
<NRadio value="nodes">
Split by Nodes
</NRadio>
</NRadioGroup>
</n-space>
<n-space justify="center" align="baseline" mb-2>
<NRadioGroup v-model:value="strategy">
<NRadio value="maxSize">
Max Chunk size (KB):
</NRadio>
<NRadio value="fixedCount">
Fixed Count:
</NRadio>
<NRadio value="chunkCount">
Chunk Count:
</NRadio>
</NRadioGroup>
<NInputNumber v-model:value="count" :min="1" placeholder="Strategy value..." style="width: 150px" />
</n-space>
<div style="flex: 0 0 100%" mb-2>
<div mx-auto max-w-600px>
<c-file-upload
title="Drag and drop a TXT, JSON or XML file here, or click to select a file"
accept=".txt,.json,.xml"
@file-upload="onFileUploaded"
/>
</div>
</div>
<div mb-1 flex justify-center>
<n-checkbox v-model:checked="preview">
Preview chunks?
</n-checkbox>
</div>
<div mb-2 flex justify-center>
<NButton size="small" @click="prepareChunks()">
Process chunks
</NButton>
</div>
<n-alert v-if="error">
{{ error }}
</n-alert>
<c-card v-if="chunks && chunks.length" :title="`Chunks (${chunks.length})`">
<div flex justify-center>
<NButton size="small" @click="downloadAllChunks()">
Download all
</NButton>
</div>
<div v-if="preview">
<template
v-for="(chunk, index) in chunks"
:key="index"
>
<n-divider>Chunk {{ index + 1 }}</n-divider>
<textarea-copyable :value="chunk" :language="extension" style="max-height: 10em" multiline word-wrap mb-1 />
<div flex justify-center>
<NButton size="small" @click="downloadChunk(index)">
Download
</NButton>
</div>
</template>
</div>
</c-card>
</div>
</template>
+13
View File
@@ -0,0 +1,13 @@
import { Cut } from '@vicons/tabler';
import { defineTool } from '../tool';
export const tool = defineTool({
name: 'File Splitter',
path: '/file-splitter',
description: 'Split JSON, XML or text file either by group of nodes or by group of lines',
keywords: ['file', 'json', 'xml', 'text', 'splitter'],
component: () => import('./file-splitter.vue'),
icon: Cut,
createdAt: new Date('2025-11-11'),
category: 'Development',
});