mirror of
https://github.com/sharevb/it-tools.git
synced 2026-11-22 22:02:38 +03:00
feat(Unicode Search): improve unicode search loading speed (#97)
Improved a section of the initializeIndices function which was causing slow loading times for really large datasets. This specific section alone was responsible for more than 75% of the loading time of the Unicode search page (Section index load speed: 3300 ms -> 400 ms) --------- Co-authored-by: ShareVB <sharevb@gmail.com>
This commit is contained in:
1 parent
9186038c6f
commit
f868b97cd7
2 files changed
+51
-8
No files matched your search
Vendored
+4
@@ -310,6 +310,10 @@ declare module '@vue/runtime-core' {
|
|||||||
NConfigProvider: typeof import('naive-ui')['NConfigProvider']
|
NConfigProvider: typeof import('naive-ui')['NConfigProvider']
|
||||||
NDivider: typeof import('naive-ui')['NDivider']
|
NDivider: typeof import('naive-ui')['NDivider']
|
||||||
NEllipsis: typeof import('naive-ui')['NEllipsis']
|
NEllipsis: typeof import('naive-ui')['NEllipsis']
|
||||||
|
NFormItem: typeof import('naive-ui')['NFormItem']
|
||||||
|
NConfigProvider: typeof import('naive-ui')['NConfigProvider']
|
||||||
|
NEllipsis: typeof import('naive-ui')['NEllipsis']
|
||||||
|
NFormItem: typeof import('naive-ui')['NFormItem']
|
||||||
NginxFormatter: typeof import('./src/tools/nginx-formatter/nginx-formatter.vue')['default']
|
NginxFormatter: typeof import('./src/tools/nginx-formatter/nginx-formatter.vue')['default']
|
||||||
NginxMemo: typeof import('./src/tools/nginx-memo/nginx-memo.vue')['default']
|
NginxMemo: typeof import('./src/tools/nginx-memo/nginx-memo.vue')['default']
|
||||||
'NginxMemo.content': typeof import('./src/tools/nginx-memo/nginx-memo.content.md')['default']
|
'NginxMemo.content': typeof import('./src/tools/nginx-memo/nginx-memo.content.md')['default']
|
||||||
|
|||||||
@@ -85,22 +85,59 @@ export function useFlexSearch<Data extends Record<string, any>>({
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
// Initialize indices with data
|
// Initialize indices with data
|
||||||
const initializeIndices = () => {
|
const initializeIndices = async () => {
|
||||||
dataMap.value.clear();
|
dataMap.value.clear();
|
||||||
indices.forEach(({ index }) => index.clear());
|
indices.forEach(({ index }) => index.clear());
|
||||||
|
|
||||||
data.forEach((item, idx) => {
|
// Pre-calculate all the values to avoid repeated nested property access
|
||||||
|
const itemData = data.map((item, idx) => {
|
||||||
const itemKey = getItemKey(item, idx);
|
const itemKey = getItemKey(item, idx);
|
||||||
dataMap.value.set(itemKey, item);
|
const values = new Map<string, string>();
|
||||||
|
|
||||||
indices.forEach(({ key, index }) => {
|
indices.forEach(({ key }) => {
|
||||||
// Support nested keys (e.g., 'foo.bar')
|
|
||||||
const value = key.split('.').reduce((obj, path) => obj?.[path], item);
|
const value = key.split('.').reduce((obj, path) => obj?.[path], item);
|
||||||
if (value) {
|
if (value) {
|
||||||
index.add(itemKey, String(value));
|
values.set(key, String(value));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
return { itemKey, item, values };
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Batch add to dataMap
|
||||||
|
itemData.forEach(({ itemKey, item }) => {
|
||||||
|
dataMap.value.set(itemKey, item);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Batch add to indices - parallel processing with chunking
|
||||||
|
await Promise.all(
|
||||||
|
indices.map(async ({ key, index }) => {
|
||||||
|
// Filter items that have values for this key upfront
|
||||||
|
const itemsForThisKey = itemData
|
||||||
|
.map(({ itemKey, values }) => ({ itemKey, value: values.get(key) }))
|
||||||
|
.filter(({ value }) => value);
|
||||||
|
|
||||||
|
// Process in chunks to avoid blocking the main thread
|
||||||
|
const chunkSize = 1000; // Adjust based on your data size
|
||||||
|
const chunks = [];
|
||||||
|
for (let i = 0; i < itemsForThisKey.length; i += chunkSize) {
|
||||||
|
chunks.push(itemsForThisKey.slice(i, i + chunkSize));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process each chunk asynchronously
|
||||||
|
for (const chunk of chunks) {
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
// Use setTimeout to yield control back to the browser
|
||||||
|
setTimeout(() => {
|
||||||
|
chunk.forEach(({ itemKey, value }) => {
|
||||||
|
index.add(itemKey, value!);
|
||||||
|
});
|
||||||
|
resolve();
|
||||||
|
}, 0);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
// Initialize on creation
|
// Initialize on creation
|
||||||
@@ -109,8 +146,10 @@ export function useFlexSearch<Data extends Record<string, any>>({
|
|||||||
// Watch for data changes (deep watch for array content)
|
// Watch for data changes (deep watch for array content)
|
||||||
watch(
|
watch(
|
||||||
() => data,
|
() => data,
|
||||||
initializeIndices,
|
() => {
|
||||||
{ deep: true },
|
initializeIndices();
|
||||||
|
},
|
||||||
|
{ immediate: true, deep: true },
|
||||||
);
|
);
|
||||||
|
|
||||||
// Function to search across all indices with weight consideration
|
// Function to search across all indices with weight consideration
|
||||||
|
|||||||
Reference in new issue
Block a user