Improved performance on the Unicode Search page (#75)

* Improved performance on the Unicode Search page:
- Created new flexSearch component (Faster than fuzzySearch). For more information, see https://github.com/nextapps-de/flexsearch
- Replaced fuzzySearch with the flexSearch component on the unicode search page
- Fixed a bug where the search limit counter didn't have any effect
- Reduced the default search limit from 100 to 40 to reduce lag from loading all the ui elements

* Added clarification to comment

* Attempt to fix pnpm-lock merge conflict

* Another attempt to fix pnpm-lock

* Try deleting lock file because regenerating doesn't make a difference

* Try regenerating from scratch

* fix: pnpm-lock.yaml

---------

Co-authored-by: sharevb <sharevb@gmail.com>
This commit is contained in:
Dark_Phoenix_andsharevb authored and GitHub committed 2025-05-24 16:11:42 +02:00
1 parent 48854592b0
commit 2355232aed
5 files changed
+149 -41

No files matched your search

+3 -22
View File
@@ -116,9 +116,7 @@ declare module '@vue/runtime-core' {
DiffViewer: typeof import('./src/tools/json-diff/diff-viewer/diff-viewer.vue')['default']
DistinctWords: typeof import('./src/tools/distinct-words/distinct-words.vue')['default']
DkimDnsGenerator: typeof import('./src/tools/dkim-dns-generator/dkim-dns-generator.vue')['default']
'Dmarc-Generator': typeof import('./src/tools/dmarc-dns-generator/dmarc--generator.vue')['default']
DmarcDnsGenerator: typeof import('./src/tools/dmarc-dns-generator/dmarc-dns-generator.vue')['default']
DmarcGenerator: typeof import('./src/tools/dmarc-dns-generator/dmarc-generator.vue')['default']
DmarcReportAnalyzer: typeof import('./src/tools/dmarc-report-analyzer/dmarc-report-analyzer.vue')['default']
DnsQueries: typeof import('./src/tools/dns-queries/dns-queries.vue')['default']
Docker: typeof import('./src/tools/docker-memo/docker.md')['default']
@@ -294,40 +292,25 @@ declare module '@vue/runtime-core' {
MorseConverter: typeof import('./src/tools/morse-converter/morse-converter.vue')['default']
MultiLinkDownloader: typeof import('./src/tools/multi-link-downloader/multi-link-downloader.vue')['default']
MyIp: typeof import('./src/tools/my-ip/my-ip.vue')['default']
NA: typeof import('naive-ui')['NA']
NanoidGenerator: typeof import('./src/tools/nanoid-generator/nanoid-generator.vue')['default']
NanoMemo: typeof import('./src/tools/nano-memo/nano-memo.vue')['default']
'NanoMemo.content': typeof import('./src/tools/nano-memo/nano-memo.content.md')['default']
NavbarButtons: typeof import('./src/components/NavbarButtons.vue')['default']
NButton: typeof import('naive-ui')['NButton']
NCard: typeof import('naive-ui')['NCard']
NCheckbox: typeof import('naive-ui')['NCheckbox']
NCheckboxGroup: typeof import('naive-ui')['NCheckboxGroup']
NCode: typeof import('naive-ui')['NCode']
NCollapseTransition: typeof import('naive-ui')['NCollapseTransition']
NConfigProvider: typeof import('naive-ui')['NConfigProvider']
NDivider: typeof import('naive-ui')['NDivider']
NDynamicTags: typeof import('naive-ui')['NDynamicTags']
NDataTable: typeof import('naive-ui')['NDataTable']
NEllipsis: typeof import('naive-ui')['NEllipsis']
NForm: typeof import('naive-ui')['NForm']
NFormItem: typeof import('naive-ui')['NFormItem']
NginxFormatter: typeof import('./src/tools/nginx-formatter/nginx-formatter.vue')['default']
NH1: typeof import('naive-ui')['NH1']
NH3: typeof import('naive-ui')['NH3']
NIcon: typeof import('naive-ui')['NIcon']
NInput: typeof import('naive-ui')['NInput']
NInputNumber: typeof import('naive-ui')['NInputNumber']
NLayout: typeof import('naive-ui')['NLayout']
NLayoutSider: typeof import('naive-ui')['NLayoutSider']
NMenu: typeof import('naive-ui')['NMenu']
NP: typeof import('naive-ui')['NP']
NRadio: typeof import('naive-ui')['NRadio']
NRadioGroup: typeof import('naive-ui')['NRadioGroup']
NScrollbar: typeof import('naive-ui')['NScrollbar']
NSelect: typeof import('naive-ui')['NSelect']
NSpace: typeof import('naive-ui')['NSpace']
NSwitch: typeof import('naive-ui')['NSwitch']
NText: typeof import('naive-ui')['NText']
NTooltip: typeof import('naive-ui')['NTooltip']
NTable: typeof import('naive-ui')['NTable']
NumberInput: typeof import('./src/tools/pomodoro-timer/app/components/NumberInput.vue')['default']
NumeronymGenerator: typeof import('./src/tools/numeronym-generator/numeronym-generator.vue')['default']
OcrImage: typeof import('./src/tools/ocr-image/ocr-image.vue')['default']
@@ -363,7 +346,6 @@ declare module '@vue/runtime-core' {
PressureConverter: typeof import('./src/tools/pressure-converter/pressure-converter.vue')['default']
ProgressBar: typeof import('./src/tools/pomodoro-timer/app/components/ProgressBar.vue')['default']
PropertiesConverter: typeof import('./src/tools/properties-converter/properties-converter.vue')['default']
PropertiesToYaml: typeof import('./src/tools/properties-to-yaml/properties-to-yaml.vue')['default']
PunycodeConverter: typeof import('./src/tools/punycode-converter/punycode-converter.vue')['default']
QrCodeDecoder: typeof import('./src/tools/qr-code-decoder/qr-code-decoder.vue')['default']
QrCodeGenerator: typeof import('./src/tools/qr-code-generator/qr-code-generator.vue')['default']
@@ -400,7 +382,6 @@ declare module '@vue/runtime-core' {
SoftwareLicenceCompatibility: typeof import('./src/tools/software-licence-compatibility/software-licence-compatibility.vue')['default']
SpanCopyable: typeof import('./src/components/SpanCopyable.vue')['default']
SpfDnsGenerator: typeof import('./src/tools/spf-dns-generator/spf-dns-generator.vue')['default']
SpfGenerator: typeof import('./src/tools/spf-dns-generator/spf-generator.vue')['default']
SplashScreen: typeof import('./src/tools/pomodoro-timer/app/views/SplashScreen.vue')['default']
SqlMinifier: typeof import('./src/tools/sql-minifier/sql-minifier.vue')['default']
SqlPrettify: typeof import('./src/tools/sql-prettify/sql-prettify.vue')['default']
+1
View File
@@ -136,6 +136,7 @@
"figue": "^1.2.0",
"file-type": "^20.1.0",
"flatten-anything": "^4.0.1",
"flexsearch": "^0.8.167",
"fuse.js": "^7.1.0",
"generate-schema": "^2.6.0",
"get-timezone-offset": "^1.0.5",
+7
View File
@@ -305,6 +305,9 @@ dependencies:
flatten-anything:
specifier: ^4.0.1
version: 4.0.1
flexsearch:
specifier: ^0.8.167
version: 0.8.167
fuse.js:
specifier: ^7.1.0
version: 7.1.0
@@ -11203,6 +11206,10 @@ packages:
is-what: 5.0.2
dev: false
/flexsearch@0.8.167:
resolution: {integrity: sha512-iCBbY9npWhNlTCiHha2pwGqtbhpSEHx8+m5FJab+DLr/IKMneLNDgWbu2+IOvsAHo/Euo2Fu/5awrAEhN4q8Jw==}
dev: false
/follow-redirects@1.15.9:
resolution: {integrity: sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==}
engines: {node: '>=4.0'}
+119
View File
@@ -0,0 +1,119 @@
import { type MaybeRef, get } from '@vueuse/core';
import { computed, ref, watch } from 'vue';
import FlexSearch from 'flexsearch';
export function useFlexSearch<Data extends Record<string, any>>({
search,
data,
options = {
keys: [],
},
limit = -1,
}: {
search: MaybeRef<string>
data: Data[]
options?: {
keys: string[]
filterEmpty?: boolean
tokenize?: 'strict' | 'forward' | 'reverse' | 'full'
resolution?: number
optimize?: boolean
cache?: boolean | number
}
limit?: MaybeRef<number>
}) {
// Extract options
const { keys, filterEmpty = true, ...indexOptions } = options;
// Create a map to store original data items
const data_map = ref(new Map<number, Data>());
// Create separate indices for each key
const indices = keys.map((key) => {
return {
key,
index: new FlexSearch.Index({
tokenize: 'forward',
preset: 'performance',
resolution: 9,
optimize: true,
cache: 100,
fastupdate: true,
...indexOptions,
}),
};
});
// Initialize indices with data
const initialize_indices = () => {
data_map.value.clear();
data.forEach((item, idx) => {
// Store original item
data_map.value.set(idx, item);
// Add to each field index
indices.forEach(({ key, index }) => {
// Get the value for this key, potentially from nested paths
const value = key.split('.').reduce((obj, path) => obj?.[path], item);
if (value) {
index.add(idx, String(value));
}
});
});
};
// Initialize on creation
initialize_indices();
// Function to search across all indices
const search_all_indices = (query: string, search_limit: number) => {
if (!query) {
return [];
}
// Use a higher individual limit for each index to ensure we don't miss potential matches
const individual_limit = search_limit > 0 ? search_limit * 2 : undefined;
// Search each index
const result_sets = indices.map(({ index }) =>
index.search(query, individual_limit ? { limit: individual_limit } : undefined),
);
// Combine and deduplicate results
const unique_ids = new Set<number>();
result_sets.forEach((results) => {
results.forEach(id => unique_ids.add(id as number));
});
// Convert back to array of original items
const result_array = Array.from(unique_ids)
.map(id => data_map.value.get(id))
.filter(Boolean) as Data[];
// Apply the limit to the final combined results
return search_limit > 0 ? result_array.slice(0, search_limit) : result_array;
};
// Watch for data changes to rebuild indices
watch(() => data.length, initialize_indices);
// Computed results
const searchResult = computed<Data[]>(() => {
const query = get(search);
const search_limit: number = get(limit);
// Handle empty query case
if (!query && !filterEmpty) {
return search_limit > 0 ? data.slice(0, search_limit) : data;
}
if (!query) {
return [];
}
return search_all_indices(query, search_limit);
});
return { searchResult };
}
+19 -19
View File
@@ -3,7 +3,7 @@ import unicodeNames from '@unicode/unicode-15.1.0/Names/index.js';
import unicodeCategories from '@unicode/unicode-15.1.0/General_Category';
import utf8 from 'utf8';
import { useFuzzySearch } from '@/composable/fuzzySearch';
import { useFlexSearch } from '@/composable/flexSearch';
import useDebouncedRef from '@/composable/debouncedref';
function toPaddedHex(num: number) {
@@ -12,10 +12,11 @@ function toPaddedHex(num: number) {
function toUTF8(codePoint: number) {
const utf8String = utf8.encode(String.fromCodePoint(codePoint));
return [...utf8String].map(c => `\\x${c.codePointAt(0)?.toString(16).toUpperCase()}`);
const bytes = [...utf8String].map(c => `\\x${c.codePointAt(0)?.toString(16).toUpperCase()}`);
return bytes.join(''); // Join the string array into a single string to fix a bug where it would otherwise join the array items with a comma
}
const searchQuery = useDebouncedRef('', 500);
const searchQuery = useDebouncedRef('', 250);
const parsedSearchQuery = computed(() => {
const parsedRegex = /^\s*(?:\&#x(?<hex1>[\da-f]+);|\&#(?<dec>\d+);|(?:U\+|\\u)?\s*(?<hex2>[\da-f]+))\s*$/gi; // NOSONAR
const parsedQuery = parsedRegex.exec(searchQuery.value);
@@ -39,18 +40,19 @@ const unicodeSearchData = [...unicodeNames].map(([codePoint, characterName]) =>
};
});
const limit = ref(100);
const { searchResult } = useFuzzySearch({
const limit = ref(40);
// Using FlexSearch instead of FuzzySearch for better performance
const { searchResult } = useFlexSearch({
search: parsedSearchQuery,
data: unicodeSearchData,
options: {
keys: ['characterName', 'hex'],
threshold: 0.2,
isCaseSensitive: false,
minMatchCharLength: 3,
useExtendedSearch: true,
filterEmpty: false,
cache: true,
resolution: 9,
optimize: true,
},
limit: limit.value,
limit,
});
</script>
@@ -71,12 +73,7 @@ const { searchResult } = useFuzzySearch({
</div>
<div v-if="searchQuery.trim().length > 0">
<div
v-if="searchResult.length === 0"
mt-4
text-20px
font-bold
>
<div v-if="searchResult.length === 0" mt-4 text-20px font-bold>
No results
</div>
@@ -89,11 +86,11 @@ const { searchResult } = useFuzzySearch({
<thead>
<th>UCOD</th>
<th>Display/UTF8</th>
<th style="width:30%">
<th style="width: 30%">
Category
</th>
<th>Html</th>
<th style="width:30%">
<th style="width: 30%">
Name
</th>
</thead>
@@ -101,7 +98,10 @@ const { searchResult } = useFuzzySearch({
<tr v-for="(result, ix) in searchResult" :key="ix">
<td>
<input-copyable :value="`U+${toPaddedHex(result.codePoint)}`" mb-1 />
<!-- //NOSONAR --><n-a :href="`https://unicodeplus.com/U+${toPaddedHex(result.codePoint)}`" target="_blank">
<!-- //NOSONAR --><n-a
:href="`https://unicodeplus.com/U+${toPaddedHex(result.codePoint)}`"
target="_blank"
>
&gt; More info
</n-a>
</td>