mirror of
https://github.com/sharevb/it-tools.git
synced 2026-11-20 21:23:08 +03:00
Improved performance on the Unicode Search page (#75)
* Improved performance on the Unicode Search page: - Created new flexSearch component (Faster than fuzzySearch). For more information, see https://github.com/nextapps-de/flexsearch - Replaced fuzzySearch with the flexSearch component on the unicode search page - Fixed a bug where the search limit counter didn't have any effect - Reduced the default search limit from 100 to 40 to reduce lag from loading all the ui elements * Added clarification to comment * Attempt to fix pnpm-lock merge conflict * Another attempt to fix pnpm-lock * Try deleting lock file because regenerating doesn't make a difference * Try regenerating from scratch * fix: pnpm-lock.yaml --------- Co-authored-by: sharevb <sharevb@gmail.com>
This commit is contained in:
1 parent
48854592b0
commit
2355232aed
5 files changed
+149
-41
No files matched your search
Vendored
+3
-22
@@ -116,9 +116,7 @@ declare module '@vue/runtime-core' {
|
||||
DiffViewer: typeof import('./src/tools/json-diff/diff-viewer/diff-viewer.vue')['default']
|
||||
DistinctWords: typeof import('./src/tools/distinct-words/distinct-words.vue')['default']
|
||||
DkimDnsGenerator: typeof import('./src/tools/dkim-dns-generator/dkim-dns-generator.vue')['default']
|
||||
'Dmarc-Generator': typeof import('./src/tools/dmarc-dns-generator/dmarc--generator.vue')['default']
|
||||
DmarcDnsGenerator: typeof import('./src/tools/dmarc-dns-generator/dmarc-dns-generator.vue')['default']
|
||||
DmarcGenerator: typeof import('./src/tools/dmarc-dns-generator/dmarc-generator.vue')['default']
|
||||
DmarcReportAnalyzer: typeof import('./src/tools/dmarc-report-analyzer/dmarc-report-analyzer.vue')['default']
|
||||
DnsQueries: typeof import('./src/tools/dns-queries/dns-queries.vue')['default']
|
||||
Docker: typeof import('./src/tools/docker-memo/docker.md')['default']
|
||||
@@ -294,40 +292,25 @@ declare module '@vue/runtime-core' {
|
||||
MorseConverter: typeof import('./src/tools/morse-converter/morse-converter.vue')['default']
|
||||
MultiLinkDownloader: typeof import('./src/tools/multi-link-downloader/multi-link-downloader.vue')['default']
|
||||
MyIp: typeof import('./src/tools/my-ip/my-ip.vue')['default']
|
||||
NA: typeof import('naive-ui')['NA']
|
||||
NanoidGenerator: typeof import('./src/tools/nanoid-generator/nanoid-generator.vue')['default']
|
||||
NanoMemo: typeof import('./src/tools/nano-memo/nano-memo.vue')['default']
|
||||
'NanoMemo.content': typeof import('./src/tools/nano-memo/nano-memo.content.md')['default']
|
||||
NavbarButtons: typeof import('./src/components/NavbarButtons.vue')['default']
|
||||
NButton: typeof import('naive-ui')['NButton']
|
||||
NCard: typeof import('naive-ui')['NCard']
|
||||
NCheckbox: typeof import('naive-ui')['NCheckbox']
|
||||
NCheckboxGroup: typeof import('naive-ui')['NCheckboxGroup']
|
||||
NCode: typeof import('naive-ui')['NCode']
|
||||
NCollapseTransition: typeof import('naive-ui')['NCollapseTransition']
|
||||
NConfigProvider: typeof import('naive-ui')['NConfigProvider']
|
||||
NDivider: typeof import('naive-ui')['NDivider']
|
||||
NDynamicTags: typeof import('naive-ui')['NDynamicTags']
|
||||
NDataTable: typeof import('naive-ui')['NDataTable']
|
||||
NEllipsis: typeof import('naive-ui')['NEllipsis']
|
||||
NForm: typeof import('naive-ui')['NForm']
|
||||
NFormItem: typeof import('naive-ui')['NFormItem']
|
||||
NginxFormatter: typeof import('./src/tools/nginx-formatter/nginx-formatter.vue')['default']
|
||||
NH1: typeof import('naive-ui')['NH1']
|
||||
NH3: typeof import('naive-ui')['NH3']
|
||||
NIcon: typeof import('naive-ui')['NIcon']
|
||||
NInput: typeof import('naive-ui')['NInput']
|
||||
NInputNumber: typeof import('naive-ui')['NInputNumber']
|
||||
NLayout: typeof import('naive-ui')['NLayout']
|
||||
NLayoutSider: typeof import('naive-ui')['NLayoutSider']
|
||||
NMenu: typeof import('naive-ui')['NMenu']
|
||||
NP: typeof import('naive-ui')['NP']
|
||||
NRadio: typeof import('naive-ui')['NRadio']
|
||||
NRadioGroup: typeof import('naive-ui')['NRadioGroup']
|
||||
NScrollbar: typeof import('naive-ui')['NScrollbar']
|
||||
NSelect: typeof import('naive-ui')['NSelect']
|
||||
NSpace: typeof import('naive-ui')['NSpace']
|
||||
NSwitch: typeof import('naive-ui')['NSwitch']
|
||||
NText: typeof import('naive-ui')['NText']
|
||||
NTooltip: typeof import('naive-ui')['NTooltip']
|
||||
NTable: typeof import('naive-ui')['NTable']
|
||||
NumberInput: typeof import('./src/tools/pomodoro-timer/app/components/NumberInput.vue')['default']
|
||||
NumeronymGenerator: typeof import('./src/tools/numeronym-generator/numeronym-generator.vue')['default']
|
||||
OcrImage: typeof import('./src/tools/ocr-image/ocr-image.vue')['default']
|
||||
@@ -363,7 +346,6 @@ declare module '@vue/runtime-core' {
|
||||
PressureConverter: typeof import('./src/tools/pressure-converter/pressure-converter.vue')['default']
|
||||
ProgressBar: typeof import('./src/tools/pomodoro-timer/app/components/ProgressBar.vue')['default']
|
||||
PropertiesConverter: typeof import('./src/tools/properties-converter/properties-converter.vue')['default']
|
||||
PropertiesToYaml: typeof import('./src/tools/properties-to-yaml/properties-to-yaml.vue')['default']
|
||||
PunycodeConverter: typeof import('./src/tools/punycode-converter/punycode-converter.vue')['default']
|
||||
QrCodeDecoder: typeof import('./src/tools/qr-code-decoder/qr-code-decoder.vue')['default']
|
||||
QrCodeGenerator: typeof import('./src/tools/qr-code-generator/qr-code-generator.vue')['default']
|
||||
@@ -400,7 +382,6 @@ declare module '@vue/runtime-core' {
|
||||
SoftwareLicenceCompatibility: typeof import('./src/tools/software-licence-compatibility/software-licence-compatibility.vue')['default']
|
||||
SpanCopyable: typeof import('./src/components/SpanCopyable.vue')['default']
|
||||
SpfDnsGenerator: typeof import('./src/tools/spf-dns-generator/spf-dns-generator.vue')['default']
|
||||
SpfGenerator: typeof import('./src/tools/spf-dns-generator/spf-generator.vue')['default']
|
||||
SplashScreen: typeof import('./src/tools/pomodoro-timer/app/views/SplashScreen.vue')['default']
|
||||
SqlMinifier: typeof import('./src/tools/sql-minifier/sql-minifier.vue')['default']
|
||||
SqlPrettify: typeof import('./src/tools/sql-prettify/sql-prettify.vue')['default']
|
||||
|
||||
@@ -136,6 +136,7 @@
|
||||
"figue": "^1.2.0",
|
||||
"file-type": "^20.1.0",
|
||||
"flatten-anything": "^4.0.1",
|
||||
"flexsearch": "^0.8.167",
|
||||
"fuse.js": "^7.1.0",
|
||||
"generate-schema": "^2.6.0",
|
||||
"get-timezone-offset": "^1.0.5",
|
||||
|
||||
Generated
+7
@@ -305,6 +305,9 @@ dependencies:
|
||||
flatten-anything:
|
||||
specifier: ^4.0.1
|
||||
version: 4.0.1
|
||||
flexsearch:
|
||||
specifier: ^0.8.167
|
||||
version: 0.8.167
|
||||
fuse.js:
|
||||
specifier: ^7.1.0
|
||||
version: 7.1.0
|
||||
@@ -11203,6 +11206,10 @@ packages:
|
||||
is-what: 5.0.2
|
||||
dev: false
|
||||
|
||||
/flexsearch@0.8.167:
|
||||
resolution: {integrity: sha512-iCBbY9npWhNlTCiHha2pwGqtbhpSEHx8+m5FJab+DLr/IKMneLNDgWbu2+IOvsAHo/Euo2Fu/5awrAEhN4q8Jw==}
|
||||
dev: false
|
||||
|
||||
/follow-redirects@1.15.9:
|
||||
resolution: {integrity: sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==}
|
||||
engines: {node: '>=4.0'}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { type MaybeRef, get } from '@vueuse/core';
|
||||
import { computed, ref, watch } from 'vue';
|
||||
import FlexSearch from 'flexsearch';
|
||||
|
||||
export function useFlexSearch<Data extends Record<string, any>>({
|
||||
search,
|
||||
data,
|
||||
options = {
|
||||
keys: [],
|
||||
},
|
||||
limit = -1,
|
||||
}: {
|
||||
search: MaybeRef<string>
|
||||
data: Data[]
|
||||
options?: {
|
||||
keys: string[]
|
||||
filterEmpty?: boolean
|
||||
tokenize?: 'strict' | 'forward' | 'reverse' | 'full'
|
||||
resolution?: number
|
||||
optimize?: boolean
|
||||
cache?: boolean | number
|
||||
}
|
||||
limit?: MaybeRef<number>
|
||||
}) {
|
||||
// Extract options
|
||||
const { keys, filterEmpty = true, ...indexOptions } = options;
|
||||
|
||||
// Create a map to store original data items
|
||||
const data_map = ref(new Map<number, Data>());
|
||||
|
||||
// Create separate indices for each key
|
||||
const indices = keys.map((key) => {
|
||||
return {
|
||||
key,
|
||||
index: new FlexSearch.Index({
|
||||
tokenize: 'forward',
|
||||
preset: 'performance',
|
||||
resolution: 9,
|
||||
optimize: true,
|
||||
cache: 100,
|
||||
fastupdate: true,
|
||||
...indexOptions,
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
// Initialize indices with data
|
||||
const initialize_indices = () => {
|
||||
data_map.value.clear();
|
||||
|
||||
data.forEach((item, idx) => {
|
||||
// Store original item
|
||||
data_map.value.set(idx, item);
|
||||
|
||||
// Add to each field index
|
||||
indices.forEach(({ key, index }) => {
|
||||
// Get the value for this key, potentially from nested paths
|
||||
const value = key.split('.').reduce((obj, path) => obj?.[path], item);
|
||||
if (value) {
|
||||
index.add(idx, String(value));
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
// Initialize on creation
|
||||
initialize_indices();
|
||||
|
||||
// Function to search across all indices
|
||||
const search_all_indices = (query: string, search_limit: number) => {
|
||||
if (!query) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Use a higher individual limit for each index to ensure we don't miss potential matches
|
||||
const individual_limit = search_limit > 0 ? search_limit * 2 : undefined;
|
||||
|
||||
// Search each index
|
||||
const result_sets = indices.map(({ index }) =>
|
||||
index.search(query, individual_limit ? { limit: individual_limit } : undefined),
|
||||
);
|
||||
|
||||
// Combine and deduplicate results
|
||||
const unique_ids = new Set<number>();
|
||||
result_sets.forEach((results) => {
|
||||
results.forEach(id => unique_ids.add(id as number));
|
||||
});
|
||||
|
||||
// Convert back to array of original items
|
||||
const result_array = Array.from(unique_ids)
|
||||
.map(id => data_map.value.get(id))
|
||||
.filter(Boolean) as Data[];
|
||||
|
||||
// Apply the limit to the final combined results
|
||||
return search_limit > 0 ? result_array.slice(0, search_limit) : result_array;
|
||||
};
|
||||
|
||||
// Watch for data changes to rebuild indices
|
||||
watch(() => data.length, initialize_indices);
|
||||
|
||||
// Computed results
|
||||
const searchResult = computed<Data[]>(() => {
|
||||
const query = get(search);
|
||||
const search_limit: number = get(limit);
|
||||
|
||||
// Handle empty query case
|
||||
if (!query && !filterEmpty) {
|
||||
return search_limit > 0 ? data.slice(0, search_limit) : data;
|
||||
}
|
||||
|
||||
if (!query) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return search_all_indices(query, search_limit);
|
||||
});
|
||||
|
||||
return { searchResult };
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import unicodeNames from '@unicode/unicode-15.1.0/Names/index.js';
|
||||
import unicodeCategories from '@unicode/unicode-15.1.0/General_Category';
|
||||
import utf8 from 'utf8';
|
||||
|
||||
import { useFuzzySearch } from '@/composable/fuzzySearch';
|
||||
import { useFlexSearch } from '@/composable/flexSearch';
|
||||
import useDebouncedRef from '@/composable/debouncedref';
|
||||
|
||||
function toPaddedHex(num: number) {
|
||||
@@ -12,10 +12,11 @@ function toPaddedHex(num: number) {
|
||||
|
||||
function toUTF8(codePoint: number) {
|
||||
const utf8String = utf8.encode(String.fromCodePoint(codePoint));
|
||||
return [...utf8String].map(c => `\\x${c.codePointAt(0)?.toString(16).toUpperCase()}`);
|
||||
const bytes = [...utf8String].map(c => `\\x${c.codePointAt(0)?.toString(16).toUpperCase()}`);
|
||||
return bytes.join(''); // Join the string array into a single string to fix a bug where it would otherwise join the array items with a comma
|
||||
}
|
||||
|
||||
const searchQuery = useDebouncedRef('', 500);
|
||||
const searchQuery = useDebouncedRef('', 250);
|
||||
const parsedSearchQuery = computed(() => {
|
||||
const parsedRegex = /^\s*(?:\&#x(?<hex1>[\da-f]+);|\&#(?<dec>\d+);|(?:U\+|\\u)?\s*(?<hex2>[\da-f]+))\s*$/gi; // NOSONAR
|
||||
const parsedQuery = parsedRegex.exec(searchQuery.value);
|
||||
@@ -39,18 +40,19 @@ const unicodeSearchData = [...unicodeNames].map(([codePoint, characterName]) =>
|
||||
};
|
||||
});
|
||||
|
||||
const limit = ref(100);
|
||||
const { searchResult } = useFuzzySearch({
|
||||
const limit = ref(40);
|
||||
// Using FlexSearch instead of FuzzySearch for better performance
|
||||
const { searchResult } = useFlexSearch({
|
||||
search: parsedSearchQuery,
|
||||
data: unicodeSearchData,
|
||||
options: {
|
||||
keys: ['characterName', 'hex'],
|
||||
threshold: 0.2,
|
||||
isCaseSensitive: false,
|
||||
minMatchCharLength: 3,
|
||||
useExtendedSearch: true,
|
||||
filterEmpty: false,
|
||||
cache: true,
|
||||
resolution: 9,
|
||||
optimize: true,
|
||||
},
|
||||
limit: limit.value,
|
||||
limit,
|
||||
});
|
||||
</script>
|
||||
|
||||
@@ -71,12 +73,7 @@ const { searchResult } = useFuzzySearch({
|
||||
</div>
|
||||
|
||||
<div v-if="searchQuery.trim().length > 0">
|
||||
<div
|
||||
v-if="searchResult.length === 0"
|
||||
mt-4
|
||||
text-20px
|
||||
font-bold
|
||||
>
|
||||
<div v-if="searchResult.length === 0" mt-4 text-20px font-bold>
|
||||
No results
|
||||
</div>
|
||||
|
||||
@@ -89,11 +86,11 @@ const { searchResult } = useFuzzySearch({
|
||||
<thead>
|
||||
<th>UCOD</th>
|
||||
<th>Display/UTF8</th>
|
||||
<th style="width:30%">
|
||||
<th style="width: 30%">
|
||||
Category
|
||||
</th>
|
||||
<th>Html</th>
|
||||
<th style="width:30%">
|
||||
<th style="width: 30%">
|
||||
Name
|
||||
</th>
|
||||
</thead>
|
||||
@@ -101,7 +98,10 @@ const { searchResult } = useFuzzySearch({
|
||||
<tr v-for="(result, ix) in searchResult" :key="ix">
|
||||
<td>
|
||||
<input-copyable :value="`U+${toPaddedHex(result.codePoint)}`" mb-1 />
|
||||
<!-- //NOSONAR --><n-a :href="`https://unicodeplus.com/U+${toPaddedHex(result.codePoint)}`" target="_blank">
|
||||
<!-- //NOSONAR --><n-a
|
||||
:href="`https://unicodeplus.com/U+${toPaddedHex(result.codePoint)}`"
|
||||
target="_blank"
|
||||
>
|
||||
> More info
|
||||
</n-a>
|
||||
</td>
|
||||
|
||||
Reference in new issue
Block a user