mirror of
https://github.com/sharevb/it-tools.git
synced 2026-11-20 21:23:08 +03:00
feat(Text to Unicode/Binary): encoding enhancements
Fix part of #1447 Fix #1425, #1072 Fix #1082, #1081, #1175
This commit is contained in:
1 parent
f3814e9a5b
commit
7f1f39dda4
7 files changed
+310
-345
No files matched your search
@@ -3,37 +3,77 @@ import { convertTextToUtf8Binary, convertUtf8BinaryToText } from './text-to-bina
|
||||
|
||||
describe('text-to-binary', () => {
|
||||
const utf8Tests = [
|
||||
{ text: '文字', binary: '11100110 10010110 10000111 11100101 10101101 10010111' },
|
||||
{ text: '💩', binary: '11110000 10011111 10010010 10101001' },
|
||||
{
|
||||
text: '文字',
|
||||
binary: '11100110 10010110 10000111 11100101 10101101 10010111',
|
||||
decimal: '',
|
||||
octal: '',
|
||||
hex: '',
|
||||
},
|
||||
{
|
||||
text: '💩',
|
||||
binary: '11110000 10011111 10010010 10101001',
|
||||
decimal: '',
|
||||
octal: '',
|
||||
hex: '',
|
||||
},
|
||||
];
|
||||
|
||||
describe('convertTextToUtf8Binary', () => {
|
||||
it('a text string is converted to its UTF-8 binary representation', () => {
|
||||
expect(convertTextToUtf8Binary('A')).toBe('01000001');
|
||||
expect(convertTextToUtf8Binary('A', { base: 8 })).toBe('0101');
|
||||
expect(convertTextToUtf8Binary('A', { base: 10 })).toBe('65');
|
||||
expect(convertTextToUtf8Binary('A', { base: 16 })).toBe('41');
|
||||
expect(convertTextToUtf8Binary('hello')).toBe('01101000 01100101 01101100 01101100 01101111');
|
||||
expect(convertTextToUtf8Binary('hello', { base: 8 })).toBe('0150 0145 0154 0154 0157');
|
||||
expect(convertTextToUtf8Binary('hello', { base: 10 })).toBe('104 101 108 108 111');
|
||||
expect(convertTextToUtf8Binary('hello', { base: 16 })).toBe('68 65 6c 6c 6f');
|
||||
expect(convertTextToUtf8Binary('')).toBe('');
|
||||
});
|
||||
it('the separator between octets can be changed', () => {
|
||||
expect(convertTextToUtf8Binary('hello', { separator: '' })).toBe('0110100001100101011011000110110001101111');
|
||||
expect(convertTextToUtf8Binary('hello', { separator: '-' })).toBe('01101000-01100101-01101100-01101100-01101111');
|
||||
expect(convertTextToUtf8Binary('hello', { separator: '-', base: 16 })).toBe('68-65-6c-6c-6f');
|
||||
});
|
||||
it('works with non-ASCII input', () => {
|
||||
for (const { text, binary } of utf8Tests) {
|
||||
const converted = convertTextToUtf8Binary(text);
|
||||
expect(converted).toBe(binary);
|
||||
}
|
||||
|
||||
expect(convertTextToUtf8Binary('💩 A', { base: 2 })).toBe('11110000 10011111 10010010 10101001 00100000 01000001');
|
||||
expect(convertTextToUtf8Binary('💩 A', { base: 8 })).toBe('0360 0237 0222 0251 040 0101');
|
||||
expect(convertTextToUtf8Binary('💩 A', { base: 10 })).toBe('240 159 146 169 32 65');
|
||||
expect(convertTextToUtf8Binary('💩 A', { base: 16 })).toBe('f0 9f 92 a9 20 41');
|
||||
});
|
||||
});
|
||||
|
||||
describe('convertUtf8BinaryToText', () => {
|
||||
it('an ascii binary string is converted to its text representation', () => {
|
||||
expect(convertUtf8BinaryToText('01101000 01100101 01101100 01101100 01101111')).toBe('hello');
|
||||
expect(convertUtf8BinaryToText('01101000 01100101 01101100 01101100 01101111', { base: 2 })).toBe('hello');
|
||||
expect(convertUtf8BinaryToText('0150 0145 0154 0154 0157', { base: 8 })).toBe('hello');
|
||||
expect(convertUtf8BinaryToText('104 101 108 108 111', { base: 10 })).toBe('hello');
|
||||
expect(convertUtf8BinaryToText('68 65 6c 6c 6f', { base: 16 })).toBe('hello');
|
||||
|
||||
expect(convertUtf8BinaryToText('11110000 10011111 10010010 10101001 00100000 01000001', { base: 2 })).toBe('💩 A');
|
||||
expect(convertUtf8BinaryToText('0360 0237 0222 0251 040 0101', { base: 8 })).toBe('💩 A');
|
||||
expect(convertUtf8BinaryToText('240 159 146 169 32 65', { base: 10 })).toBe('💩 A');
|
||||
expect(convertUtf8BinaryToText('f0 9f 92 a9 20 41', { base: 16 })).toBe('💩 A');
|
||||
|
||||
expect(convertUtf8BinaryToText('')).toBe('');
|
||||
|
||||
expect(convertUtf8BinaryToText('01000001')).toBe('A');
|
||||
expect(convertTextToUtf8Binary('')).toBe('');
|
||||
expect(convertUtf8BinaryToText('0101', { base: 8 })).toBe('A');
|
||||
expect(convertUtf8BinaryToText('65', { base: 10 })).toBe('A');
|
||||
expect(convertUtf8BinaryToText('41', { base: 16 })).toBe('A');
|
||||
});
|
||||
|
||||
it('the given binary string is cleaned before conversion', () => {
|
||||
it('the given string is cleaned before conversion', () => {
|
||||
expect(convertUtf8BinaryToText(' 01000 001garbage')).toBe('A');
|
||||
expect(convertUtf8BinaryToText(' 65garbage', { base: 10 })).toBe('A');
|
||||
expect(convertUtf8BinaryToText(' 41xxxx', { base: 16 })).toBe('A');
|
||||
});
|
||||
|
||||
it('throws an error if the given binary string is not an integer number of complete octets', () => {
|
||||
@@ -46,6 +86,13 @@ describe('text-to-binary', () => {
|
||||
expect(() => convertUtf8BinaryToText('11111111')).toThrow();
|
||||
});
|
||||
|
||||
it('the given string is cleaned from prefix before conversion', () => {
|
||||
expect(convertUtf8BinaryToText('0b01000001')).toBe('A');
|
||||
expect(convertUtf8BinaryToText('0x41', { base: 16 })).toBe('A');
|
||||
expect(convertUtf8BinaryToText('0x68 0x65 0x6c 0x6c 0x6f', { base: 16 })).toBe('hello');
|
||||
expect(convertUtf8BinaryToText('\\x68\\x65\\x6c\\x6c\\x6f', { base: 16 })).toBe('hello');
|
||||
});
|
||||
|
||||
it('works with non-ASCII input', () => {
|
||||
for (const { text, binary } of utf8Tests) {
|
||||
const reverted = convertUtf8BinaryToText(binary);
|
||||
|
||||
@@ -1,19 +1,67 @@
|
||||
export { convertTextToUtf8Binary, convertUtf8BinaryToText };
|
||||
|
||||
function convertTextToUtf8Binary(text: string, { separator = ' ' }: { separator?: string } = {}): string {
|
||||
return [...new TextEncoder().encode(text)].map(x => x.toString(2).padStart(8, '0')).join(separator);
|
||||
export type EncodingBase = 2 | 8 | 10 | 16;
|
||||
|
||||
function convertTextToUtf8Binary(text: string, { separator = ' ', base = 2 }: { separator?: string; base?: EncodingBase } = {}): string {
|
||||
if (!text?.trim()) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return [...new TextEncoder().encode(text)].map((char) => {
|
||||
const charInBase = char.toString(base);
|
||||
if (base === 2) {
|
||||
return charInBase.padStart(8, '0');
|
||||
}
|
||||
if (base === 8) {
|
||||
return `0${charInBase}`;
|
||||
}
|
||||
if (base === 16) {
|
||||
return charInBase.padStart(2, '0');
|
||||
}
|
||||
|
||||
return charInBase;
|
||||
}).join(separator);
|
||||
}
|
||||
|
||||
function convertUtf8BinaryToText(binary: string): string {
|
||||
const cleanBinary = binary.replace(/[^01]+/g, '');
|
||||
function convertUtf8BinaryToText(binary: string, { base = 2 }: { base?: EncodingBase } = {}): string {
|
||||
if (!binary?.trim()) {
|
||||
return '';
|
||||
}
|
||||
|
||||
if (cleanBinary.length % 8) {
|
||||
throw new Error('Invalid binary string');
|
||||
let codepoints: number[] = [];
|
||||
if (base === 2) {
|
||||
const cleanBinary = binary.replace(/0b/g, '').replace(/[^01]/g, '').trim();
|
||||
|
||||
if (cleanBinary.length % 8) {
|
||||
throw new Error('Invalid binary string');
|
||||
}
|
||||
|
||||
codepoints = cleanBinary
|
||||
.split(/([01]{8})/)
|
||||
.filter(Boolean)
|
||||
.map(binary => Number.parseInt(binary, 2));
|
||||
}
|
||||
else if (base === 16) {
|
||||
const cleanBinary = binary.replace(/0x|\\x/g, '').replace(/[^0-9A-Fa-f]/g, '');
|
||||
|
||||
if (cleanBinary.length % 2) {
|
||||
throw new Error('Invalid hexadecimal string');
|
||||
}
|
||||
|
||||
codepoints = cleanBinary
|
||||
.split(/([0-9A-Fa-f]{2})/)
|
||||
.filter(Boolean)
|
||||
.map(binary => Number.parseInt(binary, 16));
|
||||
}
|
||||
else {
|
||||
const cleanBinary = binary.replace(/0o/g, '').replace(/[^\d\s]/g, '');
|
||||
codepoints = cleanBinary
|
||||
.split(/\s/)
|
||||
.filter(Boolean)
|
||||
.map(binary => Number.parseInt(binary, base));
|
||||
}
|
||||
|
||||
return new TextDecoder(undefined, { fatal: true }).decode(
|
||||
Uint8Array.from({ length: cleanBinary.length / 8 }, (_, i) =>
|
||||
Number.parseInt(cleanBinary.slice(i * 8, (i + 1) * 8), 2),
|
||||
),
|
||||
Uint8Array.from(codepoints),
|
||||
);
|
||||
}
|
||||
@@ -1,15 +1,17 @@
|
||||
<script setup lang="ts">
|
||||
import { convertTextToUtf8Binary, convertUtf8BinaryToText } from './text-to-binary.models';
|
||||
import { type EncodingBase, convertTextToUtf8Binary, convertUtf8BinaryToText } from './text-to-binary.models';
|
||||
import { withDefaultOnError } from '@/utils/defaults';
|
||||
import { useCopy } from '@/composable/copy';
|
||||
import { isNotThrowing } from '@/utils/boolean';
|
||||
import { useQueryParamOrStorage } from '@/composable/queryParams';
|
||||
|
||||
const base = useQueryParamOrStorage({ name: 'base', storageName: 'txt-bin:base', defaultValue: '2' });
|
||||
const inputText = ref('');
|
||||
const binaryFromText = computed(() => convertTextToUtf8Binary(inputText.value));
|
||||
const binaryFromText = computed(() => convertTextToUtf8Binary(inputText.value, { base: Number(base.value) as EncodingBase }));
|
||||
const { copy: copyBinary } = useCopy({ source: binaryFromText });
|
||||
|
||||
const inputBinary = ref('');
|
||||
const textFromBinary = computed(() => withDefaultOnError(() => convertUtf8BinaryToText(inputBinary.value), ''));
|
||||
const textFromBinary = computed(() => withDefaultOnError(() => convertUtf8BinaryToText(inputBinary.value, { base: Number(base.value) as EncodingBase }), ''));
|
||||
const inputBinaryValidationRules = [
|
||||
{
|
||||
validator: (value: string) => isNotThrowing(() => convertUtf8BinaryToText(value)),
|
||||
@@ -20,59 +22,74 @@ const { copy: copyText } = useCopy({ source: textFromBinary });
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<c-card title="Text to UTF-8 binary">
|
||||
<c-input-text
|
||||
v-model:value="inputText"
|
||||
multiline
|
||||
placeholder="e.g. 'Hello world'"
|
||||
label="Enter text to convert to binary"
|
||||
autosize
|
||||
autofocus
|
||||
raw-text
|
||||
test-id="text-to-binary-input"
|
||||
<div>
|
||||
<c-select
|
||||
v-model:value="base"
|
||||
label="Conversion Base:"
|
||||
label-position="left"
|
||||
mb-2
|
||||
:options="[
|
||||
{ value: '2', label: 'Binary' },
|
||||
{ value: '8', label: 'Octal' },
|
||||
{ value: '10', label: 'Decimal' },
|
||||
{ value: '16', label: 'Hexadecimal' },
|
||||
]"
|
||||
/>
|
||||
<c-input-text
|
||||
v-model:value="binaryFromText"
|
||||
label="Binary from your text"
|
||||
multiline
|
||||
raw-text
|
||||
readonly
|
||||
mt-2
|
||||
placeholder="The binary representation of your text will be here"
|
||||
test-id="text-to-binary-output"
|
||||
/>
|
||||
<div mt-2 flex justify-center>
|
||||
<c-button :disabled="!binaryFromText" @click="copyBinary()">
|
||||
Copy binary to clipboard
|
||||
</c-button>
|
||||
</div>
|
||||
</c-card>
|
||||
|
||||
<c-card title="UTF-8 binary to text">
|
||||
<c-input-text
|
||||
v-model:value="inputBinary"
|
||||
multiline
|
||||
placeholder="e.g. '01001000 01100101 01101100 01101100 01101111'"
|
||||
label="Enter binary to convert to text"
|
||||
autosize
|
||||
raw-text
|
||||
:validation-rules="inputBinaryValidationRules"
|
||||
test-id="binary-to-text-input"
|
||||
/>
|
||||
<c-input-text
|
||||
v-model:value="textFromBinary"
|
||||
label="Text from your binary"
|
||||
multiline
|
||||
raw-text
|
||||
readonly
|
||||
mt-2
|
||||
placeholder="The text representation of your binary will be here"
|
||||
test-id="binary-to-text-output"
|
||||
/>
|
||||
<div mt-2 flex justify-center>
|
||||
<c-button :disabled="!textFromBinary" @click="copyText()">
|
||||
Copy text to clipboard
|
||||
</c-button>
|
||||
</div>
|
||||
</c-card>
|
||||
<c-card title="Text to UTF-8 binary">
|
||||
<c-input-text
|
||||
v-model:value="inputText"
|
||||
multiline
|
||||
placeholder="e.g. 'Hello world'"
|
||||
label="Enter text to convert to binary"
|
||||
autosize
|
||||
autofocus
|
||||
raw-text
|
||||
test-id="text-to-binary-input"
|
||||
/>
|
||||
<c-input-text
|
||||
v-model:value="binaryFromText"
|
||||
label="Binary from your text"
|
||||
multiline
|
||||
raw-text
|
||||
readonly
|
||||
mt-2
|
||||
placeholder="The binary representation of your text will be here"
|
||||
test-id="text-to-binary-output"
|
||||
/>
|
||||
<div mt-2 flex justify-center>
|
||||
<c-button :disabled="!binaryFromText" @click="copyBinary()">
|
||||
Copy binary to clipboard
|
||||
</c-button>
|
||||
</div>
|
||||
</c-card>
|
||||
|
||||
<c-card title="UTF-8 binary to text">
|
||||
<c-input-text
|
||||
v-model:value="inputBinary"
|
||||
multiline
|
||||
placeholder="e.g. '01001000 01100101 01101100 01101100 01101111'"
|
||||
label="Enter binary to convert to text"
|
||||
autosize
|
||||
raw-text
|
||||
:validation-rules="inputBinaryValidationRules"
|
||||
test-id="binary-to-text-input"
|
||||
/>
|
||||
<c-input-text
|
||||
v-model:value="textFromBinary"
|
||||
label="Text from your binary"
|
||||
multiline
|
||||
raw-text
|
||||
readonly
|
||||
mt-2
|
||||
placeholder="The text representation of your binary will be here"
|
||||
test-id="binary-to-text-output"
|
||||
/>
|
||||
<div mt-2 flex justify-center>
|
||||
<c-button :disabled="!textFromBinary" @click="copyText()">
|
||||
Copy text to clipboard
|
||||
</c-button>
|
||||
</div>
|
||||
</c-card>
|
||||
</div>
|
||||
</template>
|
||||
@@ -14,7 +14,7 @@ test.describe('Tool - Text to Unicode', () => {
|
||||
const unicode = await page.getByTestId('text-to-unicode-output').inputValue();
|
||||
|
||||
// eslint-disable-next-line unicorn/escape-case
|
||||
expect(unicode).toEqual(String.raw`\u0022it-tools\u0022 \u6587\u5b57`);
|
||||
expect(unicode).toEqual(String.raw`"it-tools" 文字`);
|
||||
});
|
||||
|
||||
test('Unicode to text conversion', async ({ page }) => {
|
||||
|
||||
@@ -1,14 +1,22 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { type ConverterId, SKIP_ASCII_HTML, SKIP_ASCII_JS, converters } from './text-to-unicode.service';
|
||||
|
||||
describe('text-to-unicode (legacy tests)', () => {
|
||||
const convertTextToUnicode = (text: string) => converters.decimalEntities.escape(text, false);
|
||||
const convertUnicodeToText = (escaped: string) => converters.decimalEntities.unescape(escaped);
|
||||
import { convertTextToUnicode, convertUnicodeToText } from './text-to-unicode.service';
|
||||
|
||||
describe('text-to-unicode', () => {
|
||||
describe('convertTextToUnicode', () => {
|
||||
it('a text string is converted to unicode representation', () => {
|
||||
expect(convertTextToUnicode('A')).toBe('A');
|
||||
expect(convertTextToUnicode('linke the string convert to unicode')).toBe('linke the string convert to unicode');
|
||||
expect(convertTextToUnicode('💩 AĀ')).toBe('💩 AĀ');
|
||||
expect(convertTextToUnicode('💩 AĀ', { encoding: 'antiuni' })).toBe('\\u1f4a9\\u20\\u41\\u0100');
|
||||
expect(convertTextToUnicode('💩 AĀ', { encoding: 'css' })).toBe('\\01f4a9\\000020\\000041\\000100');
|
||||
expect(convertTextToUnicode('💩 AĀ', { encoding: 'htmldec' })).toBe('💩 AĀ');
|
||||
expect(convertTextToUnicode('💩 AĀ', { encoding: 'htmlhex' })).toBe('💩 AĀ');
|
||||
expect(convertTextToUnicode('💩 AĀ', { encoding: 'uniplus' })).toBe('U+1f4a9 U+00020 U+00041 U+00100');
|
||||
expect(convertTextToUnicode('💩 AĀ', { encoding: 'python' })).toBe('\\U1f4a9\\x20\\x41\\u0100');
|
||||
expect(convertTextToUnicode('💩 AĀ', { encoding: 'js' })).toBe('\\u{1f4a9}\\u0020\\u0041\\u0100');
|
||||
expect(convertTextToUnicode('💩 AĀ', { encoding: 'utf16' })).toBe('\\ud83d\\udca9\\u0020\\u0041\\u0100');
|
||||
expect(convertTextToUnicode('💩 hello AĀ', { skipAscii: true })).toBe('💩 hello AĀ');
|
||||
expect(convertTextToUnicode('linke the string convert to unicode')).toBe(
|
||||
'linke the string convert to unicode');
|
||||
expect(convertTextToUnicode('')).toBe('');
|
||||
});
|
||||
});
|
||||
@@ -16,105 +24,18 @@ describe('text-to-unicode (legacy tests)', () => {
|
||||
describe('convertUnicodeToText', () => {
|
||||
it('an unicode string is converted to its text representation', () => {
|
||||
expect(convertUnicodeToText('A')).toBe('A');
|
||||
expect(convertUnicodeToText('\\u1f4a9\\u20\\u41\\u0100')).toBe('💩 AĀ');
|
||||
expect(convertUnicodeToText('\\01f4a9\\000020\\000041\\000100')).toBe('💩 AĀ');
|
||||
expect(convertUnicodeToText('💩 AĀ')).toBe('💩 AĀ');
|
||||
expect(convertUnicodeToText('💩 AĀ')).toBe('💩 AĀ');
|
||||
expect(convertUnicodeToText('U+1f4a9 U+00020 U+00041 U+00100')).toBe('💩 AĀ');
|
||||
expect(convertUnicodeToText('\\U1f4a9\\x20\\x41\\u0100')).toBe('💩 AĀ');
|
||||
expect(convertUnicodeToText('\\u{1f4a9}\\u0020\\u0041\\u0100')).toBe('💩 AĀ');
|
||||
expect(convertUnicodeToText('\\ud83d\\udca9\\u0020\\u0041\\u0100')).toBe('💩 AĀ');
|
||||
expect(convertUnicodeToText('\\01f4a9 AĀ')).toBe('💩 AĀ');
|
||||
expect(convertUnicodeToText('💩 hello AĀ')).toBe('💩 hello AĀ');
|
||||
expect(convertUnicodeToText('linke the string convert to unicode')).toBe('linke the string convert to unicode');
|
||||
expect(convertUnicodeToText('')).toBe('');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const ALL_PRINTABLE_ASCII = ' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~';
|
||||
|
||||
describe('text-to-unicode regexes', () => {
|
||||
// eslint-disable-next-line prefer-regex-literals
|
||||
const skipAsciiJs = new RegExp(String.raw`([[ -~]--['"\\]]+)`, 'gv');
|
||||
// eslint-disable-next-line prefer-regex-literals
|
||||
const skipAsciiHtml = new RegExp(String.raw`([[ -~]--[<>&'"]]+)`, 'gv');
|
||||
|
||||
it('regexes are equivalent to `v`-flag versions', () => {
|
||||
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/unicodeSets
|
||||
// regexes in `text-to-unicode.service.ts` can be replaced with `v`-flag versions once unicodeSets reaches
|
||||
// sufficient in-browser support
|
||||
expect(ALL_PRINTABLE_ASCII.match(skipAsciiJs)).toStrictEqual(ALL_PRINTABLE_ASCII.match(SKIP_ASCII_JS));
|
||||
expect(ALL_PRINTABLE_ASCII.match(skipAsciiHtml)).toStrictEqual(ALL_PRINTABLE_ASCII.match(SKIP_ASCII_HTML));
|
||||
});
|
||||
});
|
||||
|
||||
describe('text-to-unicode', () => {
|
||||
interface TestConfig {
|
||||
text: string
|
||||
results: Record<ConverterId, string>
|
||||
skipAscii?: boolean
|
||||
};
|
||||
const tests: TestConfig[] = [
|
||||
{
|
||||
text: 'ABC',
|
||||
results: {
|
||||
fullUnicode: String.raw`\u0041\u0042\u0043`,
|
||||
utf16: String.raw`\u0041\u0042\u0043`,
|
||||
hexEntities: String.raw`ABC`,
|
||||
decimalEntities: String.raw`ABC`,
|
||||
},
|
||||
},
|
||||
{
|
||||
text: 'ABC',
|
||||
skipAscii: true,
|
||||
results: {
|
||||
fullUnicode: 'ABC',
|
||||
utf16: 'ABC',
|
||||
hexEntities: 'ABC',
|
||||
decimalEntities: 'ABC',
|
||||
},
|
||||
},
|
||||
{
|
||||
text: ALL_PRINTABLE_ASCII,
|
||||
skipAscii: true,
|
||||
results: {
|
||||
// eslint-disable-next-line unicorn/escape-case
|
||||
fullUnicode: String.raw` !\u0022#$%&\u0027()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\u005c]^_${'`'}abcdefghijklmnopqrstuvwxyz{|}~`,
|
||||
// eslint-disable-next-line unicorn/escape-case
|
||||
utf16: String.raw` !\u0022#$%&\u0027()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\u005c]^_${'`'}abcdefghijklmnopqrstuvwxyz{|}~`,
|
||||
hexEntities: String.raw` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_${'`'}abcdefghijklmnopqrstuvwxyz{|}~`,
|
||||
decimalEntities: String.raw` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_${'`'}abcdefghijklmnopqrstuvwxyz{|}~`,
|
||||
},
|
||||
},
|
||||
{
|
||||
text: '文字',
|
||||
results: {
|
||||
// eslint-disable-next-line unicorn/escape-case
|
||||
fullUnicode: String.raw`\u6587\u5b57`,
|
||||
// eslint-disable-next-line unicorn/escape-case
|
||||
utf16: String.raw`\u6587\u5b57`,
|
||||
hexEntities: String.raw`文字`,
|
||||
decimalEntities: String.raw`文字`,
|
||||
},
|
||||
},
|
||||
{
|
||||
text: 'a 💩 b',
|
||||
skipAscii: true,
|
||||
results: {
|
||||
// eslint-disable-next-line unicorn/escape-case
|
||||
fullUnicode: String.raw`a \u{1f4a9} b`,
|
||||
// eslint-disable-next-line unicorn/escape-case
|
||||
utf16: String.raw`a \ud83d\udca9 b`,
|
||||
hexEntities: String.raw`a 💩 b`,
|
||||
decimalEntities: String.raw`a 💩 b`,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
for (const { text, skipAscii, results } of tests) {
|
||||
describe(`${text} (skipAscii=${skipAscii})`, () => {
|
||||
for (const [key, result] of Object.entries(results)) {
|
||||
describe(key, () => {
|
||||
const converter = converters[key as ConverterId];
|
||||
it('Escaping', () => {
|
||||
expect(converter.escape(text, skipAscii ?? false)).toBe(result);
|
||||
});
|
||||
it('Unescaping', () => {
|
||||
expect(converter.unescape(result)).toBe(text);
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -1,86 +1,66 @@
|
||||
// regex that never matches
|
||||
const SKIP_NOTHING_RE = /(\b\B)/;
|
||||
export const SKIP_ASCII_JS = /([ -!#-&(-\[\]-~]+)/g;
|
||||
export const SKIP_ASCII_HTML = /([ -!#-%(-;=?-~]+)/g;
|
||||
export type Encoding = 'htmldec' | 'htmlhex' | 'uniplus' | 'antiuni' | 'css' | 'python' | 'js' | 'utf16';
|
||||
|
||||
function codeUnits(text: string): number[] {
|
||||
return text.split('').map(char => char.codePointAt(0)!);
|
||||
}
|
||||
const ALL_PRINTABLE_ASCII = ' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~';
|
||||
|
||||
function codePoints(text: string): number[] {
|
||||
return [...text].map(char => char.codePointAt(0)!);
|
||||
}
|
||||
|
||||
interface ConverterConfig {
|
||||
name: string
|
||||
escape: {
|
||||
charValues?(text: string): number[]
|
||||
mapper(charValue: number): string
|
||||
/** @prop regular expression for default content to skip. Must have exactly 1 capture group. */
|
||||
asciiSkipper: RegExp
|
||||
function convertTextToUnicode(text: string, { encoding = 'htmldec', skipAscii = false }: { encoding?: Encoding; skipAscii?: boolean } = {}): string {
|
||||
let prefix: (value: number) => string;
|
||||
let suffix: (value: number) => string = () => '';
|
||||
let base = 16;
|
||||
let padding: (value: number) => number = () => 0;
|
||||
let separator = '';
|
||||
let codepoints = [...text];
|
||||
if (encoding === 'htmldec') {
|
||||
prefix = () => '&#';
|
||||
base = 10;
|
||||
suffix = () => ';';
|
||||
}
|
||||
unescape: {
|
||||
regex: RegExp
|
||||
radix: number
|
||||
else if (encoding === 'htmlhex') {
|
||||
prefix = () => '&#x';
|
||||
suffix = () => ';';
|
||||
}
|
||||
}
|
||||
class Converter {
|
||||
constructor(public config: ConverterConfig) {}
|
||||
|
||||
escape(text: string, skipAscii: boolean): string {
|
||||
const { asciiSkipper, charValues, mapper } = this.config.escape;
|
||||
const getCharValues = charValues ?? codePoints;
|
||||
|
||||
return text
|
||||
.split(skipAscii ? asciiSkipper : SKIP_NOTHING_RE)
|
||||
.flatMap((x, i) => i % 2 ? x : getCharValues(x).map(mapper))
|
||||
.join('');
|
||||
else if (encoding === 'uniplus') {
|
||||
prefix = () => 'U+';
|
||||
padding = () => 5;
|
||||
separator = ' ';
|
||||
}
|
||||
else if (encoding === 'antiuni') {
|
||||
prefix = () => '\\u';
|
||||
padding = (value: number) => value < 256 ? 2 : 4;
|
||||
}
|
||||
else if (encoding === 'utf16') {
|
||||
prefix = () => '\\u';
|
||||
padding = () => 4;
|
||||
codepoints = text.split('');
|
||||
}
|
||||
else if (encoding === 'python') {
|
||||
prefix = (value: number) => value < 256 ? '\\x' : (value < 65536 ? '\\u' : '\\U');
|
||||
padding = (value: number) => value < 256 ? 2 : 4;
|
||||
}
|
||||
else if (encoding === 'js') {
|
||||
prefix = (value: number) => value < 65536 ? '\\u' : '\\u{';
|
||||
suffix = (value: number) => value < 65536 ? '' : '}';
|
||||
padding = () => 4;
|
||||
}
|
||||
else if (encoding === 'css') {
|
||||
prefix = () => '\\';
|
||||
padding = () => 6;
|
||||
}
|
||||
|
||||
unescape(escaped: string): string {
|
||||
const { regex, radix } = this.config.unescape;
|
||||
|
||||
return escaped.replace(regex, (match) => {
|
||||
return String.fromCodePoint(Number.parseInt(match.replace(/\P{AHex}/gu, ''), radix));
|
||||
});
|
||||
}
|
||||
return codepoints.map((value) => {
|
||||
if (skipAscii && ALL_PRINTABLE_ASCII.includes(value)) {
|
||||
return value;
|
||||
}
|
||||
const charCode = value.codePointAt(0) || 0xFF;
|
||||
return `${prefix(charCode)}${charCode.toString(base).padStart(padding(charCode), '0')}${suffix(charCode)}`;
|
||||
}).join(separator);
|
||||
}
|
||||
|
||||
export type ConverterId = keyof typeof converters;
|
||||
const converters = {
|
||||
fullUnicode: new Converter({
|
||||
name: 'Full Unicode',
|
||||
escape: { mapper: convertCodePointToUnicode, asciiSkipper: SKIP_ASCII_JS },
|
||||
unescape: { regex: /\\u\p{AHex}{4}|\\u\{\p{AHex}{1,6}\}/gu, radix: 16 },
|
||||
}),
|
||||
utf16: new Converter({
|
||||
name: 'UTF-16 Code Units',
|
||||
escape: { charValues: codeUnits, mapper: convertCodePointToUnicode, asciiSkipper: SKIP_ASCII_JS },
|
||||
unescape: { regex: /\\u\p{AHex}{4}/gu, radix: 16 },
|
||||
}),
|
||||
hexEntities: new Converter({
|
||||
name: 'HTML Entities (Hex)',
|
||||
escape: { mapper: toHexEntities, asciiSkipper: SKIP_ASCII_HTML },
|
||||
unescape: { regex: /&#x\p{AHex}{1,6};/gu, radix: 16 },
|
||||
}),
|
||||
decimalEntities: new Converter({
|
||||
name: 'HTML Entities (Decimal)',
|
||||
escape: { mapper: toDecimalEntities, asciiSkipper: SKIP_ASCII_HTML },
|
||||
unescape: { regex: /&#\d+;/gu, radix: 10 },
|
||||
}),
|
||||
} satisfies Record<string, Converter>;
|
||||
|
||||
function convertCodePointToUnicode(codePoint: number): string {
|
||||
const hex = codePoint.toString(16);
|
||||
return hex.length > 4 ? String.raw`\u{${hex}}` : String.raw`\u${hex.padStart(4, '0')}`;
|
||||
function convertUnicodeToText(unicodeStr: string): string {
|
||||
return unicodeStr
|
||||
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(dec))
|
||||
.replace(/&#[xX]([0-9A-Fa-f]+);/g, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
|
||||
.replace(/\\u\{([0-9A-Fa-f]+)\}/g, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
|
||||
.replace(/(?:\\[uUx]|\\|\s*U\+)([0-9A-Fa-f]+)/g, (match, hex) => String.fromCodePoint(Number.parseInt(hex, 16))); // NOSONAR
|
||||
}
|
||||
|
||||
function toHexEntities(codePoint: number): string {
|
||||
return `&#x${codePoint.toString(16)};`;
|
||||
}
|
||||
|
||||
function toDecimalEntities(codePoint: number): string {
|
||||
return `&#${codePoint};`;
|
||||
}
|
||||
|
||||
export { converters };
|
||||
export { convertTextToUnicode, convertUnicodeToText };
|
||||
@@ -1,110 +1,62 @@
|
||||
<script setup lang="ts">
|
||||
import { type ConverterId, converters } from './text-to-unicode.service';
|
||||
import { type Encoding, convertTextToUnicode, convertUnicodeToText } from './text-to-unicode.service';
|
||||
import { useQueryParamOrStorage } from '@/composable/queryParams';
|
||||
import { useCopy } from '@/composable/copy';
|
||||
|
||||
const converterId = ref<ConverterId>('fullUnicode');
|
||||
const skipAscii = ref(true);
|
||||
const encoding = useQueryParamOrStorage({ name: 'enc', storageName: 'txt-uni:enc', defaultValue: 'htmldec' });
|
||||
const skipAscii = useQueryParamOrStorage({ name: 'skipAscii', storageName: 'txt-uni:asc', defaultValue: true });
|
||||
|
||||
const inputText = ref('');
|
||||
const unicodeFromText = computed(() =>
|
||||
inputText.value.trim() === ''
|
||||
? ''
|
||||
: converters[converterId.value].escape(inputText.value, skipAscii.value),
|
||||
);
|
||||
const unicodeFromText = computed(() => inputText.value.trim() === ''
|
||||
? ''
|
||||
: convertTextToUnicode(inputText.value, { encoding: encoding.value as Encoding, skipAscii: skipAscii.value }));
|
||||
const { copy: copyUnicode } = useCopy({ source: unicodeFromText });
|
||||
|
||||
const inputUnicode = ref('');
|
||||
const textFromUnicode = computed(() =>
|
||||
inputUnicode.value.trim() === '' ? '' : converters[converterId.value].unescape(inputUnicode.value),
|
||||
);
|
||||
const textFromUnicode = computed(() => inputUnicode.value.trim() === ''
|
||||
? ''
|
||||
: convertUnicodeToText(inputUnicode.value));
|
||||
const { copy: copyText } = useCopy({ source: textFromUnicode });
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="outer" flex flex-col gap-6>
|
||||
<div class="controls">
|
||||
<c-select
|
||||
v-model:value="converterId"
|
||||
searchable
|
||||
label="Conversion type:"
|
||||
:options="Object.entries(converters).map(([key, val]) => ({ label: val.config.name, value: key }))"
|
||||
/>
|
||||
<c-card title="Text to Unicode">
|
||||
<c-select
|
||||
v-model:value="encoding"
|
||||
label="Unicode encoding"
|
||||
:options="[
|
||||
{ value: 'htmldec', label: 'HTML Decimal (&#160;)' },
|
||||
{ value: 'htmlhex', label: 'HTML Hexadecimal (&#xA0;)' },
|
||||
{ value: 'uniplus', label: 'U+00A0' },
|
||||
{ value: 'antiuni', label: '\\u00A0' },
|
||||
{ value: 'css', label: 'CSS (\\0000A0)' },
|
||||
{ value: 'python', label: 'Python (\\xA0, \\u00A0, \\U100A0)' },
|
||||
{ value: 'js', label: 'Python (\\u00A0; \\u{100A0})' },
|
||||
{ value: 'utf16', label: 'UTF 16 (with surrogates)' },
|
||||
]"
|
||||
/>
|
||||
<n-form-item>
|
||||
<n-checkbox v-model:checked="skipAscii">
|
||||
Skip Ascii characters
|
||||
</n-checkbox>
|
||||
</n-form-item>
|
||||
|
||||
<c-input-text v-model:value="inputText" multiline placeholder="e.g. 'Hello Avengers'" label="Enter text to convert to unicode" autosize autofocus raw-text test-id="text-to-unicode-input" />
|
||||
<c-input-text v-model:value="unicodeFromText" label="Unicode from your text" multiline raw-text readonly mt-2 placeholder="The unicode representation of your text will be here" test-id="text-to-unicode-output" />
|
||||
<div mt-2 flex justify-center>
|
||||
<c-button :disabled="!unicodeFromText" @click="copyUnicode()">
|
||||
Copy unicode to clipboard
|
||||
</c-button>
|
||||
</div>
|
||||
<c-card class="card" title="Text to Unicode">
|
||||
<c-input-text
|
||||
v-model:value="inputText"
|
||||
multiline
|
||||
placeholder="e.g. 'Hello Avengers'"
|
||||
label="Enter text to convert to Unicode"
|
||||
autosize
|
||||
autofocus
|
||||
raw-text
|
||||
test-id="text-to-unicode-input"
|
||||
/>
|
||||
<c-input-text
|
||||
v-model:value="unicodeFromText"
|
||||
label="Unicode from your text"
|
||||
multiline
|
||||
raw-text
|
||||
readonly
|
||||
mt-2
|
||||
placeholder="The unicode representation of your text will be here"
|
||||
test-id="text-to-unicode-output"
|
||||
/>
|
||||
<div mt-2 flex justify-start>
|
||||
<n-form-item label="Skip ASCII chars with no special meaning?" :show-feedback="false" label-placement="left">
|
||||
<n-switch v-model:value="skipAscii" />
|
||||
</n-form-item>
|
||||
</div>
|
||||
<div mt-2 flex justify-center>
|
||||
<c-button :disabled="!unicodeFromText" @click="copyUnicode()">
|
||||
Copy unicode to clipboard
|
||||
</c-button>
|
||||
</div>
|
||||
</c-card>
|
||||
<c-card class="card" title="Unicode to Text">
|
||||
<c-input-text
|
||||
v-model:value="inputUnicode"
|
||||
multiline
|
||||
placeholder="Input Unicode"
|
||||
label="Enter unicode to convert to text"
|
||||
autosize
|
||||
raw-text
|
||||
test-id="unicode-to-text-input"
|
||||
/>
|
||||
<c-input-text
|
||||
v-model:value="textFromUnicode"
|
||||
label="Text from your Unicode"
|
||||
multiline
|
||||
raw-text
|
||||
readonly
|
||||
mt-2
|
||||
placeholder="The text representation of your unicode will be here"
|
||||
test-id="unicode-to-text-output"
|
||||
/>
|
||||
<div mt-2 flex justify-center>
|
||||
<c-button :disabled="!textFromUnicode" @click="copyText()">
|
||||
Copy text to clipboard
|
||||
</c-button>
|
||||
</div>
|
||||
</c-card>
|
||||
</div>
|
||||
</c-card>
|
||||
|
||||
<c-card title="Unicode to Text">
|
||||
<c-input-text v-model:value="inputUnicode" multiline placeholder="Input Unicode" label="Enter unicode to convert to text" autosize raw-text test-id="unicode-to-text-input" />
|
||||
<c-input-text v-model:value="textFromUnicode" label="Text from your Unicode" multiline raw-text readonly mt-2 placeholder="The text representation of your unicode will be here" test-id="unicode-to-text-output" />
|
||||
<div mt-2 flex justify-center>
|
||||
<c-button :disabled="!textFromUnicode" @click="copyText()">
|
||||
Copy text to clipboard
|
||||
</c-button>
|
||||
</div>
|
||||
</c-card>
|
||||
</template>
|
||||
|
||||
<style lang="less" scoped>
|
||||
.outer {
|
||||
flex: 0 1 1200px;
|
||||
margin-inline: 50px;
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.controls {
|
||||
flex: 0 1 100%;
|
||||
}
|
||||
|
||||
.card {
|
||||
flex: 1 0 max(40%, 500px);
|
||||
}
|
||||
</style>
|
||||
Reference in new issue
Block a user