feat(Text to Unicode/Binary): encoding enhancements

Fix part of #1447
Fix #1425, #1072
Fix #1082, #1081, #1175
This commit is contained in:
sharevb authored and ShareVB committed 2025-03-23 19:47:35 +01:00
1 parent f3814e9a5b
commit 7f1f39dda4
7 files changed
+310 -345

No files matched your search

@@ -3,37 +3,77 @@ import { convertTextToUtf8Binary, convertUtf8BinaryToText } from './text-to-bina
describe('text-to-binary', () => {
const utf8Tests = [
{ text: '文字', binary: '11100110 10010110 10000111 11100101 10101101 10010111' },
{ text: '💩', binary: '11110000 10011111 10010010 10101001' },
{
text: '文字',
binary: '11100110 10010110 10000111 11100101 10101101 10010111',
decimal: '',
octal: '',
hex: '',
},
{
text: '💩',
binary: '11110000 10011111 10010010 10101001',
decimal: '',
octal: '',
hex: '',
},
];
describe('convertTextToUtf8Binary', () => {
it('a text string is converted to its UTF-8 binary representation', () => {
expect(convertTextToUtf8Binary('A')).toBe('01000001');
expect(convertTextToUtf8Binary('A', { base: 8 })).toBe('0101');
expect(convertTextToUtf8Binary('A', { base: 10 })).toBe('65');
expect(convertTextToUtf8Binary('A', { base: 16 })).toBe('41');
expect(convertTextToUtf8Binary('hello')).toBe('01101000 01100101 01101100 01101100 01101111');
expect(convertTextToUtf8Binary('hello', { base: 8 })).toBe('0150 0145 0154 0154 0157');
expect(convertTextToUtf8Binary('hello', { base: 10 })).toBe('104 101 108 108 111');
expect(convertTextToUtf8Binary('hello', { base: 16 })).toBe('68 65 6c 6c 6f');
expect(convertTextToUtf8Binary('')).toBe('');
});
it('the separator between octets can be changed', () => {
expect(convertTextToUtf8Binary('hello', { separator: '' })).toBe('0110100001100101011011000110110001101111');
expect(convertTextToUtf8Binary('hello', { separator: '-' })).toBe('01101000-01100101-01101100-01101100-01101111');
expect(convertTextToUtf8Binary('hello', { separator: '-', base: 16 })).toBe('68-65-6c-6c-6f');
});
it('works with non-ASCII input', () => {
for (const { text, binary } of utf8Tests) {
const converted = convertTextToUtf8Binary(text);
expect(converted).toBe(binary);
}
expect(convertTextToUtf8Binary('💩 A', { base: 2 })).toBe('11110000 10011111 10010010 10101001 00100000 01000001');
expect(convertTextToUtf8Binary('💩 A', { base: 8 })).toBe('0360 0237 0222 0251 040 0101');
expect(convertTextToUtf8Binary('💩 A', { base: 10 })).toBe('240 159 146 169 32 65');
expect(convertTextToUtf8Binary('💩 A', { base: 16 })).toBe('f0 9f 92 a9 20 41');
});
});
describe('convertUtf8BinaryToText', () => {
it('an ascii binary string is converted to its text representation', () => {
expect(convertUtf8BinaryToText('01101000 01100101 01101100 01101100 01101111')).toBe('hello');
expect(convertUtf8BinaryToText('01101000 01100101 01101100 01101100 01101111', { base: 2 })).toBe('hello');
expect(convertUtf8BinaryToText('0150 0145 0154 0154 0157', { base: 8 })).toBe('hello');
expect(convertUtf8BinaryToText('104 101 108 108 111', { base: 10 })).toBe('hello');
expect(convertUtf8BinaryToText('68 65 6c 6c 6f', { base: 16 })).toBe('hello');
expect(convertUtf8BinaryToText('11110000 10011111 10010010 10101001 00100000 01000001', { base: 2 })).toBe('💩 A');
expect(convertUtf8BinaryToText('0360 0237 0222 0251 040 0101', { base: 8 })).toBe('💩 A');
expect(convertUtf8BinaryToText('240 159 146 169 32 65', { base: 10 })).toBe('💩 A');
expect(convertUtf8BinaryToText('f0 9f 92 a9 20 41', { base: 16 })).toBe('💩 A');
expect(convertUtf8BinaryToText('')).toBe('');
expect(convertUtf8BinaryToText('01000001')).toBe('A');
expect(convertTextToUtf8Binary('')).toBe('');
expect(convertUtf8BinaryToText('0101', { base: 8 })).toBe('A');
expect(convertUtf8BinaryToText('65', { base: 10 })).toBe('A');
expect(convertUtf8BinaryToText('41', { base: 16 })).toBe('A');
});
it('the given binary string is cleaned before conversion', () => {
it('the given string is cleaned before conversion', () => {
expect(convertUtf8BinaryToText(' 01000 001garbage')).toBe('A');
expect(convertUtf8BinaryToText(' 65garbage', { base: 10 })).toBe('A');
expect(convertUtf8BinaryToText(' 41xxxx', { base: 16 })).toBe('A');
});
it('throws an error if the given binary string is not an integer number of complete octets', () => {
@@ -46,6 +86,13 @@ describe('text-to-binary', () => {
expect(() => convertUtf8BinaryToText('11111111')).toThrow();
});
it('the given string is cleaned from prefix before conversion', () => {
expect(convertUtf8BinaryToText('0b01000001')).toBe('A');
expect(convertUtf8BinaryToText('0x41', { base: 16 })).toBe('A');
expect(convertUtf8BinaryToText('0x68 0x65 0x6c 0x6c 0x6f', { base: 16 })).toBe('hello');
expect(convertUtf8BinaryToText('\\x68\\x65\\x6c\\x6c\\x6f', { base: 16 })).toBe('hello');
});
it('works with non-ASCII input', () => {
for (const { text, binary } of utf8Tests) {
const reverted = convertUtf8BinaryToText(binary);
@@ -1,19 +1,67 @@
export { convertTextToUtf8Binary, convertUtf8BinaryToText };
function convertTextToUtf8Binary(text: string, { separator = ' ' }: { separator?: string } = {}): string {
return [...new TextEncoder().encode(text)].map(x => x.toString(2).padStart(8, '0')).join(separator);
export type EncodingBase = 2 | 8 | 10 | 16;
function convertTextToUtf8Binary(text: string, { separator = ' ', base = 2 }: { separator?: string; base?: EncodingBase } = {}): string {
if (!text?.trim()) {
return '';
}
return [...new TextEncoder().encode(text)].map((char) => {
const charInBase = char.toString(base);
if (base === 2) {
return charInBase.padStart(8, '0');
}
if (base === 8) {
return `0${charInBase}`;
}
if (base === 16) {
return charInBase.padStart(2, '0');
}
return charInBase;
}).join(separator);
}
function convertUtf8BinaryToText(binary: string): string {
const cleanBinary = binary.replace(/[^01]+/g, '');
function convertUtf8BinaryToText(binary: string, { base = 2 }: { base?: EncodingBase } = {}): string {
if (!binary?.trim()) {
return '';
}
if (cleanBinary.length % 8) {
throw new Error('Invalid binary string');
let codepoints: number[] = [];
if (base === 2) {
const cleanBinary = binary.replace(/0b/g, '').replace(/[^01]/g, '').trim();
if (cleanBinary.length % 8) {
throw new Error('Invalid binary string');
}
codepoints = cleanBinary
.split(/([01]{8})/)
.filter(Boolean)
.map(binary => Number.parseInt(binary, 2));
}
else if (base === 16) {
const cleanBinary = binary.replace(/0x|\\x/g, '').replace(/[^0-9A-Fa-f]/g, '');
if (cleanBinary.length % 2) {
throw new Error('Invalid hexadecimal string');
}
codepoints = cleanBinary
.split(/([0-9A-Fa-f]{2})/)
.filter(Boolean)
.map(binary => Number.parseInt(binary, 16));
}
else {
const cleanBinary = binary.replace(/0o/g, '').replace(/[^\d\s]/g, '');
codepoints = cleanBinary
.split(/\s/)
.filter(Boolean)
.map(binary => Number.parseInt(binary, base));
}
return new TextDecoder(undefined, { fatal: true }).decode(
Uint8Array.from({ length: cleanBinary.length / 8 }, (_, i) =>
Number.parseInt(cleanBinary.slice(i * 8, (i + 1) * 8), 2),
),
Uint8Array.from(codepoints),
);
}
+73 -56
View File
@@ -1,15 +1,17 @@
<script setup lang="ts">
import { convertTextToUtf8Binary, convertUtf8BinaryToText } from './text-to-binary.models';
import { type EncodingBase, convertTextToUtf8Binary, convertUtf8BinaryToText } from './text-to-binary.models';
import { withDefaultOnError } from '@/utils/defaults';
import { useCopy } from '@/composable/copy';
import { isNotThrowing } from '@/utils/boolean';
import { useQueryParamOrStorage } from '@/composable/queryParams';
const base = useQueryParamOrStorage({ name: 'base', storageName: 'txt-bin:base', defaultValue: '2' });
const inputText = ref('');
const binaryFromText = computed(() => convertTextToUtf8Binary(inputText.value));
const binaryFromText = computed(() => convertTextToUtf8Binary(inputText.value, { base: Number(base.value) as EncodingBase }));
const { copy: copyBinary } = useCopy({ source: binaryFromText });
const inputBinary = ref('');
const textFromBinary = computed(() => withDefaultOnError(() => convertUtf8BinaryToText(inputBinary.value), ''));
const textFromBinary = computed(() => withDefaultOnError(() => convertUtf8BinaryToText(inputBinary.value, { base: Number(base.value) as EncodingBase }), ''));
const inputBinaryValidationRules = [
{
validator: (value: string) => isNotThrowing(() => convertUtf8BinaryToText(value)),
@@ -20,59 +22,74 @@ const { copy: copyText } = useCopy({ source: textFromBinary });
</script>
<template>
<c-card title="Text to UTF-8 binary">
<c-input-text
v-model:value="inputText"
multiline
placeholder="e.g. 'Hello world'"
label="Enter text to convert to binary"
autosize
autofocus
raw-text
test-id="text-to-binary-input"
<div>
<c-select
v-model:value="base"
label="Conversion Base:"
label-position="left"
mb-2
:options="[
{ value: '2', label: 'Binary' },
{ value: '8', label: 'Octal' },
{ value: '10', label: 'Decimal' },
{ value: '16', label: 'Hexadecimal' },
]"
/>
<c-input-text
v-model:value="binaryFromText"
label="Binary from your text"
multiline
raw-text
readonly
mt-2
placeholder="The binary representation of your text will be here"
test-id="text-to-binary-output"
/>
<div mt-2 flex justify-center>
<c-button :disabled="!binaryFromText" @click="copyBinary()">
Copy binary to clipboard
</c-button>
</div>
</c-card>
<c-card title="UTF-8 binary to text">
<c-input-text
v-model:value="inputBinary"
multiline
placeholder="e.g. '01001000 01100101 01101100 01101100 01101111'"
label="Enter binary to convert to text"
autosize
raw-text
:validation-rules="inputBinaryValidationRules"
test-id="binary-to-text-input"
/>
<c-input-text
v-model:value="textFromBinary"
label="Text from your binary"
multiline
raw-text
readonly
mt-2
placeholder="The text representation of your binary will be here"
test-id="binary-to-text-output"
/>
<div mt-2 flex justify-center>
<c-button :disabled="!textFromBinary" @click="copyText()">
Copy text to clipboard
</c-button>
</div>
</c-card>
<c-card title="Text to UTF-8 binary">
<c-input-text
v-model:value="inputText"
multiline
placeholder="e.g. 'Hello world'"
label="Enter text to convert to binary"
autosize
autofocus
raw-text
test-id="text-to-binary-input"
/>
<c-input-text
v-model:value="binaryFromText"
label="Binary from your text"
multiline
raw-text
readonly
mt-2
placeholder="The binary representation of your text will be here"
test-id="text-to-binary-output"
/>
<div mt-2 flex justify-center>
<c-button :disabled="!binaryFromText" @click="copyBinary()">
Copy binary to clipboard
</c-button>
</div>
</c-card>
<c-card title="UTF-8 binary to text">
<c-input-text
v-model:value="inputBinary"
multiline
placeholder="e.g. '01001000 01100101 01101100 01101100 01101111'"
label="Enter binary to convert to text"
autosize
raw-text
:validation-rules="inputBinaryValidationRules"
test-id="binary-to-text-input"
/>
<c-input-text
v-model:value="textFromBinary"
label="Text from your binary"
multiline
raw-text
readonly
mt-2
placeholder="The text representation of your binary will be here"
test-id="binary-to-text-output"
/>
<div mt-2 flex justify-center>
<c-button :disabled="!textFromBinary" @click="copyText()">
Copy text to clipboard
</c-button>
</div>
</c-card>
</div>
</template>
@@ -14,7 +14,7 @@ test.describe('Tool - Text to Unicode', () => {
const unicode = await page.getByTestId('text-to-unicode-output').inputValue();
// eslint-disable-next-line unicorn/escape-case
expect(unicode).toEqual(String.raw`\u0022it-tools\u0022 \u6587\u5b57`);
expect(unicode).toEqual(String.raw`"it-tools" &#25991;&#23383;`);
});
test('Unicode to text conversion', async ({ page }) => {
@@ -1,14 +1,22 @@
import { describe, expect, it } from 'vitest';
import { type ConverterId, SKIP_ASCII_HTML, SKIP_ASCII_JS, converters } from './text-to-unicode.service';
describe('text-to-unicode (legacy tests)', () => {
const convertTextToUnicode = (text: string) => converters.decimalEntities.escape(text, false);
const convertUnicodeToText = (escaped: string) => converters.decimalEntities.unescape(escaped);
import { convertTextToUnicode, convertUnicodeToText } from './text-to-unicode.service';
describe('text-to-unicode', () => {
describe('convertTextToUnicode', () => {
it('a text string is converted to unicode representation', () => {
expect(convertTextToUnicode('A')).toBe('&#65;');
expect(convertTextToUnicode('linke the string convert to unicode')).toBe('&#108;&#105;&#110;&#107;&#101;&#32;&#116;&#104;&#101;&#32;&#115;&#116;&#114;&#105;&#110;&#103;&#32;&#99;&#111;&#110;&#118;&#101;&#114;&#116;&#32;&#116;&#111;&#32;&#117;&#110;&#105;&#99;&#111;&#100;&#101;');
expect(convertTextToUnicode('💩 AĀ')).toBe('&#128169;&#32;&#65;&#256;');
expect(convertTextToUnicode('💩 AĀ', { encoding: 'antiuni' })).toBe('\\u1f4a9\\u20\\u41\\u0100');
expect(convertTextToUnicode('💩 AĀ', { encoding: 'css' })).toBe('\\01f4a9\\000020\\000041\\000100');
expect(convertTextToUnicode('💩 AĀ', { encoding: 'htmldec' })).toBe('&#128169;&#32;&#65;&#256;');
expect(convertTextToUnicode('💩 AĀ', { encoding: 'htmlhex' })).toBe('&#x1f4a9;&#x20;&#x41;&#x100;');
expect(convertTextToUnicode('💩 AĀ', { encoding: 'uniplus' })).toBe('U+1f4a9 U+00020 U+00041 U+00100');
expect(convertTextToUnicode('💩 AĀ', { encoding: 'python' })).toBe('\\U1f4a9\\x20\\x41\\u0100');
expect(convertTextToUnicode('💩 AĀ', { encoding: 'js' })).toBe('\\u{1f4a9}\\u0020\\u0041\\u0100');
expect(convertTextToUnicode('💩 AĀ', { encoding: 'utf16' })).toBe('\\ud83d\\udca9\\u0020\\u0041\\u0100');
expect(convertTextToUnicode('💩 hello AĀ', { skipAscii: true })).toBe('&#128169; hello A&#256;');
expect(convertTextToUnicode('linke the string convert to unicode')).toBe(
'&#108;&#105;&#110;&#107;&#101;&#32;&#116;&#104;&#101;&#32;&#115;&#116;&#114;&#105;&#110;&#103;&#32;&#99;&#111;&#110;&#118;&#101;&#114;&#116;&#32;&#116;&#111;&#32;&#117;&#110;&#105;&#99;&#111;&#100;&#101;');
expect(convertTextToUnicode('')).toBe('');
});
});
@@ -16,105 +24,18 @@ describe('text-to-unicode (legacy tests)', () => {
describe('convertUnicodeToText', () => {
it('an unicode string is converted to its text representation', () => {
expect(convertUnicodeToText('&#65;')).toBe('A');
expect(convertUnicodeToText('\\u1f4a9\\u20\\u41\\u0100')).toBe('💩 AĀ');
expect(convertUnicodeToText('\\01f4a9\\000020\\000041\\000100')).toBe('💩 AĀ');
expect(convertUnicodeToText('&#128169;&#32;&#65;&#256;')).toBe('💩 AĀ');
expect(convertUnicodeToText('&#x1f4a9;&#x20;&#x41;&#x100;')).toBe('💩 AĀ');
expect(convertUnicodeToText('U+1f4a9 U+00020 U+00041 U+00100')).toBe('💩 AĀ');
expect(convertUnicodeToText('\\U1f4a9\\x20\\x41\\u0100')).toBe('💩 AĀ');
expect(convertUnicodeToText('\\u{1f4a9}\\u0020\\u0041\\u0100')).toBe('💩 AĀ');
expect(convertUnicodeToText('\\ud83d\\udca9\\u0020\\u0041\\u0100')).toBe('💩 AĀ');
expect(convertUnicodeToText('\\01f4a9 AĀ')).toBe('💩 AĀ');
expect(convertUnicodeToText('&#128169; hello A&#256;')).toBe('💩 hello AĀ');
expect(convertUnicodeToText('&#108;&#105;&#110;&#107;&#101;&#32;&#116;&#104;&#101;&#32;&#115;&#116;&#114;&#105;&#110;&#103;&#32;&#99;&#111;&#110;&#118;&#101;&#114;&#116;&#32;&#116;&#111;&#32;&#117;&#110;&#105;&#99;&#111;&#100;&#101;')).toBe('linke the string convert to unicode');
expect(convertUnicodeToText('')).toBe('');
});
});
});
const ALL_PRINTABLE_ASCII = ' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~';
describe('text-to-unicode regexes', () => {
// eslint-disable-next-line prefer-regex-literals
const skipAsciiJs = new RegExp(String.raw`([[ -~]--['"\\]]+)`, 'gv');
// eslint-disable-next-line prefer-regex-literals
const skipAsciiHtml = new RegExp(String.raw`([[ -~]--[<>&'"]]+)`, 'gv');
it('regexes are equivalent to `v`-flag versions', () => {
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/unicodeSets
// regexes in `text-to-unicode.service.ts` can be replaced with `v`-flag versions once unicodeSets reaches
// sufficient in-browser support
expect(ALL_PRINTABLE_ASCII.match(skipAsciiJs)).toStrictEqual(ALL_PRINTABLE_ASCII.match(SKIP_ASCII_JS));
expect(ALL_PRINTABLE_ASCII.match(skipAsciiHtml)).toStrictEqual(ALL_PRINTABLE_ASCII.match(SKIP_ASCII_HTML));
});
});
describe('text-to-unicode', () => {
interface TestConfig {
text: string
results: Record<ConverterId, string>
skipAscii?: boolean
};
const tests: TestConfig[] = [
{
text: 'ABC',
results: {
fullUnicode: String.raw`\u0041\u0042\u0043`,
utf16: String.raw`\u0041\u0042\u0043`,
hexEntities: String.raw`&#x41;&#x42;&#x43;`,
decimalEntities: String.raw`&#65;&#66;&#67;`,
},
},
{
text: 'ABC',
skipAscii: true,
results: {
fullUnicode: 'ABC',
utf16: 'ABC',
hexEntities: 'ABC',
decimalEntities: 'ABC',
},
},
{
text: ALL_PRINTABLE_ASCII,
skipAscii: true,
results: {
// eslint-disable-next-line unicorn/escape-case
fullUnicode: String.raw` !\u0022#$%&\u0027()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\u005c]^_${'`'}abcdefghijklmnopqrstuvwxyz{|}~`,
// eslint-disable-next-line unicorn/escape-case
utf16: String.raw` !\u0022#$%&\u0027()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\u005c]^_${'`'}abcdefghijklmnopqrstuvwxyz{|}~`,
hexEntities: String.raw` !&#x22;#$%&#x26;&#x27;()*+,-./0123456789:;&#x3c;=&#x3e;?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_${'`'}abcdefghijklmnopqrstuvwxyz{|}~`,
decimalEntities: String.raw` !&#34;#$%&#38;&#39;()*+,-./0123456789:;&#60;=&#62;?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_${'`'}abcdefghijklmnopqrstuvwxyz{|}~`,
},
},
{
text: '文字',
results: {
// eslint-disable-next-line unicorn/escape-case
fullUnicode: String.raw`\u6587\u5b57`,
// eslint-disable-next-line unicorn/escape-case
utf16: String.raw`\u6587\u5b57`,
hexEntities: String.raw`&#x6587;&#x5b57;`,
decimalEntities: String.raw`&#25991;&#23383;`,
},
},
{
text: 'a 💩 b',
skipAscii: true,
results: {
// eslint-disable-next-line unicorn/escape-case
fullUnicode: String.raw`a \u{1f4a9} b`,
// eslint-disable-next-line unicorn/escape-case
utf16: String.raw`a \ud83d\udca9 b`,
hexEntities: String.raw`a &#x1f4a9; b`,
decimalEntities: String.raw`a &#128169; b`,
},
},
];
for (const { text, skipAscii, results } of tests) {
describe(`${text} (skipAscii=${skipAscii})`, () => {
for (const [key, result] of Object.entries(results)) {
describe(key, () => {
const converter = converters[key as ConverterId];
it('Escaping', () => {
expect(converter.escape(text, skipAscii ?? false)).toBe(result);
});
it('Unescaping', () => {
expect(converter.unescape(result)).toBe(text);
});
});
}
});
}
});
@@ -1,86 +1,66 @@
// regex that never matches
const SKIP_NOTHING_RE = /(\b\B)/;
export const SKIP_ASCII_JS = /([ -!#-&(-\[\]-~]+)/g;
export const SKIP_ASCII_HTML = /([ -!#-%(-;=?-~]+)/g;
export type Encoding = 'htmldec' | 'htmlhex' | 'uniplus' | 'antiuni' | 'css' | 'python' | 'js' | 'utf16';
function codeUnits(text: string): number[] {
return text.split('').map(char => char.codePointAt(0)!);
}
const ALL_PRINTABLE_ASCII = ' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~';
function codePoints(text: string): number[] {
return [...text].map(char => char.codePointAt(0)!);
}
interface ConverterConfig {
name: string
escape: {
charValues?(text: string): number[]
mapper(charValue: number): string
/** @prop regular expression for default content to skip. Must have exactly 1 capture group. */
asciiSkipper: RegExp
function convertTextToUnicode(text: string, { encoding = 'htmldec', skipAscii = false }: { encoding?: Encoding; skipAscii?: boolean } = {}): string {
let prefix: (value: number) => string;
let suffix: (value: number) => string = () => '';
let base = 16;
let padding: (value: number) => number = () => 0;
let separator = '';
let codepoints = [...text];
if (encoding === 'htmldec') {
prefix = () => '&#';
base = 10;
suffix = () => ';';
}
unescape: {
regex: RegExp
radix: number
else if (encoding === 'htmlhex') {
prefix = () => '&#x';
suffix = () => ';';
}
}
class Converter {
constructor(public config: ConverterConfig) {}
escape(text: string, skipAscii: boolean): string {
const { asciiSkipper, charValues, mapper } = this.config.escape;
const getCharValues = charValues ?? codePoints;
return text
.split(skipAscii ? asciiSkipper : SKIP_NOTHING_RE)
.flatMap((x, i) => i % 2 ? x : getCharValues(x).map(mapper))
.join('');
else if (encoding === 'uniplus') {
prefix = () => 'U+';
padding = () => 5;
separator = ' ';
}
else if (encoding === 'antiuni') {
prefix = () => '\\u';
padding = (value: number) => value < 256 ? 2 : 4;
}
else if (encoding === 'utf16') {
prefix = () => '\\u';
padding = () => 4;
codepoints = text.split('');
}
else if (encoding === 'python') {
prefix = (value: number) => value < 256 ? '\\x' : (value < 65536 ? '\\u' : '\\U');
padding = (value: number) => value < 256 ? 2 : 4;
}
else if (encoding === 'js') {
prefix = (value: number) => value < 65536 ? '\\u' : '\\u{';
suffix = (value: number) => value < 65536 ? '' : '}';
padding = () => 4;
}
else if (encoding === 'css') {
prefix = () => '\\';
padding = () => 6;
}
unescape(escaped: string): string {
const { regex, radix } = this.config.unescape;
return escaped.replace(regex, (match) => {
return String.fromCodePoint(Number.parseInt(match.replace(/\P{AHex}/gu, ''), radix));
});
}
return codepoints.map((value) => {
if (skipAscii && ALL_PRINTABLE_ASCII.includes(value)) {
return value;
}
const charCode = value.codePointAt(0) || 0xFF;
return `${prefix(charCode)}${charCode.toString(base).padStart(padding(charCode), '0')}${suffix(charCode)}`;
}).join(separator);
}
export type ConverterId = keyof typeof converters;
const converters = {
fullUnicode: new Converter({
name: 'Full Unicode',
escape: { mapper: convertCodePointToUnicode, asciiSkipper: SKIP_ASCII_JS },
unescape: { regex: /\\u\p{AHex}{4}|\\u\{\p{AHex}{1,6}\}/gu, radix: 16 },
}),
utf16: new Converter({
name: 'UTF-16 Code Units',
escape: { charValues: codeUnits, mapper: convertCodePointToUnicode, asciiSkipper: SKIP_ASCII_JS },
unescape: { regex: /\\u\p{AHex}{4}/gu, radix: 16 },
}),
hexEntities: new Converter({
name: 'HTML Entities (Hex)',
escape: { mapper: toHexEntities, asciiSkipper: SKIP_ASCII_HTML },
unescape: { regex: /&#x\p{AHex}{1,6};/gu, radix: 16 },
}),
decimalEntities: new Converter({
name: 'HTML Entities (Decimal)',
escape: { mapper: toDecimalEntities, asciiSkipper: SKIP_ASCII_HTML },
unescape: { regex: /&#\d+;/gu, radix: 10 },
}),
} satisfies Record<string, Converter>;
function convertCodePointToUnicode(codePoint: number): string {
const hex = codePoint.toString(16);
return hex.length > 4 ? String.raw`\u{${hex}}` : String.raw`\u${hex.padStart(4, '0')}`;
function convertUnicodeToText(unicodeStr: string): string {
return unicodeStr
.replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(dec))
.replace(/&#[xX]([0-9A-Fa-f]+);/g, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
.replace(/\\u\{([0-9A-Fa-f]+)\}/g, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
.replace(/(?:\\[uUx]|\\|\s*U\+)([0-9A-Fa-f]+)/g, (match, hex) => String.fromCodePoint(Number.parseInt(hex, 16))); // NOSONAR
}
function toHexEntities(codePoint: number): string {
return `&#x${codePoint.toString(16)};`;
}
function toDecimalEntities(codePoint: number): string {
return `&#${codePoint};`;
}
export { converters };
export { convertTextToUnicode, convertUnicodeToText };
+48 -96
View File
@@ -1,110 +1,62 @@
<script setup lang="ts">
import { type ConverterId, converters } from './text-to-unicode.service';
import { type Encoding, convertTextToUnicode, convertUnicodeToText } from './text-to-unicode.service';
import { useQueryParamOrStorage } from '@/composable/queryParams';
import { useCopy } from '@/composable/copy';
const converterId = ref<ConverterId>('fullUnicode');
const skipAscii = ref(true);
const encoding = useQueryParamOrStorage({ name: 'enc', storageName: 'txt-uni:enc', defaultValue: 'htmldec' });
const skipAscii = useQueryParamOrStorage({ name: 'skipAscii', storageName: 'txt-uni:asc', defaultValue: true });
const inputText = ref('');
const unicodeFromText = computed(() =>
inputText.value.trim() === ''
? ''
: converters[converterId.value].escape(inputText.value, skipAscii.value),
);
const unicodeFromText = computed(() => inputText.value.trim() === ''
? ''
: convertTextToUnicode(inputText.value, { encoding: encoding.value as Encoding, skipAscii: skipAscii.value }));
const { copy: copyUnicode } = useCopy({ source: unicodeFromText });
const inputUnicode = ref('');
const textFromUnicode = computed(() =>
inputUnicode.value.trim() === '' ? '' : converters[converterId.value].unescape(inputUnicode.value),
);
const textFromUnicode = computed(() => inputUnicode.value.trim() === ''
? ''
: convertUnicodeToText(inputUnicode.value));
const { copy: copyText } = useCopy({ source: textFromUnicode });
</script>
<template>
<div class="outer" flex flex-col gap-6>
<div class="controls">
<c-select
v-model:value="converterId"
searchable
label="Conversion type:"
:options="Object.entries(converters).map(([key, val]) => ({ label: val.config.name, value: key }))"
/>
<c-card title="Text to Unicode">
<c-select
v-model:value="encoding"
label="Unicode encoding"
:options="[
{ value: 'htmldec', label: 'HTML Decimal (&amp;#160;)' },
{ value: 'htmlhex', label: 'HTML Hexadecimal (&amp;#xA0;)' },
{ value: 'uniplus', label: 'U+00A0' },
{ value: 'antiuni', label: '\\u00A0' },
{ value: 'css', label: 'CSS (\\0000A0)' },
{ value: 'python', label: 'Python (\\xA0, \\u00A0, \\U100A0)' },
{ value: 'js', label: 'Python (\\u00A0; \\u{100A0})' },
{ value: 'utf16', label: 'UTF 16 (with surrogates)' },
]"
/>
<n-form-item>
<n-checkbox v-model:checked="skipAscii">
Skip Ascii characters
</n-checkbox>
</n-form-item>
<c-input-text v-model:value="inputText" multiline placeholder="e.g. 'Hello Avengers'" label="Enter text to convert to unicode" autosize autofocus raw-text test-id="text-to-unicode-input" />
<c-input-text v-model:value="unicodeFromText" label="Unicode from your text" multiline raw-text readonly mt-2 placeholder="The unicode representation of your text will be here" test-id="text-to-unicode-output" />
<div mt-2 flex justify-center>
<c-button :disabled="!unicodeFromText" @click="copyUnicode()">
Copy unicode to clipboard
</c-button>
</div>
<c-card class="card" title="Text to Unicode">
<c-input-text
v-model:value="inputText"
multiline
placeholder="e.g. 'Hello Avengers'"
label="Enter text to convert to Unicode"
autosize
autofocus
raw-text
test-id="text-to-unicode-input"
/>
<c-input-text
v-model:value="unicodeFromText"
label="Unicode from your text"
multiline
raw-text
readonly
mt-2
placeholder="The unicode representation of your text will be here"
test-id="text-to-unicode-output"
/>
<div mt-2 flex justify-start>
<n-form-item label="Skip ASCII chars with no special meaning?" :show-feedback="false" label-placement="left">
<n-switch v-model:value="skipAscii" />
</n-form-item>
</div>
<div mt-2 flex justify-center>
<c-button :disabled="!unicodeFromText" @click="copyUnicode()">
Copy unicode to clipboard
</c-button>
</div>
</c-card>
<c-card class="card" title="Unicode to Text">
<c-input-text
v-model:value="inputUnicode"
multiline
placeholder="Input Unicode"
label="Enter unicode to convert to text"
autosize
raw-text
test-id="unicode-to-text-input"
/>
<c-input-text
v-model:value="textFromUnicode"
label="Text from your Unicode"
multiline
raw-text
readonly
mt-2
placeholder="The text representation of your unicode will be here"
test-id="unicode-to-text-output"
/>
<div mt-2 flex justify-center>
<c-button :disabled="!textFromUnicode" @click="copyText()">
Copy text to clipboard
</c-button>
</div>
</c-card>
</div>
</c-card>
<c-card title="Unicode to Text">
<c-input-text v-model:value="inputUnicode" multiline placeholder="Input Unicode" label="Enter unicode to convert to text" autosize raw-text test-id="unicode-to-text-input" />
<c-input-text v-model:value="textFromUnicode" label="Text from your Unicode" multiline raw-text readonly mt-2 placeholder="The text representation of your unicode will be here" test-id="unicode-to-text-output" />
<div mt-2 flex justify-center>
<c-button :disabled="!textFromUnicode" @click="copyText()">
Copy text to clipboard
</c-button>
</div>
</c-card>
</template>
<style lang="less" scoped>
.outer {
flex: 0 1 1200px;
margin-inline: 50px;
display: flex;
flex-direction: row;
flex-wrap: wrap;
}
.controls {
flex: 0 1 100%;
}
.card {
flex: 1 0 max(40%, 500px);
}
</style>