feat(new tool): GPT Token Encoder/Decoder

Fix #1109, #1334
This commit is contained in:
ShareVB committed 2025-08-22 13:30:08 +02:00
1 parent c031d57f41
commit dfe13de18b
3 files changed
+109 -1

No files matched your search

@@ -0,0 +1,90 @@
<script setup lang="ts">
import type { TiktokenModel } from 'js-tiktoken';
import { GPTTokens } from '../gpt-token-estimator/gpt-tokens.service';
import TextareaCopyable from '@/components/TextareaCopyable.vue';
import { useQueryParamOrStorage } from '@/composable/queryParams';
const models = GPTTokens.supportModels;
const model = useQueryParamOrStorage({ name: 'model', storageName: 'gpt-token-encoder:model', defaultValue: 'gpt-3.5-turbo-1106' });
const decodedInput = ref('');
const encodedOutput = computed(
() => {
try {
return GPTTokens.encode(model.value as TiktokenModel, decodedInput.value).join(' ');
}
catch (e: any) {
return e.toString();
}
},
);
const encodedInput = ref('');
const decodedOutput = computed(
() => {
try {
return GPTTokens.decode(model.value as TiktokenModel, encodedInput.value.split(/\s+/).map(Number));
}
catch (e: any) {
return e.toString();
}
},
);
</script>
<template>
<div max-w-600>
<c-select
v-model:value="model"
label="GPT IA Model"
:options="models"
mb-2
/>
<c-card title="Encode text to GPT Tokens">
<c-input-text
v-model:value="decodedInput"
placeholder="Put your text to encode here..."
label="Text to encode"
raw-text
multiline
rows="5"
mb-5
/>
<n-divider />
<TextareaCopyable
label="Encoded tokens"
:value="encodedOutput"
multiline
readonly
rows="5"
mb-5
/>
</c-card>
<c-card title="Decode GPT Tokens to text" mt-5>
<c-input-text
v-model:value="encodedInput"
placeholder="Put your encoded tokens here..."
label="Tokens to decode"
raw-text
multiline
rows="5"
mb-5
/>
<n-divider />
<TextareaCopyable
label="Decoded text"
:value="decodedOutput"
multiline
readonly
rows="5"
mb-5
/>
</c-card>
</div>
</template>
+13
View File
@@ -0,0 +1,13 @@
import { FileText } from '@vicons/tabler';
import { defineTool } from '../tool';
export const tool = defineTool({
name: 'GPT Token Encoder/Decoder',
path: '/gpt-token-encoder',
description: 'Encode text to GPT tokens and decode GPT tokens back to text',
keywords: ['gpt', 'llm', 'openai', 'token', 'encode', 'decode'],
component: () => import('./gpt-token-encoder.vue'),
icon: FileText,
createdAt: new Date('2025-08-15'),
category: 'Text',
});
@@ -3,7 +3,6 @@ import { useI18n } from 'vue-i18n';
import JSON5 from 'json5';
import type { TiktokenModel } from 'js-tiktoken';
import { GPTTokens } from './gpt-tokens.service';
import TextareaCopyable from '@/components/TextareaCopyable.vue';
import { useValidation } from '@/composable/validation';
import { useQueryParamOrStorage } from '@/composable/queryParams';
@@ -162,6 +161,12 @@ const outputTokenCosts = computed(() => {
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-completion-tokens')" label-placement="left">
<InputCopyable :value="outputTokenCosts.completionUsedTokens" />
</n-form-item>
<n-p mt-3>
<n-a href="https://platform.openai.com/docs/pricing" target="_blank">
See OpenAPI Pricings
</n-a>
</n-p>
</div>
</div>
</template>