feat(GPT Token Estimator): expose all models of tiktoken (and remove price thus)

Fix #1109, #1334
This commit is contained in:
ShareVB committed 2025-08-22 10:46:37 +02:00
1 parent 2b06b5b64c
commit c031d57f41
5 files changed
+207 -32

No files matched your search

+2 -1
View File
@@ -153,7 +153,6 @@
"fuse.js": "^7.1.0",
"generate-schema": "^2.6.0",
"get-timezone-offset": "^1.0.5",
"gpt-tokens": "^1.3.14",
"hash-wasm": "^4.12.0",
"haversine": "^1.1.1",
"heic-convert": "^2.1.0",
@@ -184,6 +183,7 @@
"jq-wasm": "^1.0.0",
"js-base64": "^3.7.7",
"js-beautify": "^1.15.4",
"js-tiktoken": "^1.0.21",
"jsbarcode": "^3.12.1",
"json-analyzer": "^1.2.2",
"json-editor-vue": "^0.18.0",
@@ -236,6 +236,7 @@
"netstack.js": "^2.1.2",
"nginx-config-formatter": "^1.4.5",
"node-forge": "^1.3.1",
"openai-chat-tokens": "^0.2.8",
"openpgp": "^6.2.0",
"oui-data": "^1.1.391",
"parse-duration": "^2.1.2",
+10 -15
View File
@@ -347,9 +347,6 @@ dependencies:
get-timezone-offset:
specifier: ^1.0.5
version: 1.0.5
gpt-tokens:
specifier: ^1.3.14
version: 1.3.14
hash-wasm:
specifier: ^4.12.0
version: 4.12.0
@@ -440,6 +437,9 @@ dependencies:
js-beautify:
specifier: ^1.15.4
version: 1.15.4
js-tiktoken:
specifier: ^1.0.21
version: 1.0.21
jsbarcode:
specifier: ^3.12.1
version: 3.12.1
@@ -596,6 +596,9 @@ dependencies:
node-forge:
specifier: ^1.3.1
version: 1.3.1
openai-chat-tokens:
specifier: ^0.2.8
version: 0.2.8
openpgp:
specifier: ^6.2.0
version: 6.2.0
@@ -10541,7 +10544,7 @@ packages:
'@one-ini/wasm': 0.1.1
commander: 10.0.0
minimatch: 9.0.1
semver: 7.7.1
semver: 7.7.2
dev: false
/ejs@3.1.10:
@@ -12198,14 +12201,6 @@ packages:
resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==}
engines: {node: '>= 0.4'}
/gpt-tokens@1.3.14:
resolution: {integrity: sha512-cFNErQQYGWRwYmew0wVqhCBZxTvGNr96/9pMwNXqSNu9afxqB5PNHOKHlWtUC/P4UW6Ne2UQHHaO2PaWWLpqWQ==}
dependencies:
decimal.js: 10.4.3
js-tiktoken: 1.0.16
openai-chat-tokens: 0.2.8
dev: false
/graceful-fs@4.2.11:
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
@@ -13410,8 +13405,8 @@ packages:
resolution: {integrity: sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==}
dev: false
/js-tiktoken@1.0.16:
resolution: {integrity: sha512-nUVdO5k/M9llWpiaZlBBDdtmr6qWXwSD6fgaDu2zM8UP+OXxx9V37lFkI6w0/1IuaDx7WffZ37oYd9KvcWKElg==}
/js-tiktoken@1.0.21:
resolution: {integrity: sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==}
dependencies:
base64-js: 1.5.1
dev: false
@@ -15171,7 +15166,7 @@ packages:
/openai-chat-tokens@0.2.8:
resolution: {integrity: sha512-nW7QdFDIZlAYe6jsCT/VPJ/Lam3/w2DX9oxf/5wHpebBT49KI3TN43PPhYlq1klq2ajzXWKNOLY6U4FNZM7AoA==}
dependencies:
js-tiktoken: 1.0.16
js-tiktoken: 1.0.21
dev: false
/opencollective-postinstall@2.0.3:
@@ -1,8 +1,8 @@
<script setup lang="ts">
import { useI18n } from 'vue-i18n';
import { GPTTokens } from 'gpt-tokens';
import JSON5 from 'json5';
import type { supportModelType } from 'gpt-tokens';
import type { TiktokenModel } from 'js-tiktoken';
import { GPTTokens } from './gpt-tokens.service';
import TextareaCopyable from '@/components/TextareaCopyable.vue';
import { useValidation } from '@/composable/validation';
import { useQueryParamOrStorage } from '@/composable/queryParams';
@@ -60,21 +60,19 @@ const outputTokenCosts = computed(() => {
return {
error: '',
usedTokens: '0',
usedUSD: '0',
promptUsedTokens: '0',
completionUsedTokens: '0',
};
}
const tokens = new GPTTokens({
model: model.value as supportModelType,
model: model.value as TiktokenModel,
messages: messagesArray,
tools: toolsArray,
});
return {
error: '',
usedTokens: tokens.usedTokens.toString(),
usedUSD: tokens.usedUSD.toString(),
promptUsedTokens: tokens.promptUsedTokens.toString(),
completionUsedTokens: tokens.completionUsedTokens.toString(),
};
@@ -83,7 +81,6 @@ const outputTokenCosts = computed(() => {
return {
error: e.toString(),
usedTokens: '',
usedUSD: '',
promptUsedTokens: '',
completionUsedTokens: '',
};
@@ -156,17 +153,14 @@ const outputTokenCosts = computed(() => {
</c-alert>
<div v-if="!outputTokenCosts.error">
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-used-tokens')">
<TextareaCopyable :value="outputTokenCosts.usedTokens" />
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-used-tokens')" label-placement="left">
<InputCopyable :value="outputTokenCosts.usedTokens" />
</n-form-item>
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-prompt-tokens')">
<TextareaCopyable :value="outputTokenCosts.promptUsedTokens" />
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-prompt-tokens')" label-placement="left">
<InputCopyable :value="outputTokenCosts.promptUsedTokens" />
</n-form-item>
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-completion-tokens')">
<TextareaCopyable :value="outputTokenCosts.completionUsedTokens" />
</n-form-item>
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-used-usd')">
<TextareaCopyable :value="outputTokenCosts.usedUSD" />
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-completion-tokens')" label-placement="left">
<InputCopyable :value="outputTokenCosts.completionUsedTokens" />
</n-form-item>
</div>
</div>
@@ -0,0 +1,185 @@
import type { Tiktoken, TiktokenModel } from 'js-tiktoken';
import { encodingForModel } from 'js-tiktoken';
import { promptTokensEstimate } from 'openai-chat-tokens';
interface MessageItem {
name?: string
role: 'system' | 'user' | 'assistant'
content: string
}
export class GPTTokens {
protected static modelEncodingCache = new Map<TiktokenModel, Tiktoken>();
protected static getEncodingForModelCached(model: TiktokenModel): Tiktoken {
const modelEncodingCache = GPTTokens.modelEncodingCache;
if (!modelEncodingCache.has(model)) {
try {
modelEncodingCache.set(model, encodingForModel(model));
}
catch (e: any) {
throw new Error(`Model '${model}' not found: ${e.toString()}`);
}
}
return modelEncodingCache.get(model)!;
}
constructor(options: {
model?: TiktokenModel
messages?: GPTTokens['messages']
tools?: GPTTokens['tools']
debug?: boolean
}) {
const {
model,
messages,
tools,
debug = false,
} = options;
this.model = model as TiktokenModel;
this.debug = debug;
this.messages = messages;
this.tools = tools;
this.checkOptions();
}
private checkOptions() {
if (!this.messages && !this.tools) {
throw new Error('Must set one of messages | function');
}
if (this.tools && !this.messages) {
throw new Error('Function must set messages');
}
}
public static get supportModels() {
return ['davinci-002', 'babbage-002', 'text-davinci-003', 'text-davinci-002', 'text-davinci-001', 'text-curie-001', 'text-babbage-001', 'text-ada-001', 'davinci', 'curie', 'babbage', 'ada', 'code-davinci-002', 'code-davinci-001', 'code-cushman-002', 'code-cushman-001', 'davinci-codex', 'cushman-codex', 'text-davinci-edit-001', 'code-davinci-edit-001', 'text-embedding-ada-002', 'text-embedding-3-small', 'text-embedding-3-large', 'text-similarity-davinci-001', 'text-similarity-curie-001', 'text-similarity-babbage-001', 'text-similarity-ada-001', 'text-search-davinci-doc-001', 'text-search-curie-doc-001', 'text-search-babbage-doc-001', 'text-search-ada-doc-001', 'code-search-babbage-code-001', 'code-search-ada-code-001', 'gpt2', 'gpt-3.5-turbo', 'gpt-35-turbo', 'gpt-3.5-turbo-0301', 'gpt-3.5-turbo-0613', 'gpt-3.5-turbo-1106', 'gpt-3.5-turbo-0125', 'gpt-3.5-turbo-16k', 'gpt-3.5-turbo-16k-0613', 'gpt-3.5-turbo-instruct', 'gpt-3.5-turbo-instruct-0914', 'gpt-4', 'gpt-4-0314', 'gpt-4-0613', 'gpt-4-32k', 'gpt-4-32k-0314', 'gpt-4-32k-0613', 'gpt-4-turbo', 'gpt-4-turbo-2024-04-09', 'gpt-4-turbo-preview', 'gpt-4-1106-preview', 'gpt-4-0125-preview', 'gpt-4-vision-preview', 'gpt-4o', 'gpt-4o-2024-05-13', 'gpt-4o-2024-08-06', 'gpt-4o-2024-11-20', 'gpt-4o-mini-2024-07-18', 'gpt-4o-mini', 'gpt-4o-search-preview', 'gpt-4o-search-preview-2025-03-11', 'gpt-4o-mini-search-preview', 'gpt-4o-mini-search-preview-2025-03-11', 'gpt-4o-audio-preview', 'gpt-4o-audio-preview-2024-12-17', 'gpt-4o-audio-preview-2024-10-01', 'gpt-4o-mini-audio-preview', 'gpt-4o-mini-audio-preview-2024-12-17', 'o1', 'o1-2024-12-17', 'o1-mini', 'o1-mini-2024-09-12', 'o1-preview', 'o1-preview-2024-09-12', 'o1-pro', 'o1-pro-2025-03-19', 'o3', 'o3-2025-04-16', 'o3-mini', 'o3-mini-2025-01-31', 'o4-mini', 'o4-mini-2025-04-16', 'chatgpt-4o-latest', 'gpt-4o-realtime', 'gpt-4o-realtime-preview-2024-10-01', 'gpt-4o-realtime-preview-2024-12-17', 'gpt-4o-mini-realtime-preview', 'gpt-4o-mini-realtime-preview-2024-12-17', 'gpt-4.1', 'gpt-4.1-2025-04-14', 'gpt-4.1-mini', 'gpt-4.1-mini-2025-04-14', 'gpt-4.1-nano', 'gpt-4.1-nano-2025-04-14', 'gpt-4.5-preview', 'gpt-4.5-preview-2025-02-27', 'gpt-5', 'gpt-5-2025-08-07', 'gpt-5-nano', 'gpt-5-nano-2025-08-07', 'gpt-5-mini', 'gpt-5-mini-2025-08-07', 'gpt-5-chat-latest'];
}
public readonly debug;
public readonly model;
public readonly messages?: MessageItem [];
public readonly tools?: {
type: 'function'
function: {
name: string
description?: string
parameters: Record<string, unknown>
}
} [];
// Used Tokens (total)
public get usedTokens(): number {
if (this.tools) {
return promptTokensEstimate({
messages: this.messages!,
functions: this.tools.map(item => item.function),
});
}
if (this.messages) {
return this.promptUsedTokens + this.completionUsedTokens;
}
return 0;
}
// Used Tokens (prompt)
public get promptUsedTokens() {
return GPTTokens.num_tokens_from_messages(this.promptMessages, this.model);
}
// Used Tokens (completion)
public get completionUsedTokens() {
return this.completionMessage
? GPTTokens.contentUsedTokens(this.model, this.completionMessage)
: 0;
}
public static contentUsedTokens(model: TiktokenModel, content: string) {
const encoding = GPTTokens.getEncodingForModelCached(model);
return encoding.encode(content).length;
}
private get lastMessage() {
return this.messages![this.messages!.length - 1];
}
private get promptMessages() {
return this.lastMessage.role === 'assistant' ? this.messages!.slice(0, -1) : this.messages!;
}
private get completionMessage() {
return this.lastMessage.role === 'assistant'
? this.lastMessage.content
: '';
}
/**
* Return the number of tokens in a list of messages.
* @param messages A list of messages.
* @param model The model to use for encoding.
* @returns The number of tokens in the messages.
* @throws If the model is not supported.
*/
private static num_tokens_from_messages(messages: MessageItem [], model: TiktokenModel) {
let tokens_per_message!: number;
let tokens_per_name !: number;
let num_tokens = 0;
if (model === 'gpt-3.5-turbo-0301') {
tokens_per_message = 4;
tokens_per_name = -1;
}
else {
tokens_per_message = 3;
tokens_per_name = 1;
}
const encoding = GPTTokens.getEncodingForModelCached(model);
// This is a port of the Python code from
//
// Python => Typescript by gpt-4
//
// https://notebooks.githubusercontent.com/view/ipynb?browser=edge&bypass_fastly=true&color_mode=dark&commit=d67c4181abe9dfd871d382930bb778b7014edc66&device=unknown_device&docs_host=https%3A%2F%2Fdocs.github.com&enc_url=68747470733a2f2f7261772e67697468756275736572636f6e74656e742e636f6d2f6f70656e61692f6f70656e61692d636f6f6b626f6f6b2f643637633431383161626539646664383731643338323933306262373738623730313465646336362f6578616d706c65732f486f775f746f5f636f756e745f746f6b656e735f776974685f74696b746f6b656e2e6970796e62&logged_in=true&nwo=openai%2Fopenai-cookbook&path=examples%2FHow_to_count_tokens_with_tiktoken.ipynb&platform=mac&repository_id=468576060&repository_type=Repository&version=114#6d8d98eb-e018-4e1f-8c9e-19b152a97aaf
for (const message of messages) {
num_tokens += tokens_per_message;
for (const [key, value] of Object.entries(message)) {
if (typeof value !== 'string') {
continue;
}
num_tokens += encoding.encode(value as string).length;
if (key === 'name') {
num_tokens += tokens_per_name;
}
}
}
// Supplementary
// encoding.free()
// every reply is primed with <|start|>assistant<|message|>
return num_tokens + 3;
}
public static encode(model: TiktokenModel, text: string) {
const encoding = GPTTokens.getEncodingForModelCached(model);
return encoding.encode(text);
}
public static decode(model: TiktokenModel, tokens: number[]) {
const encoding = GPTTokens.getEncodingForModelCached(model);
return encoding.decode(tokens);
}
}
+1 -1
View File
@@ -6,7 +6,7 @@ export const tool = defineTool({
name: t('tools.gpt-token-estimator.title'),
path: '/gpt-token-estimator',
description: t('tools.gpt-token-estimator.description'),
keywords: ['gpt', 'token', 'estimator'],
keywords: ['gpt', 'llm', 'openai', 'token', 'estimator'],
component: () => import('./gpt-token-estimator.vue'),
icon: CurrencyDollar,
createdAt: new Date('2024-08-15'),