mirror of
https://github.com/sharevb/it-tools.git
synced 2026-11-20 21:23:08 +03:00
feat(GPT Token Estimator): expose all models of tiktoken (and remove price thus)
Fix #1109, #1334
This commit is contained in:
1 parent
2b06b5b64c
commit
c031d57f41
5 files changed
+207
-32
No files matched your search
+2
-1
@@ -153,7 +153,6 @@
|
||||
"fuse.js": "^7.1.0",
|
||||
"generate-schema": "^2.6.0",
|
||||
"get-timezone-offset": "^1.0.5",
|
||||
"gpt-tokens": "^1.3.14",
|
||||
"hash-wasm": "^4.12.0",
|
||||
"haversine": "^1.1.1",
|
||||
"heic-convert": "^2.1.0",
|
||||
@@ -184,6 +183,7 @@
|
||||
"jq-wasm": "^1.0.0",
|
||||
"js-base64": "^3.7.7",
|
||||
"js-beautify": "^1.15.4",
|
||||
"js-tiktoken": "^1.0.21",
|
||||
"jsbarcode": "^3.12.1",
|
||||
"json-analyzer": "^1.2.2",
|
||||
"json-editor-vue": "^0.18.0",
|
||||
@@ -236,6 +236,7 @@
|
||||
"netstack.js": "^2.1.2",
|
||||
"nginx-config-formatter": "^1.4.5",
|
||||
"node-forge": "^1.3.1",
|
||||
"openai-chat-tokens": "^0.2.8",
|
||||
"openpgp": "^6.2.0",
|
||||
"oui-data": "^1.1.391",
|
||||
"parse-duration": "^2.1.2",
|
||||
|
||||
Generated
+10
-15
@@ -347,9 +347,6 @@ dependencies:
|
||||
get-timezone-offset:
|
||||
specifier: ^1.0.5
|
||||
version: 1.0.5
|
||||
gpt-tokens:
|
||||
specifier: ^1.3.14
|
||||
version: 1.3.14
|
||||
hash-wasm:
|
||||
specifier: ^4.12.0
|
||||
version: 4.12.0
|
||||
@@ -440,6 +437,9 @@ dependencies:
|
||||
js-beautify:
|
||||
specifier: ^1.15.4
|
||||
version: 1.15.4
|
||||
js-tiktoken:
|
||||
specifier: ^1.0.21
|
||||
version: 1.0.21
|
||||
jsbarcode:
|
||||
specifier: ^3.12.1
|
||||
version: 3.12.1
|
||||
@@ -596,6 +596,9 @@ dependencies:
|
||||
node-forge:
|
||||
specifier: ^1.3.1
|
||||
version: 1.3.1
|
||||
openai-chat-tokens:
|
||||
specifier: ^0.2.8
|
||||
version: 0.2.8
|
||||
openpgp:
|
||||
specifier: ^6.2.0
|
||||
version: 6.2.0
|
||||
@@ -10541,7 +10544,7 @@ packages:
|
||||
'@one-ini/wasm': 0.1.1
|
||||
commander: 10.0.0
|
||||
minimatch: 9.0.1
|
||||
semver: 7.7.1
|
||||
semver: 7.7.2
|
||||
dev: false
|
||||
|
||||
/ejs@3.1.10:
|
||||
@@ -12198,14 +12201,6 @@ packages:
|
||||
resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
/gpt-tokens@1.3.14:
|
||||
resolution: {integrity: sha512-cFNErQQYGWRwYmew0wVqhCBZxTvGNr96/9pMwNXqSNu9afxqB5PNHOKHlWtUC/P4UW6Ne2UQHHaO2PaWWLpqWQ==}
|
||||
dependencies:
|
||||
decimal.js: 10.4.3
|
||||
js-tiktoken: 1.0.16
|
||||
openai-chat-tokens: 0.2.8
|
||||
dev: false
|
||||
|
||||
/graceful-fs@4.2.11:
|
||||
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
|
||||
|
||||
@@ -13410,8 +13405,8 @@ packages:
|
||||
resolution: {integrity: sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==}
|
||||
dev: false
|
||||
|
||||
/js-tiktoken@1.0.16:
|
||||
resolution: {integrity: sha512-nUVdO5k/M9llWpiaZlBBDdtmr6qWXwSD6fgaDu2zM8UP+OXxx9V37lFkI6w0/1IuaDx7WffZ37oYd9KvcWKElg==}
|
||||
/js-tiktoken@1.0.21:
|
||||
resolution: {integrity: sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==}
|
||||
dependencies:
|
||||
base64-js: 1.5.1
|
||||
dev: false
|
||||
@@ -15171,7 +15166,7 @@ packages:
|
||||
/openai-chat-tokens@0.2.8:
|
||||
resolution: {integrity: sha512-nW7QdFDIZlAYe6jsCT/VPJ/Lam3/w2DX9oxf/5wHpebBT49KI3TN43PPhYlq1klq2ajzXWKNOLY6U4FNZM7AoA==}
|
||||
dependencies:
|
||||
js-tiktoken: 1.0.16
|
||||
js-tiktoken: 1.0.21
|
||||
dev: false
|
||||
|
||||
/opencollective-postinstall@2.0.3:
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
<script setup lang="ts">
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import { GPTTokens } from 'gpt-tokens';
|
||||
import JSON5 from 'json5';
|
||||
import type { supportModelType } from 'gpt-tokens';
|
||||
import type { TiktokenModel } from 'js-tiktoken';
|
||||
import { GPTTokens } from './gpt-tokens.service';
|
||||
import TextareaCopyable from '@/components/TextareaCopyable.vue';
|
||||
import { useValidation } from '@/composable/validation';
|
||||
import { useQueryParamOrStorage } from '@/composable/queryParams';
|
||||
@@ -60,21 +60,19 @@ const outputTokenCosts = computed(() => {
|
||||
return {
|
||||
error: '',
|
||||
usedTokens: '0',
|
||||
usedUSD: '0',
|
||||
promptUsedTokens: '0',
|
||||
completionUsedTokens: '0',
|
||||
};
|
||||
}
|
||||
|
||||
const tokens = new GPTTokens({
|
||||
model: model.value as supportModelType,
|
||||
model: model.value as TiktokenModel,
|
||||
messages: messagesArray,
|
||||
tools: toolsArray,
|
||||
});
|
||||
return {
|
||||
error: '',
|
||||
usedTokens: tokens.usedTokens.toString(),
|
||||
usedUSD: tokens.usedUSD.toString(),
|
||||
promptUsedTokens: tokens.promptUsedTokens.toString(),
|
||||
completionUsedTokens: tokens.completionUsedTokens.toString(),
|
||||
};
|
||||
@@ -83,7 +81,6 @@ const outputTokenCosts = computed(() => {
|
||||
return {
|
||||
error: e.toString(),
|
||||
usedTokens: '',
|
||||
usedUSD: '',
|
||||
promptUsedTokens: '',
|
||||
completionUsedTokens: '',
|
||||
};
|
||||
@@ -156,17 +153,14 @@ const outputTokenCosts = computed(() => {
|
||||
</c-alert>
|
||||
|
||||
<div v-if="!outputTokenCosts.error">
|
||||
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-used-tokens')">
|
||||
<TextareaCopyable :value="outputTokenCosts.usedTokens" />
|
||||
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-used-tokens')" label-placement="left">
|
||||
<InputCopyable :value="outputTokenCosts.usedTokens" />
|
||||
</n-form-item>
|
||||
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-prompt-tokens')">
|
||||
<TextareaCopyable :value="outputTokenCosts.promptUsedTokens" />
|
||||
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-prompt-tokens')" label-placement="left">
|
||||
<InputCopyable :value="outputTokenCosts.promptUsedTokens" />
|
||||
</n-form-item>
|
||||
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-completion-tokens')">
|
||||
<TextareaCopyable :value="outputTokenCosts.completionUsedTokens" />
|
||||
</n-form-item>
|
||||
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-used-usd')">
|
||||
<TextareaCopyable :value="outputTokenCosts.usedUSD" />
|
||||
<n-form-item :label="t('tools.gpt-token-estimator.texts.label-completion-tokens')" label-placement="left">
|
||||
<InputCopyable :value="outputTokenCosts.completionUsedTokens" />
|
||||
</n-form-item>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
import type { Tiktoken, TiktokenModel } from 'js-tiktoken';
|
||||
import { encodingForModel } from 'js-tiktoken';
|
||||
import { promptTokensEstimate } from 'openai-chat-tokens';
|
||||
|
||||
interface MessageItem {
|
||||
name?: string
|
||||
role: 'system' | 'user' | 'assistant'
|
||||
content: string
|
||||
}
|
||||
|
||||
export class GPTTokens {
|
||||
protected static modelEncodingCache = new Map<TiktokenModel, Tiktoken>();
|
||||
|
||||
protected static getEncodingForModelCached(model: TiktokenModel): Tiktoken {
|
||||
const modelEncodingCache = GPTTokens.modelEncodingCache;
|
||||
if (!modelEncodingCache.has(model)) {
|
||||
try {
|
||||
modelEncodingCache.set(model, encodingForModel(model));
|
||||
}
|
||||
catch (e: any) {
|
||||
throw new Error(`Model '${model}' not found: ${e.toString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
return modelEncodingCache.get(model)!;
|
||||
}
|
||||
|
||||
constructor(options: {
|
||||
model?: TiktokenModel
|
||||
messages?: GPTTokens['messages']
|
||||
tools?: GPTTokens['tools']
|
||||
debug?: boolean
|
||||
}) {
|
||||
const {
|
||||
model,
|
||||
messages,
|
||||
tools,
|
||||
debug = false,
|
||||
} = options;
|
||||
|
||||
this.model = model as TiktokenModel;
|
||||
this.debug = debug;
|
||||
this.messages = messages;
|
||||
this.tools = tools;
|
||||
|
||||
this.checkOptions();
|
||||
}
|
||||
|
||||
private checkOptions() {
|
||||
if (!this.messages && !this.tools) {
|
||||
throw new Error('Must set one of messages | function');
|
||||
}
|
||||
|
||||
if (this.tools && !this.messages) {
|
||||
throw new Error('Function must set messages');
|
||||
}
|
||||
}
|
||||
|
||||
public static get supportModels() {
|
||||
return ['davinci-002', 'babbage-002', 'text-davinci-003', 'text-davinci-002', 'text-davinci-001', 'text-curie-001', 'text-babbage-001', 'text-ada-001', 'davinci', 'curie', 'babbage', 'ada', 'code-davinci-002', 'code-davinci-001', 'code-cushman-002', 'code-cushman-001', 'davinci-codex', 'cushman-codex', 'text-davinci-edit-001', 'code-davinci-edit-001', 'text-embedding-ada-002', 'text-embedding-3-small', 'text-embedding-3-large', 'text-similarity-davinci-001', 'text-similarity-curie-001', 'text-similarity-babbage-001', 'text-similarity-ada-001', 'text-search-davinci-doc-001', 'text-search-curie-doc-001', 'text-search-babbage-doc-001', 'text-search-ada-doc-001', 'code-search-babbage-code-001', 'code-search-ada-code-001', 'gpt2', 'gpt-3.5-turbo', 'gpt-35-turbo', 'gpt-3.5-turbo-0301', 'gpt-3.5-turbo-0613', 'gpt-3.5-turbo-1106', 'gpt-3.5-turbo-0125', 'gpt-3.5-turbo-16k', 'gpt-3.5-turbo-16k-0613', 'gpt-3.5-turbo-instruct', 'gpt-3.5-turbo-instruct-0914', 'gpt-4', 'gpt-4-0314', 'gpt-4-0613', 'gpt-4-32k', 'gpt-4-32k-0314', 'gpt-4-32k-0613', 'gpt-4-turbo', 'gpt-4-turbo-2024-04-09', 'gpt-4-turbo-preview', 'gpt-4-1106-preview', 'gpt-4-0125-preview', 'gpt-4-vision-preview', 'gpt-4o', 'gpt-4o-2024-05-13', 'gpt-4o-2024-08-06', 'gpt-4o-2024-11-20', 'gpt-4o-mini-2024-07-18', 'gpt-4o-mini', 'gpt-4o-search-preview', 'gpt-4o-search-preview-2025-03-11', 'gpt-4o-mini-search-preview', 'gpt-4o-mini-search-preview-2025-03-11', 'gpt-4o-audio-preview', 'gpt-4o-audio-preview-2024-12-17', 'gpt-4o-audio-preview-2024-10-01', 'gpt-4o-mini-audio-preview', 'gpt-4o-mini-audio-preview-2024-12-17', 'o1', 'o1-2024-12-17', 'o1-mini', 'o1-mini-2024-09-12', 'o1-preview', 'o1-preview-2024-09-12', 'o1-pro', 'o1-pro-2025-03-19', 'o3', 'o3-2025-04-16', 'o3-mini', 'o3-mini-2025-01-31', 'o4-mini', 'o4-mini-2025-04-16', 'chatgpt-4o-latest', 'gpt-4o-realtime', 'gpt-4o-realtime-preview-2024-10-01', 'gpt-4o-realtime-preview-2024-12-17', 'gpt-4o-mini-realtime-preview', 'gpt-4o-mini-realtime-preview-2024-12-17', 'gpt-4.1', 'gpt-4.1-2025-04-14', 'gpt-4.1-mini', 'gpt-4.1-mini-2025-04-14', 'gpt-4.1-nano', 'gpt-4.1-nano-2025-04-14', 'gpt-4.5-preview', 'gpt-4.5-preview-2025-02-27', 'gpt-5', 'gpt-5-2025-08-07', 'gpt-5-nano', 'gpt-5-nano-2025-08-07', 'gpt-5-mini', 'gpt-5-mini-2025-08-07', 'gpt-5-chat-latest'];
|
||||
}
|
||||
|
||||
public readonly debug;
|
||||
public readonly model;
|
||||
public readonly messages?: MessageItem [];
|
||||
|
||||
public readonly tools?: {
|
||||
type: 'function'
|
||||
function: {
|
||||
name: string
|
||||
description?: string
|
||||
parameters: Record<string, unknown>
|
||||
}
|
||||
} [];
|
||||
|
||||
// Used Tokens (total)
|
||||
public get usedTokens(): number {
|
||||
if (this.tools) {
|
||||
return promptTokensEstimate({
|
||||
messages: this.messages!,
|
||||
functions: this.tools.map(item => item.function),
|
||||
});
|
||||
}
|
||||
|
||||
if (this.messages) {
|
||||
return this.promptUsedTokens + this.completionUsedTokens;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Used Tokens (prompt)
|
||||
public get promptUsedTokens() {
|
||||
return GPTTokens.num_tokens_from_messages(this.promptMessages, this.model);
|
||||
}
|
||||
|
||||
// Used Tokens (completion)
|
||||
public get completionUsedTokens() {
|
||||
return this.completionMessage
|
||||
? GPTTokens.contentUsedTokens(this.model, this.completionMessage)
|
||||
: 0;
|
||||
}
|
||||
|
||||
public static contentUsedTokens(model: TiktokenModel, content: string) {
|
||||
const encoding = GPTTokens.getEncodingForModelCached(model);
|
||||
|
||||
return encoding.encode(content).length;
|
||||
}
|
||||
|
||||
private get lastMessage() {
|
||||
return this.messages![this.messages!.length - 1];
|
||||
}
|
||||
|
||||
private get promptMessages() {
|
||||
return this.lastMessage.role === 'assistant' ? this.messages!.slice(0, -1) : this.messages!;
|
||||
}
|
||||
|
||||
private get completionMessage() {
|
||||
return this.lastMessage.role === 'assistant'
|
||||
? this.lastMessage.content
|
||||
: '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the number of tokens in a list of messages.
|
||||
* @param messages A list of messages.
|
||||
* @param model The model to use for encoding.
|
||||
* @returns The number of tokens in the messages.
|
||||
* @throws If the model is not supported.
|
||||
*/
|
||||
private static num_tokens_from_messages(messages: MessageItem [], model: TiktokenModel) {
|
||||
let tokens_per_message!: number;
|
||||
let tokens_per_name !: number;
|
||||
|
||||
let num_tokens = 0;
|
||||
|
||||
if (model === 'gpt-3.5-turbo-0301') {
|
||||
tokens_per_message = 4;
|
||||
tokens_per_name = -1;
|
||||
}
|
||||
else {
|
||||
tokens_per_message = 3;
|
||||
tokens_per_name = 1;
|
||||
}
|
||||
|
||||
const encoding = GPTTokens.getEncodingForModelCached(model);
|
||||
|
||||
// This is a port of the Python code from
|
||||
//
|
||||
// Python => Typescript by gpt-4
|
||||
//
|
||||
// https://notebooks.githubusercontent.com/view/ipynb?browser=edge&bypass_fastly=true&color_mode=dark&commit=d67c4181abe9dfd871d382930bb778b7014edc66&device=unknown_device&docs_host=https%3A%2F%2Fdocs.github.com&enc_url=68747470733a2f2f7261772e67697468756275736572636f6e74656e742e636f6d2f6f70656e61692f6f70656e61692d636f6f6b626f6f6b2f643637633431383161626539646664383731643338323933306262373738623730313465646336362f6578616d706c65732f486f775f746f5f636f756e745f746f6b656e735f776974685f74696b746f6b656e2e6970796e62&logged_in=true&nwo=openai%2Fopenai-cookbook&path=examples%2FHow_to_count_tokens_with_tiktoken.ipynb&platform=mac&repository_id=468576060&repository_type=Repository&version=114#6d8d98eb-e018-4e1f-8c9e-19b152a97aaf
|
||||
|
||||
for (const message of messages) {
|
||||
num_tokens += tokens_per_message;
|
||||
|
||||
for (const [key, value] of Object.entries(message)) {
|
||||
if (typeof value !== 'string') {
|
||||
continue;
|
||||
}
|
||||
|
||||
num_tokens += encoding.encode(value as string).length;
|
||||
if (key === 'name') {
|
||||
num_tokens += tokens_per_name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Supplementary
|
||||
// encoding.free()
|
||||
|
||||
// every reply is primed with <|start|>assistant<|message|>
|
||||
return num_tokens + 3;
|
||||
}
|
||||
|
||||
public static encode(model: TiktokenModel, text: string) {
|
||||
const encoding = GPTTokens.getEncodingForModelCached(model);
|
||||
return encoding.encode(text);
|
||||
}
|
||||
|
||||
public static decode(model: TiktokenModel, tokens: number[]) {
|
||||
const encoding = GPTTokens.getEncodingForModelCached(model);
|
||||
return encoding.decode(tokens);
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ export const tool = defineTool({
|
||||
name: t('tools.gpt-token-estimator.title'),
|
||||
path: '/gpt-token-estimator',
|
||||
description: t('tools.gpt-token-estimator.description'),
|
||||
keywords: ['gpt', 'token', 'estimator'],
|
||||
keywords: ['gpt', 'llm', 'openai', 'token', 'estimator'],
|
||||
component: () => import('./gpt-token-estimator.vue'),
|
||||
icon: CurrencyDollar,
|
||||
createdAt: new Date('2024-08-15'),
|
||||
|
||||
Reference in new issue
Block a user