From a8a2a0d3f099530fa9cce7d4ec14a5f3b5976eed Mon Sep 17 00:00:00 2001 From: sharevb Date: Sun, 12 Jan 2025 22:31:11 +0100 Subject: [PATCH] feat(new tool): GPT Token Counter Fix #1334 --- components.d.ts | 1 + package.json | 1 + pnpm-lock.yaml | 23 +++ .../gpt-token-estimator.vue | 170 ++++++++++++++++++ src/tools/gpt-token-estimator/index.ts | 12 ++ src/tools/index.ts | 2 + 6 files changed, 209 insertions(+) create mode 100644 src/tools/gpt-token-estimator/gpt-token-estimator.vue create mode 100644 src/tools/gpt-token-estimator/index.ts diff --git a/components.d.ts b/components.d.ts index 91cc5e15..0d005c65 100644 --- a/components.d.ts +++ b/components.d.ts @@ -257,6 +257,7 @@ declare module '@vue/runtime-core' { NLayoutSider: typeof import('naive-ui')['NLayoutSider'] NMenu: typeof import('naive-ui')['NMenu'] NScrollbar: typeof import('naive-ui')['NScrollbar'] + NScrollbar: typeof import('naive-ui')['NScrollbar'] NumeronymGenerator: typeof import('./src/tools/numeronym-generator/numeronym-generator.vue')['default'] OcrImage: typeof import('./src/tools/ocr-image/ocr-image.vue')['default'] Option43Generator: typeof import('./src/tools/option43-generator/option43-generator.vue')['default'] diff --git a/package.json b/package.json index ac188e2c..1b8fb2b1 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,7 @@ "file-type": "^19.6.0", "flatten-anything": "^4.0.1", "fuse.js": "^6.6.2", + "gpt-tokens": "^1.3.13", "generate-schema": "^2.6.0", "get-timezone-offset": "^1.0.5", "hash-wasm": "^4.11.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f83d86d8..f6e8278e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -260,6 +260,9 @@ dependencies: get-timezone-offset: specifier: ^1.0.5 version: 1.0.5 + gpt-tokens: + specifier: ^1.3.13 + version: 1.3.13 hash-wasm: specifier: ^4.11.0 version: 4.12.0 @@ -10657,6 +10660,14 @@ packages: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} + /gpt-tokens@1.3.13: + resolution: {integrity: sha512-f1s9yc7jn2XCuy4jdeDJ5eaLwg1/fYKaLCxSE9osCQbQeBmoULr8XLNYYz4fr5ZzXzJH25m/9QwgUY0DCkUagA==} + dependencies: + decimal.js: 10.4.3 + js-tiktoken: 1.0.16 + openai-chat-tokens: 0.2.8 + dev: false + /graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -11833,6 +11844,12 @@ packages: resolution: {integrity: sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==} dev: false + /js-tiktoken@1.0.16: + resolution: {integrity: sha512-nUVdO5k/M9llWpiaZlBBDdtmr6qWXwSD6fgaDu2zM8UP+OXxx9V37lFkI6w0/1IuaDx7WffZ37oYd9KvcWKElg==} + dependencies: + base64-js: 1.5.1 + dev: false + /js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -13292,6 +13309,12 @@ packages: is-wsl: 2.2.0 dev: true + /openai-chat-tokens@0.2.8: + resolution: {integrity: sha512-nW7QdFDIZlAYe6jsCT/VPJ/Lam3/w2DX9oxf/5wHpebBT49KI3TN43PPhYlq1klq2ajzXWKNOLY6U4FNZM7AoA==} + dependencies: + js-tiktoken: 1.0.16 + dev: false + /opencollective-postinstall@2.0.3: resolution: {integrity: sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==} hasBin: true diff --git a/src/tools/gpt-token-estimator/gpt-token-estimator.vue b/src/tools/gpt-token-estimator/gpt-token-estimator.vue new file mode 100644 index 00000000..e7fa304d --- /dev/null +++ b/src/tools/gpt-token-estimator/gpt-token-estimator.vue @@ -0,0 +1,170 @@ + + + diff --git a/src/tools/gpt-token-estimator/index.ts b/src/tools/gpt-token-estimator/index.ts new file mode 100644 index 00000000..3476fa07 --- /dev/null +++ b/src/tools/gpt-token-estimator/index.ts @@ -0,0 +1,12 @@ +import { CurrencyDollar } from '@vicons/tabler'; +import { defineTool } from '../tool'; + +export const tool = defineTool({ + name: 'GPT Token Estimator', + path: '/gpt-token-estimator', + description: 'OpenAI GPT Token Estimator', + keywords: ['gpt', 'token', 'estimator'], + component: () => import('./gpt-token-estimator.vue'), + icon: CurrencyDollar, + createdAt: new Date('2024-08-15'), +}); diff --git a/src/tools/index.ts b/src/tools/index.ts index 264c8c5a..0cb13ba6 100644 --- a/src/tools/index.ts +++ b/src/tools/index.ts @@ -25,6 +25,7 @@ import { tool as angleConverter } from './angle-converter'; import { tool as floatingPointNumberConverter } from './floating-point-number-converter'; import { tool as snowflakeIdExtractor } from './snowflake-id-extractor'; import { tool as emailNormalizer } from './email-normalizer'; +import { tool as gptTokenEstimator } from './gpt-token-estimator'; import { tool as myIp } from './my-ip'; import { tool as geoDistanceCalculator } from './geo-distance-calculator'; import { tool as xVrSpamcauseDecoder } from './x-vr-spamcause-decoder'; @@ -461,6 +462,7 @@ export const toolsByCategory: ToolCategory[] = [ components: [ gitMemo, nanoMemo, + gptTokenEstimator, ], }, {