token-count Count LLM tokens (real BPE) and context usage
Flags
--encoding select: cl100k_base | o200k_base default: cl100k_base
Examples
Count exact tokens in a system prompt
Usage
"You are a helpful assistant. Answer the user's question conc..." | token-countCheck real token usage for a ChatGPT prompt
Usage
"Write a detailed product description for a wireless Bluetoot..." | token-countMeasure API token cost for a batch of prompts
Usage
"Summarize the following meeting notes and extract action ite..." | token-countView source
async (input, opts = {})=>{
if (!input.trim()) return "(empty input)";
const { getEncoder } = await import('./tokenizer_qMtbZfTQ.mjs').then(async (m)=>{
await m.__tla;
return m;
});
const encoding = opts.encoding || "cl100k_base";
const enc = await getEncoder(encoding);
const tokens = enc.encode(input).length;
const chars = input.length;
const words = input.trim().split(/\s+/).filter(Boolean).length;
const bytes = new Blob([
input
]).size;
const ratio = tokens > 0 ? (chars / tokens).toFixed(1) : "—";
const pct = (ctx)=>{
const p = (tokens / ctx) * 100;
return p < 0.01 ? "<0.01%" : p < 1 ? p.toFixed(2) + "%" : p.toFixed(1) + "%";
};
return [
` Tokens: ${tokens.toLocaleString()} (${encoding})`,
` Chars: ${chars.toLocaleString()}`,
` Words: ${words.toLocaleString()}`,
` Bytes: ${bytes.toLocaleString()}`,
` Ratio: ${ratio} chars/tok`,
``,
` ── Context window usage ──`,
` GPT-4o 128K ${pct(128000).padStart(7)}`,
` Claude 3.5 200K ${pct(200000).padStart(7)}`,
` Gemini 1.5 1M ${pct(1000000).padStart(7)}`,
` Llama 3 128K ${pct(128000).padStart(7)}`
].join("\n");
}