token-truncate Truncate text to fit a token budget (real BPE)
Flags
--max number default: 4096
--encoding select: cl100k_base | o200k_base default: cl100k_base
Examples
Truncate a system prompt to fit a 20-token budget
Usage
"You are a helpful assistant that answers questions about pro..." | token-truncateTrim text to exactly 10 tokens for an API call
Usage
"The quick brown fox jumps over the lazy dog. Pack my box wit..." | token-truncateTruncate a stack trace to fit an LLM error-analysis prompt
Usage
"Error: ENOENT: no such file or directory, open '/usr/local/e..." | token-truncateView source
async (input, opts = {})=>{
if (!input.trim()) return "(empty input)";
const { getEncoder } = await import('./tokenizer_qMtbZfTQ.mjs').then(async (m)=>{
await m.__tla;
return m;
});
const encoding = opts.encoding || "cl100k_base";
const max = Math.max(1, parseInt(opts.max, 10) || 4096);
const enc = await getEncoder(encoding);
const ids = enc.encode(input);
if (ids.length <= max) return input;
const truncated = enc.decode(ids.slice(0, max));
return new TextDecoder("utf-8", {
fatal: false
}).decode(truncated);
}