Keep every request inside a token budget

How to do it

Set the input budget per feature well under the model's context window, such as 30,000 tokens. Set max_tokens or max_output_tokens on every call. Count tokens with the provider's counting endpoint or tokenizer, such as Anthropic's count_tokens, not by characters. When over budget, drop the oldest conversation turns first, then the lowest-ranked retrieved chunks. Never cut the instructions or the user's latest message. Keep the full history in your database, so trimming changes only what the model sees. Log how many tokens were trimmed.

Example

export async function fitToBudget(request: ModelRequest, budget: number): Promise<ModelRequest> {
  const trimmed = { ...request, history: [...request.history], chunks: [...request.chunks] };
  while ((await countTokens(trimmed)) > budget) {
    if (trimmed.history.length > 0) trimmed.history.shift();
    else if (trimmed.chunks.length > 0) trimmed.chunks.pop();
    else throw new InputTooLongError();
  }
  return trimmed;
}

const countTokens = async (r: ModelRequest) =>
  (await anthropic.messages.countTokens({ model: config.model, system: r.instructions, messages: toMessages(r) })).input_tokens;

def fit_to_budget(request: ModelRequest, budget: int) -> ModelRequest:
    trimmed = request.model_copy(update={"history": list(request.history), "chunks": list(request.chunks)})
    while count_tokens(trimmed) > budget:
        if trimmed.history:
            trimmed.history.pop(0)
        elif trimmed.chunks:
            trimmed.chunks.pop()
        else:
            raise InputTooLongError()
    return trimmed

def count_tokens(request: ModelRequest) -> int:
    return anthropic_client.messages.count_tokens(
        model=settings.model, system=request.instructions, messages=to_messages(request)).input_tokens

How to check

A test sends a conversation of 200 long turns. The gateway sends at most the budget, the instructions and the latest message are intact, and the log shows the trimmed count. A search finds no model call without an output limit.