Generative AI projects often start with predictable prototyping budgets. However, moving pipelines to production introduces unpredictable user input volumes, unconstrained multi turn conversations, agent loop cascades, and recursive retrieval steps. Without strict runtime cost controls, spending can escalate very quickly.
Managing costs effectively requires proactive, dynamic guardrails embedded into client SDK wrappers and gateway components. This blog post provides technical patterns using Python and TypeScript to implement token caps, cost estimation middleware, request throttling, and model fallback routing in Azure AI Foundry environments.
Anatomy of LLM Cost Drivers
Unexpected billing spikes generally trace back to three root causes:
Prompt Inflation: System instructions and full conversation histories attached to every turn without context truncation.
Agentic Loop Cascades: Autonomous agents hitting edge conditions and looping continuously against model endpoints.
Over-provisioned Model Selection: Routing basic text routing or classification tasks to large models like GPT-4o instead of smaller models like GPT-4o-mini.
import sys
import tiktoken
from azure.ai.inference import ChatCompletionsClient
from azure.ai.inference.models import SystemMessage, UserMessage
from azure.core.credentials import AzureKeyCredential
class CostGuardedFoundryClient:
def __init__(self, primary_endpoint: str, primary_key: str, fallback_endpoint: str, fallback_key: str):
self.primary_client = ChatCompletionsClient(
endpoint=primary_endpoint, credential=AzureKeyCredential(primary_key)
)
self.fallback_client = ChatCompletionsClient(
endpoint=fallback_endpoint, credential=AzureKeyCredential(fallback_key)
)
self.tokenizer = tiktoken.get_encoding("cl100k_base")
# Pricing constants per 1k tokens (example estimation rates)
self.PRIMARY_INPUT_COST = 0.0025
self.FALLBACK_INPUT_COST = 0.00015
self.MAX_ALLOWED_PROMPT_TOKENS = 4000
def _count_tokens(self, text: str) -> int:
return len(self.tokenizer.encode(text))
def generate_guarded_completion(self, system_prompt: str, user_prompt: str, max_cost_limit_usd: float):
total_input_tokens = self._count_tokens(system_prompt) + self._count_tokens(user_prompt)
# Guardrail 1: Enforce Token Truncation
if total_input_tokens > self.MAX_ALLOWED_PROMPT_TOKENS:
raise ValueError(f"Prompt token count ({total_input_tokens}) exceeds maximum limit ({self.MAX_ALLOWED_PROMPT_TOKENS}).")
# Guardrail 2: Dynamic Routing Based on Budget & Token Count
estimated_primary_cost = (total_input_tokens / 1000.0) * self.PRIMARY_INPUT_COST
if estimated_primary_cost > max_cost_limit_usd:
# Fallback to lower-cost model deployment
print(f"Primary model estimated cost (${estimated_primary_cost:.4f}) exceeds target limit (${max_cost_limit_usd:.4f}). Routing to fallback model.")
response = self.fallback_client.complete(
messages=[
SystemMessage(content=system_prompt),
UserMessage(content=user_prompt)
],
model="gpt-4o-mini"
)
else:
response = self.primary_client.complete(
messages=[
SystemMessage(content=system_prompt),
UserMessage(content=user_prompt)
],
model="gpt-4o"
)
return response
TypeScript
import { AzureKeyCredential } from "@azure/core-auth";
import ModelClient from "@azure-rest/ai-inference";
interface ChatMessage {
role: "system" | "user" | "assistant";
content: string;
}
export class TokenBudgetManager {
private maxTokens: number;
constructor(maxTokens: number = 3000) {
this.maxTokens = maxTokens;
}
// Basic token estimation heuristic (approx 4 chars per token)
private estimateTokens(text: string): int {
return Math.ceil(text.length / 4);
}
public truncateHistory(systemPrompt: string, history: ChatMessage[]): ChatMessage[] {
let currentTokens = this.estimateTokens(systemPrompt);
const truncated: ChatMessage[] = [];
for (let i = history.length - 1; i >= 0; i--) {
const msgTokens = this.estimateTokens(history[i].content);
if (currentTokens + msgTokens > this.maxTokens) {
break;
}
currentTokens += msgTokens;
truncated.unshift(history[i]);
}
return [{ role: "system", content: systemPrompt }, ...truncated];
}
}

No comments:
Post a Comment