Agent Cost Optimization

Classification ("is this about billing or technical?"): gpt-4o-mini ($0.15/1M)
Simple extraction ("extract the date from this email"):   gpt-4o-mini
Tool argument generation ("what args for search?"):       gpt-4o
Complex reasoning ("analyze these three contracts"):      claude-sonnet-4 ($3/1M)
Final synthesis ("write the executive summary"):          gpt-4o

def get_model_for_step(step_type: str, task_complexity: str):
    routing = {
        "classification":    "gpt-4o-mini",
        "extraction":        "gpt-4o-mini",
        "tool_args":         "gpt-4o",
        "reasoning":         "claude-sonnet-4" if task_complexity == "high" else "gpt-4o",
        "synthesis":         "gpt-4o",
        "safety_check":      "gpt-4o-mini",  # Safety checks are pattern matching
    }
    return routing.get(step_type, "gpt-4o")

Without caching:
  Turn 1: 5K tokens (system + tools + user message) → full price
  Turn 2: 5K tokens → full price again
  Turn 3: 5K tokens → full price again

With caching:
  Turn 1: 5K tokens → 4K cached (system+tools), 1K new → cache miss cost
  Turn 2: 5K tokens → 4K cache hit (90% discount), 1K new → ~90% cheaper
  Turn 3: 5K tokens → 4K cache hit, 1K new → ~90% cheaper

# Anthropic: explicit breakpoint on system prompt
response = client.messages.create(
    model="claude-sonnet-4-20250514",
    max_tokens=1024,
    system=[
        {
            "type": "text",
            "text": "You are an expert code reviewer...",  # 2K tokens of instructions
            "cache_control": {"type": "ephemeral"}
        }
    ],
    messages=[{"role": "user", "content": user_query}]
)
# Second call with same system prompt: 2K tokens at 0.1x price

# OpenAI: automatic — no code changes. Monitor cache hits.
response = client.responses.create(
    model="gpt-4o",
    input=[{"role": "system", "content": long_system_prompt},
           {"role": "user", "content": user_query}]
)
print(response.usage.input_tokens_details.cached_tokens)  # Monitor hits

def run_agent_with_budget(agent, task, max_turns=8, max_tokens_per_turn=4096, total_budget=50000):
    total_tokens = 0

    for turn in range(max_turns):
        response = agent.run(
            task,
            max_tokens=min(max_tokens_per_turn, total_budget - total_tokens)
        )
        total_tokens += response.usage.total_tokens

        if total_tokens >= total_budget:
            return {"status": "budget_exceeded",
                    "partial_result": response.output,
                    "tokens_used": total_tokens}

        if response.done:
            return {"status": "completed",
                    "result": response.output,
                    "tokens_used": total_tokens,
                    "turns": turn + 1}

    return {"status": "max_turns_reached",
            "partial_result": response.output,
            "tokens_used": total_tokens}

def should_terminate_early(response_history, confidence_threshold=0.9):
    """Stop if the agent is confident enough or if it's going in circles."""

    # Rule 1: Agent explicitly signals completion
    if "FINAL_ANSWER" in response_history[-1] or "task complete" in response_history[-1].lower():
        return True, "agent_signaled_completion"

    # Rule 2: Last 3 steps produced no new information
    if len(response_history) >= 3:
        last_three = response_history[-3:]
        if all(len(r) < 200 for r in last_three):
            return True, "diminishing_returns"

    # Rule 3: Agent is repeating itself (circular loop)
    if len(response_history) >= 2:
        if response_history[-1][:200] == response_history[-2][:200]:
            return True, "repeating_output"

    # Rule 4: LLM estimates high confidence (expensive check)
    if len(response_history) > 2:
        score = estimate_confidence(response_history)
        if score > confidence_threshold:
            return True, f"high_confidence_{score:.2f}"

    return False, None

Without compaction:
  Turn 50: Context = 45K tokens (full history)
  Turn 100: Context window exceeded → oldest messages dropped

With compaction:
  Turn 50: Context = 15K (last 10 turns + summary of earlier 40)
  Turn 100: Context = 15K (last 10 turns + updated summary)

def compact_conversation(messages, keep_last=10):
    """Summarize older messages, keep recent ones in full."""
    if len(messages) <= keep_last:
        return messages

    to_summarize = messages[:-keep_last]
    recent = messages[-keep_last:]

    summary_prompt = f"Summarize this conversation in 3 bullet points:\n\n{to_summarize}"
    summary = llm.generate(summary_prompt, max_tokens=200)

    return [{"role": "system", "content": f"Earlier conversation summary: {summary}"}] + recent

class OptimizedAgent:
    def __init__(self):
        self.router = ModelRouter()            # Lever 1: model tiering
        self.cache = PromptCache()             # Lever 2: prompt caching
        self.budget = TokenBudget(50000)       # Lever 3: token budgeting
        self.terminator = EarlyTerminator()    # Lever 4: early termination
        self.compactor = ContextCompactor()    # Lever 5: compaction

    def run(self, task):
        while not self.is_done():
            model = self.router.get_model(self.current_step_type())
            response = self.call_llm(model, self.build_prompt())
            self.budget.spend(response.usage.total_tokens)

            if self.terminator.should_stop(self.history):
                break
            if self.budget.exhausted():
                return self.partial_result()
            if self.context_too_long():
                self.compactor.compress()

        return self.final_result()

def track_agent_metrics(run):
    return {
        "task_id": run.task_id,
        "success": run.success,
        "turns": len(run.steps),
        "input_tokens": sum(s.usage.input_tokens for s in run.steps),
        "cached_tokens": sum(s.usage.cached_tokens for s in run.steps),
        "output_tokens": sum(s.usage.output_tokens for s in run.steps),
        "total_cost": sum(s.usage.cost for s in run.steps),
        "model_breakdown": Counter(s.model for s in run.steps),
        "tool_calls": sum(len(s.tool_calls) for s in run.steps),
        "premature_termination": run.terminated_early,
        "compaction_events": run.compaction_count
    }