Stop using one model for everything: a 60-line Python cost audit that cut our test bill 98%

"""cost_audit.py: cheapest model that still passes, per task."""
import json, os, time
from openai import OpenAI

client = OpenAI(base_url="https://api.qubax.ai/v1",
                api_key=os.environ["QUBAX_API_KEY"])

# USD per 1M tokens (input, output). Pull live prices from GET /v1/models.
PRICES = {
    "claude-opus-5.5": (1.21458, 6.07292),
    "claude-sonnet-5": (0.3835, 1.5339),
    "glm-5.3-flash": (0.011278, 0.0426),
    "deepseek-v4.1-flash": (0.014706, 0.058824),
}

def as_json(out):
    return json.loads(out[out.find("{"): out.rfind("}") + 1] or "{}")

TASKS = [
  {"name": "classify_ticket", "max_tokens": 400,
   "messages": [{"role": "user", "content":
     "Classify this support ticket as one of billing, bug, feature, other. "
     "Reply with the label only.\n\nTicket: I was charged twice for my "
     "October invoice."}],
   "check": lambda o: o.strip().lower().strip(".") == "billing"},
  {"name": "extract_json", "max_tokens": 600,
   "messages": [{"role": "user", "content":
     "Extract name, company and plan as JSON with keys name, company, plan. "
     "JSON only.\n\nHi, this is Maria Lopez from Northwind. We want to move "
     "from the Starter plan to Pro next month."}],
   "check": lambda o: (lambda d: d.get("name") == "Maria Lopez"
       and d.get("company") == "Northwind"
       and "pro" in str(d.get("plan", "")).lower())(as_json(o))},
  {"name": "summarize", "max_tokens": 600,
   "messages": [{"role": "user", "content":
     "Summarize in one sentence of at most 25 words: Our Q3 revenue grew 18% "
     "to $4.2M, driven by enterprise deals, while churn fell from 3.1% to "
     "2.4%. Hiring is paused until Q1."}],
   "check": lambda o: 0 < len(o.split()) <= 30 and "18" in o},
]

def run(model, task):
    t0 = time.time()
    r = client.chat.completions.create(model=model, temperature=0,
        messages=task["messages"], max_tokens=task["max_tokens"])
    out = (r.choices[0].message.content or "").strip()
    try: ok = bool(task["check"](out))
    except Exception: ok = False
    pin, pout = PRICES[model]
    usd = (r.usage.prompt_tokens * pin + r.usage.completion_tokens * pout) / 1e6
    return {"model": model, "ok": ok, "usd": usd,
            "ms": int((time.time() - t0) * 1000)}

route = {}
for task in TASKS:
    rows = sorted((run(m, task) for m in PRICES), key=lambda x: x["usd"])
    print(f"\n{task['name']}")
    for x in rows:
        print(f"  {x['model']:<22} {'PASS' if x['ok'] else 'FAIL'}"
              f"  ${x['usd']:.6f}  {x['ms']} ms")
    passing = [x for x in rows if x["ok"]]
    route[task["name"]] = passing[0]["model"] if passing else "claude-opus-5.5"

print("\nRouting table:", json.dumps(route, indent=2))

ROUTE = json.load(open("route.json"))   # output of the audit

def llm(task, messages, **kw):
    return client.chat.completions.create(
        model=ROUTE.get(task, "claude-sonnet-5"), messages=messages, **kw)