import os, json
from openai import OpenAI

incumbent = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
radium = OpenAI(api_key=os.environ["RADIUM_API_KEY"], base_url="https://api.radium.cloud/v1")

def compare(prompt: str, incumbent_model: str, radium_model: str):
    r1 = incumbent.chat.completions.create(model=incumbent_model, messages=[{"role":"user","content":prompt}])
    r2 = radium.chat.completions.create(model=radium_model, messages=[{"role":"user","content":prompt}])
    return {
        "incumbent": r1.choices[0].message.content,
        "radium": r2.choices[0].message.content,
        "incumbent_tokens": r1.usage.total_tokens,
        "radium_tokens": r2.usage.total_tokens,
    }

results = []
for prompt in open("test-prompts.txt").readlines():
    results.append(compare(prompt.strip(), "gpt-4o", "hal-1.0"))

json.dump(results, open("comparison-results.json", "w"), indent=2)

model_list:
  - model_name: gpt-4o
    litellm_params:
      model: openai/gpt-4o
      api_key: os.environ/OPENAI_API_KEY
  - model_name: radium-hal
    litellm_params:
      model: openai/hal-1.0
      api_base: https://api.radium.cloud/v1
      api_key: os.environ/RADIUM_API_KEY

router_settings:
  routing_strategy: simple-shuffle
  model_group_alias:
    production: gpt-4o
    canary: radium-hal

import os
from openai import OpenAI

client = OpenAI(
    api_key=os.environ["LITELLM_PROXY_KEY"],
    base_url="http://localhost:4000"
)

# 95% hits gpt-4o, 5% hits hal-1.0
response = client.chat.completions.create(
    model="production",
    messages=[{"role": "user", "content": "Hello"}]
)

import random

def pick_provider():
    return "radium" if random.random() < 0.25 else "openai"

def chat(messages, provider="radium"):
    if provider == "radium":
        client = radium_client
        model = "hal-1.0"
    else:
        client = incumbent_client
        model = "gpt-4o"
    return client.chat.completions.create(model=model, messages=messages)