Llama 3.1 70B Instruct

export ANTHROPIC_BASE_URL="https://deference.si"
export ANTHROPIC_AUTH_TOKEN="$DEFERENCE_API_KEY"
export CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY="1"
claude --model meta-llama/llama-3.1-70b-instruct

model = "meta-llama/llama-3.1-70b-instruct"
model_provider = "deference"
web_search = "disabled"

[model_providers.deference]
name = "Deference"
base_url = "https://deference.si/v1"
wire_api = "responses"
env_key = "DEFERENCE_API_KEY"

# Cursor › Settings › Models
OpenAI API Key:            your sk-df-… key
Override OpenAI Base URL:  https://deference.si/v1
Add model:                 meta-llama/llama-3.1-70b-instruct

{
  "provider": {
    "deference": {
      "npm": "@ai-sdk/openai-compatible",
      "options": { "baseURL": "https://deference.si/v1", "apiKey": "{env:DEFERENCE_API_KEY}" },
      "models": { "meta-llama/llama-3.1-70b-instruct": { "limit": { "context": 131072, "output": 16384 } } }
    }
  }
}

curl https://deference.si/v1/chat/completions \
  -H "Authorization: Bearer $DEFERENCE_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/llama-3.1-70b-instruct",
    "messages": [{ "role": "user", "content": "Say hello in one sentence." }]
  }'

import os
from openai import OpenAI
client = OpenAI(base_url="https://deference.si/v1", api_key=os.environ["DEFERENCE_API_KEY"])
completion = client.chat.completions.create(
    model="meta-llama/llama-3.1-70b-instruct",
    messages=[{"role": "user", "content": "Say hello in one sentence."}],
)
print(completion.choices[0].message.content)

import OpenAI from "openai";

const client = new OpenAI({ baseURL: "https://deference.si/v1", apiKey: process.env.DEFERENCE_API_KEY });
const completion = await client.chat.completions.create({
  model: "meta-llama/llama-3.1-70b-instruct",
  messages: [{ role: "user", content: "Say hello in one sentence." }],
});
console.log(completion.choices[0].message.content);