OpenAI Realtime API Transport (WebSocket vs WebRTC vs SIP) Errors

# Backend — mints ephemeral client secret and returns to browser
from fastapi import FastAPI, HTTPException
import os
import httpx

api = FastAPI()

OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]


@api.post("/realtime/session")
async def create_realtime_session():
    """Mint an ephemeral client_secret for a browser WebRTC session."""
    async with httpx.AsyncClient(timeout=30) as http:
        resp = await http.post(
            "https://api.openai.com/v1/realtime/client_secrets",
            headers={
                "Authorization": f"Bearer {OPENAI_API_KEY}",
                "Content-Type": "application/json",
            },
            json={
                "session": {
                    "type": "realtime",
                    "model": "gpt-realtime",       # current GA model family
                    "instructions": (
                        "You are a helpful voice assistant. Speak concisely."
                    ),
                    "audio": {
                        "output": {"voice": "cedar"},
                        "input": {
                            "transcription": {"model": "whisper-1", "language": "en"},
                        },
                    },
                    "turn_detection": {
                        "type": "server_vad",
                        "threshold": 0.5,
                        "prefix_padding_ms": 300,
                        "silence_duration_ms": 200,
                    },
                },
            },
        )
        if resp.status_code != 200:
            raise HTTPException(500, f"mint failed: {resp.text}")

        data = resp.json()
        # data.value is the client_secret string (short-lived)
        # data.expires_at is a Unix timestamp
        return {
            "client_secret": data["value"],
            "expires_at": data["expires_at"],
            "model": "gpt-realtime",
        }

<!-- Browser — WebRTC connection using the ephemeral secret -->
<script>
async function connectVoice() {
    // 1. Get ephemeral secret from your backend
    const sessionResp = await fetch("/realtime/session", { method: "POST" });
    const { client_secret, model } = await sessionResp.json();

    // 2. Set up local peer connection
    const pc = new RTCPeerConnection();

    // 3. Handle remote audio (model speech)
    pc.ontrack = (event) => {
        const audioEl = document.getElementById("assistantAudio");
        audioEl.srcObject = event.streams[0];
        audioEl.play();
    };

    // 4. Add local mic to the connection
    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
    stream.getTracks().forEach(t => pc.addTrack(t, stream));

    // 5. Data channel for events (session updates, function calls)
    const dc = pc.createDataChannel("oai-events");
    dc.onmessage = (e) => {
        const event = JSON.parse(e.data);
        console.log("event:", event.type, event);
    };

    // 6. Create SDP offer and exchange with OpenAI
    const offer = await pc.createOffer();
    await pc.setLocalDescription(offer);

    const sdpResp = await fetch(
        `https://api.openai.com/v1/realtime?model=${model}`,
        {
            method: "POST",
            body: offer.sdp,
            headers: {
                "Authorization": `Bearer ${client_secret}`,
                "Content-Type": "application/sdp",
            },
        }
    );
    const answer = { type: "answer", sdp: await sdpResp.text() };
    await pc.setRemoteDescription(answer);
}

document.getElementById("startBtn").addEventListener("click", connectVoice);
</script>
<audio id="assistantAudio" autoplay></audio>
<button id="startBtn">Start voice</button>

import asyncio
import json
import os
import base64

import websockets

OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]


async def connect_realtime_ws():
    """Server-to-server WebSocket connection to Realtime API."""
    url = "wss://api.openai.com/v1/realtime?model=gpt-realtime"

    async with websockets.connect(
        url,
        additional_headers={
            "Authorization": f"Bearer {OPENAI_API_KEY}",
            "OpenAI-Beta": "realtime=v1",              # some versions still expect this
        },
        max_size=None,                                  # audio frames can be large
    ) as ws:

        # 1. Configure the session
        await ws.send(json.dumps({
            "type": "session.update",
            "session": {
                "instructions": "You are a helpful voice assistant. Be concise.",
                "audio": {
                    "input":  {"format": "pcm16"},      # 24kHz PCM16 mono
                    "output": {"format": "pcm16", "voice": "cedar"},
                },
                "turn_detection": {
                    "type": "server_vad",
                    "threshold": 0.5,
                    "silence_duration_ms": 200,
                },
                "tools": [],                            # add function tools here
            },
        }))

        # 2. Send audio input (from your source — phone, mic-capture proxy, etc.)
        async def send_audio(audio_bytes: bytes):
            """audio_bytes is raw PCM16 24kHz mono."""
            await ws.send(json.dumps({
                "type": "input_audio_buffer.append",
                "audio": base64.b64encode(audio_bytes).decode("ascii"),
            }))

        # 3. Listen for server events
        async for message in ws:
            event = json.loads(message)
            t = event["type"]

            if t == "session.created":
                print(f"session {event['session']['id']} created")

            elif t == "session.updated":
                print("session configured")

            elif t == "input_audio_buffer.speech_started":
                print("user started speaking")

            elif t == "input_audio_buffer.speech_stopped":
                print("user stopped speaking")

            elif t == "response.output_audio.delta":
                # base64 audio chunk to send to the user's speakers
                audio = base64.b64decode(event["delta"])
                await your_playback_sink(audio)

            elif t == "response.output_audio_transcript.delta":
                # Model's spoken transcript — good for captions/logging
                print(f"[model says] {event['delta']}", end="", flush=True)

            elif t == "response.function_call_arguments.done":
                # Full tool call args
                args = json.loads(event["arguments"])
                result = await handle_tool_call(event["name"], args)
                # Return the result
                await ws.send(json.dumps({
                    "type": "conversation.item.create",
                    "item": {
                        "type": "function_call_output",
                        "call_id": event["call_id"],
                        "output": json.dumps(result),
                    },
                }))
                await ws.send(json.dumps({"type": "response.create"}))

            elif t == "error":
                print(f"ERROR: {event['error']['message']}")


# ✅ Full-duplex loop — feed mic, play speaker
async def full_duplex():
    ws_task = asyncio.create_task(connect_realtime_ws())
    # ... your audio-capture task feeds bytes via send_audio ...
    await ws_task


# ❌ ANTI-PATTERN — running this from browser code
# Long-lived API key would be exposed to every user.
# Use WebRTC from browser instead (see Fix #1).

# ✅ CURRENT GA session shape
GA_SESSION_CONFIG = {
    "type": "session.update",
    "session": {
        "type": "realtime",
        "model": "gpt-realtime",                       # or gpt-realtime-2 depending on rollout
        "instructions": "You are a voice concierge for a hotel. Be warm and concise.",
        "audio": {
            "input": {
                "format": "pcm16",                     # 24kHz mono little-endian
                "transcription": {
                    "model": "whisper-1",
                    "language": "en",
                },
                "noise_reduction": {"type": "near_field"},
            },
            "output": {
                "format": "pcm16",                     # or "g711_ulaw", "g711_alaw" for telephony
                "voice": "cedar",                      # cedar, marin, alloy, coral, echo, etc.
                "speed": 1.0,                          # 0.25 - 2.0
            },
        },
        "output_modalities": ["audio", "text"],        # what the model can respond with
        "turn_detection": {
            "type": "server_vad",                      # or "semantic_vad" or None (manual)
            "threshold": 0.5,
            "prefix_padding_ms": 300,
            "silence_duration_ms": 200,
            "create_response": True,                   # auto-respond after user stops
            "interrupt_response": True,                # allow user to interrupt model
        },
        "tools": [
            {
                "type": "function",
                "name": "check_availability",
                "description": "Check room availability for given dates",
                "parameters": {
                    "type": "object",
                    "properties": {
                        "check_in":  {"type": "string", "format": "date"},
                        "check_out": {"type": "string", "format": "date"},
                    },
                    "required": ["check_in", "check_out"],
                },
            },
        ],
        "tool_choice": "auto",
        "temperature": 0.7,                            # Realtime models still accept temp
    },
}


# ❌ OLD (preview) shape — will be ignored or rejected
OLD_SESSION_CONFIG = {
    "type": "session.update",
    "session": {
        "modalities": ["audio", "text"],               # old — now output_modalities
        "voice": "alloy",                              # old — now audio.output.voice
        "input_audio_format": "pcm16",                 # old — now audio.input.format
        "output_audio_format": "pcm16",                # old — now audio.output.format
        "instructions": "...",
    },
}


# ✅ Event names — GA prefixes with output_ for audio outputs
NEW_EVENTS = {
    "response.output_audio.delta",                     # was response.audio.delta
    "response.output_audio_transcript.delta",          # was response.audio_transcript.delta
    "response.output_text.delta",                      # was response.text.delta
    "response.function_call_arguments.delta",          # unchanged
    "input_audio_buffer.speech_started",
    "input_audio_buffer.speech_stopped",
}

# When consuming events, handle both shapes during migration if needed
def normalize_event_type(event_type: str) -> str:
    mapping = {
        "response.audio.delta":            "response.output_audio.delta",
        "response.audio_transcript.delta": "response.output_audio_transcript.delta",
        "response.text.delta":             "response.output_text.delta",
    }
    return mapping.get(event_type, event_type)


# ✅ Compatibility helper — accept old or new keys, normalize to new
def normalize_session_config(cfg: dict) -> dict:
    session = cfg.get("session", {})
    audio = session.setdefault("audio", {"input": {}, "output": {}})

    if "voice" in session:
        audio["output"].setdefault("voice", session.pop("voice"))
    if "input_audio_format" in session:
        audio["input"].setdefault("format", session.pop("input_audio_format"))
    if "output_audio_format" in session:
        audio["output"].setdefault("format", session.pop("output_audio_format"))
    if "modalities" in session:
        session.setdefault("output_modalities", session.pop("modalities"))

    return cfg