OpenAI Realtime API Audio Format (PCM16, G.711) Encoding Errors

// Browser — capture mic, resample to 24kHz PCM16, send as base64
// Assumes an open WebSocket `ws` to Realtime API (or via your proxy)

const TARGET_SAMPLE_RATE = 24000;

async function startCapture(ws) {
    const stream = await navigator.mediaDevices.getUserMedia({
        audio: {
            channelCount: 1,
            sampleRate: TARGET_SAMPLE_RATE,   // hint; browser may not honor
            echoCancellation: true,
            noiseSuppression: true,
        },
    });

    // AudioContext with explicit sample rate
    const ac = new AudioContext({ sampleRate: TARGET_SAMPLE_RATE });
    await ac.audioWorklet.addModule("pcm-worklet.js");

    const src = ac.createMediaStreamSource(stream);
    const worklet = new AudioWorkletNode(ac, "pcm-worklet");
    src.connect(worklet);

    worklet.port.onmessage = (ev) => {
        // ev.data is Int16Array of PCM16 samples at 24kHz
        const bytes = new Uint8Array(ev.data.buffer);
        const b64 = btoa(String.fromCharCode(...bytes));
        ws.send(JSON.stringify({
            type: "input_audio_buffer.append",
            audio: b64,
        }));
    };
}

// pcm-worklet.js — AudioWorkletProcessor
/*
class PCMWorklet extends AudioWorkletProcessor {
    process(inputs) {
        const input = inputs[0];
        if (input.length === 0) return true;
        const channel = input[0];               // mono channel

        // Convert Float32 [-1..1] to Int16 [-32768..32767]
        const int16 = new Int16Array(channel.length);
        for (let i = 0; i < channel.length; i++) {
            const s = Math.max(-1, Math.min(1, channel[i]));
            int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
        }
        this.port.postMessage(int16);
        return true;
    }
}
registerProcessor("pcm-worklet", PCMWorklet);
*/

# Server — resample and encode PCM16 for Realtime input
import base64
import numpy as np
from scipy import signal


TARGET_SR = 24_000     # Realtime pcm16 sample rate


def to_realtime_pcm16(samples: np.ndarray, source_sr: int) -> bytes:
    """
    Convert audio samples (Float32 in [-1, 1] or Int16) at source_sr
    into raw PCM16 bytes at 24kHz mono, little-endian.
    """
    # 1. Convert to Float32 in [-1, 1]
    if samples.dtype == np.int16:
        samples = samples.astype(np.float32) / 32768.0
    elif samples.dtype == np.int32:
        samples = samples.astype(np.float32) / 2147483648.0
    elif samples.dtype != np.float32:
        samples = samples.astype(np.float32)

    # 2. Downmix stereo → mono
    if samples.ndim == 2:
        samples = samples.mean(axis=1)          # or axis=0 depending on layout

    # 3. Resample to 24kHz
    if source_sr != TARGET_SR:
        new_length = int(len(samples) * TARGET_SR / source_sr)
        samples = signal.resample(samples, new_length).astype(np.float32)

    # 4. Convert to Int16 little-endian
    samples = np.clip(samples, -1.0, 1.0)
    int16 = (samples * 32767).astype(np.int16)

    # numpy int16 is native byte order; ensure little-endian
    int16 = int16.astype("<i2")

    return int16.tobytes()


def send_audio(ws, audio_bytes: bytes):
    """Send raw PCM16 bytes as a Realtime input_audio_buffer.append event."""
    import json
    ws.send(json.dumps({
        "type": "input_audio_buffer.append",
        "audio": base64.b64encode(audio_bytes).decode("ascii"),
    }))


# ✅ Read a WAV file and stream to Realtime — strip WAV header
import wave

def stream_wav_to_realtime(ws, path: str, chunk_ms: int = 100):
    with wave.open(path, "rb") as wf:
        source_sr = wf.getframerate()
        channels = wf.getnchannels()
        frames_per_chunk = int(source_sr * chunk_ms / 1000)

        while True:
            raw = wf.readframes(frames_per_chunk)
            if not raw:
                break

            samples = np.frombuffer(raw, dtype=np.int16)
            if channels == 2:
                samples = samples.reshape(-1, 2)

            pcm = to_realtime_pcm16(samples, source_sr)
            send_audio(ws, pcm)


# ❌ ANTI-PATTERN — sending WAV bytes directly (includes 44-byte header)
# with open("audio.wav", "rb") as f:
#     ws.send(json.dumps({
#         "type": "input_audio_buffer.append",
#         "audio": base64.b64encode(f.read()).decode(),   # BUG: header will be interpreted as audio
#     }))

import asyncio
import base64
import json
import websockets


# ✅ Session config for telephony — G.711 μ-law both directions
TELEPHONY_SESSION = {
    "type": "session.update",
    "session": {
        "instructions": "You are a customer support voice agent for a bank. Verify identity before discussing accounts.",
        "audio": {
            "input":  {"format": "g711_ulaw"},   # 8kHz μ-law input from carrier
            "output": {"format": "g711_ulaw", "voice": "marin"},
        },
        "output_modalities": ["audio", "text"],
        "turn_detection": {"type": "server_vad", "silence_duration_ms": 300},
    },
}


# ✅ Twilio Media Streams — receives μ-law 8kHz base64 already
# Just forward directly to Realtime, no conversion needed
async def bridge_twilio_to_realtime(twilio_ws, realtime_ws):
    """Bidirectional bridge between Twilio Media Stream and Realtime API."""

    await realtime_ws.send(json.dumps(TELEPHONY_SESSION))

    async def twilio_to_openai():
        async for msg in twilio_ws:
            data = json.loads(msg)
            if data["event"] == "media":
                # Twilio sends payload as base64 μ-law 8kHz
                await realtime_ws.send(json.dumps({
                    "type": "input_audio_buffer.append",
                    "audio": data["media"]["payload"],    # already base64
                }))

    async def openai_to_twilio():
        stream_sid = None
        async for msg in realtime_ws:
            event = json.loads(msg)
            if event["type"] == "response.output_audio.delta":
                # Realtime sends base64 μ-law → forward to Twilio
                await twilio_ws.send(json.dumps({
                    "event": "media",
                    "streamSid": stream_sid,
                    "media": {"payload": event["delta"]},
                }))
            elif event["type"] == "input_audio_buffer.speech_started":
                # User started talking — tell Twilio to clear its playback buffer
                # (implements barge-in)
                if stream_sid:
                    await twilio_ws.send(json.dumps({
                        "event": "clear",
                        "streamSid": stream_sid,
                    }))

    await asyncio.gather(twilio_to_openai(), openai_to_twilio())


# ✅ Manual μ-law encoding when source is PCM
def pcm16_to_ulaw(pcm_samples: bytes) -> bytes:
    """Convert PCM16 to G.711 μ-law. Uses standard ITU-T G.711 lookup."""
    import audioop
    return audioop.lin2ulaw(pcm_samples, 2)     # width=2 bytes for PCM16

def ulaw_to_pcm16(ulaw_bytes: bytes) -> bytes:
    import audioop
    return audioop.ulaw2lin(ulaw_bytes, 2)


# ✅ Same helpers for A-law (European telephony)
def pcm16_to_alaw(pcm_samples: bytes) -> bytes:
    import audioop
    return audioop.lin2alaw(pcm_samples, 2)


# ✅ Rate check — 8kHz means 8000 samples/second → 1 byte per sample for μ-law
# 100ms chunk = 800 bytes μ-law, or 1600 bytes PCM16
BYTES_PER_100MS_ULAW = 800
BYTES_PER_100MS_PCM16 = 3200                    # 24kHz * 2 bytes * 0.1s

# Use these to validate chunk sizes are sensible before sending

import base64
import json


CHUNK_MS = 100                      # send audio every 100ms
SAMPLE_WIDTH_PCM16 = 2              # bytes per sample
SR_PCM16 = 24_000
SAMPLES_PER_CHUNK = int(SR_PCM16 * CHUNK_MS / 1000)                     # 2400 samples
BYTES_PER_CHUNK = SAMPLES_PER_CHUNK * SAMPLE_WIDTH_PCM16               # 4800 bytes


async def send_pcm_stream(ws, pcm_bytes: bytes):
    """
    Stream PCM16 bytes to Realtime in aligned chunks.
    Each chunk is a multiple of SAMPLE_WIDTH_PCM16 to avoid mid-sample splits.
    """
    if len(pcm_bytes) % SAMPLE_WIDTH_PCM16 != 0:
        # Trim to even byte count
        pcm_bytes = pcm_bytes[:len(pcm_bytes) - (len(pcm_bytes) % SAMPLE_WIDTH_PCM16)]

    for i in range(0, len(pcm_bytes), BYTES_PER_CHUNK):
        chunk = pcm_bytes[i:i + BYTES_PER_CHUNK]
        # Chunk always ends on an even byte because BYTES_PER_CHUNK is even
        await ws.send(json.dumps({
            "type": "input_audio_buffer.append",
            "audio": base64.b64encode(chunk).decode("ascii"),
        }))


# ✅ Manual turn detection — you commit the buffer when user is done
MANUAL_TURN_SESSION = {
    "type": "session.update",
    "session": {
        "audio": {"input": {"format": "pcm16"}, "output": {"format": "pcm16"}},
        "turn_detection": None,        # manual mode — no server VAD
    },
}


async def push_to_talk(ws, mic_capture_generator):
    """User holds a button, we send audio, on release we commit."""
    await ws.send(json.dumps(MANUAL_TURN_SESSION))

    # Wait for user to press button... start capture...
    async for pcm_chunk in mic_capture_generator:
        await send_pcm_stream(ws, pcm_chunk)

    # User released the button — commit the buffer
    await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))

    # Ask the model to respond
    await ws.send(json.dumps({"type": "response.create"}))


# ✅ Server VAD mode — auto-commit on silence, no manual step needed
SERVER_VAD_SESSION = {
    "type": "session.update",
    "session": {
        "audio": {"input": {"format": "pcm16"}, "output": {"format": "pcm16"}},
        "turn_detection": {
            "type": "server_vad",
            "silence_duration_ms": 200,     # commit after 200ms of silence
            "create_response": True,        # auto-generate response after commit
        },
    },
}


# ✅ Buffer overflow protection — clear buffer if too much unsent audio
async def send_with_backpressure(ws, pcm_bytes: bytes, max_buffer_seconds: float = 10.0):
    """Clear buffer if approaching size limit."""
    seconds = len(pcm_bytes) / (SR_PCM16 * SAMPLE_WIDTH_PCM16)
    if seconds > max_buffer_seconds:
        await ws.send(json.dumps({"type": "input_audio_buffer.clear"}))
    await send_pcm_stream(ws, pcm_bytes)


# ✅ Playback — decode base64 audio deltas and buffer for playback
class AudioPlayback:
    def __init__(self):
        self.buffer = bytearray()

    def handle_delta(self, b64_audio: str):
        self.buffer += base64.b64decode(b64_audio)

    def flush(self):
        """Called on response.output_audio.done — feed to audio output."""
        audio = bytes(self.buffer)
        self.buffer.clear()
        # Write to your audio sink here
        # (For WebRTC, playback is automatic via the peer connection's audio track;
        # for WebSocket, you need to pipe to a speaker/file)
        return audio