LangServe — route registration, streaming, and playground failures

from fastapi import FastAPI
from langserve import add_routes
from langchain_openai import ChatOpenAI
from langchain_anthropic import ChatAnthropic
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from pydantic import BaseModel

app = FastAPI(title="MyApp LLM Service")

# Chain 1: OpenAI-backed general QA
qa_prompt = ChatPromptTemplate.from_template("Answer briefly: {question}")
qa_chain = qa_prompt | ChatOpenAI(model="gpt-4o-mini") | StrOutputParser()

# Chain 2: Claude-backed summarisation
summarise_prompt = ChatPromptTemplate.from_template(
    "Summarise this in 3 bullet points:\n\n{text}"
)
summarise_chain = (
    summarise_prompt | ChatAnthropic(model="claude-haiku-4-5-20251001") | StrOutputParser()
)

# Explicit input schemas for the playground UI
class QAInput(BaseModel):
    question: str

class SummariseInput(BaseModel):
    text: str

# Register each chain at a unique path
add_routes(app, qa_chain, path="/qa", input_type=QAInput, output_type=str)
add_routes(app, summarise_chain, path="/summarise", input_type=SummariseInput, output_type=str)

# For the app to run:
# pip install "langserve[all]" fastapi uvicorn
# uvicorn langserve_app:app --host 0.0.0.0 --port 8000

# Endpoints created:
#   POST /qa/invoke
#   POST /qa/batch
#   POST /qa/stream
#   GET  /qa/playground
#   POST /summarise/invoke
#   ...

from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from langserve import add_routes

app = FastAPI()

# CORS — allow the playground origin (and any other client)
app.add_middleware(
    CORSMiddleware,
    allow_origins=[
        "https://app.example.com",       # your prod frontend
        "http://localhost:5173",         # local dev
    ],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
    expose_headers=["*"],
)

add_routes(app, my_chain, path="/chain")

# Run with uvicorn — do NOT enable response buffering
# uvicorn langserve_app:app \
#     --host 0.0.0.0 --port 8000 \
#     --workers 4 \
#     --loop uvloop \
#     --http h11 \
#     --limit-concurrency 200 \
#     # NO --proxy-headers unless you actually have a proxy

# nginx site block for LangServe

server {
    listen 443 ssl http2;
    server_name api.example.com;

    location /chain/ {
        proxy_pass http://langserve:8000;
        proxy_http_version 1.1;

        # === CRITICAL for streaming ===
        proxy_buffering off;              # do not buffer response
        proxy_cache off;                  # do not cache streamed
        proxy_read_timeout 300s;          # long timeout for slow streams
        proxy_connect_timeout 10s;
        proxy_send_timeout 300s;

        # Forward SSE / event-stream properly
        proxy_set_header Connection "";
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        # === Increase body size for large RAG payloads ===
        client_max_body_size 32M;
        proxy_read_timeout 300s;
    }
}

from langchain_core.runnables import ConfigurableFieldSpec
from langchain_core.runnables.history import RunnableWithMessageHistory
from langserve import add_routes

# ... your chat chain and history factory here ...

chain_with_history = RunnableWithMessageHistory(
    chain,
    get_session_history,
    input_messages_key="input",
    history_messages_key="history",
    history_factory_config=[
        ConfigurableFieldSpec(
            id="user_id", annotation=str, name="User ID",
            description="Unique user identifier", default="",
        ),
        ConfigurableFieldSpec(
            id="conversation_id", annotation=str, name="Conversation ID",
            description="Conversation ID within the user's history", default="",
        ),
    ],
)

# Register with LangServe — configurable fields become playground form inputs
add_routes(
    app,
    chain_with_history,
    path="/chat",
    per_req_config_modifier=lambda config, req: {
        **config,
        "configurable": {
            "user_id": req.headers.get("x-user-id", ""),
            "conversation_id": req.headers.get("x-conversation-id", ""),
        }
    },
)

# The per_req_config_modifier lets you pull config from HTTP headers/auth
# instead of trusting the client body. Critical for multi-tenant apps.