LlamaIndex as_chat_engine() — mode selection matters more than tutorials suggest

from llama_index.core.memory import ChatMemoryBuffer
from llama_index.core import VectorStoreIndex

# ... index built already ...

# Bounded memory — prevents unbounded context growth
memory = ChatMemoryBuffer.from_defaults(token_limit=4000)

chat_engine = index.as_chat_engine(
    chat_mode="condense_plus_context",
    memory=memory,
    similarity_top_k=6,
    system_prompt=(
        "You are a helpful assistant with access to a knowledge base. "
        "When you can, cite the source document."
    ),
    verbose=True,                     # dev-time: see the rewrite + retrieval
)

# --- Multi-turn conversation ---
response = chat_engine.chat("What is the refund policy?")
print(response)
# Refunds within 30 days, minus shipping costs. [source: refund_policy.pdf]

response = chat_engine.chat("What about damaged items?")
print(response)
# For damaged items, refunds are processed within 5 business days
# and cover full shipping cost. [source: refund_policy.pdf, section 3.2]

# Access chat history
for msg in chat_engine.chat_history:
    print(f"{msg.role}: {msg.content[:80]}")

# Reset when switching topics or on user request
chat_engine.reset()

from llama_index.core.memory import ChatMemoryBuffer, ChatSummaryMemoryBuffer
from llama_index.llms.openai import OpenAI
from llama_index.core.llms import ChatMessage, MessageRole

# --- Simple buffer with token limit ---
memory = ChatMemoryBuffer.from_defaults(token_limit=3000)

# --- Summary memory for very long conversations ---
llm = OpenAI(model="gpt-4o-mini")
summary_memory = ChatSummaryMemoryBuffer.from_defaults(
    token_limit=3000,
    summarize_prompt="Summarize the conversation so far, preserving all facts.",
    llm=llm,
)

# --- Load prior conversation from your DB ---
def load_chat_history(user_id: str, conversation_id: str) -> list[ChatMessage]:
    # Fetch from Postgres/Redis/etc.
    rows = fetch_history_rows(user_id, conversation_id)
    return [ChatMessage(role=r["role"], content=r["content"]) for r in rows]

# --- Reconstruct the chat engine with loaded history ---
prior_msgs = load_chat_history("u_alice", "c_20260803")

memory = ChatMemoryBuffer.from_defaults(
    token_limit=3000,
    chat_history=prior_msgs,
)

chat_engine = index.as_chat_engine(
    chat_mode="condense_plus_context",
    memory=memory,
    similarity_top_k=6,
)

# Continue where the user left off
response = chat_engine.chat("Can you elaborate on your last answer?")
print(response)

# --- Save updated history back after the turn ---
def save_chat_history(user_id, conversation_id, messages):
    for msg in messages:
        upsert_row(user_id, conversation_id, msg.role, msg.content)

save_chat_history("u_alice", "c_20260803", chat_engine.chat_history)

from llama_index.core.chat_engine import CondensePlusContextChatEngine
from llama_index.core.memory import ChatMemoryBuffer
from llama_index.llms.openai import OpenAI

llm = OpenAI(model="gpt-4o-mini")
memory = ChatMemoryBuffer.from_defaults(token_limit=3000)

# --- Custom condense prompt for better follow-up handling ---
CUSTOM_CONDENSE_PROMPT = """
Given the following conversation and a follow-up question, rewrite the follow-up
question to be a standalone question that captures all relevant context.

Rules:
1. If the follow-up refers to something in prior turns (e.g. "what about X?"),
   INCLUDE the original topic in the rewrite.
2. If the follow-up is a fresh topic, use it as-is.
3. Preserve every named entity, product, and number mentioned in prior turns
   that's relevant to the follow-up.

Chat history:
{chat_history}

Follow-up: {question}

Standalone question:
"""

from llama_index.core.prompts import PromptTemplate

chat_engine = CondensePlusContextChatEngine.from_defaults(
    retriever=index.as_retriever(similarity_top_k=6),
    llm=llm,
    memory=memory,
    condense_prompt=PromptTemplate(CUSTOM_CONDENSE_PROMPT),
    verbose=True,     # prints the condensed question every turn
)

# --- Follow-up handling test ---
chat_engine.chat("What is the refund policy?")
# Bot: Refunds within 30 days...

response = chat_engine.chat("What about for damaged items?")
# You will see the printed rewrite:
# > "What is the refund policy for damaged items?"
# instead of
# > "What is the damaged items policy?"

# --- Manually inspect the condensed question ---
# For deeper debugging, subclass and log every step
class DebugChatEngine(CondensePlusContextChatEngine):
    def _condense_question(self, chat_history, question):
        standalone = super()._condense_question(chat_history, question)
        print(f"[condense] {question!r} -> {standalone!r}")
        return standalone