LlamaIndex query_engine.query() — empty response or wrong nodes retrieved

from llama_index.core import VectorStoreIndex

query_engine = index.as_query_engine(similarity_top_k=6)

# --- Step 1: check what the retriever alone returns ---
retriever = query_engine.retriever
nodes = retriever.retrieve("What is the refund policy?")
print(f"Retriever returned {len(nodes)} nodes")
for i, node in enumerate(nodes):
    print(f"  {i}: score={node.score:.3f}")
    print(f"     source: {node.metadata.get('source', '?')}")
    print(f"     text: {node.text[:100]}...")
    print()

# --- If 0 nodes: retrieval is the problem ---
# Check: is the index actually populated?
print(f"Index size: {len(index.docstore.docs)} docs")

# Check: does a very-broad query find anything?
broad = retriever.retrieve("anything")
print(f"Broad query returned {len(broad)} nodes")

# Check: is embed_model the same for the query as the index?
print(f"Index embed_model: {index._embed_model}")
print(f"Retriever embed_model: {retriever._embed_model}")

# --- Step 2: if retrieval works but response is bad, look at synthesizer ---
from llama_index.core.response_synthesizers import get_response_synthesizer

synth = get_response_synthesizer(response_mode="compact")
response = synth.synthesize("What is the refund policy?", nodes)
print(response)

# Now you can vary response_mode without affecting retrieval
for mode in ["compact", "refine", "tree_summarize"]:
    synth = get_response_synthesizer(response_mode=mode)
    r = synth.synthesize("...", nodes)
    print(f"[{mode}] {r.response[:150]}")

from llama_index.core.postprocessor import (
    SimilarityPostprocessor,
    MetadataReplacementPostProcessor,
)
from llama_index.core.vector_stores import (
    MetadataFilters,
    MetadataFilter,
    FilterOperator,
)

# --- Metadata filters push down to the vector store ---
filters = MetadataFilters(filters=[
    MetadataFilter(key="category", value="policies", operator=FilterOperator.EQ),
    MetadataFilter(key="year", value=2025, operator=FilterOperator.GTE),
])

query_engine = index.as_query_engine(
    similarity_top_k=10,           # retrieve broadly
    filters=filters,
    node_postprocessors=[
        # Drop weakly-scored nodes
        SimilarityPostprocessor(similarity_cutoff=0.3),
        # Cross-encoder rerank the top-10 to top-5
        # (requires: pip install llama-index-postprocessor-cohere-rerank)
        # CohereRerank(top_n=5, model="rerank-english-v3.0", api_key="..."),
    ],
    response_mode="compact",
)

response = query_engine.query("What is the refund policy for policies from 2025 onward?")
print(response)

# For introspection
for node in response.source_nodes:
    print(f"score={node.score:.3f} source={node.metadata.get('source')}")

# --- Measure hit rate on a labeled test set ---
test_queries = [
    ("What is the refund policy?", ["doc_refund", "doc_terms"]),
    ("Password reset process?", ["doc_password_kb", "doc_account_faq"]),
    # ...
]

hits = 0
for query, expected_sources in test_queries:
    resp = query_engine.query(query)
    retrieved_sources = {n.metadata.get("source") for n in resp.source_nodes}
    if any(exp in retrieved_sources for exp in expected_sources):
        hits += 1
print(f"Hit rate: {hits}/{len(test_queries)} = {hits/len(test_queries):.1%}")

# --- compact: default; concatenates as much as fits ---
# Best for: <=10 nodes of ~500 tokens each
qe_compact = index.as_query_engine(
    similarity_top_k=6,
    response_mode="compact",
)

# --- refine: process one node at a time, refining the answer ---
# Best for: 15-50 nodes; longer LLM cost but higher-quality synthesis
qe_refine = index.as_query_engine(
    similarity_top_k=20,
    response_mode="refine",
)

# --- tree_summarize: hierarchical summarize for very large context ---
# Best for: 50+ nodes; recursive summarization
qe_tree = index.as_query_engine(
    similarity_top_k=50,
    response_mode="tree_summarize",
    use_async=True,               # tree_summarize parallelizes well
)

# --- accumulate: answer PER node, then combine (for comparative queries) ---
# Best for: "compare X across all documents"
qe_accum = index.as_query_engine(
    similarity_top_k=10,
    response_mode="accumulate",
)

# --- Pick based on context and question type ---
def choose_response_mode(context_size_estimate: int, question_type: str) -> str:
    if question_type == "compare":
        return "accumulate"
    if context_size_estimate < 4000:
        return "compact"
    if context_size_estimate < 20000:
        return "refine"
    return "tree_summarize"

# --- Common trap: refine mode + streaming ---
# refine mode does NOT stream naturally (produces refined answers per node)
# use compact if you need streaming to end users
qe_streaming = index.as_query_engine(
    similarity_top_k=6,
    response_mode="compact",
    streaming=True,                # stream the final answer
)

response = qe_streaming.query("...")
for chunk in response.response_gen:
    print(chunk, end="", flush=True)