LangChain VectorStoreRetriever — empty results, wrong docs, filters ignored

from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma

embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

vs = Chroma(
    collection_name="my_docs",
    embedding_function=embeddings,
    persist_directory="./chroma_db",
)

# Baseline: plain similarity, k=10
retriever = vs.as_retriever(
    search_type="similarity",
    search_kwargs={"k": 10},
)

# Measure with a labeled test set
test_queries = [
    ("What is the refund policy?", ["doc_refund_policy_v3", "doc_terms_of_service"]),
    ("How do I reset my password?", ["doc_password_reset_kb", "doc_account_faq"]),
    # ...
]

hits = 0
misses = 0
for query, expected_ids in test_queries:
    docs = retriever.invoke(query)
    retrieved_ids = {d.metadata.get("id") for d in docs}
    if any(exp in retrieved_ids for exp in expected_ids):
        hits += 1
    else:
        misses += 1
        print(f"MISS: {query!r}")
        print(f"  expected any of: {expected_ids}")
        print(f"  retrieved: {list(retrieved_ids)[:5]}")

print(f"Hit rate: {hits}/{hits+misses} = {hits/(hits+misses):.1%}")

# Then experiment with variations and see which lifts hit rate

# --- Pinecone: MongoDB-style operators ---
from langchain_pinecone import PineconeVectorStore
retriever = PineconeVectorStore(index_name="...").as_retriever(
    search_kwargs={
        "k": 10,
        "filter": {"category": {"$eq": "policy"}, "date": {"$gte": "2025-01-01"}},
    }
)

# --- Chroma: simple dict (LangChain wrapper translates) ---
from langchain_community.vectorstores import Chroma
retriever = Chroma(...).as_retriever(
    search_kwargs={
        "k": 10,
        "filter": {"category": "policy"},   # equality
    }
)

# --- Qdrant: native filter format ---
from langchain_qdrant import QdrantVectorStore
from qdrant_client.models import Filter, FieldCondition, MatchValue
retriever = QdrantVectorStore(...).as_retriever(
    search_kwargs={
        "k": 10,
        "filter": Filter(must=[
            FieldCondition(key="category", match=MatchValue(value="policy"))
        ]),
    }
)

# --- Milvus / Zilliz: boolean expression string ---
from langchain_milvus import Milvus
retriever = Milvus(...).as_retriever(
    search_kwargs={
        "k": 10,
        "expr": "category == 'policy' and date >= '2025-01-01'",
    }
)

# --- FAISS: no native filter — post-filter in Python ---
from langchain_community.vectorstores import FAISS
from langchain_core.documents import Document

def faiss_with_filter(retriever, query: str, filter_fn) -> list[Document]:
    docs = retriever.invoke(query)
    return [d for d in docs if filter_fn(d)]

filtered = faiss_with_filter(
    faiss_retriever,
    "policy question",
    lambda d: d.metadata.get("category") == "policy",
)

# When to reach for MMR:
#   - Corpus has many near-duplicate documents
#   - Same passage indexed multiple times (different pages, versions)
#   - Similar-worded but distinct concepts

retriever_mmr = vs.as_retriever(
    search_type="mmr",
    search_kwargs={
        "k": 5,                # final number of results
        "fetch_k": 30,         # candidate pool BEFORE re-ranking (must be > k)
        "lambda_mult": 0.6,    # 0.5-0.7 is a sensible starting range
                              # closer to 1 = more relevance
                              # closer to 0 = more diversity
    }
)

# For A/B testing MMR vs plain similarity on your test set:
def compare_retrievers(similarity_retriever, mmr_retriever, test_queries):
    sim_hit_rate = evaluate(similarity_retriever, test_queries)
    mmr_hit_rate = evaluate(mmr_retriever, test_queries)
    print(f"similarity hit rate: {sim_hit_rate:.1%}")
    print(f"mmr hit rate:        {mmr_hit_rate:.1%}")

# Common finding: MMR HURTS hit rate but IMPROVES answer quality (diverse context
# reduces one-sided answers). Measure end-to-end answer quality, not just retrieval.

# For score-threshold retrieval, tune with real data — do not guess:
retriever_thresh = vs.as_retriever(
    search_type="similarity_score_threshold",
    search_kwargs={"k": 10, "score_threshold": 0.4},
    # Start LOW (0.3-0.4) and raise only if you see too much irrelevant context.
)