LangChain DocumentSplitter — chunks lose semantic units or exceed token limits

from langchain_text_splitters import RecursiveCharacterTextSplitter
import tiktoken

# Use the encoder for the embedding model you'll use downstream
encoder = tiktoken.encoding_for_model("text-embedding-3-small")

def token_len(text: str) -> int:
    return len(encoder.encode(text))

splitter = RecursiveCharacterTextSplitter(
    chunk_size=800,             # target tokens per chunk
    chunk_overlap=150,          # tokens of overlap between adjacent chunks
    length_function=token_len,  # count in tokens
    separators=["\n\n", "\n", ". ", " ", ""],  # tried in order
    add_start_index=True,       # metadata: where in the source the chunk starts
)

# Read a document
with open("big_doc.txt") as f:
    text = f.read()

chunks = splitter.split_text(text)
print(f"Created {len(chunks)} chunks")
for i, chunk in enumerate(chunks[:3]):
    print(f"\n--- chunk {i} ({token_len(chunk)} tokens) ---")
    print(chunk[:200] + "...")

# For a Document object with metadata, use split_documents
from langchain_core.documents import Document

docs = [Document(page_content=text, metadata={"source": "big_doc.txt"})]
chunked_docs = splitter.split_documents(docs)
# Each chunk inherits source metadata; start_index added automatically

from langchain_text_splitters import (
    MarkdownHeaderTextSplitter,
    RecursiveCharacterTextSplitter,
    HTMLHeaderTextSplitter,
    Language,
)

# --- Markdown ---
md_text = """
# Introduction
Some intro text.

## Section A
Content of A.

## Section B
Content of B.
"""

# First split by headers (creates chunks with header metadata)
md_header_splitter = MarkdownHeaderTextSplitter(
    headers_to_split_on=[
        ("#", "Header 1"),
        ("##", "Header 2"),
        ("###", "Header 3"),
    ]
)
md_docs = md_header_splitter.split_text(md_text)

# Then optionally sub-split large sections with the recursive splitter
text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=500, chunk_overlap=100, length_function=len
)
final_chunks = text_splitter.split_documents(md_docs)

# --- HTML ---
html_splitter = HTMLHeaderTextSplitter(
    headers_to_split_on=[("h1", "H1"), ("h2", "H2"), ("h3", "H3")]
)
html_chunks = html_splitter.split_text(html_content)

# --- Source code ---
python_splitter = RecursiveCharacterTextSplitter.from_language(
    language=Language.PYTHON,
    chunk_size=1500,
    chunk_overlap=100,
)
py_chunks = python_splitter.split_text(python_source)
# Splits on class/function boundaries first, then indentation

# Same pattern for JS, TS, Go, Java, C++, Rust, etc.
js_splitter = RecursiveCharacterTextSplitter.from_language(
    language=Language.JS, chunk_size=1500, chunk_overlap=100
)

from langchain_experimental.text_splitter import SemanticChunker
from langchain_openai import OpenAIEmbeddings

embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

semantic_splitter = SemanticChunker(
    embeddings=embeddings,
    breakpoint_threshold_type="percentile",  # or "standard_deviation", "interquartile"
    breakpoint_threshold_amount=95,          # split at top 5% of shifts
)

docs = semantic_splitter.create_documents([text])
print(f"Created {len(docs)} semantic chunks")

# Trade-offs:
# + Chunks align with topic boundaries, not arbitrary character positions
# + Great for transcripts, meeting notes, unstructured reports
# - Slower to build (one embedding call per sentence-pair distance)
# - Cost proportional to document size
# - Chunk sizes are variable; may exceed embedding limits without a cap

# Combine with a hard token cap
from langchain_text_splitters import RecursiveCharacterTextSplitter
import tiktoken

def token_len(text: str) -> int:
    return len(tiktoken.encoding_for_model("text-embedding-3-small").encode(text))

semantic_chunks = semantic_splitter.create_documents([text])

# Apply a hard-cap split on any semantic chunks that grew too large
capper = RecursiveCharacterTextSplitter(
    chunk_size=1500, chunk_overlap=150, length_function=token_len
)
final_docs = capper.split_documents(semantic_chunks)