使用LangChain构建RAG管道：从文档处理到向量搜索

用户查询 → 检索 → 上下文 + 查询 → LLM → 回答

from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import TokenTextSplitter

# 加载文档
loader = PyPDFLoader("knowledge_base.pdf")
documents = loader.load()

# 分割为块
text_splitter = TokenTextSplitter(
    chunk_size=500,
    chunk_overlap=50,
    encoding_name="cl100k_base"  # OpenAI分词器
)

chunks = text_splitter.split_documents(documents)

from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import Redis

# 初始化嵌入
embeddings = OpenAIEmbeddings()

# 创建向量存储
vector_store = Redis.from_documents(
    documents=chunks,
    embedding=embeddings,
    redis_url="redis://localhost:6379",
    index_name="knowledge_base"
)

from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors import LLMChainExtractor

# 基本相似性搜索
retriever = vector_store.as_retriever(
    search_type="similarity",
    search_kwargs={"k": 5}
)

# 高级：上下文压缩
compressor = LLMChainExtractor.from_llm(llm)
compression_retriever = ContextualCompressionRetriever(
    base_compressor=compressor,
    base_retriever=retriever
)

from langchain.prompts import ChatPromptTemplate

system_prompt = """你是一位农业专家。仅基于提供的上下文回答问题。如果上下文没有包含足够的信息，请说"我没有足够的信息来回答这个问题。"

上下文：{context}
"""

prompt = ChatPromptTemplate.from_messages([
    ("system", system_prompt),
    ("human", "{input}")
])

from fastapi import FastAPI
from fastapi.responses import StreamingResponse
import asyncio

app = FastAPI()

@app.post("/chat")
async def chat(query: str):
    async def generate():
        async for chunk in chain.astream({"input": query}):
            yield f"data: {chunk.content}\n\n"
            await asyncio.sleep(0.01)  # 模拟打字效果
        yield "data: [DONE]\n\n"
    
    return StreamingResponse(
        generate(),
        media_type="text/event-stream"
    )

try:
    response = await chain.ainvoke({"input": query})
except Exception as e:
    logger.error(f"RAG错误: {e}")
    return {"error": "生成响应失败"}

from langchain.cache import RedisCache

cache = RedisCache(redis_url="redis://localhost:6379")
llm = OpenAI(cache=cache)