LlamaIndex节点索引生成与存储实战指南

Documents 转成 Nodes 这事儿，真挺费时费力的。所以聪明的做法是把 Nodes 存起来，下次直接加载，省得每次重走 files → documents → nodes 那条老路。

说到存储，先要搞明白 LlamaIndex 里“索引”到底是个啥。别被这个词唬住，它跟你日常翻的书籍目录完全不是一回事。书籍目录是从目录找内容页码，但 LlamaIndex 的索引更像一个容器——好比一本书，你把 Nodes 一页页塞进去，它自己就给你生成目录。所以理解 LlamaIndex 的索引时，脑子里要有个容器的画面，而不是传统目录。常见的索引类型包括：

一般索引：
- SummaryIndex
- DocumentSummaryIndex
- TreeIndex
向量索引：
- VectorStoreIndex

存索引就两条路子：要么直接存成文件，要么扔进向量数据库。前一种是一般索引的常规操作，后一种只有 VectorStoreIndex 才能玩得转。

今天拿 chromadb 这个玩具级向量库搭个 demo 试试。先导包：

from datetime import datetime
import os
import chromadb
from dotenv import load_dotenv
from llama_index.core import Document, SimpleDirectoryReader, VectorStoreIndex, StorageContext, Settings
from llama_index.core.node_parser import SemanticSplitterNodeParser
from llama_index.core.schema import BaseNode
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
from llama_index.llms.dashscope import DashScope
from llama_index.vector_stores.chroma import ChromaVectorStore

然后写个保存函数：

def sa ve_to_vector_db(nodes: list[BaseNode], path: str, collection_name: str) -> None:
    """sa ve nodes to vector db
    :param nodes: nodes
    :param path: path
    :param collection_name: collection name
    :return:
    """
    # 定义一个embed_model
    embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5" # 中文友好的模型，也可以换成其他)
    # client - 生成一个client
    chroma_client = chromadb.PersistentClient(path=path)
    # collection - 定义一个集合，即索引存在哪个集合里
    collection = chroma_client.get_or_create_collection(collection_name)
    # vector store - 定义一个store，说明存在哪里
    vector_store = ChromaVectorStore(chroma_collection=collection)
    # storage context － 定义一个上下文
    storage_context = StorageContext.from_defaults(vector_store=vector_store)
    # sa ve to db - 保存进
    VectorStoreIndex(
        nodes,
        embed_model = embed_model, # 注意这里，需要指定embed_model
        storage_context=storage_context,
        show_progress=True
    )

为什么 VectorStoreIndex 要单独指定 embed_model？因为向量化是文本进向量数据库的前提，文本不能直接往里塞。而且 embed_model 还要负责后续的相似性查询，找出跟用户查询最相关的 nodes。不过实践里更常见的做法是把 embed_model 挂在全局 Settings 上，省得每个地方都传一遍：

Settings.embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5")

这样一来，VectorStoreIndex 实例化时就能省掉 embed_model 参数：

VectorStoreIndex(
    nodes,
    storage_context=storage_context,
    show_progress=True
)

将所有前面的代码组合起来进行测试

from datetime import datetime
import os
import chromadb
from dotenv import load_dotenv
from llama_index.core import Document, SimpleDirectoryReader, VectorStoreIndex, StorageContext, Settings
from llama_index.core.node_parser import SemanticSplitterNodeParser
from llama_index.core.schema import BaseNode
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
from llama_index.llms.dashscope import DashScope
from llama_index.vector_stores.chroma import ChromaVectorStore
from build_documents.directory import PDFReader

def load_documents(path: str) -> list[Document]:
    """load documents from path
    :param path: path that contains documents
    :return: documents
    """
    parser = PDFReader()
    return SimpleDirectoryReader(path, file_extractor={".pdf": parser}).load_data()

def transform_to_nodes(documents: list[Document]) -> list[BaseNode]:
    """transform documents to nodes
    :param documents: documents
    :return: nodes
    """
    embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5", device="cpu")
    splitter = SemanticSplitterNodeParser(
        embed_model=embed_model,
        buffer_size=1,
        # 根据文档特性调整
        breakpoint_percentile_threshold=85,  # 85 或 95
    )
    return splitter.get_nodes_from_documents(documents)

def create_llm():
    return DashScope(
        api_base=os.environ.get("OPENAI_API_BASE"),
        api_key=os.environ.get("OPENAI_API_KEY"),
        model=os.environ.get('OPENAI_MODEL_NAME'),
        is_chat_model=True,
        is_function_calling_model=True,
        enable_thinking=False,
        temperature=0.8
    )

def sa ve_to_vector_db(nodes: list[BaseNode], path: str, collection_name: str) -> None:
    """sa ve nodes to vector db
    :param nodes: nodes
    :param path: path
    :param collection_name: collection name
    :return:
    """
    # client
    chroma_client = chromadb.PersistentClient(path=path)
    # collection
    collection = chroma_client.get_or_create_collection(collection_name)
    # vector store
    vector_store = ChromaVectorStore(chroma_collection=collection)
    # storage context
    storage_context = StorageContext.from_defaults(vector_store=vector_store)
    # sa ve to db
    VectorStoreIndex(
        nodes,
        storage_context=storage_context,
        show_progress=True
    )

if __name__ == "__main__":
    load_dotenv()
    Settings.embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5")
    print(f'{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}: start loading documents')
    documents = load_documents("./files/")
    print(f'{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}: complete loading documents')
    print(f'{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}: start transforming documents to nodes')
    nodes = transform_to_nodes(documents)
    print(f'{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}: complete transforming documents to nodes')
    print(f'{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}: start sa ving nodes to db')
    sa ve_to_vector_db(nodes, path="./data/db", collection_name="postgresql-docs")
    print(f'{datetime.now().strftime("%Y-%m-%d %H:%M:%S")}: complete sa ving nodes to db')

来源：互联网