召回效果差时，无法判断是embedding 模型的问题（文本向量化时语义丢失），还是检索策略的问题（索引 / 参数设置不合理）； 文档漏召时，不知道目标文档的向量在空间中处于什么位置，是否与查询向量属于同一聚类； 出现误召时，无法解释为什么无关文档的向量会与查询向量产生高相似度，是文本拆分问题还是向量分布问题。
下载docker-compose.ymlwget https://github.com/milvus-io/milvus/releases/download/v2.6.8/milvus-standalone-docker-compose.yml -O docker-compose.yml启动Milvus（检查端口映射：19530:19530）docker-compose up -d验证服务启动docker ps | grep milvus应该看到3个容器：milvus-standalone, milvus-etcd, milvus-minio
from pymilvus import MilvusClientfrom pymilvus.milvus_client.index import IndexParamsfrom openai import OpenAIfrom langchain_text_splitters import RecursiveCharacterTextSplitterimport umapfrom sklearn.neighbors import NearestNeighborsimport jsonimport numpy as npimport osimport glob--- CONFIG ---MILVUS_URI = "http://localhost:19530"COLLECTION_NAME = "golem_memories"JSON_OUTPUT_PATH = "./golem_cortex.json"数据文件夹（用户把 md 文件放在这里）DATA_DIR = "./data"OpenAI Embedding ConfigOPENAI_API_KEY = os.getenv("OPENAI_API_KEY")OPENAI_BASE_URL = "https://api.openai.com/v1" #OPENAI_EMBEDDING_MODEL = "text-embedding-3-small"1536 dimensionsEMBEDDING_DIM = 1536颜色映射（自动轮转分配颜色）COLORS = [[0.29, 0.87, 0.50],Green[0.22, 0.74, 0.97],Blue[0.60, 0.20, 0.80],Purple[0.94, 0.94, 0.20],Gold[0.98, 0.55, 0.00],Orange[0.90, 0.30, 0.40],Red[0.40, 0.90, 0.90],Cyan[0.95, 0.50, 0.90],MAgenta]def get_embeddings(texts):"""Batch embedding using OpenAI API"""client = OpenAI(api_key=OPENAI_API_KEY, base_url=OPENAI_BASE_URL)embeddings = []batch_size = 100OpenAI allows multiple texts per requestfor i in range(0, len(texts), batch_size):batch = texts[i:i + batch_size]response = client.embeddings.create(model=OPENAI_EMBEDDING_MODEL,input=batch)embeddings.extend([item.embedding for item in response.data])print(f" ↳ Embedded {min(i + batch_size, len(texts))}/{len(texts)}...")return np.array(embeddings)def load_markdown_files(data_dir):"""Load all markdown files from the data directory"""md_files = glob.glob(os.path.join(data_dir, "**/*.md"), recursive=True)if not md_files:print(f" ❌ ERROR: No .md files found in '{data_dir}'")print(f" 👉 Create a '{data_dir}' folder and put your markdown files there.")print(f" 👉 Example: {data_dir}/doc1.md, {data_dir}/docs/doc2.md")return Nonedocs = []print(f"\n📚 FOUND {len(md_files)} MARKDOWN FILES:")for i, file_path in enumerate(md_files):filename = os.path.basename(file_path)相对于 data_dir 的路径作为类别rel_path = os.path.relpath(file_path, data_dir)category = os.path.dirname(rel_path) if os.path.dirname(rel_path) else "default"with open(file_path, 'r', encoding='utf-8') as f:content = f.read()docs.append({"title": filename,"text": content,"cat": category,"path": file_path})print(f" {i+1}. [{category}] {filename}")return docsdef ingest_dense():print(f"🧠 PROJECT GOLEM - NEURAL MEMORY BUILDER")print(f"=" * 50)if not OPENAI_API_KEY:print(" ❌ ERROR: OPENAI_API_KEY environment variable not set!")print(" 👉 Run: export OPENAI_API_KEY='your-key-here'")returnprint(f" ↳ Using OpenAI Embedding: {OPENAI_EMBEDDING_MODEL}")print(f" ↳ Embedding Dimension: {EMBEDDING_DIM}")print(f" ↳ Data Directory: {DATA_DIR}")1. Load local markdown filesdocs = load_markdown_files(DATA_DIR)if docs is None:return2. Split documents into chunksprint(f"\n📦 PROCESSING DOCUMENTS...")splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=50)chunks = []raw_texts = []colors = []chunk_titles = []categories = []for doc in docs:doc_chunks = splitter.create_documents([doc['text']])cat_index = hash(doc['cat']) % len(COLORS)for i, chunk in enumerate(doc_chunks):chunks.append({"text": chunk.page_content,"title": doc['title'],"cat": doc['cat']})raw_texts.append(chunk.page_content)colors.append(COLORS[cat_index])chunk_titles.append(f"{doc['title']} (chunk {i+1})")categories.append(doc['cat'])print(f" ↳ Created {len(chunks)} text chunks from {len(docs)} documents")3. Generate embeddingsprint(f"\n🔮 GENERATING EMBEDDINGS...")vectors = get_embeddings(raw_texts)4. 3D Projection (UMAP)print("\n🎨 CALCULATING 3D MANIFOLD...")reducer = umap.UMAP(n_components=3, n_neighbors=30, min_dist=0.1, metric='cosine')embeddings_3d = reducer.fit_transform(vectors)5. Wiring (KNN)print(" ↳ Wiring Synapses (finding connections)...")nbrs = NearestNeighbors(n_neighbors=8, metric='cosine').fit(vectors)distances, indices = nbrs.kneighbors(vectors)6. Prepare output datacortex_data = []milvus_data = []for i in range(len(chunks)):cortex_data.append({"id": i,"title": chunk_titles[i],"cat": categories[i],"pos": embeddings_3d[i].tolist(),"col": colors[i],"nbs": indices[i][1:].tolist()})milvus_data.append({"id": i,"text": chunks[i]['text'],"title": chunk_titles[i],"category": categories[i],"vector": vectors[i].tolist()})with open(JSON_OUTPUT_PATH, 'w') as f:json.dump(cortex_data, f)7. Store vectors in Milvusprint("\n💾 STORING IN MILVUS...")client = MilvusClient(uri=MILVUS_URI)Drop existing collection if it existsif client.has_collection(COLLECTION_NAME):print(f" ↳ Dropping existing collection '{COLLECTION_NAME}'...")client.drop_collection(COLLECTION_NAME)Create new collectionprint(f" ↳ Creating collection '{COLLECTION_NAME}' (dim={EMBEDDING_DIM})...")client.create_collection(collection_name=COLLECTION_NAME,dimension=EMBEDDING_DIM)Insert dataprint(f" ↳ Inserting {len(milvus_data)} vectors...")client.insert(collection_name=COLLECTION_NAME,data=milvus_data)Create index for faster searchprint(" ↳ Creating index...")index_params = IndexParams()index_params.add_index(field_name="vector",index_type="AUTOINDEX",metric_type="COSINE")client.create_index(collection_name=COLLECTION_NAME,index_params=index_params)print(f"\n✅ CORTEX GENERATED SUCCESSFULLY!")print(f" 📊 {len(chunks)} memory nodes stored in Milvus")print(f" 📁 Cortex data saved to: {JSON_OUTPUT_PATH}")print(f" 🚀 Run 'python GolemServer.py' to start the server")if __name__ == "__main__":ingest_dense()from flask import Flask, request, jsonify, send_from_directoryfrom openai import OpenAIfrom pymilvus import MilvusClientimport jsonimport osimport sys--- CONFIG ---Explicitly set the folder to where this script is locatedBASE_DIR = os.path.dirname(os.path.abspath(__file__))OpenAI Embedding ConfigOPENAI_API_KEY = os.getenv("OPENAI_API_KEY")OPENAI_BASE_URL = "https://api.openai.com/v1"OPENAI_EMBEDDING_MODEL = "text-embedding-3-small"Milvus ConfigMILVUS_URI = "http://localhost:19530"COLLECTION_NAME = "golem_memories"These match the files generated by ingest.pyJSON_FILE = "golem_cortex.json"UPDATED: Matches your new repo filenameHTML_FILE = "index.html"app = Flask(__name__, static_folder=BASE_DIR)print(f"\n🧠 PROJECT GOLEM SERVER")print(f" 📂 Serving from: {BASE_DIR}")--- DIAGNOSTICS ---Check if files exist before startingmissing_files = []if not os.path.exists(os.path.join(BASE_DIR, JSON_FILE)):missing_files.append(JSON_FILE)if not os.path.exists(os.path.join(BASE_DIR, HTML_FILE)):missing_files.append(HTML_FILE)if missing_files:print(f" ❌ CRITICAL ERROR: Missing files in this folder:")for f in missing_files:print(f" - {f}")print(" 👉 Did you run 'python ingest.py' successfully?")sys.exit(1)else:print(f" ✅ Files Verified: Cortex Map found.")Check API Keyif not OPENAI_API_KEY:print(f" ❌ CRITICAL ERROR: OPENAI_API_KEY environment variable not set!")print(" 👉 Run: export OPENAI_API_KEY='your-key-here'")sys.exit(1)print(f" ↳ Using OpenAI Embedding: {OPENAI_EMBEDDING_MODEL}")print(" ↳ Connecting to Milvus...")milvus_client = MilvusClient(uri=MILVUS_URI)Verify collection existsif not milvus_client.has_collection(COLLECTION_NAME):print(f" ❌ CRITICAL ERROR: Collection '{COLLECTION_NAME}' not found in Milvus.")print(" 👉 Did you run 'python ingest.py' successfully?")sys.exit(1)Initialize OpenAI clientopenai_client = OpenAI(api_key=OPENAI_API_KEY, base_url=OPENAI_BASE_URL)--- ROUTES ('/')def root():Force serve the specific HTML filereturn send_from_directory(BASE_DIR, HTML_FILE)@app.route('/')def serve_static(filename):return send_from_directory(BASE_DIR, filename)@app.route('/query', methods=['POST'])def query_brain():data = request.jsontext = data.get('query', '')if not text: return jsonify({"indices": []})print(f"🔎 Query: {text}")Get query embedding from OpenAIresponse = openai_client.embeddings.create(model=OPENAI_EMBEDDING_MODEL,input=text)query_vec = response.data[0].embeddingSearch in Milvusresults = milvus_client.search(collection_name=COLLECTION_NAME,data=[query_vec],limit=50,output_fields=["id"])Extract indices and scoresindices = [r['id'] for r in results[0]]scores = [r['distance'] for r in results[0]]return jsonify({"indices": indices,"scores": scores})if __name__ == '__main__':print(" ✅ SYSTEM ONLINE: http://localhost:8000")app.run(port=8000)3.下载数据集存放指定目录
https://github.com/milvus-io/milvus-docs/tree/v2.6.x/site/en
python ingest.py
python GolemServer.py
3D 空间中标记为“INDEXES”类别的红色簇中，约 15 个节点亮度显著增强（2-3 倍）
命中节点包括index_types.md、hnsw_index.md、ivf_index.md等文档的 chunk 前端绘制从查询向量位置到这些节点的半透明连线，镜头平滑聚焦到红色簇区域
空间中所有节点保持原色，仅有微弱的尺寸波动（<1.1 倍）
命中节点分散在多个不同颜色的簇中，无明显聚集模式
摄像机未触发聚焦行为（因未达到阈值 0.5）
阅读推荐Milvus+印度最大电商平台，如何打造服务两亿月活用户的商品比价系统Claude通过Cowork实现模型主动记忆，要如何复现？我们还需要RAG吗？MCP好在哪儿？如何用Milvus-Skills 搭建知识库" data-itemshowtype="0" linktype="text" data-linktype="2">Skills 比MCP好在哪儿？如何用Milvus-Skills 搭建知识库熠智AI+Milvus:从Embedding 到数据处理、问题重写，电商AI客服架构怎么搭？官宣，Milvus开源语义高亮模型：告别饱和检索，帮RAG、agent剪枝80%上下文都有混合检索与智能路由了，谁还在给RAG赛博哭坟？
