[{"data":1,"prerenderedAt":314},["ShallowReactive",2],{"post-videoagent_note0\u002Fmain":3},{"id":4,"title":5,"body":6,"cover":300,"date":301,"description":284,"draft":302,"extension":303,"meta":304,"navigation":305,"path":306,"seo":307,"stem":308,"summary":309,"tags":310,"__hash__":313},"posts\u002Fposts\u002Fvideoagent_note0\u002Fmain.md","VideoAgent 开发笔记0: 基于 Mem0 的实时视觉问答 Agent",{"type":7,"value":8,"toc":283},"minimark",[9,14,18,35,39,44,69,73,80,130,134,137,168,172,176,190,194,206,210,265,269],[10,11,13],"h2",{"id":12},"_1-项目目标-project-objective","1. 项目目标 (Project Objective)",[15,16,17],"p",{},"构建一个能够实时处理视频流输入的智能 Agent。该 Agent 需具备以下能力：",[19,20,21,29],"ul",{},[22,23,24,28],"li",{},[25,26,27],"strong",{},"实时感知","：理解当前的视觉（视频）信息。",[22,30,31,34],{},[25,32,33],{},"长期记忆","：通过检索历史交互和视频内容，结合当前上下文进行连贯问答。",[10,36,38],{"id":37},"_2-核心架构-core-architecture","2. 核心架构 (Core Architecture)",[40,41,43],"h3",{"id":42},"_21-技术栈-tech-stack","2.1 技术栈 (Tech Stack)",[19,45,46,52,63],{},[22,47,48,51],{},[25,49,50],{},"Captioning Model & Main Inference Model",": 豆包 (Doubao) API。",[22,53,54,57,58,62],{},[25,55,56],{},"Embedding Model",": OpenAI ",[59,60,61],"code",{},"text-embedding-3-small","。",[22,64,65,68],{},[25,66,67],{},"Memory Framework",": Mem0。",[40,70,72],{"id":71},"_22-记忆模块-memory-module","2.2 记忆模块 (Memory Module)",[15,74,75,76,79],{},"采用 ",[25,77,78],{},"mem0"," 作为记忆管理框架，负责视频内容的语义化存储。",[19,81,82,96],{},[22,83,84,87,88,91,92,95],{},[25,85,86],{},"写入机制","：采用",[25,89,90],{},"固定窗口","策略（0-10s, 10-20s...），每 ",[25,93,94],{},"10秒"," 截取一个视频片段，通过多模态模型转换为文本描述 (Caption)，写入记忆库。",[22,97,98,101,102,105,106,109,110,113,114,117,118],{},[25,99,100],{},"存储痛点","：mem0 设计了 ",[59,103,104],{},"add"," (新增), ",[59,107,108],{},"update"," (更新), ",[59,111,112],{},"delete"," (删除), ",[59,115,116],{},"noop"," (无操作) 四种状态。\n",[19,119,120],{},[22,121,122,126,127,129],{},[123,124,125],"em",{},"现状","：由于视频 Caption 内容随时间变化大，且带有时间戳信息，导致系统几乎总是触发 ",[59,128,104],{}," 操作，造成记忆库快速膨胀。",[40,131,133],{"id":132},"_23-推理管线-inference-pipeline","2.3 推理管线 (Inference Pipeline)",[15,135,136],{},"当接收到用户 Query 时，构建如下 Context 输入给大模型：",[138,139,140,146,156,162],"ol",{},[22,141,142,145],{},[25,143,144],{},"用户指令"," (User Prompt \u002F System Prompt)",[22,147,148,151,152,155],{},[25,149,150],{},"当前视觉信息","：最近的 ",[25,153,154],{},"4帧"," 图像（作为实时视觉感知的锚点）。",[22,157,158,161],{},[25,159,160],{},"相关记忆 (RAG)","：根据 Query 从 mem0 中检索出的 Top-3 历史视频 Caption。",[22,163,164,167],{},[25,165,166],{},"当前问题"," (Current Query)。",[10,169,171],{"id":170},"_3-当前挑战与局限-challenges-limitations","3. 当前挑战与局限 (Challenges & Limitations)",[40,173,175],{"id":174},"_31-上下文覆盖不足-context-coverage","3.1 上下文覆盖不足 (Context Coverage)",[19,177,178,184],{},[22,179,180,183],{},[25,181,182],{},"问题","：固定的 Top-3 RAG 检索仅能覆盖约 30秒 (3 x 10s) 的历史跨度。",[22,185,186,189],{},[25,187,188],{},"后果","：对于需要长跨度信息的“总结性问题”无能为力。单纯增加检索数量 (Top-K) 会导致上下文过长，引起注意力稀疏 (Sparse Attention)，反而降低回答性能。",[40,191,193],{"id":192},"_32-记忆干扰与噪声-memory-interference","3.2 记忆干扰与噪声 (Memory Interference)",[19,195,196,201],{},[22,197,198,200],{},[25,199,182],{},"：对于仅需“所见即所得”的实时问题（如“现在画面里有什么？”），引入历史记忆反而造成干扰。",[22,202,203,205],{},[25,204,188],{},"：模型可能混淆“当前画面”与“记忆中的过时信息”，导致回答错误。",[40,207,209],{"id":208},"_33-语义推导与记忆路由失效-failure-in-reasoning-based-memory-routing","3.3 语义推导与记忆路由失效 (Failure in Reasoning-based Memory Routing)",[19,211,212,225,231,259],{},[22,213,214,216,217,220,221,224],{},[25,215,182],{},"：当前的 RAG 机制仅依赖",[25,218,219],{},"直接语义相似度","，无法处理任何需要",[25,222,223],{},"逻辑推导","才能定位目标记忆的问题（不仅限于时间维度）。",[22,226,227,230],{},[25,228,229],{},"核心缺陷","：系统缺失“先分析意图，再制定检索策略”的能力。",[22,232,233,236,237],{},[25,234,235],{},"案例","：\n",[19,238,239,249],{},[22,240,241,244,245,248],{},[25,242,243],{},"隐式时间约束","：用户问“刚刚那本书叫什么”，系统无法理解“刚刚”意味着需要检索",[123,246,247],{},"最新","的记录，只能机械地检索与“书”语义最接近的片段（可能是很久以前的《哈利波特》）。",[22,250,251,254,255,258],{},[25,252,253],{},"逻辑\u002F因果链条","：若问“他做完饭后干了什么？”，系统倾向于检索包含“做饭”画面的记忆，而非“做饭”",[123,256,257],{},"之后","的时间片段。",[22,260,261,264],{},[25,262,263],{},"结论","：单纯的 Semantic Search 无法承担复杂的记忆路由 (Memory Routing) 任务。",[40,266,268],{"id":267},"_34-混合推理策略失效-strategic-failure","3.4 混合推理策略失效 (Strategic Failure)",[19,270,271,277],{},[22,272,273,276],{},[25,274,275],{},"尝试方案","：设计两阶段推理 —— 先让模型看最近几帧尝试回答；若无法回答，再请求访问记忆。",[22,278,279,282],{},[25,280,281],{},"失败原因","：模型倾向于“偷懒”或自信地产生幻觉 (Hallucination)，不愿承认“不知道”并调用记忆工具。Prompt 优化目前尚未奏效。",{"title":284,"searchDepth":285,"depth":285,"links":286},"",2,[287,288,294],{"id":12,"depth":285,"text":13},{"id":37,"depth":285,"text":38,"children":289},[290,292,293],{"id":42,"depth":291,"text":43},3,{"id":71,"depth":291,"text":72},{"id":132,"depth":291,"text":133},{"id":170,"depth":285,"text":171,"children":295},[296,297,298,299],{"id":174,"depth":291,"text":175},{"id":192,"depth":291,"text":193},{"id":208,"depth":291,"text":209},{"id":267,"depth":291,"text":268},"\u002Fimages\u002Fposts\u002Fvideoagent_note0\u002Fcover.jpg","2026-01-04",false,"md",{},true,"\u002Fposts\u002Fvideoagent_note0\u002Fmain",{"title":5,"description":284},"posts\u002Fvideoagent_note0\u002Fmain","构建一个具备长期记忆能力的实时视觉问答 Agent，探索基于 mem0 的视频流语义化存储与检索方案。",[311,312],"notes","开发","z52pJfNjljIo2XQ9eAOU950mK5sQ4n7hOh1Ozsm_xVk",1782672216867]