[{"data":1,"prerenderedAt":488},["ShallowReactive",2],{"post-memory_note0\u002Fmemory-note-0":3},{"id":4,"title":5,"body":6,"cover":474,"date":475,"description":12,"draft":476,"extension":477,"meta":478,"navigation":479,"path":480,"seo":481,"stem":482,"summary":483,"tags":484,"__hash__":487},"posts\u002Fposts\u002FMemory_note0\u002Fmemory-note-0.md","Memory调研笔记0",{"type":7,"value":8,"toc":456},"minimark",[9,13,16,31,34,45,50,53,56,60,65,68,71,91,95,98,101,121,124,132,135,143,146,151,155,158,161,164,175,178,186,189,191,199,202,210,214,220,223,245,248,256,258,266,268,273,277,281,284,287,290,294,297,300,314,317,321,324,327,338,341,352,355,359,362,366,369,375,383,389,397,403,411,417,425,429,434,437,442],[10,11,12],"p",{},"现在最落地的 memory，多半是外部存储 + 人工规则\u002F工具 API，llm agent也可能参与memory的管理和查询（mem0 \u002F MIRIX \u002F DVD \u002F WorldMM）。",[10,14,15],{},"新趋势是“让模型学会管记忆”，可以分两派：",[17,18,19,23],"ul",{},[20,21,22],"li",{},"固定操作集下学策略（Memory-R1：ADD\u002FUPDATE\u002FDELETE\u002FNOOP）",[20,24,25,26,30],{},"直接把记忆揉进推理过程（MEM1：内部状态 ",[27,28,29],"code",{},"\u003CIS>"," 迭代压缩）",[10,32,33],{},"视频领域基本在复刻这两派：",[17,35,36,39,42],{},[20,37,38],{},"结构化外存 + 自动回忆工具链（DeepVideoDiscovery）",[20,40,41],{},"层级记忆 + 先粗后细 + 多 agent（VideoLucy）",[20,43,44],{},"端到端带记忆看视频，即MEM1用到视频上（VideoMem：SFT+RL，但训练很贵，3200 A100 hours）",[46,47,49],"h2",{"id":48},"_1memory-reseach的背景","1.Memory Reseach的背景",[10,51,52],{},"LLM的长上下文限制已经是总所周知的问题了，不仅是上下文窗口塞不下，attention计算量增加，还会存在注意力稀疏关注不到重点的问题（个人试过用kimi-k2、deepseek等api做qq聊天记录总结，会发现超长上下文情况会有很大幻觉。也尝试过用Qwen3-VL-8B搭流视频理解，发现长视频token下提问容易被过时事件干扰）。比较古典的解决方法是用RAG，直接暴力用问题检索向量数据库，但这方法从数据库搭建（碎片化）到检索内容（因果推理，数字等语意理解极度low level）哪哪都是问题，噪声很大。",[10,54,55],{},"所以自然需要研究如何从数据库的组织方法、内容的检索策略上做优化，现在我们一般用memory这个概念来称呼这类研究（",[46,57,59],{"id":58},"_2memory代表工作介绍","2.memory代表工作介绍",[61,62,64],"h3",{"id":63},"mem0arxiv-250419413","mem0（arxiv 2504.19413）",[10,66,67],{},"目前最热门memory开源项目，方法为 Agent 用固定 API 操作向量数据库，维护一堆碎片记忆条目。",[10,69,70],{},"核心做法：",[17,72,73,76,79,82,85,88],{},[20,74,75],{},"LLM 先从对话里识别关键信息（facts）",[20,77,78],{},"再通过固定 API 做 增删改查（ADD\u002FUPDATE\u002FDELETE\u002F…）",[20,80,81],{},"记忆形态偏零散、轻量、工程可落地\n优点：简单、容易开源落地、可控性强。\n缺点：",[20,83,84],{},"“识别什么该记”很吃 prompt\u002F启发式",[20,86,87],{},"本质只是个用llm赋能优化了向量数据库维护的RAG系统，碎片多了以后，检索质量和一致性很容易漂",[20,89,90],{},"长期连贯性不靠“理解”，更像“记笔记的草稿本”",[61,92,94],{"id":93},"mirixarxiv-250707957","MIRIX（arxiv 2507.07957）",[10,96,97],{},"也是个很热门的开源项目，把记忆做成“操作系统”，设计结构化记忆用multi-agent管理，自动路由。",[10,99,100],{},"共设计了六类记忆：",[17,102,103,106,109,112,115,118],{},[20,104,105],{},"核心记忆：user profile（最高优先级）",[20,107,108],{},"情景记忆：日记式",[20,110,111],{},"语义记忆：概念\u002F实体\u002F关系抽象",[20,113,114],{},"程序记忆：流程\u002F技能",[20,116,117],{},"资源记忆：长文档\u002F大文件",[20,119,120],{},"敏感记忆：密码\u002FAPI key（注意合规与隔离）",[10,122,123],{},"管理方式：Multi-agent",[17,125,126,129],{},[20,127,128],{},"存储：Meta Memory Manager 从上下文识别要存的东西，分发给 6 个 sub-agent",[20,130,131],{},"检索：Chat Agent 先看 6 类 summary → 决定查哪类 → 用向量（会自己重写query）\u002FBM25\u002F精确匹配等查",[10,133,134],{},"优点：",[17,136,137,140],{},[20,138,139],{},"结构化强，工程可控，适合复杂任务\u002F企业场景",[20,141,142],{},"“记忆类型”本身就是一种 inductive bias（归纳偏置）",[10,144,145],{},"缺点：",[17,147,148],{},[20,149,150],{},"设计重、维护成本高",[61,152,154],{"id":153},"memory-r1arxiv-250819828","Memory-R1（arxiv 2508.19828）",[10,156,157],{},"llm没有训练过如何管理记忆，导致有时“存\u002F更新\u002F删\u002F不管”的行为不合理，那就考虑把 mem0 的记忆 API 操作，变成可以用 RL 学出来的策略，使得记忆维护的更精确。",[10,159,160],{},"模块 1：Memory Manager",[10,162,163],{},"对每条新 fact，从 {ADD, UPDATE, DELETE, NOOP} 里选一个\nRL 流程：",[17,165,166,169,172],{},[20,167,168],{},"管理器操作 → 新记忆库状态",[20,170,171],{},"冻结的 Answer Agent 用新记忆回答问题",[20,173,174],{},"与 GT 比较（EM）→ 给奖励",[10,176,177],{},"模块 2：Answer Agent",[17,179,180,183],{},[20,181,182],{},"从 RAG top-k（很多，60条）候选里，CoT推理出一小部分最有用的，然后回答",[20,184,185],{},"按答案给奖励",[10,187,188],{},"（Answer Agent RL有点丑陋，本质像在训一个更会做过滤的模型）",[10,190,134],{},[17,192,193,196],{},[20,194,195],{},"把“记忆管理”从 prompt 手艺活，推进到可学习策略",[20,197,198],{},"操作集小，训练更稳",[10,200,201],{},"优化的方向：",[17,203,204,207],{},[20,205,206],{},"固定操作集限制表达力（“记忆结构”很难涌现）",[20,208,209],{},"奖励信号似乎比较粗粒度",[61,211,213],{"id":212},"mem1iclr26-score8666","MEM1（ICLR26 score8666）",[10,215,216,217,219],{},"不搞外部库，直接让 Agent 学会把过去压缩进一个新的总结文本 ",[27,218,29],{},"，迭代更新。",[10,221,222],{},"核心机制：",[17,224,225,231,238],{},[20,226,227,228,230],{},"用文本 ",[27,229,29],{}," 维护“内部状态\u002F记忆”",[20,232,233,234,237],{},"每轮 t：生成新的 ",[27,235,236],{},"\u003CIS_t>","，总结旧记忆 + 新观察 + 推理",[20,239,240,241,244],{},"更新完得到 ",[27,242,243],{},"\u003CIS_(t+1)>"," 后，丢弃旧标签，防止 prompt 变长，逼模型学压缩与整合",[10,246,247],{},"RL 设计：",[17,249,250,253],{},[20,251,252],{},"PPO 端到端，只用任务成功做奖励",[20,254,255],{},"通过改mask限制注意力：每个 token 只能看当时仍保留的 token，逼它学会在当前memory下推理",[10,257,134],{},[17,259,260,263],{},[20,261,262],{},"自由度最高：理论上“记忆结构”可能自己长出来",[20,264,265],{},"内存使用可控（cache 类的极致）",[10,267,145],{},[17,269,270],{},[20,271,272],{},"少量的memory状态在超长上下文下最终也得丢失信息",[46,274,276],{"id":275},"_3视频理解方向的memory工作","3.视频理解方向的memory工作",[61,278,280],{"id":279},"videomemarxiv-251204540","VideoMem（arxiv 2512.04540）",[10,282,283],{},"MEM1 + Video工作",[10,285,286],{},"让 agent 带着问题一段段看视频，边看边更记忆，SFT + RL。由于长视频数据每个视频就一个问题，RL监督力度不够，改了下GRPO，每看一段视频片段都尝试回答一下问题然后RL。",[10,288,289],{},"训练很烧（3200 A100 hours），没开源。\n定位：cache 类（精简记忆进上下文）在视频任务上的版本。\n隐含代价：视频领域的“观察”太大太噪，端到端学压缩更贵。",[61,291,293],{"id":292},"videolucyneurips25","VideoLucy（NeurIPS25）",[10,295,296],{},"固定三层层级记忆 + 先粗看后细看的策略 + 回溯 loop（multi agent，为了用强推理模型）",[10,298,299],{},"四个 agent：",[17,301,302,305,308,311],{},[20,303,304],{},"Captioning Agent：看视频片段 → 产文本描述（系统的眼睛）",[20,306,307],{},"Localization Agent：根据问题 + 当前记忆 → 找最相关时间段（导航与过滤）",[20,309,310],{},"Instruction Agent：判断还缺啥信息 → 生成更具体的指令给 Captioning（告诉眼睛看哪里）",[20,312,313],{},"Answering Agent：判断是否足够回答；不够就触发下一轮回溯",[10,315,316],{},"经验结论：\n这种“自动回忆\u002F回溯”很吃主控推理引擎质量（Deepseek-R1 \u002F o3 ），所以才整了个VL看视频转述给llm推理的pipeline，实验也提到换弱一点开源模型性能会明显掉。",[61,318,320],{"id":319},"deepvideodiscoveryneurips25结构化记忆-工具链式回忆globalclipframe-三层","DeepVideoDiscovery（NeurIPS25）——“结构化记忆 + 工具链式回忆（Global\u002FClip\u002FFrame 三层）”",[10,322,323],{},"把视频记忆做成三层索引体系，然后让强主 agent 通过工具一步步查。",[10,325,326],{},"三层记忆结构：",[17,328,329,332,335],{},[20,330,331],{},"Global：主体注册表（Subject Registry）——谁出现、啥特征、在哪些时间段",[20,333,334],{},"Clip（5 秒一段）：caption + embedding，便于语义检索",[20,336,337],{},"Frame（比如 2fps）：需要细节时做 VQA\u002F检查（车牌号、物体交接等）\n记忆最后大概组织成：{Global, {Clip, Frame}_i}",[10,339,340],{},"工具（由主 agent 调）：",[17,342,343,346,349],{},[20,344,345],{},"Global Browse：宏观概览",[20,347,348],{},"Clip Search：按 query 检索相关 clip caption",[20,350,351],{},"Frame Inspect：指定 query 或 time_range，对帧做 VQA",[10,353,354],{},"定位：磁盘类 + 自动回忆（工具链），工程味很浓，也很实用，也是VL看视频转述给llm推理，依赖强推理引擎",[61,356,358],{"id":357},"worldmmarxiv-251202425多模态世界记忆情景语义图-视觉向量库","WorldMM（arxiv 2512.02425）——“多模态世界记忆：情景\u002F语义（图）+ 视觉（向量库）”",[10,360,361],{},"把记忆拆成三块：情景记忆图、语义记忆图、视觉记忆向量库。跟DVD比基本就是记忆结构设计不同，加强版",[46,363,365],{"id":364},"_4memory分类坐标系","4.memory分类坐标系",[10,367,368],{},"按目前调研来看，可以用4个纬度来分类memory：",[370,371,372],"ol",{},[20,373,374],{},"存哪里？",[17,376,377,380],{},[20,378,379],{},"外部“磁盘类”：向量库\u002F文档库\u002F图数据库，推理时按需检索（mem0、MIRIX、Memory-R1、DVD、WorldMM、DeepVideoDiscovery）",[20,381,382],{},"内部“缓存类”：直接维护一个精简 state ，推理时全放进上下文（MEM1、VideoMem）",[370,384,386],{"start":385},2,[20,387,388],{},"长什么样？",[17,390,391,394],{},[20,392,393],{},"零碎化（碎片 facts）",[20,395,396],{},"结构化（profile \u002F 层级 \u002F 图 \u002F registry）",[370,398,400],{"start":399},3,[20,401,402],{},"谁来管？",[17,404,405,408],{},[20,406,407],{},"人工策略\u002F固定路由（成熟落地多）",[20,409,410],{},"可学习管理（RL \u002F 端到端）",[370,412,414],{"start":413},4,[20,415,416],{},"怎么回忆？",[17,418,419,422],{},[20,420,421],{},"固定检索（top-k 向量相似度 + BM25 + 精确匹配）",[20,423,424],{},"自动回忆（automatic agent，先粗看再细看、工具调用、回溯循环、强推理模型主控）",[46,426,428],{"id":427},"_5个人一些启发性的insight","5.个人一些启发性的insight",[370,430,431],{},[20,432,433],{},"记忆是感知的先验，人看视频可能会根据先前情节决定后面更关注什么内容，但“用问题当先验”可能太强\nVideoMem 用 question 引导记录什么——有效但强条件：真实应用里问题不一定先给你。而且导致拿到问题后才能开始构架记忆，延时高，特别是流式场景下",[10,435,436],{},"现实更像：实时被动观察 → 形成可复用记忆 → 遇到问题马上就可以回忆\u002F检索。\n所以可考虑从“无问题条件下的记忆组织”入手",[370,438,439],{"start":385},[20,440,441],{},"从调研看记忆可分为磁盘形记忆（记很多，按需检索一部分进上下文）和缓存型记忆（记很少，但每次推理都全带着），那感觉可以结合两者提出混合型：",[17,443,444,447,450,453],{},[20,445,446],{},"cache：高频、近期、强相关的“工作集（working set）”",[20,448,449],{},"disk：长尾细节、可回溯证据、资源文档",[20,451,452],{},"是否可以让模型学习记忆要放在“缓存”还是“磁盘”，使得大部分情况可以直接回答，小部分情况进一步检索也能回答？",[20,454,455],{},"核心难点：让模型学会“该放缓存还是该落盘”（并且能自我纠错）",{"title":457,"searchDepth":385,"depth":385,"links":458},"",[459,460,466,472,473],{"id":48,"depth":385,"text":49},{"id":58,"depth":385,"text":59,"children":461},[462,463,464,465],{"id":63,"depth":399,"text":64},{"id":93,"depth":399,"text":94},{"id":153,"depth":399,"text":154},{"id":212,"depth":399,"text":213},{"id":275,"depth":385,"text":276,"children":467},[468,469,470,471],{"id":279,"depth":399,"text":280},{"id":292,"depth":399,"text":293},{"id":319,"depth":399,"text":320},{"id":357,"depth":399,"text":358},{"id":364,"depth":385,"text":365},{"id":427,"depth":385,"text":428},"\u002Fimages\u002Fposts\u002FMemory_note0\u002Fcover.jpg","2025-12-30",false,"md",{},true,"\u002Fposts\u002Fmemory_note0\u002Fmemory-note-0",{"title":5,"description":12},"posts\u002FMemory_note0\u002Fmemory-note-0","近期memory工作调研，记忆系统到底在帮 Agent 干什么？",[485,486],"notes","调研","qgfh4opOh8czZIxDnmxyW3S3FSVDM9gAJyKRgAnLi8w",1782672216867]