From 7904fa3139d2a13366610e634232c30e2e380d63 Mon Sep 17 00:00:00 2001 From: Darssal <110779014+Darssal@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:56:59 +0800 Subject: [PATCH] Make research collection a canonical project index --- .github/workflows/checks.yml | 47 +- .gitignore | 4 - CONTRIBUTING.md | 24 +- README.md | 62 +- THIRD_PARTY.md | 19 +- docs/ENGINEERING_NOTES.md | 22 +- docs/PUBLICATION_SCOPE.md | 32 +- projects/job-text-classification/.env.example | 2 - projects/job-text-classification/.gitignore | 18 - projects/job-text-classification/README.md | 73 - .../job-text-classification/pyproject.toml | 23 - .../src/job_text_classification/__init__.py | 4 - .../src/job_text_classification/__main__.py | 3 - .../src/job_text_classification/baseline.py | 84 - .../src/job_text_classification/bert.py | 103 - .../src/job_text_classification/cli.py | 130 - .../src/job_text_classification/data.py | 99 - .../examples/synthetic_jobs.jsonl | 24 - .../job_text_classification/http_adapter.py | 84 - .../tests/test_regressions.py | 190 -- projects/memlab/.env.example | 15 - projects/memlab/.gitignore | 23 - projects/memlab/README.md | 38 - projects/memlab/THIRD_PARTY.md | 7 - projects/memlab/benchmarks/__init__.py | 1 - .../memlab/benchmarks/public_main/__init__.py | 1 - .../public_main/longmemeval/__init__.py | 1 - .../public_main/longmemeval/prepare.py | 127 - projects/memlab/configs/memlab.default.yaml | 68 - .../memlab/configs/postgres/postgresql.conf | 17 - .../memlab/configs/qdrant/production.yaml | 9 - projects/memlab/configs/redis/redis.conf | 13 - projects/memlab/configs/redis/redis_lfu.conf | 13 - projects/memlab/configs/redis/redis_lru.conf | 13 - projects/memlab/docs/ARCHITECTURE.md | 20 - projects/memlab/docs/SETUP.md | 42 - projects/memlab/docs/TESTING.md | 26 - projects/memlab/infra/docker-compose.yml | 58 - projects/memlab/memlab/__init__.py | 5 - projects/memlab/memlab/analysis/__init__.py | 1 - .../benchmark_discriminativeness_gates.py | 478 ---- .../memlab/memlab/analysis/branch_audit.py | 636 ----- .../analysis/public_track_design_audit.py | 277 -- .../memlab/analysis/public_track_gate.py | 463 --- projects/memlab/memlab/backends/__init__.py | 7 - .../memlab/memlab/backends/object_store.py | 56 - .../memlab/memlab/backends/postgres_store.py | 208 -- .../memlab/memlab/backends/qdrant_store.py | 109 - .../memlab/memlab/backends/redis_store.py | 94 - projects/memlab/memlab/bench/__init__.py | 1 - .../memlab/memlab/bench/postgres_bench.py | 37 - projects/memlab/memlab/bench/qdrant_bench.py | 58 - projects/memlab/memlab/bench/redis_bench.py | 45 - projects/memlab/memlab/checkpoint/__init__.py | 1 - .../memlab/memlab/checkpoint/langgraph_pg.py | 22 - projects/memlab/memlab/config.py | 131 - projects/memlab/memlab/envcheck.py | 347 --- projects/memlab/memlab/event_logger.py | 27 - projects/memlab/memlab/io.py | 41 - projects/memlab/memlab/memory/__init__.py | 17 - projects/memlab/memlab/memory/api.py | 35 - projects/memlab/memlab/memory/manager.py | 7 - projects/memlab/memlab/memory/types.py | 56 - .../memlab/memlab/memory_manager/__init__.py | 17 - projects/memlab/memlab/memory_manager/l0.py | 156 -- projects/memlab/memlab/memory_manager/l1.py | 27 - projects/memlab/memlab/memory_manager/l2.py | 36 - projects/memlab/memlab/memory_manager/l3.py | 22 - .../memlab/memlab/memory_manager/manager.py | 1116 -------- .../memlab/memlab/memory_manager/vector.py | 39 - projects/memlab/memlab/metrics/__init__.py | 16 - projects/memlab/memlab/metrics/eventlog.py | 15 - projects/memlab/memlab/metrics/manifest.py | 126 - projects/memlab/memlab/metrics/stats.py | 34 - .../memlab/memlab/metrics/summary_builder.py | 689 ----- projects/memlab/memlab/microbench.py | 142 - projects/memlab/memlab/pipeline/__init__.py | 5 - .../memlab/memlab/pipeline/control_plane.py | 550 ---- .../memlab/memlab/pipeline/model_profile.py | 219 -- projects/memlab/memlab/runner_core.py | 912 ------ projects/memlab/memlab/runner_e2e.py | 2470 ----------------- projects/memlab/memlab/runner_stub.py | 351 --- projects/memlab/memlab/runners/__init__.py | 1 - projects/memlab/memlab/runners/common.py | 146 - .../memlab/memlab/runners/deerflow_adapter.py | 45 - projects/memlab/memlab/runners/e2e_runner.py | 7 - projects/memlab/memlab/runners/llm.py | 69 - .../memlab/memlab/runners/trace_replay.py | 7 - projects/memlab/memlab/runtime_paths.py | 10 - projects/memlab/memlab/session_logging.py | 18 - projects/memlab/memlab/tools/__init__.py | 1 - projects/memlab/memlab/tools/doctor.py | 38 - projects/memlab/memlab/tools/embedding.py | 29 - projects/memlab/memlab/types.py | 36 - projects/memlab/memlab/workloads/__init__.py | 1 - .../memlab/memlab/workloads/gen_corpus.py | 104 - projects/memlab/memlab/workloads/gen_trace.py | 93 - projects/memlab/pyproject.toml | 21 - projects/memlab/scripts/_infra_common.ps1 | 39 - projects/memlab/scripts/check_docker.ps1 | 15 - projects/memlab/scripts/check_infra.ps1 | 93 - .../scripts/deerflow_embedded_worker.py | 153 - projects/memlab/scripts/down_infra.ps1 | 7 - projects/memlab/scripts/run_microbench.ps1 | 17 - projects/memlab/scripts/run_stub.ps1 | 45 - .../memlab/scripts/switch_redis_policy.ps1 | 28 - projects/memlab/scripts/up_infra.ps1 | 35 - projects/memlab/sql/001_schema.sql | 28 - ...test_benchmark_discriminativeness_gates.py | 150 - projects/memlab/tests/test_branch_audit.py | 214 -- projects/memlab/tests/test_control_plane.py | 113 - .../tests/test_infra_probe_configuration.py | 50 - .../tests/test_memory_manager_policy.py | 80 - .../memlab/tests/test_metrics_denominator.py | 88 - .../tests/test_pipeline_model_profile.py | 33 - .../memlab/tests/test_public_qa_packing.py | 152 - .../tests/test_public_track_design_audit.py | 118 - .../memlab/tests/test_public_track_gate.py | 207 -- .../tests/test_runtime_error_privacy.py | 135 - .../tests/test_stub_event_attribution.py | 86 - .../memlab/tests/test_system_rebaseline.py | 78 - publication-policy.json | 152 +- tests/test_publication_policy.py | 53 +- tools/check_index.py | 101 + 124 files changed, 331 insertions(+), 14205 deletions(-) delete mode 100644 projects/job-text-classification/.env.example delete mode 100644 projects/job-text-classification/.gitignore delete mode 100644 projects/job-text-classification/README.md delete mode 100644 projects/job-text-classification/pyproject.toml delete mode 100644 projects/job-text-classification/src/job_text_classification/__init__.py delete mode 100644 projects/job-text-classification/src/job_text_classification/__main__.py delete mode 100644 projects/job-text-classification/src/job_text_classification/baseline.py delete mode 100644 projects/job-text-classification/src/job_text_classification/bert.py delete mode 100644 projects/job-text-classification/src/job_text_classification/cli.py delete mode 100644 projects/job-text-classification/src/job_text_classification/data.py delete mode 100644 projects/job-text-classification/src/job_text_classification/examples/synthetic_jobs.jsonl delete mode 100644 projects/job-text-classification/src/job_text_classification/http_adapter.py delete mode 100644 projects/job-text-classification/tests/test_regressions.py delete mode 100644 projects/memlab/.env.example delete mode 100644 projects/memlab/.gitignore delete mode 100644 projects/memlab/README.md delete mode 100644 projects/memlab/THIRD_PARTY.md delete mode 100644 projects/memlab/benchmarks/__init__.py delete mode 100644 projects/memlab/benchmarks/public_main/__init__.py delete mode 100644 projects/memlab/benchmarks/public_main/longmemeval/__init__.py delete mode 100644 projects/memlab/benchmarks/public_main/longmemeval/prepare.py delete mode 100644 projects/memlab/configs/memlab.default.yaml delete mode 100644 projects/memlab/configs/postgres/postgresql.conf delete mode 100644 projects/memlab/configs/qdrant/production.yaml delete mode 100644 projects/memlab/configs/redis/redis.conf delete mode 100644 projects/memlab/configs/redis/redis_lfu.conf delete mode 100644 projects/memlab/configs/redis/redis_lru.conf delete mode 100644 projects/memlab/docs/ARCHITECTURE.md delete mode 100644 projects/memlab/docs/SETUP.md delete mode 100644 projects/memlab/docs/TESTING.md delete mode 100644 projects/memlab/infra/docker-compose.yml delete mode 100644 projects/memlab/memlab/__init__.py delete mode 100644 projects/memlab/memlab/analysis/__init__.py delete mode 100644 projects/memlab/memlab/analysis/benchmark_discriminativeness_gates.py delete mode 100644 projects/memlab/memlab/analysis/branch_audit.py delete mode 100644 projects/memlab/memlab/analysis/public_track_design_audit.py delete mode 100644 projects/memlab/memlab/analysis/public_track_gate.py delete mode 100644 projects/memlab/memlab/backends/__init__.py delete mode 100644 projects/memlab/memlab/backends/object_store.py delete mode 100644 projects/memlab/memlab/backends/postgres_store.py delete mode 100644 projects/memlab/memlab/backends/qdrant_store.py delete mode 100644 projects/memlab/memlab/backends/redis_store.py delete mode 100644 projects/memlab/memlab/bench/__init__.py delete mode 100644 projects/memlab/memlab/bench/postgres_bench.py delete mode 100644 projects/memlab/memlab/bench/qdrant_bench.py delete mode 100644 projects/memlab/memlab/bench/redis_bench.py delete mode 100644 projects/memlab/memlab/checkpoint/__init__.py delete mode 100644 projects/memlab/memlab/checkpoint/langgraph_pg.py delete mode 100644 projects/memlab/memlab/config.py delete mode 100644 projects/memlab/memlab/envcheck.py delete mode 100644 projects/memlab/memlab/event_logger.py delete mode 100644 projects/memlab/memlab/io.py delete mode 100644 projects/memlab/memlab/memory/__init__.py delete mode 100644 projects/memlab/memlab/memory/api.py delete mode 100644 projects/memlab/memlab/memory/manager.py delete mode 100644 projects/memlab/memlab/memory/types.py delete mode 100644 projects/memlab/memlab/memory_manager/__init__.py delete mode 100644 projects/memlab/memlab/memory_manager/l0.py delete mode 100644 projects/memlab/memlab/memory_manager/l1.py delete mode 100644 projects/memlab/memlab/memory_manager/l2.py delete mode 100644 projects/memlab/memlab/memory_manager/l3.py delete mode 100644 projects/memlab/memlab/memory_manager/manager.py delete mode 100644 projects/memlab/memlab/memory_manager/vector.py delete mode 100644 projects/memlab/memlab/metrics/__init__.py delete mode 100644 projects/memlab/memlab/metrics/eventlog.py delete mode 100644 projects/memlab/memlab/metrics/manifest.py delete mode 100644 projects/memlab/memlab/metrics/stats.py delete mode 100644 projects/memlab/memlab/metrics/summary_builder.py delete mode 100644 projects/memlab/memlab/microbench.py delete mode 100644 projects/memlab/memlab/pipeline/__init__.py delete mode 100644 projects/memlab/memlab/pipeline/control_plane.py delete mode 100644 projects/memlab/memlab/pipeline/model_profile.py delete mode 100644 projects/memlab/memlab/runner_core.py delete mode 100644 projects/memlab/memlab/runner_e2e.py delete mode 100644 projects/memlab/memlab/runner_stub.py delete mode 100644 projects/memlab/memlab/runners/__init__.py delete mode 100644 projects/memlab/memlab/runners/common.py delete mode 100644 projects/memlab/memlab/runners/deerflow_adapter.py delete mode 100644 projects/memlab/memlab/runners/e2e_runner.py delete mode 100644 projects/memlab/memlab/runners/llm.py delete mode 100644 projects/memlab/memlab/runners/trace_replay.py delete mode 100644 projects/memlab/memlab/runtime_paths.py delete mode 100644 projects/memlab/memlab/session_logging.py delete mode 100644 projects/memlab/memlab/tools/__init__.py delete mode 100644 projects/memlab/memlab/tools/doctor.py delete mode 100644 projects/memlab/memlab/tools/embedding.py delete mode 100644 projects/memlab/memlab/types.py delete mode 100644 projects/memlab/memlab/workloads/__init__.py delete mode 100644 projects/memlab/memlab/workloads/gen_corpus.py delete mode 100644 projects/memlab/memlab/workloads/gen_trace.py delete mode 100644 projects/memlab/pyproject.toml delete mode 100644 projects/memlab/scripts/_infra_common.ps1 delete mode 100644 projects/memlab/scripts/check_docker.ps1 delete mode 100644 projects/memlab/scripts/check_infra.ps1 delete mode 100644 projects/memlab/scripts/deerflow_embedded_worker.py delete mode 100644 projects/memlab/scripts/down_infra.ps1 delete mode 100644 projects/memlab/scripts/run_microbench.ps1 delete mode 100644 projects/memlab/scripts/run_stub.ps1 delete mode 100644 projects/memlab/scripts/switch_redis_policy.ps1 delete mode 100644 projects/memlab/scripts/up_infra.ps1 delete mode 100644 projects/memlab/sql/001_schema.sql delete mode 100644 projects/memlab/tests/test_benchmark_discriminativeness_gates.py delete mode 100644 projects/memlab/tests/test_branch_audit.py delete mode 100644 projects/memlab/tests/test_control_plane.py delete mode 100644 projects/memlab/tests/test_infra_probe_configuration.py delete mode 100644 projects/memlab/tests/test_memory_manager_policy.py delete mode 100644 projects/memlab/tests/test_metrics_denominator.py delete mode 100644 projects/memlab/tests/test_pipeline_model_profile.py delete mode 100644 projects/memlab/tests/test_public_qa_packing.py delete mode 100644 projects/memlab/tests/test_public_track_design_audit.py delete mode 100644 projects/memlab/tests/test_public_track_gate.py delete mode 100644 projects/memlab/tests/test_runtime_error_privacy.py delete mode 100644 projects/memlab/tests/test_stub_event_attribution.py delete mode 100644 projects/memlab/tests/test_system_rebaseline.py create mode 100644 tools/check_index.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 88c074d..803315a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,4 +1,4 @@ -name: Code and publication checks +name: Index and publication checks on: push: @@ -7,39 +7,24 @@ on: permissions: contents: read -jobs: - publication: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: '3.12' - - run: python tools/check_publication.py - - run: python -m unittest discover -s tests -v +env: + PYTHONDONTWRITEBYTECODE: '1' - classification: - runs-on: ubuntu-latest - defaults: - run: - working-directory: projects/job-text-classification - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: '3.12' - - run: python -m pip install -e '.[dev]' - - run: python -m pytest -q - - memory: +jobs: + index: runs-on: ubuntu-latest - defaults: - run: - working-directory: projects/memlab steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' - - run: python -m pip install -e '.[dev]' - - run: python -m pytest -q + - name: Index inventory and canonical links + run: python tools/check_index.py --remote + env: + GH_TOKEN: ${{ github.token }} + - name: Scan clean documentation and maintenance source + run: python tools/check_publication.py + - name: Scan actual Git index blobs + run: python tools/check_publication.py --staged + - name: Publication boundary regressions + run: python -m unittest discover -s tests -v diff --git a/.gitignore b/.gitignore index 4018a8d..2f471aa 100644 --- a/.gitignore +++ b/.gitignore @@ -58,10 +58,6 @@ temp/ *.rar *.7z -# Downloaded benchmark content must stay local, including older CLI paths -projects/memlab/benchmarks/public_main/longmemeval/*.json -projects/memlab/benchmarks/public_main/longmemeval/*.jsonl - # Operating system Thumbs.db .DS_Store diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c235164..cba84b3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,25 +1,23 @@ # 贡献与维护 -每次修改针对一个项目和一个明确的问题。先阅读该项目 README，再运行它提供的合成示例和离线测试。 +本仓库维护项目导航、跨项目设计笔记、来源和公开检查工具。实现、依赖、样例和项目测试的修改应提交到 README 对应的独立仓库；本索引不接受第二份项目源码。 ## 提交前 -逐个选择本次修改的文件并加入 Git 暂存区，再检查将要提交的内容： +1. 核对链接指向项目当前维护入口；历史链接须明确标注为追溯用途。 +2. 明确选择本次改动的文件并暂存，检查 `git diff --cached` 和文件清单。 +3. 执行以下检查。报告写到仓库外；默认全目录检查适用于无缓存、无本地环境的干净快照。 ```bash -python tools/check_publication.py --staged -python -m unittest discover -s tests -v -git diff --check -git diff --cached --stat -git diff --cached +python -B tools/check_index.py +python -B -m unittest discover -s tests -v +python -B tools/check_publication.py +python -B tools/check_publication.py --staged +git diff --cached --check ``` -`--staged` 检查暂存区的实际文件内容，本地未暂存的环境、缓存和运行输出不影响结果。无参数模式严格检查整个工作目录，适合首次发布或检查干净的代码快照；若目录中有本地虚拟环境或缓存，它们会按规则报错。 - -使用子项目 README 中的命令运行相关测试。需要数据、GPU、在线模型或外部服务的检查应单独记录，不能用离线单元测试通过代替真实实验成功。 +`python -B tools/check_index.py --remote` 另用已安装的 GitHub CLI 只读核对独立仓库的默认分支和链接文件。CI 以只读 token 执行此检查，不需要扩大账号权限。 ## 变更说明 -Pull Request 应写清触发问题、修改后的行为、实际运行的检查和未验证的依赖。模型效果数字需要对应数据划分、运行配置和可审查的结果来源；没有这些依据时，只描述代码实现与已运行的验证。 - -外部依赖保留原项目名称与来源。新增第三方代码前检查其许可证和署名要求。本仓库未统一授予开源许可证，不能把依赖项目的许可证推定为本仓库全部代码的许可证。 +Pull Request 写清链接或说明的问题、改后的阅读入口及实际检查结果。模型效果需要对应实验依据，不能把单元测试数量或合成例子的指标写成研究结论。保留外部框架、模型与参考实现来源，不将上游许可证推定为本仓库的统一许可证。 diff --git a/README.md b/README.md index 5f5cc76..5c3d127 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,28 @@ -# Research Code Collection +# Research Project Index -**Agent 多层记忆与中文文本分类实验工程。** 两个独立 Python 子项目，覆盖上下文组装、异步会话、实验有效性检查、分类基线与服务批处理。 +公开研究工程的导航与设计笔记。每个项目在各自独立仓库维护源码、安装说明、测试和来源；本仓库只维护索引与跨项目工程说明。 -[![Code and publication checks](https://github.com/Darssal/Research-Code-Collection/actions/workflows/checks.yml/badge.svg)](https://github.com/Darssal/Research-Code-Collection/actions/workflows/checks.yml) +[![Index and publication checks](https://github.com/Darssal/Research-Code-Collection/actions/workflows/checks.yml/badge.svg)](https://github.com/Darssal/Research-Code-Collection/actions/workflows/checks.yml) -[MemLab](#memlab--agent-多层记忆) · [文本分类](#招聘文本三分类) · [工程笔记](docs/ENGINEERING_NOTES.md) · [验证与来源](#验证与来源) +## Canonical Repositories -## MemLab / Agent 多层记忆 - -面向并行会话，分离近期上下文、热缓存、持久化元数据与向量检索，提供统一记忆接口和实验控制组件。 - -| 技术问题 | 实现机制 | 入口 | +| Project | 实现与当前范围 | 入口 | | --- | --- | --- | -| 记忆如何放置与取回 | L0–L3 分层、重要度准入、大对象指针、异步批量写入 | [记忆管理器](projects/memlab/memlab/memory_manager/manager.py) | -| 检索内容如何进入上下文 | 相关性筛选、版本与文本去重、注入预算、直接证据绕过 | [上下文策略](projects/memlab/memlab/memory_manager/l0.py) | -| 并发会话如何保留归属 | ContextVar 隔离日志上下文，记录缓存与降级路径 | [会话日志](projects/memlab/memlab/session_logging.py) | -| 哪些运行可进入汇总 | 有效性条件、失效阶段回滚和重跑计划 | [实验控制](projects/memlab/memlab/pipeline/control_plane.py) | - -`Python 3.12` · `asyncio` · `Redis` · `PostgreSQL` · `Qdrant` - -[项目与安装](projects/memlab/) · [架构](projects/memlab/docs/ARCHITECTURE.md) · [53 项离线测试的覆盖范围](projects/memlab/docs/TESTING.md) - -## 招聘文本三分类 - -识别环保、人工智能与其它招聘文本（其它在接口中使用标签 `否`），提供本地基线、可选 BERT 和外部聊天服务适配器。 - -- **数据与评测**：CSV / JSONL 接口、固定种子分层划分、重复文本检查；先划分再拟合 TF-IDF，按实际类别顺序对齐概率与指标。 -- **服务批处理**：超时、有界重试、非临时错误快速结束、并发保序；默认预测产物只保留 ID、标签与错误状态。 -- **可运行示例**：24 条新编写的合成文本，CPU 执行本地随机森林流程，16 项回归测试覆盖核心行为。 - -```bash -cd projects/job-text-classification -python -m pip install -e . -python -m job_text_classification demo -``` - -`Python` · `scikit-learn` · `TF-IDF` · `Random Forest` · `ThreadPoolExecutor` - -[数据接口与运行](projects/job-text-classification/) · [分类实现](projects/job-text-classification/src/job_text_classification/baseline.py) · [服务适配](projects/job-text-classification/src/job_text_classification/http_adapter.py) · [回归测试](projects/job-text-classification/tests/test_regressions.py) - -## 相关项目 / FracLogGPT-Lite - -[FracLogGPT-Lite](https://github.com/Darssal/FracLogGPT-Lite) 在独立仓库维护成像测井裂缝解释的多模态实验流程：LoRA-SFT、DPO / SimPO、可恢复推理与视觉对比解码。 +| **FracLogGPT-Lite** | 成像测井裂缝解释的多模态实验流程：LoRA-SFT、DPO / SimPO、可恢复推理和视觉对比解码。公开 4B 实验工程与论文 Qwen2.5-VL 7B 设置分别说明。 | [Repository](https://github.com/Darssal/FracLogGPT-Lite) · [Quickstart](https://github.com/Darssal/FracLogGPT-Lite/blob/main/docs/QUICKSTART.md) | +| **MemLab** | Agent L0–L3 多层记忆、上下文注入、异步会话归属与运行有效性检查。离线测试与真实基础设施 / 模型实验分开验证。 | [Repository](https://github.com/Darssal/MemLab) · [Architecture](https://github.com/Darssal/MemLab/blob/main/docs/ARCHITECTURE.md) · [Testing](https://github.com/Darssal/MemLab/blob/main/docs/TESTING.md) | +| **Recruitment Text Classification** | 中文招聘文本三分类：TF-IDF / 随机森林、可选 BERT 与有界 HTTP 适配。24 条虚构样例用于流程验证；未实现论文完整 RAG 链路。 | [Repository](https://github.com/Darssal/Recruitment-Text-Classification) · [Engineering Notes](https://github.com/Darssal/Recruitment-Text-Classification/blob/main/docs/ENGINEERING_NOTES.md) | +| **JSP UI Prototype** | 早期 JSP 界面和参考登录动画的独立存档。保留 13 个页面与 CodePen 来源；业务页多为占位，没有认证、数据库或 Java 业务后端。 | [Repository](https://github.com/Darssal/JSP-UI-Prototype) · [Scope and Provenance](https://github.com/Darssal/JSP-UI-Prototype/blob/main/docs/LEGACY_JSP.md) | -[项目介绍](https://github.com/Darssal/FracLogGPT-Lite) · [无 GPU 验证](https://github.com/Darssal/FracLogGPT-Lite/blob/main/docs/QUICKSTART.md) · [双分支解码](https://github.com/Darssal/FracLogGPT-Lite/blob/main/src/fraclog/vcd.py) +## Engineering Notes -## 设计与验证 +[工程实现与设计取舍](docs/ENGINEERING_NOTES.md)连接问题、机制、小例子和验证入口，覆盖阶段恢复、同步解码、上下文预算、会话隔离、分类划分与批处理失败。所有源码链接指向对应独立仓库。 -[工程笔记](docs/ENGINEERING_NOTES.md)连接问题、机制、小例子和测试，涵盖推理恢复、上下文与会话隔离、分类评测与失败处理。 +安装、运行和贡献代码请进入项目仓库。各工程的单元测试检查代码行为；合成样例、离线检查或原型完整性检查均不能替代论文效果复现、真实服务集成或后端功能验证。 -每个子项目独立安装。分类 demo 与 MemLab 单元测试可离线运行；MemLab stub / E2E 需要另行配置基础设施。持续集成检查分类、记忆组件及公开文件边界，详见 [Actions](https://github.com/Darssal/Research-Code-Collection/actions/workflows/checks.yml)。 +## Migration and Maintenance -## 验证与来源 +2026-10-19，MemLab 与分类工程从已经公开去敏的 Collection 提交 `8f143f67598f88fbb8a16d599a7f3d17c623d281` 建立独立维护入口。本仓库当前分支移除两份项目源码；原有文件仍可在 [Git 历史](https://github.com/Darssal/Research-Code-Collection/tree/8f143f67598f88fbb8a16d599a7f3d17c623d281/projects) 中追溯。后续改动只提交到独立仓库，历史快照不作为第二份维护入口。 -2026-10-18 验证记录：分类 16 项、MemLab 53 项离线测试通过。合成示例用于检查流程；BERT 训练、真实 HTTP 服务、数据库与 DeerFlow 集成、论文性能实验尚未在公开版整理中运行。 +JSP 原型从个人主页已公开的 Git blobs 迁移，来源记录保留在其独立仓库。FracLogGPT-Lite 已单独维护，本次未复制或重命名。 -源码、配置、测试与合成示例可以公开；真实语料、论文材料、权重、运行输出和凭据保留在本地。第三方框架与模型的来源见 [THIRD_PARTY.md](THIRD_PARTY.md)，提交规则见 [CONTRIBUTING.md](CONTRIBUTING.md) 与[公开范围](docs/PUBLICATION_SCOPE.md)。 +本仓库 CI 检查索引结构、文档链接、公开文件边界及实际 Git index，不安装或执行其他项目。源码和真实数据不迁回本索引。[贡献规则](CONTRIBUTING.md)、[公开范围](docs/PUBLICATION_SCOPE.md)与[来源说明](THIRD_PARTY.md)约束后续维护。 diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index ead8a36..6f6463e 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -1,13 +1,14 @@ -# 第三方来源与依赖 +# 来源与第三方依赖 -本仓库整理已有实验脚本与工程代码。它没有复制原项目中的数据、论文附件或第三方仓库历史。 +当前仓库是索引与工程说明。历史 Collection 整理过 MemLab 与招聘分类工程；它们的当前源码和依赖说明已迁入独立仓库，未复制原实验数据、论文附件或第三方 Git 历史。 -| 项目 | 依赖或上游 | 使用范围 | -| --- | --- | --- | -| FracLogGPT-Lite | [ms-swift](https://github.com/modelscope/ms-swift)、[Transformers](https://github.com/huggingface/transformers)、[PEFT](https://github.com/huggingface/peft) | 详细说明保留在该独立仓库 | -| 招聘文本分类 | [scikit-learn](https://github.com/scikit-learn/scikit-learn)、[PyTorch](https://github.com/pytorch/pytorch)、[Transformers](https://github.com/huggingface/transformers) | 文本特征、分类器与可选 BERT 模型加载；未复制其源码或模型权重 | -| memlab | [DeerFlow](https://github.com/bytedance/deer-flow)、Redis、PostgreSQL、Qdrant | 外部实验运行依赖；未包含服务数据与 DeerFlow 源码 | +| 项目 | 当前来源与依赖说明 | +| --- | --- | +| FracLogGPT-Lite | [项目说明与依赖来源](https://github.com/Darssal/FracLogGPT-Lite/blob/main/README.md)：ms-swift、Transformers、PEFT 等外部能力 | +| MemLab | [THIRD_PARTY](https://github.com/Darssal/MemLab/blob/main/THIRD_PARTY.md)：DeerFlow、Redis、PostgreSQL、Qdrant 与 LongMemEval 数据来源边界 | +| Recruitment Text Classification | [THIRD_PARTY](https://github.com/Darssal/Recruitment-Text-Classification/blob/main/THIRD_PARTY.md)：scikit-learn、NumPy、joblib、可选 PyTorch / Transformers；无上游模型文件 | +| JSP UI Prototype | [原有来源说明](https://github.com/Darssal/JSP-UI-Prototype/blob/main/docs/LEGACY_JSP.md)：保留 [CodePen — danielkvist / LYNVyPL](https://codepen.io/danielkvist/pen/LYNVyPL) 的登录动画参考地址 | -memlab 原实验工程的 DeerFlow checkout 基线为 `7de94394d4295182701ffb47e938e7c39b963091`。该版本的 [MIT 许可证](https://github.com/bytedance/deer-flow/blob/7de94394d4295182701ffb47e938e7c39b963091/LICENSE) 保留 ByteDance 与 DeerFlow Authors 署名；使用外部 checkout 时应保留并遵守其许可。本仓库不会将上游许可证套用于独立整理的实验代码。 +MemLab 原实验使用的 DeerFlow 基线为 `7de94394d4295182701ffb47e938e7c39b963091`。其 [MIT 许可证](https://github.com/bytedance/deer-flow/blob/7de94394d4295182701ffb47e938e7c39b963091/LICENSE) 保留 ByteDance Ltd. 与 DeerFlow Authors 署名；外部 checkout 应继续遵守该许可。LongMemEval 的下载内容和数据许可证需另行核验，当前索引不携带数据。 -模型标识仅用于配置示例，使用者需自行核对对应模型条款、版本和运行条件。本仓库不对提供方当前资费或服务可用性作保证。 +各工程未因迁移而获得新的统一许可证；本索引不将上游许可证套用于整理代码，也不把参考动画或框架能力宣称为个人原创。模型和服务的版本、许可及运行条件应在使用时核对。 diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index 14bcf22..5094e3c 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -1,6 +1,6 @@ # 工程实现与设计取舍 -这份说明沿着源码中的三条路径展开：多模态实验的阶段衔接与解码、Agent 记忆的上下文组装与运行控制、文本分类的数据划分与服务批处理。例子用于解释代码行为；模型效果和真实服务性能需要独立实验。 +这份索引笔记连接三个独立仓库中的实现路径：多模态实验的阶段衔接与解码、Agent 记忆的上下文组装与运行控制、文本分类的数据划分与服务批处理。例子用于解释代码行为；模型效果和真实服务性能需要独立实验。 ## 1. FracLogGPT-Lite：让阶段衔接与预测恢复有明确语义 @@ -24,16 +24,16 @@ **机制。** -- **上下文组装：** [`l0.py`](../projects/memlab/memlab/memory_manager/l0.py) 先保留近期消息，再按检索分数排序候选；过滤低相关项，按键与版本去重，并用规范化文本前 512 个字符的哈希去除重复内容。选中记忆受注入预算约束，最终组装还会从前部裁剪消息。 [`manager.py`](../projects/memlab/memlab/memory_manager/manager.py) 在已有直接工具证据时可绕过检索，并返回检索跳过原因、缓存命中和降级轨迹。 +- **上下文组装：** [`l0.py`](https://github.com/Darssal/MemLab/blob/main/memlab/memory_manager/l0.py) 先保留近期消息，再按检索分数排序候选；过滤低相关项，按键与版本去重，并用规范化文本前 512 个字符的哈希去除重复内容。选中记忆受注入预算约束，最终组装还会从前部裁剪消息。 [`manager.py`](https://github.com/Darssal/MemLab/blob/main/memlab/memory_manager/manager.py) 在已有直接工具证据时可绕过检索，并返回检索跳过原因、缓存命中和降级轨迹。 - **分层放置：** 同一管理器按重要度与对象大小选择 L0 近期上下文、L1 Redis、L2 Postgres / Qdrant 或 L3 对象存储。大对象可保存为正文加指针；异步写入队列提供批处理与背压策略。服务失败会进入降级记录，后台写入失败会向调用方暴露。 -- **并发归属：** [`session_logging.py`](../projects/memlab/memlab/session_logging.py) 安装共享操作 logger，从 `ContextVar` 读取当前 agent。会话 worker 设置上下文并在 `finally` 中恢复；A、B 会话交错执行时，各自的操作仍读取各自上下文。初始化和独立后台批次没有唯一会话归属时，字段可以为空。 -- **运行有效性：** [`control_plane.py`](../projects/memlab/memlab/pipeline/control_plane.py) 区分 accepted、invalid 和 diagnostic 等状态。新格式运行只有在已接受、汇总链有效、非诊断且未被替代时才进入正式报告；旧格式保留环境、复现和真实模型标志的兼容检查。失效运行记录异常、确定最早回滚阶段并生成重跑计划。[`summary_builder.py`](../projects/memlab/memlab/metrics/summary_builder.py) 据此排除不合格运行。 +- **并发归属：** [`session_logging.py`](https://github.com/Darssal/MemLab/blob/main/memlab/session_logging.py) 安装共享操作 logger，从 `ContextVar` 读取当前 agent。会话 worker 设置上下文并在 `finally` 中恢复；A、B 会话交错执行时，各自的操作仍读取各自上下文。初始化和独立后台批次没有唯一会话归属时，字段可以为空。 +- **运行有效性：** [`control_plane.py`](https://github.com/Darssal/MemLab/blob/main/memlab/pipeline/control_plane.py) 区分 accepted、invalid 和 diagnostic 等状态。新格式运行只有在已接受、汇总链有效、非诊断且未被替代时才进入正式报告；旧格式保留环境、复现和真实模型标志的兼容检查。失效运行记录异常、确定最早回滚阶段并生成重跑计划。[`summary_builder.py`](https://github.com/Darssal/MemLab/blob/main/memlab/metrics/summary_builder.py) 据此排除不合格运行。 **小例子。** 近期消息已经包含所需工具证据时，策略可以直接组装上下文，并记录 `direct_tool_evidence_present`；无需为了“使用记忆”再次检索。另一条运行即使生成了答案，若 manifest 被标记为 diagnostic，也不会成为正式对比表中的有效样本。 **设计取舍。** token 数按 `ceil(len(text) / 4)` 估算，属于启发式预算；它不是模型 tokenizer 的精确计数，单条过长的最终用户消息也不会被该裁剪循环删除。文本去重只比较规范化后的前缀，并非语义去重。默认 `hash_embedding` 用于确定性实验，不等同于经过训练的语义嵌入模型。`ContextVar` 解决异步调用上下文的日志归属，没有使跨服务写入成为事务，也不会为所有后台工作自动补上会话身份。有效性门禁约束报告输入，不能替代实验设计与人工判断。 -**测试入口：** [检索绕过策略](../projects/memlab/tests/test_memory_manager_policy.py) · [L3 指针化与降级轨迹](../projects/memlab/tests/test_system_rebaseline.py) · [交错会话日志归属](../projects/memlab/tests/test_stub_event_attribution.py) · [失效运行与回滚](../projects/memlab/tests/test_control_plane.py) · [汇总排除与作用域](../projects/memlab/tests/test_branch_audit.py) · [指标分母](../projects/memlab/tests/test_metrics_denominator.py) +**测试入口：** [检索绕过策略](https://github.com/Darssal/MemLab/blob/main/tests/test_memory_manager_policy.py) · [L3 指针化与降级轨迹](https://github.com/Darssal/MemLab/blob/main/tests/test_system_rebaseline.py) · [交错会话日志归属](https://github.com/Darssal/MemLab/blob/main/tests/test_stub_event_attribution.py) · [失效运行与回滚](https://github.com/Darssal/MemLab/blob/main/tests/test_control_plane.py) · [汇总排除与作用域](https://github.com/Darssal/MemLab/blob/main/tests/test_branch_audit.py) · [指标分母](https://github.com/Darssal/MemLab/blob/main/tests/test_metrics_denominator.py) ## 3. 招聘文本分类：隔离训练信息，明确批处理失败 @@ -51,12 +51,16 @@ flowchart LR **机制。** -- **先 split，后 fit：** [`data.py`](../projects/job-text-classification/src/job_text_classification/data.py) 检查标签，拒绝去空白、大小写规范化后相同的文本，再用固定种子按类别划分。 [`cli.py`](../projects/job-text-classification/src/job_text_classification/cli.py) 只把训练子集交给 `fit`；验证文本仅经过已拟合流水线的 `predict`。字符二至四元组 TF-IDF 与随机森林封装在 scikit-learn `Pipeline` 中。 -- **按 `classes_` 对齐概率：** [`baseline.py`](../projects/job-text-classification/src/job_text_classification/baseline.py) 将 `predict_proba` 的每列先映射到分类器的真实类别，再按固定公共标签顺序输出。未参与训练的类别填入 `0.0`；指标计算还检查输入记录与预测的 ID 顺序一致。 -- **有界 HTTP 调用：** [`http_adapter.py`](../projects/job-text-classification/src/job_text_classification/http_adapter.py) 为请求配置 timeout 和最大 attempts，失败后有限退避；除 429 外的 4xx 直接结束。响应大小和标签范围受检查。`ThreadPoolExecutor.map` 保留输入顺序，每条失败记录也会返回状态；CLI 写出结果后在有失败时返回退出码 2。输出保留记录 ID、标签、状态和概率，不写入输入原文、服务回复正文或凭据。 +- **先 split，后 fit：** [`data.py`](https://github.com/Darssal/Recruitment-Text-Classification/blob/main/src/job_text_classification/data.py) 检查标签，拒绝去空白、大小写规范化后相同的文本，再用固定种子按类别划分。 [`cli.py`](https://github.com/Darssal/Recruitment-Text-Classification/blob/main/src/job_text_classification/cli.py) 只把训练子集交给 `fit`；验证文本仅经过已拟合流水线的 `predict`。字符二至四元组 TF-IDF 与随机森林封装在 scikit-learn `Pipeline` 中。 +- **按 `classes_` 对齐概率：** [`baseline.py`](https://github.com/Darssal/Recruitment-Text-Classification/blob/main/src/job_text_classification/baseline.py) 将 `predict_proba` 的每列先映射到分类器的真实类别，再按固定公共标签顺序输出。未参与训练的类别填入 `0.0`；指标计算还检查输入记录与预测的 ID 顺序一致。 +- **有界 HTTP 调用：** [`http_adapter.py`](https://github.com/Darssal/Recruitment-Text-Classification/blob/main/src/job_text_classification/http_adapter.py) 为请求配置 timeout 和最大 attempts，失败后有限退避；除 429 外的 4xx 直接结束。响应大小和标签范围受检查。`ThreadPoolExecutor.map` 保留输入顺序，每条失败记录也会返回状态；CLI 写出结果后在有失败时返回退出码 2。输出保留记录 ID、标签、状态和概率，不写入输入原文、服务回复正文或凭据。 **小例子。** 分类器内部顺序为 `[人工智能, 环保]`，返回概率 `[0.2, 0.8]` 时，公开结果仍按 `[环保, 人工智能, 否]` 输出 `{环保: 0.8, 人工智能: 0.2, 否: 0.0}`。并发处理中第一条较慢、第二条先完成，最终结果仍以第一条、第二条排列；认证失败不会用完全部重试次数。 **设计取舍。** 拒绝规范化重复文本是明确的划分约束，没有解决语义近重复、同公司岗位泄漏或时间漂移；正式研究还需要独立测试方案。固定标签映射使接口稳定，不代表三类预测已经校准。保序意味着慢请求会影响后续结果交付；当前实现收集全部结果，不提供流式完成通知。timeout 约束请求等待，不是整个任务的严格总时限。该 HTTP 适配器遵循 `message / mode / textResponse` 协议，不包含知识库、索引或检索链路。 -**测试入口：** [`test_regressions.py`](../projects/job-text-classification/tests/test_regressions.py) 覆盖训练词表隔离、固定种子与重复文本、概率列映射、记录对齐、模型保存加载、超时与非法响应、401 快速失败、并发保序及输出不含原文。运行方法见[项目 README](../projects/job-text-classification/README.md)。 +**测试入口：** [`test_regressions.py`](https://github.com/Darssal/Recruitment-Text-Classification/blob/main/tests/test_regressions.py) 覆盖训练词表隔离、固定种子与重复文本、概率列映射、记录对齐、模型保存加载、超时与非法响应、401 快速失败、并发保序及输出不含原文。运行方法见[项目 README](https://github.com/Darssal/Recruitment-Text-Classification/blob/main/README.md)。 + +## 维护入口 + +源码、测试和各项目的最新说明分别在 [FracLogGPT-Lite](https://github.com/Darssal/FracLogGPT-Lite)、[MemLab](https://github.com/Darssal/MemLab) 与 [Recruitment Text Classification](https://github.com/Darssal/Recruitment-Text-Classification) 维护。本页保留跨项目设计阅读路径；不承载可安装项目。JSP 学习原型及 CodePen 来源见 [JSP UI Prototype](https://github.com/Darssal/JSP-UI-Prototype)。 diff --git a/docs/PUBLICATION_SCOPE.md b/docs/PUBLICATION_SCOPE.md index 00382a6..ebf85d0 100644 --- a/docs/PUBLICATION_SCOPE.md +++ b/docs/PUBLICATION_SCOPE.md @@ -1,28 +1,22 @@ -# 公开范围 +# 公开与维护范围 -本仓库从原实验目录中按允许列表整理代码。原始目录保留在本地，不与公开仓库自动镜像。 +当前分支仅维护项目索引、设计说明、来源与维护检查工具，不承载可安装研究项目。原来两个项目的源码已转到独立仓库；旧版本仍在 Git 历史中保留。 ## 可以提交 -- 已审查的 Python 源码、测试、依赖声明和必要的服务配置。 -- 不含实验记录的运行说明、接口说明和来源说明。 -- 完全新编写的小型合成样例，以及测试中动态生成的临时数据。 -- 空值环境变量模板。服务地址、口令和模型选择由使用者自行填写。 +- 指向独立仓库的导航和已核对的源码、文档及测试链接。 +- 无私人信息的工程设计解释、运行边界、来源和迁移说明。 +- 检查索引结构、文档链接和公开边界所需的维护脚本及测试；夹具仅在测试时动态生成。 +- 仓库级只读 CI 配置。 -## 保留在本地 +## 项目修改的去向 -- 实验用真实数据、完整招聘文本、原始图像、表格、标注和向量索引。 -- 本地 API 凭据、服务工作区信息、私人地址和机器路径。 -- 训练权重、检查点、缓存、数据库、预测文本、运行日志和追踪记录。 -- 论文正文、图表、审稿意见、投稿材料、课程报告和合作分工材料。 -- 原 Git 历史、嵌套第三方仓库和未经确认的上游源码副本。 +README 列出的 canonical repository 是每个项目唯一的当前维护入口。源码、依赖、配置、训练与推理代码、项目数据样例及项目回归测试均提交到相应仓库，不复制回本索引。历史快照只用于追溯。 -## 新增代码时 +真实数据、论文及审稿材料、个人证明文件、API 凭据、私人地址、模型、检查点、数据库、运行输出和缓存继续保留在本地。索引维护不导入原始实验目录或私人 Git 历史。 -1. 先在公开仓库中创建分支，选择具体源文件迁移。不要将整个实验目录拖入仓库。 -2. 将路径与凭据改为显式配置；检查 notebook 输出、注释、测试夹具和文档是否夹带真实内容。 -3. 补充不依赖真实材料的最小验证，并说明尚未验证的外部依赖。 -4. 将明确选择的文件加入暂存区，运行 `python tools/check_publication.py --staged` 和相关测试，检查 `git diff --cached` 与暂存文件清单。无参数的严格全目录检查用于干净发布快照。 -5. 通过 Pull Request 保存变更与验证记录。 +## 发布检查 -公开检查规则保存在 `publication-policy.json`。扫描器不打印命中原文，避免将敏感值再次写入日志。启发式检查无法证明所有文件均不含敏感内容，允许列表与人工审查仍是发布边界的一部分。 +`publication-policy.json` 不允许 `projects/` 目录。`tools/check_index.py` 同时核对当前跟踪文件清单与文档链接；公开扫描使用工作目录或实际 Git index blobs，暂存模式读取已暂存策略。新增文件先明确暂存，再检查变更，通过 Pull Request 与 CI 合并。 + +扫描不打印命中原文，也不能证明历史、来源、许可或所有内容均无敏感信息。维护者仍需核对提交范围和文字依据。 diff --git a/projects/job-text-classification/.env.example b/projects/job-text-classification/.env.example deleted file mode 100644 index 76a6f2c..0000000 --- a/projects/job-text-classification/.env.example +++ /dev/null @@ -1,2 +0,0 @@ -JOB_CLASSIFIER_API_URL= -JOB_CLASSIFIER_API_KEY= diff --git a/projects/job-text-classification/.gitignore b/projects/job-text-classification/.gitignore deleted file mode 100644 index 05ee71e..0000000 --- a/projects/job-text-classification/.gitignore +++ /dev/null @@ -1,18 +0,0 @@ -__pycache__/ -.pytest_cache/ -*.py[cod] -*.egg-info/ -.venv/ -.env -.env.* -!.env.example -outputs/ -data/ -models/ -checkpoints/ -*.joblib -*.pkl -*.safetensors -*.bin -*.pt -*.pth diff --git a/projects/job-text-classification/README.md b/projects/job-text-classification/README.md deleted file mode 100644 index 0fdceb4..0000000 --- a/projects/job-text-classification/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# 招聘文本三分类 / 本地基线与服务批处理 - -识别环保、人工智能与其它招聘文本（接口标签分别为 `环保`、`人工智能`、`否`），提供中文字符 TF-IDF + 随机森林、可选 BERT 基线，以及外部聊天服务的 HTTP 适配器。默认示例仅在 CPU 上处理 24 条新编写的虚构文本，无需原始语料、模型权重、GPU 或 API。 - -核心实现包括训练词表隔离、固定标签概率对齐、预测输出最小化，以及带超时、有界重试和并发保序的外部服务批处理。 - -[快速运行](#快速运行) · [数据与基线](#数据接口与本地基线) · [HTTP 适配](#可选-http-服务适配) · [设计取舍](../../docs/ENGINEERING_NOTES.md) - -## 快速运行 - -在本子目录运行以下命令，推荐 Python 3.10 或更新版本： - -```powershell -python -m venv .venv -.\.venv\Scripts\Activate.ps1 -python -m pip install -e . -python -m job_text_classification demo -python -m unittest discover -s tests -v -``` - -macOS/Linux 激活环境使用 `source .venv/bin/activate`。直接执行 `python -m job_text_classification` 同样进入离线 demo。输出标明 `synthetic_fixture`、`held_out_validation` 和 `paper_metrics: false`；虚构文本上的指标只验证流程，不用于解释真实岗位分类质量。 - -## 数据接口与本地基线 - -输入支持 UTF-8 CSV 或 JSONL，使用 `text`、`label` 两个字段，可提供 `record_id`。标签固定为 `环保`、`人工智能`、`否`。训练要求标签齐全；预测时 `label` 可省略。缺失 ID 时按输入顺序生成 `row-1` 等 ID，重复 ID、空文本和未知标签会报错。 - -以下示例完全虚构： - -```json -{"record_id":"example-1","text":"虚构岗位：维护污水治理设备。","label":"环保"} -``` - -内置合成文件位于 `src/job_text_classification/examples/synthetic_jobs.jsonl`。用户自行准备的数据放入忽略目录 `data/`： - -```powershell -python -m job_text_classification train --input data/jobs.jsonl --model-output outputs/rf.joblib -python -m job_text_classification predict --input data/new_jobs.csv --model outputs/rf.joblib --output outputs/predictions.jsonl -``` - -训练先按固定种子进行分层划分，再仅在训练文本上拟合 TF-IDF。每类至少需要两条记录；标准化后重复的文本会被拒绝，避免同一文本同时进入训练与验证。字符二至四元组使中文示例无需额外分词器。验证集用于流程检查，正式研究仍需独立测试集及领域标注方案。 - -预测结果使用固定标签顺序，并按分类器实际 `classes_` 对齐概率列。JSONL 输出默认只含记录 ID、可选真实标签、预测标签、状态和概率；不包含原文。训练后的模型及输出均在 `.gitignore` 范围内。只加载自己生成或可信来源的 joblib 模型文件。 - -## 可选 BERT - -```powershell -python -m pip install -e ".[bert]" -python -m job_text_classification bert-train --input data/jobs.jsonl --model-name models/bert-base-chinese --model-output outputs/bert --epochs 1 -python -m job_text_classification bert-predict --input data/new_jobs.jsonl --model outputs/bert --output outputs/bert_predictions.jsonl -``` - -默认仅查找本机已有的模型与缓存，设备默认为 CPU。确实需要从模型平台下载时，在 `bert-train` 中显式添加 `--allow-model-download`，并自行评估网络、磁盘及算力需求。训练保存最终轮次到指定目录，预测从同一目录加载，模型配置中保存固定标签映射。此整理版没有实际下载、训练或评测 BERT，不能由代码存在推断其性能。 - -## 可选 HTTP 服务适配 - -仅 `http` 子命令会将输入文本发送到配置的服务。运行前自行配置环境变量 `JOB_CLASSIFIER_API_URL` 和 `JOB_CLASSIFIER_API_KEY`；`.env.example` 只包含空值模板，程序不会自动加载 `.env`。服务需接收 `POST` JSON `{"message":"提示及输入文本","mode":"chat"}`，并返回 `{"textResponse":"环保"}` 等合法标签。这是原脚本的服务协议，不是通用 OpenAI API，也不包含知识库构建、检索索引或向量库实现。 - -```powershell -python -m job_text_classification http --input data/authorized_jobs.jsonl --output outputs/http_predictions.jsonl --timeout 20 --attempts 3 --workers 4 -``` - -请求保留默认 HTTPS 证书校验，使用超时和有界重试；认证等非临时客户端错误直接结束，非法响应只记录错误类型。并发结果保持输入顺序，没有会因失败计数而永久等待的进度线程。失败行照常写出并使 CLI 返回退出码 2；请求体、服务回复正文和凭据不进入输出。 - -## 公开范围与验证 - -本工程基于原研究脚本的分类与批处理功能重新组织。原先名为 `bert_classifier.py` 的脚本实际使用 TF-IDF 和随机森林，公开版按真实方法命名；BERT 保留为可选路径。这里的验证针对代码流程，不提供论文效果或个人贡献归属的证明。 - -- 保留：数据接口、训练与推理代码、HTTP 适配、全新合成例子和回归测试。 -- 排除：原始岗位语料、Excel、论文及审稿材料、真实输出、服务地址、密钥、模型与既有实验指标。 -- 回归测试覆盖：训练词表隔离、概率列对齐、固定指标标签顺序、重复文本划分限制、模型保存加载、HTTP 超时/非法响应/认证失败、并发保序和输出不带原文。 -- 本地验证范围：Python 语法编译、CPU 随机森林合成 demo、模拟 HTTP。没有调用真实服务或运行 BERT 训练。 - -主要依赖来自 [scikit-learn](https://scikit-learn.org/)、可选 [PyTorch](https://pytorch.org/) 和 [Transformers](https://github.com/huggingface/transformers)。本目录未复制这些依赖的源代码或模型文件；本轮未新增开源许可证。 diff --git a/projects/job-text-classification/pyproject.toml b/projects/job-text-classification/pyproject.toml deleted file mode 100644 index 39cfd64..0000000 --- a/projects/job-text-classification/pyproject.toml +++ /dev/null @@ -1,23 +0,0 @@ -[build-system] -requires = ["setuptools>=68"] -build-backend = "setuptools.build_meta" - -[project] -name = "job-text-classification" -version = "0.1.0" -description = "Data-free job text classification examples with local baselines and an optional HTTP adapter" -requires-python = ">=3.10" -dependencies = ["numpy>=1.24", "scikit-learn>=1.3", "joblib>=1.3"] - -[project.optional-dependencies] -bert = ["torch>=2.1", "transformers>=4.40"] -dev = ["pytest>=8"] - -[project.scripts] -job-text-classification = "job_text_classification.cli:main" - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.setuptools.package-data] -job_text_classification = ["examples/*.jsonl"] diff --git a/projects/job-text-classification/src/job_text_classification/__init__.py b/projects/job-text-classification/src/job_text_classification/__init__.py deleted file mode 100644 index 4bf0dea..0000000 --- a/projects/job-text-classification/src/job_text_classification/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Public, data-free engineering extraction of local classification scripts.""" - -LABELS = ("环保", "人工智能", "否") -__version__ = "0.1.0" diff --git a/projects/job-text-classification/src/job_text_classification/__main__.py b/projects/job-text-classification/src/job_text_classification/__main__.py deleted file mode 100644 index eb53e2f..0000000 --- a/projects/job-text-classification/src/job_text_classification/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .cli import main - -raise SystemExit(main()) diff --git a/projects/job-text-classification/src/job_text_classification/baseline.py b/projects/job-text-classification/src/job_text_classification/baseline.py deleted file mode 100644 index c034afc..0000000 --- a/projects/job-text-classification/src/job_text_classification/baseline.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Character TF-IDF + random forest, fitted only on training records.""" - -from __future__ import annotations - -from pathlib import Path - -from . import LABELS -from .data import Record, public_prediction - - -class RandomForestBaseline: - def __init__(self, seed: int = 42, n_estimators: int = 100): - from sklearn.ensemble import RandomForestClassifier - from sklearn.feature_extraction.text import TfidfVectorizer - from sklearn.pipeline import Pipeline - - self.pipeline = Pipeline([ - ("tfidf", TfidfVectorizer(analyzer="char", ngram_range=(2, 4), max_features=5000)), - ("classifier", RandomForestClassifier(n_estimators=n_estimators, random_state=seed, n_jobs=1)), - ]) - - def fit(self, records: list[Record]) -> "RandomForestBaseline": - if not records or any(record.label not in LABELS for record in records): - raise ValueError("Training requires non-empty, labelled records") - self.pipeline.fit([record.text for record in records], [record.label for record in records]) - return self - - def predict(self, records: list[Record]) -> list[dict]: - if not records: - return [] - texts = [record.text for record in records] - predictions = self.pipeline.predict(texts) - probabilities = self.pipeline.predict_proba(texts) - classes = list(self.pipeline.named_steps["classifier"].classes_) - rows = [] - for record, prediction, values in zip(records, predictions, probabilities): - by_label = {label: float(value) for label, value in zip(classes, values)} - rows.append(public_prediction( - record, str(prediction), status="ok", - probabilities={label: by_label.get(label, 0.0) for label in LABELS}, - )) - return rows - - def save(self, path: str | Path) -> None: - import joblib - - destination = Path(path) - destination.parent.mkdir(parents=True, exist_ok=True) - joblib.dump(self, destination) - - @classmethod - def load(cls, path: str | Path) -> "RandomForestBaseline": - """Load only artifacts that you generated or otherwise trust.""" - import joblib - - model = joblib.load(path) - if not isinstance(model, cls): - raise ValueError("The artifact is not a RandomForestBaseline") - return model - - -def classification_metrics(records: list[Record], rows: list[dict]) -> dict: - from sklearn.metrics import accuracy_score, classification_report, confusion_matrix - - if not records or len(records) != len(rows): - raise ValueError("Metrics require equally sized, non-empty record and prediction lists") - if any(record.record_id != row.get("record_id") for record, row in zip(records, rows)): - raise ValueError("Metric rows must match the input record order") - if any(record.label not in LABELS for record in records): - raise ValueError("Metrics require labels") - if any(row.get("status") != "ok" or row.get("prediction") not in LABELS for row in rows): - raise ValueError("Metrics require successful predictions") - expected = [record.label for record in records] - predicted = [row["prediction"] for row in rows] - return { - "label_order": list(LABELS), - "records": len(records), - "accuracy": float(accuracy_score(expected, predicted)), - "classification_report": classification_report( - expected, predicted, labels=list(LABELS), target_names=list(LABELS), - output_dict=True, zero_division=0, - ), - "confusion_matrix": confusion_matrix(expected, predicted, labels=list(LABELS)).tolist(), - } diff --git a/projects/job-text-classification/src/job_text_classification/bert.py b/projects/job-text-classification/src/job_text_classification/bert.py deleted file mode 100644 index 9b7dbbf..0000000 --- a/projects/job-text-classification/src/job_text_classification/bert.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Optional BERT baseline. Loading remote weights requires explicit opt-in.""" - -from __future__ import annotations - -from pathlib import Path - -from . import LABELS -from .data import Record, public_prediction - - -class BertBaseline: - def __init__(self, tokenizer, model, *, device: str = "cpu", max_length: int = 128): - if not 8 <= max_length <= 512: - raise ValueError("max_length must be between 8 and 512") - self.tokenizer, self.model = tokenizer, model - self.device, self.max_length = device, max_length - self.model.to(device) - - @classmethod - def for_training( - cls, model_name: str = "bert-base-chinese", *, allow_download: bool = False, - device: str = "cpu", max_length: int = 128, seed: int = 42, - ) -> "BertBaseline": - import torch - from transformers import BertForSequenceClassification, BertTokenizer - - torch.manual_seed(seed) - tokenizer = BertTokenizer.from_pretrained(model_name, local_files_only=not allow_download) - model = BertForSequenceClassification.from_pretrained( - model_name, local_files_only=not allow_download, num_labels=len(LABELS), - id2label=dict(enumerate(LABELS)), label2id={label: index for index, label in enumerate(LABELS)}, - ) - return cls(tokenizer, model, device=device, max_length=max_length) - - @classmethod - def load(cls, model_dir: str | Path, *, device: str = "cpu", max_length: int = 128) -> "BertBaseline": - from transformers import BertForSequenceClassification, BertTokenizer - - if not Path(model_dir).is_dir(): - raise ValueError("Prediction requires an existing local model directory") - tokenizer = BertTokenizer.from_pretrained(model_dir, local_files_only=True) - model = BertForSequenceClassification.from_pretrained(model_dir, local_files_only=True) - actual = tuple(model.config.id2label.get(index) for index in range(len(LABELS))) - if actual != LABELS or model.config.num_labels != len(LABELS): - raise ValueError("Local checkpoint does not have this project's fixed label order") - return cls(tokenizer, model, device=device, max_length=max_length) - - def fit(self, records: list[Record], *, epochs: int = 1, batch_size: int = 8, learning_rate: float = 2e-5, seed: int = 42) -> None: - import torch - from torch.utils.data import DataLoader, TensorDataset - - if not records or any(record.label not in LABELS for record in records): - raise ValueError("BERT training requires labelled records") - if not 1 <= epochs <= 20 or not 1 <= batch_size <= 256 or learning_rate <= 0: - raise ValueError("Use epochs in [1, 20], batch_size in [1, 256], and positive learning_rate") - encoded = self.tokenizer( - [record.text for record in records], padding=True, truncation=True, - max_length=self.max_length, return_tensors="pt", - ) - labels = torch.tensor([LABELS.index(record.label) for record in records], dtype=torch.long) - dataset = TensorDataset(encoded["input_ids"], encoded["attention_mask"], labels) - loader = DataLoader(dataset, batch_size=batch_size, shuffle=True, generator=torch.Generator().manual_seed(seed)) - optimizer = torch.optim.AdamW(self.model.parameters(), lr=learning_rate) - for _ in range(epochs): - self.model.train() - for input_ids, attention_mask, targets in loader: - optimizer.zero_grad() - output = self.model( - input_ids=input_ids.to(self.device), attention_mask=attention_mask.to(self.device), - labels=targets.to(self.device), - ) - output.loss.backward() - optimizer.step() - - def predict(self, records: list[Record], *, batch_size: int = 8) -> list[dict]: - import torch - - if not 1 <= batch_size <= 256: - raise ValueError("batch_size must be between 1 and 256") - rows = [] - self.model.eval() - with torch.no_grad(): - for start in range(0, len(records), batch_size): - batch = records[start:start + batch_size] - encoded = self.tokenizer( - [record.text for record in batch], padding=True, truncation=True, - max_length=self.max_length, return_tensors="pt", - ) - output = self.model(**{key: value.to(self.device) for key, value in encoded.items()}) - values = torch.softmax(output.logits, dim=-1).cpu().tolist() - for record, probabilities in zip(batch, values): - prediction = LABELS[max(range(len(LABELS)), key=lambda index: probabilities[index])] - rows.append(public_prediction( - record, prediction, status="ok", - probabilities={label: float(probabilities[index]) for index, label in enumerate(LABELS)}, - )) - return rows - - def save(self, model_dir: str | Path) -> None: - destination = Path(model_dir) - destination.mkdir(parents=True, exist_ok=True) - self.model.save_pretrained(destination) - self.tokenizer.save_pretrained(destination) diff --git a/projects/job-text-classification/src/job_text_classification/cli.py b/projects/job-text-classification/src/job_text_classification/cli.py deleted file mode 100644 index c95ad20..0000000 --- a/projects/job-text-classification/src/job_text_classification/cli.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Explicit local demo, training, prediction and optional HTTP commands.""" - -from __future__ import annotations - -import argparse -import json -import sys -from importlib.resources import as_file, files - -from .data import load_records, stratified_split, write_jsonl - - -def parser() -> argparse.ArgumentParser: - result = argparse.ArgumentParser(description="Data-free job text classification examples") - commands = result.add_subparsers(dest="command") - demo = commands.add_parser("demo", help="Offline CPU demo using only the bundled synthetic fixture") - demo.add_argument("--output", help="Optional JSONL output, with no source text") - train = commands.add_parser("train", help="Train TF-IDF + random forest on user-supplied labelled data") - train.add_argument("--input", required=True) - train.add_argument("--model-output", required=True) - train.add_argument("--validation-fraction", type=float, default=0.25) - train.add_argument("--seed", type=int, default=42) - predict = commands.add_parser("predict", help="Predict with a trusted locally generated random forest artifact") - predict.add_argument("--input", required=True) - predict.add_argument("--model", required=True) - predict.add_argument("--output", required=True) - http = commands.add_parser("http", help="Explicit opt-in: transmit supplied texts to the configured HTTP service") - http.add_argument("--input", required=True) - http.add_argument("--output", required=True) - http.add_argument("--workers", type=int, default=4) - http.add_argument("--timeout", type=float, default=20) - http.add_argument("--attempts", type=int, default=3) - bert_train = commands.add_parser("bert-train", help="Optional BERT training; remote downloads require an explicit flag") - bert_train.add_argument("--input", required=True) - bert_train.add_argument("--model-output", required=True) - bert_train.add_argument("--model-name", default="bert-base-chinese") - bert_train.add_argument("--allow-model-download", action="store_true") - bert_train.add_argument("--epochs", type=int, default=1) - bert_train.add_argument("--batch-size", type=int, default=8) - bert_train.add_argument("--learning-rate", type=float, default=2e-5) - bert_train.add_argument("--validation-fraction", type=float, default=0.25) - bert_train.add_argument("--seed", type=int, default=42) - bert_train.add_argument("--device", choices=["cpu", "cuda"], default="cpu") - bert_predict = commands.add_parser("bert-predict", help="Optional BERT prediction from an existing local model directory") - bert_predict.add_argument("--input", required=True) - bert_predict.add_argument("--model", required=True) - bert_predict.add_argument("--output", required=True) - bert_predict.add_argument("--device", choices=["cpu", "cuda"], default="cpu") - return result - - -def main(argv: list[str] | None = None) -> int: - args = parser().parse_args(argv) - command = args.command or "demo" - try: - if command in {"demo", "train"}: - from .baseline import RandomForestBaseline, classification_metrics - - if command == "demo": - resource = files("job_text_classification").joinpath("examples/synthetic_jobs.jsonl") - with as_file(resource) as path: - records = load_records(path, require_labels=True) - seed, validation_fraction = 42, 0.25 - else: - records = load_records(args.input, require_labels=True) - seed, validation_fraction = args.seed, args.validation_fraction - train, validation = stratified_split(records, validation_fraction, seed) - model = RandomForestBaseline(seed=seed).fit(train) - rows = model.predict(validation) - if command == "train": - model.save(args.model_output) - elif getattr(args, "output", None): - write_jsonl(args.output, rows) - summary = { - "model": "character_tfidf_random_forest", "training_records": len(train), - "data_scope": "synthetic_fixture" if command == "demo" else "user_supplied", - "evaluation_scope": "held_out_validation", "paper_metrics": False, - "metrics": classification_metrics(validation, rows), - } - elif command == "predict": - from .baseline import RandomForestBaseline - - records = load_records(args.input) - rows = RandomForestBaseline.load(args.model).predict(records) - write_jsonl(args.output, rows) - summary = {"records": len(rows), "mode": "local_random_forest"} - elif command == "http": - from .http_adapter import HttpServiceAdapter - - adapter = HttpServiceAdapter.from_environment(timeout=args.timeout, attempts=args.attempts) - records = load_records(args.input) - rows = adapter.predict(records, workers=args.workers) - write_jsonl(args.output, rows) - failed = sum(row["status"] != "ok" for row in rows) - summary = {"records": len(rows), "failed": failed, "mode": "configured_http_service"} - print(json.dumps(summary, ensure_ascii=False, indent=2)) - return 2 if failed else 0 - elif command == "bert-train": - from .baseline import classification_metrics - from .bert import BertBaseline - - records = load_records(args.input, require_labels=True) - train, validation = stratified_split(records, args.validation_fraction, args.seed) - model = BertBaseline.for_training( - args.model_name, allow_download=args.allow_model_download, device=args.device, seed=args.seed, - ) - model.fit(train, epochs=args.epochs, batch_size=args.batch_size, learning_rate=args.learning_rate, seed=args.seed) - rows = model.predict(validation) - model.save(args.model_output) - summary = { - "model": "bert", "training_records": len(train), "data_scope": "user_supplied", - "evaluation_scope": "held_out_validation", "checkpoint": "final_epoch", "paper_metrics": False, - "metrics": classification_metrics(validation, rows), - } - else: - from .bert import BertBaseline - - records = load_records(args.input) - rows = BertBaseline.load(args.model, device=args.device).predict(records) - write_jsonl(args.output, rows) - summary = {"records": len(rows), "mode": "local_bert"} - print(json.dumps(summary, ensure_ascii=False, indent=2)) - return 0 - except (ValueError, OSError, ImportError) as exc: - print(f"error: {exc}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/projects/job-text-classification/src/job_text_classification/data.py b/projects/job-text-classification/src/job_text_classification/data.py deleted file mode 100644 index 10abb11..0000000 --- a/projects/job-text-classification/src/job_text_classification/data.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Common CSV/JSONL schema and deterministic, text-disjoint splitting.""" - -from __future__ import annotations - -import csv -import json -import random -from dataclasses import dataclass -from pathlib import Path - -from . import LABELS - - -@dataclass(frozen=True) -class Record: - record_id: str - text: str - label: str | None = None - - -def load_records(path: str | Path, *, require_labels: bool = False) -> list[Record]: - path = Path(path) - with path.open(encoding="utf-8-sig", newline="") as stream: - if path.suffix.lower() == ".csv": - rows = list(csv.DictReader(stream)) - elif path.suffix.lower() == ".jsonl": - rows = [json.loads(line) for line in stream if line.strip()] - else: - raise ValueError("Input must be a CSV or JSONL file") - records: list[Record] = [] - seen: set[str] = set() - for index, row in enumerate(rows, start=1): - if not isinstance(row, dict) or not isinstance(row.get("text"), str): - raise ValueError(f"Row {index}: text must be a string") - text = row["text"].strip() - if not text: - raise ValueError(f"Row {index}: text must not be empty") - record_id = row.get("record_id") or f"row-{index}" - if not isinstance(record_id, str) or record_id in seen: - raise ValueError(f"Row {index}: record_id must be a unique string") - label = row.get("label") or None - if label is not None and label not in LABELS: - raise ValueError(f"Row {index}: label is outside the fixed label set") - if require_labels and label is None: - raise ValueError(f"Row {index}: label is required") - seen.add(record_id) - records.append(Record(record_id, text, label)) - if not records: - raise ValueError("Input contains no records") - return records - - -def stratified_split( - records: list[Record], validation_fraction: float = 0.25, seed: int = 42 -) -> tuple[list[Record], list[Record]]: - """Split before any vectorizer/model fitting; reject duplicate normalized text.""" - if not 0 < validation_fraction < 1: - raise ValueError("validation_fraction must be between 0 and 1") - groups = {label: [] for label in LABELS} - seen_text: set[str] = set() - for record in records: - if record.label not in groups: - raise ValueError("Training requires a valid label for every record") - normalized = " ".join(record.text.split()).casefold() - if normalized in seen_text: - raise ValueError("Duplicate normalized text must be resolved before splitting") - seen_text.add(normalized) - groups[record.label].append(record) - rng = random.Random(seed) - train, validation = [], [] - for label in LABELS: - group = groups[label][:] - if len(group) < 2: - raise ValueError("Splitting requires at least two records per label") - rng.shuffle(group) - count = min(len(group) - 1, max(1, round(len(group) * validation_fraction))) - validation.extend(group[:count]) - train.extend(group[count:]) - rng.shuffle(train) - rng.shuffle(validation) - return train, validation - - -def write_jsonl(path: str | Path, rows: list[dict]) -> None: - destination = Path(path) - destination.parent.mkdir(parents=True, exist_ok=True) - with destination.open("w", encoding="utf-8", newline="\n") as stream: - for row in rows: - stream.write(json.dumps(row, ensure_ascii=False) + "\n") - - -def public_prediction(record: Record, prediction: str | None, **details) -> dict: - """Keep the original text out of generated prediction artifacts.""" - return { - "record_id": record.record_id, - "expected_label": record.label, - "prediction": prediction, - **details, - } diff --git a/projects/job-text-classification/src/job_text_classification/examples/synthetic_jobs.jsonl b/projects/job-text-classification/src/job_text_classification/examples/synthetic_jobs.jsonl deleted file mode 100644 index 9cf9023..0000000 --- a/projects/job-text-classification/src/job_text_classification/examples/synthetic_jobs.jsonl +++ /dev/null @@ -1,24 +0,0 @@ -{"record_id":"synthetic-env-01","text":"虚构岗位：设计污水处理流程并记录环保设备运行情况。","label":"环保"} -{"record_id":"synthetic-env-02","text":"虚构岗位：开展环境监测并整理污染治理方案。","label":"环保"} -{"record_id":"synthetic-env-03","text":"虚构岗位：维护废气处理设备，检查排放控制流程。","label":"环保"} -{"record_id":"synthetic-env-04","text":"虚构岗位：分析水质监测结果，优化污水治理工艺。","label":"环保"} -{"record_id":"synthetic-env-05","text":"虚构岗位：编制环境保护方案并安排污染源巡检。","label":"环保"} -{"record_id":"synthetic-env-06","text":"虚构岗位：负责固体废物分类与资源回收工艺设计。","label":"环保"} -{"record_id":"synthetic-env-07","text":"虚构岗位：评估生态修复措施并记录环境监测指标。","label":"环保"} -{"record_id":"synthetic-env-08","text":"虚构岗位：为工厂制定节水及污水排放治理方案。","label":"环保"} -{"record_id":"synthetic-ai-01","text":"虚构岗位：训练机器学习模型并评估人工智能算法。","label":"人工智能"} -{"record_id":"synthetic-ai-02","text":"虚构岗位：开发自然语言处理模型并维护训练代码。","label":"人工智能"} -{"record_id":"synthetic-ai-03","text":"虚构岗位：调试计算机视觉神经网络和模型推理流程。","label":"人工智能"} -{"record_id":"synthetic-ai-04","text":"虚构岗位：研究深度学习算法，验证模型训练结果。","label":"人工智能"} -{"record_id":"synthetic-ai-05","text":"虚构岗位：实现文本分类模型并改进机器学习特征。","label":"人工智能"} -{"record_id":"synthetic-ai-06","text":"虚构岗位：构建人工智能推荐模型和离线评估脚本。","label":"人工智能"} -{"record_id":"synthetic-ai-07","text":"虚构岗位：优化神经网络训练与模型推理速度。","label":"人工智能"} -{"record_id":"synthetic-ai-08","text":"虚构岗位：开发语音识别算法并分析机器学习实验。","label":"人工智能"} -{"record_id":"synthetic-other-01","text":"虚构岗位：安排门店商品陈列并整理日常收银记录。","label":"否"} -{"record_id":"synthetic-other-02","text":"虚构岗位：接待酒店住客，协调入住与退房手续。","label":"否"} -{"record_id":"synthetic-other-03","text":"虚构岗位：制作烘焙食品并维护厨房用品清单。","label":"否"} -{"record_id":"synthetic-other-04","text":"虚构岗位：为仓库商品安排打包与配送计划。","label":"否"} -{"record_id":"synthetic-other-05","text":"虚构岗位：整理会议日程并办理办公室物品采购。","label":"否"} -{"record_id":"synthetic-other-06","text":"虚构岗位：接听门店咨询电话并登记客户预约。","label":"否"} -{"record_id":"synthetic-other-07","text":"虚构岗位：核对商品订单，处理退换货登记。","label":"否"} -{"record_id":"synthetic-other-08","text":"虚构岗位：维护酒店客房清洁与物品补充记录。","label":"否"} diff --git a/projects/job-text-classification/src/job_text_classification/http_adapter.py b/projects/job-text-classification/src/job_text_classification/http_adapter.py deleted file mode 100644 index 341cf7e..0000000 --- a/projects/job-text-classification/src/job_text_classification/http_adapter.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Optional service adapter, not a retrieval/indexing implementation.""" - -from __future__ import annotations - -import json -import os -import time -import urllib.error -import urllib.parse -import urllib.request -from concurrent.futures import ThreadPoolExecutor - -from . import LABELS -from .data import Record, public_prediction - -PROMPT = "请判断以下虚构或已获授权的招聘文本，只输出人工智能、环保或否。招聘文本：" - - -class HttpServiceAdapter: - def __init__(self, url: str, api_key: str, *, timeout: float = 20, attempts: int = 3, retry_delay: float = 0.2): - parsed = urllib.parse.urlsplit(url) - if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password: - raise ValueError("API URL must be an HTTP(S) endpoint without embedded credentials") - if not api_key: - raise ValueError("An API key is required") - if timeout <= 0 or timeout > 300 or attempts < 1 or attempts > 10 or retry_delay < 0: - raise ValueError("Use timeout in (0, 300], attempts in [1, 10], and non-negative retry_delay") - self._url, self._api_key = url, api_key - self.timeout, self.attempts, self.retry_delay = timeout, attempts, retry_delay - - @classmethod - def from_environment(cls, **kwargs) -> "HttpServiceAdapter": - url, api_key = os.getenv("JOB_CLASSIFIER_API_URL", ""), os.getenv("JOB_CLASSIFIER_API_KEY", "") - if not url or not api_key: - raise ValueError("HTTP mode requires JOB_CLASSIFIER_API_URL and JOB_CLASSIFIER_API_KEY") - return cls(url, api_key, **kwargs) - - def _request(self, text: str) -> str: - body = json.dumps({"message": PROMPT + "\n" + text, "mode": "chat"}, ensure_ascii=False).encode("utf-8") - request = urllib.request.Request( - self._url, data=body, - headers={"Content-Type": "application/json"}, - method="POST", - ) - # Do not forward the credential if the endpoint redirects to another location. - request.add_unredirected_header("Authorization", "Bearer " + self._api_key) - # Default urllib TLS verification is preserved. Never log request or response bodies. - with urllib.request.urlopen(request, timeout=self.timeout) as response: - payload = response.read(1024 * 1024 + 1) - if len(payload) > 1024 * 1024: - raise ValueError("Response exceeds the configured size limit") - result = json.loads(payload.decode("utf-8")) - if not isinstance(result, dict) or not isinstance(result.get("textResponse"), str): - raise ValueError("Missing textResponse") - label = result["textResponse"].split("")[-1].strip() - if label not in LABELS: - raise ValueError("Response is outside the fixed label set") - return label - - def classify(self, record: Record) -> dict: - error = "request_failed" - for attempt in range(1, self.attempts + 1): - try: - prediction = self._request(record.text) - return public_prediction(record, prediction, status="ok", attempts=attempt) - except urllib.error.HTTPError as exc: - error = f"http_{exc.code}" - # Authentication and other non-transient client errors should not be retried. - if 400 <= exc.code < 500 and exc.code != 429: - return public_prediction(record, None, status="error", attempts=attempt, error=error) - except (urllib.error.URLError, TimeoutError, OSError): - error = "network_or_timeout" - except (ValueError, UnicodeError): - error = "invalid_response" - if attempt < self.attempts: - time.sleep(min(2.0, self.retry_delay * 2 ** (attempt - 1))) - return public_prediction(record, None, status="error", attempts=self.attempts, error=error) - - def predict(self, records: list[Record], workers: int = 4) -> list[dict]: - if not 1 <= workers <= 32: - raise ValueError("workers must be between 1 and 32") - # executor.map retains input order. No independent progress thread can wait forever. - with ThreadPoolExecutor(max_workers=workers) as executor: - return list(executor.map(self.classify, records)) diff --git a/projects/job-text-classification/tests/test_regressions.py b/projects/job-text-classification/tests/test_regressions.py deleted file mode 100644 index 037dfb4..0000000 --- a/projects/job-text-classification/tests/test_regressions.py +++ /dev/null @@ -1,190 +0,0 @@ -from __future__ import annotations - -import contextlib -import io -import json -import tempfile -import time -import unittest -import urllib.error -from pathlib import Path -from unittest.mock import patch - -from job_text_classification import LABELS -from job_text_classification.baseline import RandomForestBaseline, classification_metrics -from job_text_classification.cli import main, parser -from job_text_classification.data import Record, load_records, public_prediction, stratified_split, write_jsonl -from job_text_classification.http_adapter import HttpServiceAdapter - - -def training_records(): - return [ - Record("env-1", "污水治理环保设备", "环保"), Record("env-2", "废气污染环保监测", "环保"), - Record("ai-1", "人工智能模型训练", "人工智能"), Record("ai-2", "机器学习人工智能", "人工智能"), - Record("other-1", "门店商品收银记录", "否"), Record("other-2", "酒店住客办理入住", "否"), - ] - - -class DataTests(unittest.TestCase): - def test_csv_schema_and_missing_optional_label(self): - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "input.csv" - path.write_text("record_id,text,label\nfirst,虚构门店岗位,否\nsecond,虚构研究岗位,\n", encoding="utf-8") - records = load_records(path) - self.assertEqual([record.record_id for record in records], ["first", "second"]) - self.assertIsNone(records[1].label) - with self.assertRaises(ValueError): - load_records(path, require_labels=True) - - def test_duplicate_ids_and_unknown_labels_rejected(self): - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "input.jsonl" - for rows in [ - [{"record_id": "same", "text": "一"}, {"record_id": "same", "text": "二"}], - [{"text": "虚构描述", "label": "unknown"}], - ]: - write_jsonl(path, rows) - with self.assertRaises(ValueError): - load_records(path) - - def test_split_is_seeded_and_text_disjoint(self): - records = training_records() - train, validation = stratified_split(records, seed=17) - self.assertEqual((train, validation), stratified_split(records, seed=17)) - self.assertEqual(set(record.label for record in train), set(LABELS)) - self.assertFalse({record.text for record in train} & {record.text for record in validation}) - with self.assertRaises(ValueError): - stratified_split(records + [Record("duplicate", records[0].text, records[0].label)]) - - def test_saved_predictions_exclude_source_text(self): - source = Record("row-1", "这是一段只供本地输入使用的虚构文本", "环保") - row = public_prediction(source, "环保", status="ok") - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "prediction.jsonl" - write_jsonl(path, [row]) - result = path.read_text(encoding="utf-8") - self.assertNotIn(source.text, result) - self.assertNotIn('"text"', result) - - -class BaselineTests(unittest.TestCase): - def test_validation_text_never_changes_training_vocabulary(self): - model = RandomForestBaseline(n_estimators=10).fit(training_records()) - vocabulary = model.pipeline.named_steps["tfidf"].vocabulary_.copy() - model.predict([Record("validation", "龘龘龘龘独有验证词", "否")]) - self.assertNotIn("龘龘", vocabulary) - self.assertEqual(vocabulary, model.pipeline.named_steps["tfidf"].vocabulary_) - - def test_probability_columns_follow_estimator_classes(self): - # Deliberately train only two classes: the public result still has all three keys. - records = training_records()[:4] - model = RandomForestBaseline(n_estimators=10).fit(records) - raw = model.pipeline.predict_proba([records[0].text])[0] - classes = model.pipeline.named_steps["classifier"].classes_ - row = model.predict([records[0]])[0] - self.assertEqual(list(row["probabilities"]), list(LABELS)) - for label, probability in zip(classes, raw): - self.assertAlmostEqual(row["probabilities"][label], probability) - self.assertEqual(row["probabilities"]["否"], 0.0) - - def test_fixed_metric_order_and_row_alignment(self): - records = training_records() - rows = [public_prediction(record, record.label, status="ok") for record in records] - metrics = classification_metrics(records, rows) - self.assertEqual(metrics["label_order"], list(LABELS)) - self.assertEqual(metrics["confusion_matrix"], [[2, 0, 0], [0, 2, 0], [0, 0, 2]]) - with self.assertRaises(ValueError): - classification_metrics(records, list(reversed(rows))) - - def test_local_model_round_trip(self): - model = RandomForestBaseline(n_estimators=10).fit(training_records()) - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "model.joblib" - model.save(path) - loaded = RandomForestBaseline.load(path) - self.assertEqual(model.predict(training_records()), loaded.predict(training_records())) - - -class HttpTests(unittest.TestCase): - def adapter(self, **kwargs): - return HttpServiceAdapter("https://service.invalid/classify", "fixture", retry_delay=0, **kwargs) - - def test_repeated_failure_terminates_without_text_or_secret(self): - adapter = self.adapter(attempts=3) - source = Record("row", "只供测试的虚构文本", "否") - with patch.object(adapter, "_request", side_effect=TimeoutError) as request: - rows = adapter.predict([source], workers=1) - self.assertEqual(request.call_count, 3) - self.assertEqual(rows[0]["status"], "error") - self.assertEqual(rows[0]["attempts"], 3) - self.assertNotIn(source.text, json.dumps(rows, ensure_ascii=False)) - self.assertNotIn("fixture", json.dumps(rows)) - - def test_parallel_results_preserve_input_order_even_when_completion_does_not(self): - adapter = self.adapter() - def fake_request(text): - time.sleep(0.02 if text == "slow" else 0) - return "否" - with patch.object(adapter, "_request", side_effect=fake_request): - rows = adapter.predict([Record("first", "slow"), Record("second", "fast")], workers=2) - self.assertEqual([row["record_id"] for row in rows], ["first", "second"]) - - def test_bad_json_and_out_of_set_reply_are_bounded(self): - for payload in [b"not-json", b'{"textResponse":"unknown"}', b'{"other":"value"}']: - adapter = self.adapter(attempts=2) - response = unittest.mock.MagicMock() - response.__enter__.return_value.read.return_value = payload - with patch("urllib.request.urlopen", return_value=response) as request: - row = adapter.classify(Record("row", "虚构文本")) - self.assertEqual(request.call_count, 2) - self.assertEqual(row["error"], "invalid_response") - - def test_401_is_not_retried_and_error_does_not_include_body(self): - adapter = self.adapter(attempts=3) - error = urllib.error.HTTPError("https://service.invalid", 401, "private-body", {}, None) - with patch.object(adapter, "_request", side_effect=error) as request: - row = adapter.classify(Record("row", "虚构文本")) - self.assertEqual(request.call_count, 1) - self.assertEqual(row["error"], "http_401") - self.assertNotIn("private-body", json.dumps(row)) - - def test_protocol_cleans_think_and_passes_configured_timeout(self): - adapter = self.adapter(timeout=1.5) - response = unittest.mock.MagicMock() - response.__enter__.return_value.read.return_value = json.dumps({"textResponse": "reason环保"}).encode() - with patch("urllib.request.urlopen", return_value=response) as request: - row = adapter.classify(Record("row", "虚构文本")) - self.assertEqual(row["prediction"], "环保") - self.assertEqual(request.call_args.kwargs["timeout"], 1.5) - sent_request = request.call_args.args[0] - self.assertIn("Authorization", sent_request.unredirected_hdrs) - self.assertNotIn("Authorization", sent_request.headers) - - def test_missing_environment_and_unbounded_options_rejected(self): - with patch.dict("os.environ", {}, clear=True): - with self.assertRaises(ValueError): - HttpServiceAdapter.from_environment() - for options in [{"attempts": 0}, {"attempts": 11}, {"timeout": 0}, {"timeout": 301}]: - with self.assertRaises(ValueError): - self.adapter(**options) - - -class CliTests(unittest.TestCase): - def test_default_demo_is_offline_and_marks_synthetic_scope(self): - output = io.StringIO() - with patch("urllib.request.urlopen", side_effect=AssertionError("network must not run")), contextlib.redirect_stdout(output): - code = main([]) - summary = json.loads(output.getvalue()) - self.assertEqual(code, 0) - self.assertEqual(summary["data_scope"], "synthetic_fixture") - self.assertEqual(summary["evaluation_scope"], "held_out_validation") - self.assertFalse(summary["paper_metrics"]) - - def test_bert_download_requires_explicit_opt_in(self): - args = parser().parse_args(["bert-train", "--input", "input.jsonl", "--model-output", "outputs/model"]) - self.assertFalse(args.allow_model_download) - self.assertEqual(args.device, "cpu") - - -if __name__ == "__main__": - unittest.main() diff --git a/projects/memlab/.env.example b/projects/memlab/.env.example deleted file mode 100644 index c89b588..0000000 --- a/projects/memlab/.env.example +++ /dev/null @@ -1,15 +0,0 @@ -# Copy values into your shell or a private .env file; Python does not auto-load it. -MEMLAB_CONFIG_PATH=./configs/memlab.default.yaml -MEMLAB_OUTPUT_DIR=./runs -REDIS_URL=redis://127.0.0.1:6379/0 -PG_URI= -POSTGRES_USER= -POSTGRES_PASSWORD= -POSTGRES_DB= -QDRANT_URL=http://127.0.0.1:6333 -QDRANT_COLLECTION=memlab_docs -# External DeerFlow checkout, separate from this repository. -MEMLAB_DEERFLOW_ROOT= -OPENAI_BASE_URL= -OPENAI_API_KEY= -OPENAI_MODEL= diff --git a/projects/memlab/.gitignore b/projects/memlab/.gitignore deleted file mode 100644 index c5c5e4a..0000000 --- a/projects/memlab/.gitignore +++ /dev/null @@ -1,23 +0,0 @@ -__pycache__/ -*.py[cod] -*.egg-info/ -.pytest_cache/ -.venv/ -venv/ -.env -.env.* -!.env.example -data/ -traces/ -/workloads/ -runs/ -logs/ -results/ -artifacts/ -storage/ -deer-flow/ -benchmarks/public_main/**/raw/ -benchmarks/public_main/**/prepared/ -*.parquet -*.arrow -*.jsonl diff --git a/projects/memlab/README.md b/projects/memlab/README.md deleted file mode 100644 index 42bb266..0000000 --- a/projects/memlab/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# MemLab / Agent 多层记忆与实验控制 - -面向并行 Agent 会话，将近期上下文、热缓存、持久化元数据与向量检索组织为统一记忆接口。提供上下文预算与去重、分层放置、异步会话记录和实验有效性检查。 - -[架构](docs/ARCHITECTURE.md) · [核心记忆策略](memlab/memory_manager/manager.py) · [实验控制](memlab/pipeline/control_plane.py) · [设计取舍](../../docs/ENGINEERING_NOTES.md) · [离线测试](docs/TESTING.md) - -## 可以从代码中查看的能力 - -- **上下文与记忆分层**：L0 上下文预算、L1 Redis 缓存、L2 Postgres 元数据与 Qdrant 检索，以及可选 L3 对象存储。 -- **记忆策略**：准入和放置、直接证据识别、检索注入预算、L3 指针化，以及服务不可用时的降级记录。 -- **运行记录**：manifest、事件日志、模型配置一致性检查，以及有效性门禁。未通过检查的运行会被汇总逻辑排除。 -- **实验入口**：合成语料/访问轨迹生成器、stub runner、服务 microbench 和 E2E runner。stub runner 仍需要外部基础设施；离线体验请先运行测试。 - -## 最短阅读与验证路径 - -需要 Python 3.12 或更新版本。在本目录执行： - -```bash -python -m venv .venv -# Windows: .venv\Scripts\activate -# macOS/Linux: source .venv/bin/activate -python -m pip install -e ".[dev]" -python -m pytest -q -``` - -测试使用临时目录和构造的样例，不需要模型、数据集、在线 API 或启动数据库。服务集成、Docker 镜像、真实模型调用和性能实验没有在这次公开版整理中运行，具体边界见 [测试说明](docs/TESTING.md)。 - -建议阅读 [记忆管理器](memlab/memory_manager/manager.py)、[运行控制](memlab/pipeline/control_plane.py)、[上下文组装](memlab/memory_manager/l0.py)，再看相应 [单元测试](tests/)。[架构说明](docs/ARCHITECTURE.md) 描述各层职责；[环境配置](docs/SETUP.md) 给出外部服务的接入方式。 - -## 上游与公开范围 - -公开版本提供源码、配置和离线测试；真实对话、论文数据、实验输出与模型凭据保留在本地。性能比较需另行配置服务与数据并完成实验，具体状态见[测试边界](docs/TESTING.md)。 - -DeerFlow 来自 [ByteDance / deer-flow](https://github.com/bytedance/deer-flow)，本项目的适配参考固定版本 `7de94394d4295182701ffb47e938e7c39b963091`。公开版没有复制该上游源码，外部 checkout 和依赖需要自行准备。适配代码不代表对该上游项目的全部实现负责。 - -基准适配器只保留 LongMemEval 的准备脚本和 profile 判断逻辑；下载仅在显式运行该脚本时发生。使用数据集时应先核对来源及使用条件。原工作区的历史实验矩阵、论文汇报脚本和实验结论不属于本子工程。 - -此整理版暂未添加新的源码许可证；第三方材料仍遵循其原有许可证。详见 [第三方说明](THIRD_PARTY.md)。 diff --git a/projects/memlab/THIRD_PARTY.md b/projects/memlab/THIRD_PARTY.md deleted file mode 100644 index c25e59c..0000000 --- a/projects/memlab/THIRD_PARTY.md +++ /dev/null @@ -1,7 +0,0 @@ -# 第三方来源 - -- **DeerFlow**：上游 [bytedance/deer-flow](https://github.com/bytedance/deer-flow)，适配参考 commit [`7de94394d4295182701ffb47e938e7c39b963091`](https://github.com/bytedance/deer-flow/tree/7de94394d4295182701ffb47e938e7c39b963091)。本地上游许可证为 MIT，署名 ByteDance Ltd. 和 DeerFlow Authors。本子工程不复制其源码；自行获取时应保留并遵循上游许可证。 -- **LongMemEval**：保留准备适配代码，不分发数据。脚本中的默认数据来源为 Hugging Face `yuyijiong/LongMemEval_24k`；运行前自行核对该来源、许可证及其与原始基准的关系。 -- **Python 依赖**：Pydantic、PyYAML、HTTPX、Redis、asyncpg 等通过包管理器安装，不在本仓库 vendoring。各自的许可证与使用条件以其发行版本为准。 - -原代码工作区未提供 MemLab 自身的许可证。此整理过程不代替权利人选择许可证，也不把第三方工作表述为个人独立实现。 diff --git a/projects/memlab/benchmarks/__init__.py b/projects/memlab/benchmarks/__init__.py deleted file mode 100644 index 29be07b..0000000 --- a/projects/memlab/benchmarks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Optional benchmark preparation source; no datasets are distributed.""" diff --git a/projects/memlab/benchmarks/public_main/__init__.py b/projects/memlab/benchmarks/public_main/__init__.py deleted file mode 100644 index b0d859c..0000000 --- a/projects/memlab/benchmarks/public_main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public benchmark adapters.""" diff --git a/projects/memlab/benchmarks/public_main/longmemeval/__init__.py b/projects/memlab/benchmarks/public_main/longmemeval/__init__.py deleted file mode 100644 index f2d0ea5..0000000 --- a/projects/memlab/benchmarks/public_main/longmemeval/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""LongMemEval preparation adapter; download occurs only when explicitly run.""" diff --git a/projects/memlab/benchmarks/public_main/longmemeval/prepare.py b/projects/memlab/benchmarks/public_main/longmemeval/prepare.py deleted file mode 100644 index 58d23a9..0000000 --- a/projects/memlab/benchmarks/public_main/longmemeval/prepare.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import random -import re -from pathlib import Path -from typing import Any - - -def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w", encoding="utf-8", newline="\n") as f: - for row in rows: - f.write(json.dumps(row, ensure_ascii=False) + "\n") - - -def _resolve_longmemeval_profile(question: str, question_type: str) -> tuple[str, str]: - q = str(question or "").strip().lower() - qtype = str(question_type or "").strip().lower() - if "preference" in qtype or re.search(r"\b(recommend|suggest|prefer|would like)\b", q): - return "summary", "token_f1" - if "temporal-reasoning" in qtype and re.search(r"\b(order|first to last|chronological)\b", q): - return "summary", "token_f1" - return "short_answer", "relaxed_span" - - -def main() -> None: - # Optional dependencies are required only for the explicit download command. - import pandas as pd - from huggingface_hub import hf_hub_download - - ap = argparse.ArgumentParser() - ap.add_argument("--out-dir", type=str, default="data/benchmarks/longmemeval") - ap.add_argument("--repo-id", type=str, default="yuyijiong/LongMemEval_24k") - ap.add_argument("--filename", type=str, default="longmemeval_24k.parquet") - ap.add_argument("--max-samples", type=int, default=200) - ap.add_argument("--seed", type=int, default=42) - args = ap.parse_args() - - out_dir = Path(args.out_dir) - raw_path = Path( - hf_hub_download( - repo_id=str(args.repo_id), - repo_type="dataset", - filename=str(args.filename), - ) - ) - df = pd.read_parquet(raw_path) - rows = df.to_dict(orient="records") - rng = random.Random(int(args.seed)) - rng.shuffle(rows) - if int(args.max_samples) > 0: - rows = rows[: max(1, int(args.max_samples))] - - tasks_public: list[dict[str, Any]] = [] - tasks_internal: list[dict[str, Any]] = [] - docs_internal: list[dict[str, Any]] = [] - for i, row in enumerate(rows): - task_id = f"lme_{i:06d}" - doc_id = f"longmemeval_doc_{i:06d}" - question = str(row.get("question", "")).strip() - context = str(row.get("prompt", "")) - gold = str(row.get("answer", "")).strip() - qtype = str(row.get("question_type", "unknown")).strip() - prompt_len = int(row.get("prompt_len", len(context))) - qa_prompt_profile, answer_metric_hint = _resolve_longmemeval_profile(question, qtype) - metadata = { - "question_type": qtype, - "prompt_len": prompt_len, - "context_chars": len(context), - "task_type": "long_context_qa", - "memory_needed": True, - "answer_metric_hint": answer_metric_hint, - "qa_prompt_profile": qa_prompt_profile, - } - tasks_public.append( - { - "task_id": task_id, - "source_dataset": "longmemeval", - "question": question, - "context": context, - "dialogue": "", - "docs": "", - "gold_answer": gold, - "metadata": metadata, - } - ) - tasks_internal.append( - { - "task_id": task_id, - "doc_id": doc_id, - "doc_title_hint": f"LongMemEval {qtype}", - "question": question, - "answer_key": gold, - "gold_answer": gold, - "source_dataset": "longmemeval", - "task_type": "qa", - "answer_field": "final_answer", - "metadata": metadata, - } - ) - docs_internal.append( - { - "doc_id": doc_id, - "title": f"LongMemEval Prompt {qtype}", - "body": context, - } - ) - - _write_jsonl(out_dir / "tasks_public.jsonl", tasks_public) - _write_jsonl(out_dir / "tasks_internal.jsonl", tasks_internal) - _write_jsonl(out_dir / "docs_internal.jsonl", docs_internal) - manifest = { - "dataset": "longmemeval", - "source_repo": str(args.repo_id), - "source_file": str(args.filename), - "raw_path": str(raw_path), - "sample_count": len(tasks_public), - "seed": int(args.seed), - } - (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8") - print(json.dumps(manifest, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/configs/memlab.default.yaml b/projects/memlab/configs/memlab.default.yaml deleted file mode 100644 index 0cdc44c..0000000 --- a/projects/memlab/configs/memlab.default.yaml +++ /dev/null @@ -1,68 +0,0 @@ -experiment: - run_id: F1 - variant_id: F1 - seed: 42 - repetitions: 3 - n_agents: 50 - duration_s: 300 - llm_backend: stub - redis_policy: allkeys-lfu - summarization_on: true - output_dir: ./runs - logs_dir: ./runs - -memory: - enabled_tiers: - - L0 - - L1 - - L2 - - L3 - l0: - max_prompt_tokens: 8000 - max_memory_injection_tokens: 2000 - max_injected_tokens: 256 - keep_recent_messages: 10 - summarize_trigger_tokens: 7000 - summarize_target_tokens: 3000 - min_relevance_score: 0.20 - l1: - enabled: true - ttl_s: 300 - max_object_bytes: 65536 - reuse_threshold: 0.20 - l2: - enabled: true - vector_enabled: true - vector_backend: qdrant - qdrant_shard_number: 1 - default_top_k: 8 - collection_name: memlab_docs - write_batch_size: 512 - write_flush_interval_ms: 50 - min_write_importance: 0.70 - max_batch_per_flush: 512 - max_flush_ms: 50 - drop_queue_depth_threshold: 12000 - backpressure_drop_importance_threshold: 0.85 - embedding_cache_ttl_ms: 120000 - retrieval_cache_ttl_ms: 15000 - l3: - enabled: false - root_dir: ./storage/l3_objects - pointer_threshold_bytes: 65536 - pointer_preview_chars: 240 - policy: - packing_budget: balanced - checkpoint_policy: l2_only - direct_evidence_retrieval_bypass: true - -services: - redis_url: redis://127.0.0.1:6379/0 - pg_uri: "" # Set PG_URI in your private environment before using Postgres. - qdrant_url: http://127.0.0.1:6333 - qdrant_collection: memlab_docs - -workloads: - corpus_path: ./data/docs.jsonl - tasks_path: ./data/tasks.jsonl - trace_path: ./traces/trace_200a.jsonl diff --git a/projects/memlab/configs/postgres/postgresql.conf b/projects/memlab/configs/postgres/postgresql.conf deleted file mode 100644 index 44a9aaf..0000000 --- a/projects/memlab/configs/postgres/postgresql.conf +++ /dev/null @@ -1,17 +0,0 @@ -max_connections = 200 -listen_addresses = '*' - -shared_buffers = 2GB -work_mem = 32MB -maintenance_work_mem = 512MB -effective_cache_size = 8GB - -wal_compression = on -checkpoint_timeout = 15min -max_wal_size = 8GB -min_wal_size = 1GB - -log_min_duration_statement = 200ms -log_checkpoints = on -log_connections = on -log_disconnections = on diff --git a/projects/memlab/configs/qdrant/production.yaml b/projects/memlab/configs/qdrant/production.yaml deleted file mode 100644 index 4e6545a..0000000 --- a/projects/memlab/configs/qdrant/production.yaml +++ /dev/null @@ -1,9 +0,0 @@ -service: - http_port: 6333 - grpc_port: 6334 - -storage: - storage_path: /qdrant/storage - -telemetry: - disabled: true diff --git a/projects/memlab/configs/redis/redis.conf b/projects/memlab/configs/redis/redis.conf deleted file mode 100644 index fc5fd90..0000000 --- a/projects/memlab/configs/redis/redis.conf +++ /dev/null @@ -1,13 +0,0 @@ -bind 0.0.0.0 -port 6379 -protected-mode no - -maxmemory 4gb -maxmemory-policy allkeys-lru -maxmemory-samples 10 - -save "" -appendonly no - -tcp-keepalive 60 -timeout 0 diff --git a/projects/memlab/configs/redis/redis_lfu.conf b/projects/memlab/configs/redis/redis_lfu.conf deleted file mode 100644 index 829b301..0000000 --- a/projects/memlab/configs/redis/redis_lfu.conf +++ /dev/null @@ -1,13 +0,0 @@ -bind 0.0.0.0 -port 6379 -protected-mode no - -maxmemory 4gb -maxmemory-policy allkeys-lfu -maxmemory-samples 10 - -save "" -appendonly no - -tcp-keepalive 60 -timeout 0 diff --git a/projects/memlab/configs/redis/redis_lru.conf b/projects/memlab/configs/redis/redis_lru.conf deleted file mode 100644 index fc5fd90..0000000 --- a/projects/memlab/configs/redis/redis_lru.conf +++ /dev/null @@ -1,13 +0,0 @@ -bind 0.0.0.0 -port 6379 -protected-mode no - -maxmemory 4gb -maxmemory-policy allkeys-lru -maxmemory-samples 10 - -save "" -appendonly no - -tcp-keepalive 60 -timeout 0 diff --git a/projects/memlab/docs/ARCHITECTURE.md b/projects/memlab/docs/ARCHITECTURE.md deleted file mode 100644 index ed14c27..0000000 --- a/projects/memlab/docs/ARCHITECTURE.md +++ /dev/null @@ -1,20 +0,0 @@ -# 架构与代码导航 - -`memlab.memory_manager.manager.MultiLevelMemoryManager` 统一处理写入、检索和上下文组装。各层通过独立适配器连接，配置见 `memlab/config.py`。 - -| 部件 | 职责 | 主要实现 | -| --- | --- | --- | -| L0 | 保留最近消息、限制检索注入预算、组装上下文 | `memory_manager/l0.py` | -| L1 | Redis 缓存与批量写入队列 | `memory_manager/l1.py`、`backends/redis_store.py` | -| L2 | Postgres 元数据、Qdrant 向量检索 | `memory_manager/l2.py`、`memory_manager/vector.py` | -| L3 | 文件对象存储及大内容指针 | `memory_manager/l3.py` | -| 运行控制 | 分支标识、失效记录、验收字段和回滚阶段 | `pipeline/control_plane.py` | -| 汇总 | 从运行 manifest 和事件记录生成表格 | `metrics/summary_builder.py` | - -`runner_stub.py` 生成确定性的访问负载；`runner_e2e.py` 执行外部模型/DeerFlow 路径。它们需要真实服务配置。`envcheck.py` 检查服务与模型环境，失败时记录原因。公开版没有把环境失败自动解释为实验通过。 - -测试中的小数、延迟值、任务编号和断言值是构造输入及预期值。源码中的预算、阈值、批量大小和默认种子属于算法/运行参数，不能作为实测性能或论文结果引用。轻量 `hash_embedding` 用于确定性实验，其实现不等同于高质量语义嵌入模型。 - -历史整体 orchestrator 依赖未发布的实验矩阵和报告脚本，已从此公开子工程排除。这里保留的是组件、独立 runner 和验证逻辑。 - -共享记忆管理器只安装一次操作 logger；会话调用通过 `ContextVar` 记录当前 agent，避免并发 worker 覆盖 logger。运行初始化和独立后台批次没有单一会话归属，其 agent/session 字段可为空，不能强行分配给最近执行的会话。 diff --git a/projects/memlab/docs/SETUP.md b/projects/memlab/docs/SETUP.md deleted file mode 100644 index 54c5ad4..0000000 --- a/projects/memlab/docs/SETUP.md +++ /dev/null @@ -1,42 +0,0 @@ -# 环境配置 - -先按 README 安装并运行离线测试。以下服务路径作为手动配置入口提供，本次整理没有启动它们。 - -## 本地基础设施 - -```bash -python -m pip install -e ".[infra]" -``` - -复制 `.env.example` 为私有 `.env`，填入 `POSTGRES_USER`、`POSTGRES_PASSWORD`、`POSTGRES_DB`。Docker Compose 会读取本目录 `.env`，缺少这些值时会拒绝启动。`PG_URI` 需使用与数据库一致的连接信息。 - -Python 代码不会自动读取 `.env`；运行 Python CLI 前，须将 `PG_URI`、`REDIS_URL`、`QDRANT_URL` 等值设置到当前 shell。请自行使用合适的凭据管理方式，不要将连接串或 `.env` 提交到 Git。 - -```bash -docker compose -f infra/docker-compose.yml up -d -``` - -Compose 服务端口只绑定本机回环地址。Redis/Postgres/Qdrant 的版本来自原工程配置，此整理过程没有拉取镜像、核实镜像可用性或做服务兼容性验证。Windows 可使用 `scripts/up_infra.ps1` 和 `scripts/check_infra.ps1`。停止使用 `docker compose -f infra/docker-compose.yml down`，该命令默认保留命名卷。 - -## 合成负载 - -不需要复制原工作区数据。生成器使用固定种子构造虚拟文档与访问记录： - -```bash -python -m memlab.workloads.gen_corpus --docs 4 --tasks 4 --doc-bytes-min 64 --doc-bytes-max 128 --out data -python -m memlab.workloads.gen_trace --agents 2 --duration-s 1 --out traces/demo.jsonl -``` - -这些命令写入已忽略的目录；文件不应作为真实研究样本或性能结果发布。stub runner 是基础设施实验入口，不能因为名字含 stub 就视作无服务演示。 - -## 外部 DeerFlow 与模型 - -自行准备 [DeerFlow](https://github.com/bytedance/deer-flow) checkout，并选择版本 `7de94394d4295182701ffb47e938e7c39b963091`。按该版本的上游文档安装 backend 环境并配置服务。本子工程不分发该源码、虚拟环境或配置文件。 - -设置 `MEMLAB_DEERFLOW_ROOT` 为外部 checkout 路径；旧适配器使用 `DEERFLOW_BACKEND_PATH`，如使用该适配器还应设置到 checkout 的 `backend` 目录。设置自己的 `OPENAI_BASE_URL`、`OPENAI_MODEL`、`OPENAI_API_KEY`。E2E 运行要求显式配置这些值，公开版没有预设供应商、免费额度或默认模型。此处变量命名只是现有兼容协议约定，不要求使用特定模型供应商。 - -嵌入式 worker 仍按原工程的 Windows backend 虚拟环境布局寻找 `.venv/Scripts/python.exe`。跨平台嵌入式路径、DeerFlow API 兼容性和在线推理均未在此次整理中验证。 - -## 可选基准准备脚本 - -`benchmarks/public_main/longmemeval/prepare.py` 保留纯源码，profile 判断可离线测试。显式运行准备命令前需安装 `.[benchmarks]`，核对数据集来源、访问权限和使用条件。该命令会联网下载数据，默认输出在忽略目录中；不属于离线验证流程。 diff --git a/projects/memlab/docs/TESTING.md b/projects/memlab/docs/TESTING.md deleted file mode 100644 index 834651d..0000000 --- a/projects/memlab/docs/TESTING.md +++ /dev/null @@ -1,26 +0,0 @@ -# 验证边界 - -```bash -python -m pip install -e ".[dev]" -python -m pytest -q -``` - -本工程的离线测试覆盖配置/变体映射、上下文策略、L3 指针化、失效与验收记录、答案解析、QA profile、构造运行记录的汇总、服务版本探针的命令生成、并发会话的事件归属，以及 HTTP/服务异常回显不进入诊断记录。测试数据在代码内构造或写入临时目录，不携带原工作区的对话、任务文件或实验记录。 - -记忆管理测试关闭外部层，或把服务适配器替换为失败桩；Redis 客户端包只用于构造对象。测试不会连接 Redis/Postgres/Qdrant，不调用在线模型，也不运行下载脚本。 - -单元测试通过只支持这些逻辑分支的断言。以下尚未验证： - -- Docker 镜像拉取、启动、健康检查与持久化。 -- Redis/Postgres/Qdrant 的真实读写、吞吐和故障恢复。 -- DeerFlow 固定版本的服务适配和嵌入式 worker。 -- 外部模型 API、费用、基准下载或完整实验矩阵。 -- 论文性能数值、跨方案胜负或统计显著性。 - -主工作区缺失的 `longmemeval/prepare.py` 仅从归档恢复源码以满足 profile 测试；下载依赖已延迟到 `main()`，没有恢复任何数据。 - -并发测试对 stub 的两个 worker 交错执行，并对 E2E worker 的上下文封装使用模拟任务体。它们验证会话归属逻辑，不代表在线 E2E 流程通过。HTTP 错误只记录状态码或异常类型；外部运行的正常输出、事件记录和服务日志仍应保存在私有忽略目录，公开前另行审查。 - -## 本次整理的验证记录 - -在独立 Python 3.12.14 虚拟环境安装 `.[dev]` 后，`pytest -q -p no:cacheprovider` 为 **53 passed**。同时通过全部 Python 文件的内存编译、8 个 PowerShell 脚本的语法解析、`pip check`，以及两个 runner 的 `--help` 入口检查。验证时没有启动外部服务或调用在线 API。 diff --git a/projects/memlab/infra/docker-compose.yml b/projects/memlab/infra/docker-compose.yml deleted file mode 100644 index 97f422b..0000000 --- a/projects/memlab/infra/docker-compose.yml +++ /dev/null @@ -1,58 +0,0 @@ -services: - redis: - image: redis:8.6.1 - command: ["redis-server", "/usr/local/etc/redis/redis.conf"] - volumes: - - ../configs/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro - - redis_data:/data - ports: - - "127.0.0.1:6379:6379" - healthcheck: - test: ["CMD", "redis-cli", "-h", "127.0.0.1", "-p", "6379", "PING"] - interval: 10s - timeout: 5s - retries: 12 - - postgres: - image: postgres:18.3 - environment: - POSTGRES_USER: ${POSTGRES_USER:?Set POSTGRES_USER locally} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD locally} - POSTGRES_DB: ${POSTGRES_DB:?Set POSTGRES_DB locally} - TZ: UTC - command: ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"] - volumes: - - postgres_data:/var/lib/postgresql - - ../configs/postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro - - ../sql/001_schema.sql:/docker-entrypoint-initdb.d/001_schema.sql:ro - ports: - - "127.0.0.1:5432:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB -h 127.0.0.1 -p 5432"] - interval: 10s - timeout: 5s - retries: 12 - - qdrant: - image: qdrant/qdrant:v1.17.0 - environment: - RUN_MODE: production - volumes: - - qdrant_data:/qdrant/storage - - ../configs/qdrant/production.yaml:/qdrant/config/production.yaml:ro - ports: - - "127.0.0.1:6333:6333" - - "127.0.0.1:6334:6334" - healthcheck: - test: ["CMD-SHELL", "grep -q ':18BD' /proc/net/tcp || grep -q ':18BD' /proc/net/tcp6"] - interval: 10s - timeout: 5s - retries: 12 - -volumes: - redis_data: - name: memlab_redis_data - postgres_data: - name: memlab_postgres_data - qdrant_data: - name: memlab_qdrant_data diff --git a/projects/memlab/memlab/__init__.py b/projects/memlab/memlab/__init__.py deleted file mode 100644 index 7ef27ce..0000000 --- a/projects/memlab/memlab/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""memlab package.""" - -from .config import MemlabConfig, load_config - -__all__ = ["MemlabConfig", "load_config"] diff --git a/projects/memlab/memlab/analysis/__init__.py b/projects/memlab/memlab/analysis/__init__.py deleted file mode 100644 index 294ec92..0000000 --- a/projects/memlab/memlab/analysis/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Analysis helpers.""" diff --git a/projects/memlab/memlab/analysis/benchmark_discriminativeness_gates.py b/projects/memlab/memlab/analysis/benchmark_discriminativeness_gates.py deleted file mode 100644 index df494c3..0000000 --- a/projects/memlab/memlab/analysis/benchmark_discriminativeness_gates.py +++ /dev/null @@ -1,478 +0,0 @@ -from __future__ import annotations - -import argparse -import json -from collections import defaultdict -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -from memlab.pipeline.control_plane import accepted_for_official_reporting, infer_track_id_from_manifest - - -def _utc_now_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _read_json(path: Path) -> dict[str, Any]: - with path.open("r", encoding="utf-8-sig") as f: - return json.load(f) - - -def _safe_float(v: Any, default: float = 0.0) -> float: - try: - return float(v) - except Exception: - return default - - -def _mean(values: list[float]) -> float: - return sum(values) / max(1, len(values)) - - -def _parse_dataset(tasks_path: str) -> str: - norm = str(tasks_path or "").replace("\\", "/") - if "/public_main/" not in norm: - return "unknown" - parts = norm.split("/public_main/", 1)[1].split("/") - return parts[0] if parts else "unknown" - - -def _is_comparable(row: dict[str, Any]) -> bool: - return bool(row.get("env_valid")) and bool(row.get("repro_valid")) and bool(row.get("llm_real")) - - -def _load_null_map(path: str) -> dict[str, str]: - if not str(path or "").strip(): - return {} - raw = _read_json(Path(path)) - if not isinstance(raw, dict): - return {} - out: dict[str, str] = {} - for key, value in raw.items(): - if isinstance(value, str) and value.strip(): - out[str(key).strip()] = str(value).strip() - elif isinstance(value, dict): - workload_id = str(value.get("workload_id", "") or "").strip() - if workload_id: - out[str(key).strip()] = workload_id - return out - - -def _collect_rows( - runs_dir: Path, - *, - workload_id: str = "", - dataset_filter: str = "", - model_id_filter: str = "", - experiment_track_filter: str = "", - track_id_filter: str = "", -) -> list[dict[str, Any]]: - rows_by_key: dict[tuple[str, str, str, int], dict[str, Any]] = {} - updated_ts_by_key: dict[tuple[str, str, str, int], float] = {} - if not runs_dir.exists(): - return [] - for run_dir in runs_dir.iterdir(): - if not run_dir.is_dir(): - continue - manifest_path = run_dir / "manifest.json" - summary_path = run_dir / "summary.json" - if not manifest_path.exists() or not summary_path.exists(): - continue - try: - manifest = _read_json(manifest_path) - summary = _read_json(summary_path) - except Exception: - continue - wid = str(manifest.get("workload_id", "") or "") - if not wid or "PREFLIGHT" in wid: - continue - if workload_id and wid != workload_id: - continue - extra = manifest.get("extra", {}) if isinstance(manifest.get("extra"), dict) else {} - tasks_path = str(extra.get("tasks_path", "") or "") - dataset = str(extra.get("dataset_id", "") or "").strip() or _parse_dataset(tasks_path) - if dataset_filter and dataset != dataset_filter: - continue - model_id = str(manifest.get("model_id", "") or "") - if model_id_filter and model_id != model_id_filter: - continue - if not accepted_for_official_reporting(manifest): - continue - experiment_track = str(extra.get("experiment_track", "mainline") or "mainline") - if experiment_track_filter and experiment_track != experiment_track_filter: - continue - track_id = infer_track_id_from_manifest(manifest) - if track_id_filter and track_id != track_id_filter: - continue - repeat_idx = int(manifest.get("repeat_idx", 0)) - variant = str(manifest.get("variant_id", "") or "") - row = { - "run_dir": str(run_dir), - "mtime": float(run_dir.stat().st_mtime), - "dataset": dataset, - "workload_id": wid, - "track_id": track_id, - "variant": variant, - "repeat_idx": repeat_idx, - "seed": int(manifest.get("seed", 0)), - "model_id": model_id, - "experiment_track": experiment_track, - "env_valid": bool(manifest.get("env_valid", False)), - "repro_valid": bool(manifest.get("repro_valid", False)), - "llm_real": bool(manifest.get("llm_real", False)), - "success_rate": _safe_float(summary.get("success_rate", 0.0)), - "strict_accuracy": _safe_float(summary.get("strict_accuracy", summary.get("accuracy", 0.0))), - "relaxed_accuracy": _safe_float(summary.get("relaxed_accuracy", summary.get("accuracy", 0.0))), - "benchmark_metric": str(summary.get("benchmark_metric", "") or ""), - "benchmark_score": _safe_float( - summary.get( - "benchmark_score_on_total", - summary.get("benchmark_score", summary.get("strict_accuracy", summary.get("accuracy", 0.0))), - ) - ), - "parse_failure_rate": _safe_float(summary.get("parse_failure_rate", 0.0)), - "no_answer_rate": _safe_float(summary.get("no_answer_rate", 0.0)), - "processing_p95_ms": _safe_float(((summary.get("latency") or {}).get("processing") or {}).get("p95", 0.0)), - "end_to_end_p95_ms": _safe_float(((summary.get("latency") or {}).get("end_to_end") or {}).get("p95", 0.0)), - } - key = (wid, track_id, variant, repeat_idx) - mtime = float(run_dir.stat().st_mtime) - if mtime >= updated_ts_by_key.get(key, -1.0): - updated_ts_by_key[key] = mtime - rows_by_key[key] = row - rows = list(rows_by_key.values()) - rows.sort(key=lambda item: (str(item.get("dataset", "")), str(item.get("workload_id", "")), str(item.get("track_id", "")), str(item.get("variant", "")), int(item.get("repeat_idx", 0)))) - return rows - - -def _aggregate(rows: list[dict[str, Any]], variant: str) -> dict[str, Any]: - sub = [r for r in rows if str(r.get("variant", "")) == variant] - return { - "variant": variant, - "runs": len(sub), - "benchmark_metric": str(sub[0].get("benchmark_metric", "")) if sub else "", - "benchmark_score": _mean([_safe_float(r.get("benchmark_score", 0.0)) for r in sub]) if sub else 0.0, - "success_rate": _mean([_safe_float(r.get("success_rate", 0.0)) for r in sub]) if sub else 0.0, - "strict_accuracy": _mean([_safe_float(r.get("strict_accuracy", 0.0)) for r in sub]) if sub else 0.0, - "relaxed_accuracy": _mean([_safe_float(r.get("relaxed_accuracy", 0.0)) for r in sub]) if sub else 0.0, - "parse_failure_rate": _mean([_safe_float(r.get("parse_failure_rate", 0.0)) for r in sub]) if sub else 0.0, - "no_answer_rate": _mean([_safe_float(r.get("no_answer_rate", 0.0)) for r in sub]) if sub else 0.0, - "processing_p95_ms": _mean([_safe_float(r.get("processing_p95_ms", 0.0)) for r in sub]) if sub else 0.0, - "end_to_end_p95_ms": _mean([_safe_float(r.get("end_to_end_p95_ms", 0.0)) for r in sub]) if sub else 0.0, - "comparable_rate": _mean([1.0 if _is_comparable(r) else 0.0 for r in sub]) if sub else 0.0, - } - - -def _is_main_smoke_workload(workload_id: str, dataset: str) -> bool: - wid = str(workload_id or "").upper() - prefix = f"PUB-{str(dataset).upper()}-SMOKE" - return wid.startswith(prefix) and "NULLSANITY" not in wid and "MEMEXP" not in wid and "ALIGN" not in wid - - -def _find_latest_main_smoke_workload(rows: list[dict[str, Any]], dataset: str) -> str: - grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) - for row in rows: - if row.get("dataset") != dataset: - continue - wid = str(row.get("workload_id", "") or "") - if not _is_main_smoke_workload(wid, dataset): - continue - grouped[wid].append(row) - candidates: list[tuple[float, str, bool]] = [] - for wid, group_rows in grouped.items(): - latest_mtime = max(_safe_float(r.get("mtime", 0.0)) for r in group_rows) - comparable_variants = {str(r.get("variant", "")) for r in group_rows if _is_comparable(r)} - candidates.append((latest_mtime, wid, {"B1", "F1"}.issubset(comparable_variants))) - if not candidates: - return "" - candidates.sort(key=lambda item: (item[2], item[0], item[1]), reverse=True) - return str(candidates[0][1]) - - -def _discover_null_workload( - rows: list[dict[str, Any]], - *, - dataset: str, - mapped_workload_id: str, -) -> str: - if mapped_workload_id: - return mapped_workload_id - prefix = f"PUB-{str(dataset).upper()}-SMOKE-NULLSANITY-E550-K6" - latest_mtime = -1.0 - latest_workload = "" - for row in rows: - wid = str(row.get("workload_id", "") or "") - if not wid.startswith(prefix): - continue - if str(row.get("variant", "")) != "B1": - continue - if not _is_comparable(row): - continue - mtime = _safe_float(row.get("mtime", 0.0)) - if mtime >= latest_mtime: - latest_mtime = mtime - latest_workload = wid - return latest_workload - - -def _recommended_disposition(*, dataset: str, ceiling: bool, too_hard: bool, refusal_dominated: bool, non_memory_dependent: bool, eligible_for_pilot: bool) -> str: - if eligible_for_pilot: - return "eligible_for_new_pilot" - if ceiling or non_memory_dependent: - return "do_not_scale" - if too_hard: - return "alignment_only" - if refusal_dominated: - return "memory_exposure_only" - return "memory_exposure_only" if dataset in {"longbench_v2", "babilong"} else "alignment_only" - - -def _analyze_workload( - *, - dataset: str, - workload_id: str, - rows: list[dict[str, Any]], - all_rows: list[dict[str, Any]] | None = None, - null_workload_id: str = "", -) -> dict[str, Any]: - main_rows = [r for r in rows if str(r.get("workload_id", "")) == workload_id] - comparable_rows = [r for r in main_rows if _is_comparable(r)] - b1 = _aggregate(comparable_rows, "B1") - f1 = _aggregate(comparable_rows, "F1") - if int(b1.get("runs", 0)) == 0 or int(f1.get("runs", 0)) == 0: - return { - "dataset": dataset, - "workload_id": workload_id, - "official_smoke_workload_id": workload_id, - "rows": len(comparable_rows), - "b1": b1, - "f1": f1, - "benchmark_metric": "", - "benchmark_diff": None, - "strict_diff": None, - "no_answer_diff": None, - "success_diff": None, - "ceiling": False, - "too_hard_or_bad_adaptation": False, - "refusal_dominated": False, - "needs_null_evidence_sanity": False, - "null_evidence_available": False, - "null_workload_id": "", - "null_b1_benchmark_score": None, - "null_retention": None, - "non_memory_dependent": False, - "eligible_for_pilot": False, - "recommended_disposition": "insufficient_comparison_variants", - } - b1_score = _safe_float(b1.get("benchmark_score", 0.0)) - f1_score = _safe_float(f1.get("benchmark_score", 0.0)) - ceiling = b1_score > 0.90 - too_hard = b1_score < 0.05 and f1_score < 0.05 - refusal_dominated = _safe_float(b1.get("no_answer_rate", 0.0)) > 0.30 and _safe_float(f1.get("no_answer_rate", 0.0)) > 0.30 - needs_null_sanity = not ceiling and not too_hard and not refusal_dominated - - lookup_rows = all_rows if all_rows is not None else rows - null_rows = [r for r in lookup_rows if str(r.get("workload_id", "")) == null_workload_id] if null_workload_id else [] - null_comparable_rows = [r for r in null_rows if _is_comparable(r)] - null_b1 = _aggregate(null_comparable_rows, "B1") - null_available = bool(null_workload_id) and int(null_b1.get("runs", 0)) > 0 - null_b1_score = _safe_float(null_b1.get("benchmark_score", 0.0)) - null_retention = (null_b1_score / max(b1_score, 1e-6)) if null_available else 0.0 - non_memory_dependent = bool(null_available and null_b1_score >= 0.30 and null_retention >= 0.80) - eligible_for_pilot = bool( - int(b1.get("runs", 0)) > 0 - and int(f1.get("runs", 0)) > 0 - and not ceiling - and not too_hard - and not refusal_dominated - and (not needs_null_sanity or null_available) - and not non_memory_dependent - ) - disposition = _recommended_disposition( - dataset=dataset, - ceiling=ceiling, - too_hard=too_hard, - refusal_dominated=refusal_dominated, - non_memory_dependent=non_memory_dependent, - eligible_for_pilot=eligible_for_pilot, - ) - return { - "dataset": dataset, - "workload_id": workload_id, - "official_smoke_workload_id": workload_id, - "rows": len(comparable_rows), - "b1": b1, - "f1": f1, - "benchmark_metric": str(b1.get("benchmark_metric", "") or f1.get("benchmark_metric", "")), - "benchmark_diff": f1_score - b1_score, - "strict_diff": _safe_float(f1.get("strict_accuracy", 0.0)) - _safe_float(b1.get("strict_accuracy", 0.0)), - "no_answer_diff": _safe_float(f1.get("no_answer_rate", 0.0)) - _safe_float(b1.get("no_answer_rate", 0.0)), - "success_diff": _safe_float(f1.get("success_rate", 0.0)) - _safe_float(b1.get("success_rate", 0.0)), - "ceiling": ceiling, - "too_hard_or_bad_adaptation": too_hard, - "refusal_dominated": refusal_dominated, - "needs_null_evidence_sanity": needs_null_sanity, - "null_evidence_available": null_available, - "null_workload_id": null_workload_id if null_available else "", - "null_b1_benchmark_score": null_b1_score if null_available else None, - "null_retention": null_retention if null_available else None, - "non_memory_dependent": non_memory_dependent, - "eligible_for_pilot": eligible_for_pilot, - "recommended_disposition": disposition, - } - - -def _write_md(path: Path, results: list[dict[str, Any]], *, title: str) -> None: - lines: list[str] = [ - f"# {title}", - "", - "| dataset | official_smoke_workload_id | metric | B1 score | F1 score | B1 strict | F1 strict | B1 no_answer | F1 no_answer | B1 success | F1 success | null_b1_score | null_retention | ceiling | too_hard/bad_adaptation | refusal_dominated | non_memory_dependent | eligible_for_pilot | recommended_disposition |", - "|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|---|---|", - ] - for row in results: - b1 = row.get("b1", {}) - f1 = row.get("f1", {}) - null_b1 = row.get("null_b1_benchmark_score", None) - null_retention = row.get("null_retention", None) - lines.append( - "| {dataset} | {workload_id} | {metric} | {b1_score:.3f} | {f1_score:.3f} | {b1_strict:.3f} | {f1_strict:.3f} | {b1_no_answer:.3f} | {f1_no_answer:.3f} | {b1_success:.3f} | {f1_success:.3f} | {null_b1} | {null_retention} | {ceiling} | {too_hard} | {refusal} | {non_memory} | {eligible} | {disposition} |".format( - dataset=str(row.get("dataset", "")), - workload_id=str(row.get("official_smoke_workload_id", "")), - metric=str(row.get("benchmark_metric", "")), - b1_score=_safe_float(b1.get("benchmark_score", 0.0)), - f1_score=_safe_float(f1.get("benchmark_score", 0.0)), - b1_strict=_safe_float(b1.get("strict_accuracy", 0.0)), - f1_strict=_safe_float(f1.get("strict_accuracy", 0.0)), - b1_no_answer=_safe_float(b1.get("no_answer_rate", 0.0)), - f1_no_answer=_safe_float(f1.get("no_answer_rate", 0.0)), - b1_success=_safe_float(b1.get("success_rate", 0.0)), - f1_success=_safe_float(f1.get("success_rate", 0.0)), - null_b1=("n/a" if null_b1 is None else f"{_safe_float(null_b1):.3f}"), - null_retention=("n/a" if null_retention is None else f"{_safe_float(null_retention):.3f}"), - ceiling=str(bool(row.get("ceiling", False))).lower(), - too_hard=str(bool(row.get("too_hard_or_bad_adaptation", False))).lower(), - refusal=str(bool(row.get("refusal_dominated", False))).lower(), - non_memory=str(bool(row.get("non_memory_dependent", False))).lower(), - eligible=str(bool(row.get("eligible_for_pilot", False))).lower(), - disposition=str(row.get("recommended_disposition", "")), - ) - ) - lines.extend( - [ - "", - "## Notes", - "", - "- `score` uses `benchmark_score_on_total` when available, rather than forcing `strict_accuracy` on open-ended benchmarks.", - "- `ceiling`: `B1 score > 0.90`.", - "- `too_hard_or_bad_adaptation`: both B1/F1 score `< 0.05`.", - "- `refusal_dominated`: both B1/F1 no_answer `> 0.30`.", - "- `non_memory_dependent`: null-evidence B1 keeps strong score (`score >= 0.30` and retention `>= 0.80`).", - "- `eligible_for_pilot` requires all gate flags to stay false and, when needed, null-evidence sanity must already exist.", - ] - ) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--runs-dir", type=str, default="runs") - ap.add_argument("--dataset", type=str, default="") - ap.add_argument("--datasets", type=str, default="") - ap.add_argument("--workload-id", type=str, default="") - ap.add_argument("--stage", type=str, default="smoke") - ap.add_argument("--model-id-filter", type=str, default="") - ap.add_argument("--experiment-track-filter", type=str, default="") - ap.add_argument("--track-id-filter", type=str, default="") - ap.add_argument("--null-evidence-map-json", type=str, default="") - ap.add_argument("--output-json", type=str, required=True) - ap.add_argument("--output-md", type=str, required=True) - args = ap.parse_args() - - null_map = _load_null_map(str(args.null_evidence_map_json or "")) - rows = _collect_rows( - Path(args.runs_dir), - workload_id=str(args.workload_id or ""), - dataset_filter=str(args.dataset or ""), - model_id_filter=str(args.model_id_filter or ""), - experiment_track_filter=str(args.experiment_track_filter or ""), - track_id_filter=str(args.track_id_filter or ""), - ) - rows_all_tracks = _collect_rows( - Path(args.runs_dir), - workload_id="", - dataset_filter=str(args.dataset or ""), - model_id_filter=str(args.model_id_filter or ""), - experiment_track_filter="", - track_id_filter="", - ) - datasets = [d.strip() for d in str(args.datasets or "").split(",") if d.strip()] - if str(args.dataset or "").strip(): - datasets = [str(args.dataset).strip()] - if not datasets and not str(args.workload_id or "").strip(): - datasets = sorted({str(r.get("dataset", "")) for r in rows if str(r.get("dataset", "")) != "unknown"}) - - results: list[dict[str, Any]] = [] - if str(args.workload_id or "").strip(): - if not rows: - result = { - "dataset": str(args.dataset or "unknown"), - "workload_id": str(args.workload_id), - "stage": str(args.stage or "smoke"), - "eligible_for_pilot": False, - "recommended_disposition": "do_not_scale", - "reason": "missing rows", - } - out_json = Path(args.output_json) - out_json.parent.mkdir(parents=True, exist_ok=True) - out_json.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") - _write_md(Path(args.output_md), [result], title="Benchmark Discriminativeness Gates") - print(json.dumps(result, ensure_ascii=False)) - return - dataset = str(args.dataset or rows[0].get("dataset", "unknown")) - null_workload_id = _discover_null_workload(rows_all_tracks, dataset=dataset, mapped_workload_id=null_map.get(dataset, "")) - results = [ - _analyze_workload( - dataset=dataset, - workload_id=str(args.workload_id), - rows=rows, - all_rows=rows_all_tracks, - null_workload_id=null_workload_id, - ) - ] - else: - for dataset in datasets: - workload_id = _find_latest_main_smoke_workload(rows, dataset) - if not workload_id: - continue - null_workload_id = _discover_null_workload(rows_all_tracks, dataset=dataset, mapped_workload_id=null_map.get(dataset, "")) - results.append( - _analyze_workload( - dataset=dataset, - workload_id=workload_id, - rows=rows, - all_rows=rows_all_tracks, - null_workload_id=null_workload_id, - ) - ) - - report: dict[str, Any] = { - "generated_at_utc": _utc_now_iso(), - "model_id_filter": str(args.model_id_filter or ""), - "experiment_track_filter": str(args.experiment_track_filter or ""), - "track_id_filter": str(args.track_id_filter or ""), - "results": results, - } - if len(results) == 1: - report.update(results[0]) - - out_json = Path(args.output_json) - out_json.parent.mkdir(parents=True, exist_ok=True) - out_json.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") - _write_md(Path(args.output_md), results, title="Benchmark Discriminativeness Gates") - print(json.dumps(report if len(results) == 1 else {"results": len(results), "output_json": str(args.output_json)}, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/analysis/branch_audit.py b/projects/memlab/memlab/analysis/branch_audit.py deleted file mode 100644 index 4c2c849..0000000 --- a/projects/memlab/memlab/analysis/branch_audit.py +++ /dev/null @@ -1,636 +0,0 @@ -from __future__ import annotations - -import argparse -import json -from collections import Counter, defaultdict -from pathlib import Path -from typing import Any - -from memlab.pipeline.control_plane import ( - ControlPlane, - BranchKey, - accepted_for_official_reporting, - fingerprint_paths, - infer_branch_key_from_manifest, - infer_dataset_id_from_manifest, - infer_track_id_from_manifest, - merge_control_fields, - normalize_dataset_id, - sha1_text, -) - - -def _read_json(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) - - -def _read_jsonl(path: Path) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - with path.open("r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - rows.append(json.loads(line)) - return rows - - -def _resolve_prepared_manifest_from_run_manifest(manifest: dict[str, Any]) -> Path | None: - extra = manifest.get("extra") if isinstance(manifest.get("extra"), dict) else {} - for key in ("tasks_path", "docs_path"): - raw = str(extra.get(key, "") or "").strip() - if not raw: - continue - candidate = Path(raw).parent / "manifest.json" - if candidate.exists(): - return candidate - return None - - -def _count_raw_samples(dataset_id: str, dataset_manifest: dict[str, Any]) -> int | None: - dataset_id = normalize_dataset_id(dataset_id) - raw_path = Path(str(dataset_manifest.get("raw_path", ""))) - if not raw_path.exists(): - return None - if dataset_id in {"longbench_v2", "babilong"}: - raw = json.loads(raw_path.read_text(encoding="utf-8")) - return len(raw) if isinstance(raw, list) else None - if dataset_id == "longmemeval": - import pandas as pd - - return int(len(pd.read_parquet(raw_path))) - if dataset_id == "locomo": - import pandas as pd - - df = pd.read_parquet(raw_path) - total = 0 - for _, row in df.iterrows(): - total += min(len(list(row.get("questions", []))), len(list(row.get("answers", [])))) - return int(total) - if dataset_id == "squad_v11": - raw = json.loads(raw_path.read_text(encoding="utf-8")) - total = 0 - for article in raw.get("data", []): - for para in article.get("paragraphs", []): - for qa in para.get("qas", []): - answers = qa.get("answers", []) - if qa.get("id") and qa.get("question") and answers and str(answers[0].get("text", "")).strip(): - total += 1 - return total - return None - - -def _resolve_dataset_audit(base_dir: Path, *, max_samples: int) -> tuple[dict[str, Any], list[str]]: - manifest_path = base_dir / "manifest.json" - tasks_path = base_dir / "tasks_internal.jsonl" - docs_path = base_dir / "docs_internal.jsonl" - report: dict[str, Any] = { - "base_dir": base_dir.as_posix(), - "manifest_path": manifest_path.as_posix(), - "tasks_path": tasks_path.as_posix(), - "docs_path": docs_path.as_posix(), - "checks": {}, - "anomaly_codes": [], - } - anomalies: list[str] = [] - if not manifest_path.exists() or not tasks_path.exists() or not docs_path.exists(): - anomalies.append("DATA_COUNT_MISMATCH") - report["checks"]["required_files_present"] = False - return report, anomalies - report["checks"]["required_files_present"] = True - manifest = _read_json(manifest_path) - dataset_id = normalize_dataset_id(str(manifest.get("dataset", base_dir.name))) - tasks = _read_jsonl(tasks_path) - docs = _read_jsonl(docs_path) - manifest_count = int(manifest.get("sample_count", 0) or 0) - task_count = len(tasks) - doc_count = len(docs) - raw_count = _count_raw_samples(dataset_id, manifest) - report["dataset_id"] = dataset_id - report["raw_sample_count"] = raw_count - report["prepared_sample_count"] = manifest_count - report["task_count"] = task_count - report["doc_count"] = doc_count - report["requested_max_samples"] = int(max_samples) - expected_full = int(max_samples) <= 0 - report["full_or_slice"] = "full" if expected_full else "slice" - if manifest_count != task_count or manifest_count != doc_count: - anomalies.append("DATA_COUNT_MISMATCH") - if raw_count is not None and expected_full and manifest_count != raw_count: - anomalies.append("FULL_SLICE_MIXED") - task_ids = [str(row.get("task_id", "")).strip() for row in tasks] - doc_ids = [str(row.get("doc_id", "")).strip() for row in docs] - dup_task_ids = sorted([k for k, v in Counter(task_ids).items() if k and v > 1]) - empty_task_ids = [idx for idx, value in enumerate(task_ids) if not value] - empty_doc_ids = [idx for idx, value in enumerate(doc_ids) if not value] - empty_gold = [] - for idx, row in enumerate(tasks): - gold = str(row.get("gold_answer", row.get("answer_key", "")) or "").strip() - if not gold: - empty_gold.append(idx) - if dup_task_ids or empty_task_ids or empty_doc_ids: - anomalies.append("DUPLICATE_OR_MISSING_SAMPLE") - if empty_gold: - anomalies.append("EMPTY_GOLD_OR_LABEL") - report["checks"]["duplicate_task_ids"] = dup_task_ids[:20] - report["checks"]["empty_task_id_rows"] = empty_task_ids[:20] - report["checks"]["empty_doc_id_rows"] = empty_doc_ids[:20] - report["checks"]["empty_gold_rows"] = empty_gold[:20] - report["dataset_fingerprint"] = fingerprint_paths([manifest_path, tasks_path, docs_path]) - report["checks"]["counts_consistent"] = not {"DATA_COUNT_MISMATCH", "FULL_SLICE_MIXED"} & set(anomalies) - report["anomaly_codes"] = sorted(set(anomalies)) - return report, sorted(set(anomalies)) - - -def _find_latest_run( - runs_dir: Path, - *, - workload_id: str, - variant_id: str, - repeat_idx: int, - model_id: str = "", - track_id: str = "", -) -> Path | None: - candidates: list[tuple[float, Path]] = [] - for run_dir in runs_dir.iterdir(): - if not run_dir.is_dir(): - continue - manifest_path = run_dir / "manifest.json" - if not manifest_path.exists(): - continue - manifest = _read_json(manifest_path) - if str(manifest.get("workload_id", "")) != workload_id: - continue - if str(manifest.get("variant_id", "")) != variant_id: - continue - if int(manifest.get("repeat_idx", 0)) != int(repeat_idx): - continue - if model_id and str(manifest.get("model_id", "")) != model_id: - continue - if track_id and infer_track_id_from_manifest(manifest) != track_id: - continue - candidates.append((float(run_dir.stat().st_mtime), run_dir)) - if not candidates: - return None - candidates.sort(key=lambda item: item[0], reverse=True) - return candidates[0][1] - - -def _find_previous_matching_run( - runs_dir: Path, - *, - current_run_dir: Path, - workload_id: str, - variant_id: str, - repeat_idx: int, - dataset_id: str, - track_id: str, -) -> Path | None: - candidates: list[tuple[float, Path]] = [] - for run_dir in runs_dir.iterdir(): - if not run_dir.is_dir() or run_dir.resolve() == current_run_dir.resolve(): - continue - manifest_path = run_dir / "manifest.json" - if not manifest_path.exists(): - continue - manifest = _read_json(manifest_path) - if str(manifest.get("workload_id", "")) != workload_id: - continue - if str(manifest.get("variant_id", "")) != variant_id: - continue - if int(manifest.get("repeat_idx", 0)) != int(repeat_idx): - continue - if normalize_dataset_id(infer_dataset_id_from_manifest(manifest)) != normalize_dataset_id(dataset_id): - continue - if infer_track_id_from_manifest(manifest) != track_id: - continue - candidates.append((float(run_dir.stat().st_mtime), run_dir)) - if not candidates: - return None - candidates.sort(key=lambda item: item[0], reverse=True) - return candidates[0][1] - - -def _protocol_fingerprint(manifest: dict[str, Any]) -> str: - extra = manifest.get("extra") if isinstance(manifest.get("extra"), dict) else {} - model_lock = extra.get("model_lock") if isinstance(extra.get("model_lock"), dict) else {} - payload = { - "model_base_url": manifest.get("model_base_url"), - "model_id": manifest.get("model_id"), - "expected_base_url": model_lock.get("expected_base_url"), - "expected_model_id": model_lock.get("expected_model_id"), - "prompt_mode": extra.get("prompt_mode"), - "qa_prompt_profile": extra.get("qa_prompt_profile"), - "qa_support_mode": extra.get("qa_support_mode"), - "session_scope": extra.get("session_scope"), - "max_evidence_chars": extra.get("max_evidence_chars"), - "context_top_k": extra.get("context_top_k"), - "context_item_max_chars": extra.get("context_item_max_chars"), - "answer_format_mode": extra.get("answer_format_mode"), - "structured_evidence": extra.get("structured_evidence"), - "track_id": infer_track_id_from_manifest(manifest), - } - return sha1_text(json.dumps(payload, sort_keys=True, ensure_ascii=False)) - - -def _audit_run_outputs( - run_dir: Path, - *, - expected_prompt_mode: str = "", - expected_qa_profile: str = "", - expected_session_scope: str = "", - expected_model_id: str = "", - expected_base_url: str = "", - expected_track_id: str = "", - invalid_rate_threshold: float = 0.15, -) -> tuple[dict[str, Any], list[str], bool]: - manifest_path = run_dir / "manifest.json" - summary_path = run_dir / "summary.json" - eventlog_path = run_dir / "eventlog.jsonl" - report: dict[str, Any] = { - "run_dir": run_dir.as_posix(), - "manifest_path": manifest_path.as_posix(), - "summary_path": summary_path.as_posix(), - "eventlog_path": eventlog_path.as_posix(), - "checks": {}, - "anomaly_codes": [], - } - anomalies: list[str] = [] - if not manifest_path.exists(): - anomalies.append("SUMMARY_CHAIN_INCONSISTENT") - report["checks"]["manifest_present"] = False - return report, anomalies, False - manifest = _read_json(manifest_path) - report["run_id"] = str(manifest.get("run_id", run_dir.name)) - report["branch_key"] = as_branch_dict(infer_branch_key_from_manifest(manifest)) - report["checks"]["manifest_present"] = True - if not summary_path.exists(): - anomalies.append("SUMMARY_CHAIN_INCONSISTENT") - report["checks"]["summary_present"] = False - return report, sorted(set(anomalies)), False - report["checks"]["summary_present"] = True - summary = _read_json(summary_path) - report["checks"]["eventlog_present"] = bool(eventlog_path.exists()) - if not eventlog_path.exists(): - anomalies.append("EVENTLOG_MISSING") - extra = manifest.get("extra") if isinstance(manifest.get("extra"), dict) else {} - actual_prompt_mode = str(extra.get("prompt_mode", "")).strip() - actual_qa_profile = str(extra.get("qa_prompt_profile", "")).strip() - actual_session_scope = str(extra.get("session_scope", "")).strip() - actual_track_id = infer_track_id_from_manifest(manifest) - model_lock = extra.get("model_lock") if isinstance(extra.get("model_lock"), dict) else {} - actual_model_id = str(manifest.get("model_id", "")).strip() - actual_base_url = str(manifest.get("model_base_url", "")).strip().rstrip("/") - expected_model_id = str(expected_model_id or "").strip() - expected_base_url = str(expected_base_url or "").strip().rstrip("/") - if expected_prompt_mode and actual_prompt_mode != expected_prompt_mode: - anomalies.append("PROVIDER_PARAM_DRIFT") - if expected_qa_profile and actual_qa_profile != expected_qa_profile: - anomalies.append("PROMPT_PROFILE_MISMATCH") - if expected_session_scope and actual_session_scope != expected_session_scope: - anomalies.append("SESSION_SCOPE_DRIFT") - if expected_track_id and actual_track_id != expected_track_id: - anomalies.append("MIXED_TRACK_COMPARISON") - if expected_model_id and actual_model_id != expected_model_id: - anomalies.append("MODEL_LOCK_DRIFT") - if expected_base_url and actual_base_url != expected_base_url: - anomalies.append("MODEL_LOCK_DRIFT") - if model_lock: - if expected_model_id and str(model_lock.get("expected_model_id", "")).strip() != expected_model_id: - anomalies.append("MODEL_LOCK_DRIFT") - if expected_base_url and str(model_lock.get("expected_base_url", "")).strip().rstrip("/") != expected_base_url: - anomalies.append("MODEL_LOCK_DRIFT") - if str(summary.get("run_id", "")).strip() not in {"", str(manifest.get("run_id", ""))}: - anomalies.append("SUMMARY_CHAIN_INCONSISTENT") - if str(summary.get("variant_id", "")).strip() not in {"", str(manifest.get("variant_id", ""))}: - anomalies.append("SUMMARY_CHAIN_INCONSISTENT") - if str(summary.get("workload_id", "")).strip() not in {"", str(manifest.get("workload_id", ""))}: - anomalies.append("SUMMARY_CHAIN_INCONSISTENT") - invalid_rate = float(summary.get("parse_failure_rate_on_total", summary.get("parse_failure_rate", 0.0)) or 0.0) - no_answer_rate = float(summary.get("no_answer_rate_on_total", summary.get("no_answer_rate", 0.0)) or 0.0) - report["checks"]["invalid_rate"] = invalid_rate - report["checks"]["no_answer_rate"] = no_answer_rate - if invalid_rate > float(invalid_rate_threshold) or no_answer_rate > float(invalid_rate_threshold): - anomalies.append("INVALID_RATE_SPIKE") - report["summary_chain_valid"] = not bool({"SUMMARY_CHAIN_INCONSISTENT", "EVENTLOG_MISSING"} & set(anomalies)) - report["protocol_fingerprint"] = _protocol_fingerprint(manifest) - report["anomaly_codes"] = sorted(set(anomalies)) - diagnostic_only = set(anomalies) <= {"INVALID_RATE_SPIKE"} and bool(anomalies) - return report, sorted(set(anomalies)), diagnostic_only - - -def as_branch_dict(key: BranchKey) -> dict[str, Any]: - return { - "branch_id": key.branch_id, - "dataset_id": key.dataset_id, - "track_id": key.track_id, - "variant_id": key.variant_id, - "repeat_idx": int(key.repeat_idx), - } - - -def command_audit_dataset(args: argparse.Namespace) -> int: - key = BranchKey( - dataset_id=normalize_dataset_id(str(args.dataset_id)), - track_id=str(args.track_id), - variant_id=str(args.variant_id), - repeat_idx=int(args.repeat_idx), - ) - cp = ControlPlane(artifacts_root=args.artifacts_root) - cp.register_branch(key, metadata={"audit_type": "dataset"}) - report, anomalies = _resolve_dataset_audit(Path(args.base_dir), max_samples=int(args.max_samples)) - cp.write_audit_report(key, stage_id="audit_data", report=report) - dataset_manifest_path = Path(args.base_dir) / "manifest.json" - if dataset_manifest_path.exists(): - dataset_manifest = _read_json(dataset_manifest_path) - dataset_manifest = merge_control_fields( - dataset_manifest, - { - "dataset_fingerprint": report.get("dataset_fingerprint"), - }, - ) - Path(dataset_manifest_path).write_text(json.dumps(dataset_manifest, indent=2, ensure_ascii=False), encoding="utf-8") - if anomalies: - rerun_plan_id = cp.create_rerun_plan( - key, - rollback_root_stage="audit_data", - anomaly_codes=anomalies, - invalidated_run_ids=[], - why="Dataset audit failed.", - what="Rebuild prepared dataset artifacts and rerun downstream stages.", - ) - cp.record_anomaly( - key, - run_id="", - stage_id="audit_data", - anomaly_codes=anomalies, - reason="Dataset audit failed.", - diagnostic_only=False, - rerun_plan_id=rerun_plan_id, - ) - cp.record_stage_status( - key, - stage_id="audit_data", - status="invalid", - note="Dataset audit failed.", - anomaly_codes=anomalies, - rollback_root_stage="audit_data", - rerun_plan_id=rerun_plan_id, - ) - return 2 - cp.record_stage_status(key, stage_id="audit_data", status="accepted", note="Dataset audit passed.") - return 0 - - -def command_audit_run(args: argparse.Namespace) -> int: - runs_dir = Path(args.runs_dir) - run_dir = Path(args.run_dir) if args.run_dir else _find_latest_run( - runs_dir, - workload_id=str(args.workload_id), - variant_id=str(args.variant_id), - repeat_idx=int(args.repeat_idx), - model_id=str(args.model_id or ""), - track_id=str(args.track_id or ""), - ) - if run_dir is None: - raise SystemExit(f"No matching run found for workload={args.workload_id}, variant={args.variant_id}, repeat={args.repeat_idx}") - manifest = _read_json(run_dir / "manifest.json") - key = BranchKey( - dataset_id=normalize_dataset_id(str(args.dataset_id or infer_dataset_id_from_manifest(manifest))), - track_id=str(args.track_id or infer_track_id_from_manifest(manifest)), - variant_id=str(args.variant_id or manifest.get("variant_id", "")), - repeat_idx=int(args.repeat_idx if args.repeat_idx is not None else manifest.get("repeat_idx", 0)), - ) - cp = ControlPlane(artifacts_root=args.artifacts_root) - cp.register_branch(key, metadata={"audit_type": "run"}) - prepared_manifest_path = _resolve_prepared_manifest_from_run_manifest(manifest) - prepared_manifest = _read_json(prepared_manifest_path) if prepared_manifest_path and prepared_manifest_path.exists() else {} - dataset_fingerprint = str( - manifest.get("dataset_fingerprint") - or prepared_manifest.get("dataset_fingerprint") - or "" - ).strip() - report, anomalies, diagnostic_only = _audit_run_outputs( - run_dir, - expected_prompt_mode=str(args.expected_prompt_mode or ""), - expected_qa_profile=str(args.expected_qa_profile or ""), - expected_session_scope=str(args.expected_session_scope or ""), - expected_model_id=str(args.model_id or ""), - expected_base_url=str(args.base_url or ""), - expected_track_id=str(args.track_id or ""), - invalid_rate_threshold=float(args.invalid_rate_threshold), - ) - report["run_id"] = str(manifest.get("run_id", run_dir.name)) - report["dataset_fingerprint"] = dataset_fingerprint or None - cp.write_audit_report(key, stage_id=str(args.stage_id), report=report) - manifest_updates = { - "input_fingerprint": fingerprint_paths( - [ - str((manifest.get("extra") or {}).get("tasks_path", "")), - str((manifest.get("extra") or {}).get("docs_path", "")), - ] - ), - "protocol_fingerprint": report.get("protocol_fingerprint"), - "dataset_fingerprint": dataset_fingerprint or None, - "track_id": key.track_id, - "summary_chain_valid": bool(report.get("summary_chain_valid", False)), - } - if anomalies: - cp.annotate_manifest(run_dir / "manifest.json", **manifest_updates) - cp.invalidate_run( - run_dir / "manifest.json", - key=key, - stage_id=str(args.stage_id), - anomaly_codes=anomalies, - reason=f"Run audit failed at {args.stage_id}.", - diagnostic_only=diagnostic_only, - ) - return 2 - previous_run_dir = _find_previous_matching_run( - runs_dir, - current_run_dir=run_dir, - workload_id=str(manifest.get("workload_id", "")), - variant_id=str(manifest.get("variant_id", "")), - repeat_idx=int(manifest.get("repeat_idx", 0)), - dataset_id=key.dataset_id, - track_id=key.track_id, - ) - supersedes_run_id = "" - if previous_run_dir is not None: - previous_manifest = _read_json(previous_run_dir / "manifest.json") - supersedes_run_id = str(previous_manifest.get("run_id", previous_run_dir.name)) - cp.annotate_manifest( - previous_run_dir / "manifest.json", - acceptance_status="superseded", - superseded_by_run_id=str(manifest.get("run_id", run_dir.name)), - ) - cp.annotate_manifest( - run_dir / "manifest.json", - **manifest_updates, - acceptance_status="accepted", - diagnostic_only=False, - invalid_reason_codes=[], - anomaly_classification=[], - supersedes_run_id=(supersedes_run_id or None), - superseded_by_run_id=None, - ) - cp.record_stage_status( - key, - stage_id=str(args.stage_id), - status="accepted", - run_id=str(manifest.get("run_id", run_dir.name)), - note="Run audit passed.", - ) - return 0 - - -def command_audit_aggregate(args: argparse.Namespace) -> int: - runs_dir = Path(args.runs_dir) - cp = ControlPlane(artifacts_root=args.artifacts_root) - grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) - run_dirs: dict[tuple[str, str], list[Path]] = defaultdict(list) - for run_dir in runs_dir.iterdir(): - if not run_dir.is_dir(): - continue - manifest_path = run_dir / "manifest.json" - summary_path = run_dir / "summary.json" - if not manifest_path.exists() or not summary_path.exists(): - continue - manifest = _read_json(manifest_path) - if not accepted_for_official_reporting(manifest): - continue - if str(args.workload_id or "").strip(): - if str(manifest.get("workload_id", "") or "") != str(args.workload_id): - continue - actual_dataset_id = normalize_dataset_id(str(infer_dataset_id_from_manifest(manifest))) - actual_track_id = str(infer_track_id_from_manifest(manifest)) - if str(args.dataset_id or "").strip() and actual_dataset_id != normalize_dataset_id(str(args.dataset_id)): - continue - if str(args.track_id or "").strip() and actual_track_id != str(args.track_id): - continue - key = (actual_dataset_id, actual_track_id) - grouped[key].append( - { - "run_dir": run_dir, - "manifest": manifest, - "summary": _read_json(summary_path), - } - ) - run_dirs[key].append(run_dir) - anomalies_found = 0 - for (dataset_id, track_id), items in grouped.items(): - by_variant: dict[str, list[dict[str, Any]]] = defaultdict(list) - prompt_modes = {str((item["manifest"].get("extra") or {}).get("prompt_mode", "")) for item in items} - protocol_fingerprints = {_protocol_fingerprint(item["manifest"]) for item in items} - if len([x for x in prompt_modes if x]) > 1 or len(protocol_fingerprints) > 1: - anomalies_found += 1 - for item in items: - manifest = item["manifest"] - branch = BranchKey( - dataset_id=dataset_id, - track_id=track_id, - variant_id=str(manifest.get("variant_id", "")), - repeat_idx=int(manifest.get("repeat_idx", 0)), - ) - cp.invalidate_run( - item["run_dir"] / "manifest.json", - key=branch, - stage_id="aggregate", - anomaly_codes=["MIXED_TRACK_COMPARISON", "PROVIDER_PARAM_DRIFT"], - reason="Aggregate audit found protocol drift inside one dataset/track group.", - diagnostic_only=False, - ) - continue - for item in items: - by_variant[str(item["manifest"].get("variant_id", ""))].append(item) - base_rows = by_variant.get("B1") or by_variant.get("B0") or [] - if not base_rows: - continue - base_scores = { - int(row["manifest"].get("repeat_idx", 0)): float(row["summary"].get("benchmark_score_on_total", 0.0)) - for row in base_rows - } - for variant in ("V1", "F1", "F2"): - variant_rows = by_variant.get(variant, []) - if not variant_rows: - continue - deltas: list[float] = [] - for row in variant_rows: - repeat_idx = int(row["manifest"].get("repeat_idx", 0)) - if repeat_idx not in base_scores: - continue - delta = float(row["summary"].get("benchmark_score_on_total", 0.0)) - float(base_scores[repeat_idx]) - deltas.append(delta) - if len(deltas) >= 2 and any(delta > 0 for delta in deltas) and any(delta < 0 for delta in deltas): - anomalies_found += 1 - for row in variant_rows: - manifest = row["manifest"] - branch = BranchKey( - dataset_id=dataset_id, - track_id=track_id, - variant_id=str(manifest.get("variant_id", "")), - repeat_idx=int(manifest.get("repeat_idx", 0)), - ) - cp.invalidate_run( - row["run_dir"] / "manifest.json", - key=branch, - stage_id="aggregate", - anomaly_codes=["REPEAT_DIRECTION_CONFLICT"], - reason="Aggregate audit found conflicting delta directions across repeats.", - diagnostic_only=True, - ) - return 0 if anomalies_found == 0 else 2 - - -def build_parser() -> argparse.ArgumentParser: - ap = argparse.ArgumentParser() - sub = ap.add_subparsers(dest="cmd", required=True) - - ds = sub.add_parser("dataset") - ds.add_argument("--base-dir", type=str, required=True) - ds.add_argument("--dataset-id", type=str, required=True) - ds.add_argument("--track-id", type=str, required=True) - ds.add_argument("--variant-id", type=str, required=True) - ds.add_argument("--repeat-idx", type=int, required=True) - ds.add_argument("--max-samples", type=int, default=0) - ds.add_argument("--artifacts-root", type=str, default="artifacts") - - run = sub.add_parser("run") - run.add_argument("--runs-dir", type=str, default="runs") - run.add_argument("--run-dir", type=str, default="") - run.add_argument("--dataset-id", type=str, default="") - run.add_argument("--track-id", type=str, default="") - run.add_argument("--variant-id", type=str, required=True) - run.add_argument("--repeat-idx", type=int, required=True) - run.add_argument("--workload-id", type=str, required=True) - run.add_argument("--stage-id", type=str, required=True, choices=["smoke", "pilot", "full", "aggregate", "promote"]) - run.add_argument("--model-id", type=str, default="") - run.add_argument("--base-url", type=str, default="") - run.add_argument("--expected-prompt-mode", type=str, default="") - run.add_argument("--expected-qa-profile", type=str, default="") - run.add_argument("--expected-session-scope", type=str, default="") - run.add_argument("--invalid-rate-threshold", type=float, default=0.15) - run.add_argument("--artifacts-root", type=str, default="artifacts") - - agg = sub.add_parser("aggregate") - agg.add_argument("--runs-dir", type=str, default="runs") - agg.add_argument("--dataset-id", type=str, default="") - agg.add_argument("--track-id", type=str, default="") - agg.add_argument("--workload-id", type=str, default="") - agg.add_argument("--artifacts-root", type=str, default="artifacts") - return ap - - -def main() -> None: - args = build_parser().parse_args() - if args.cmd == "dataset": - raise SystemExit(command_audit_dataset(args)) - if args.cmd == "run": - raise SystemExit(command_audit_run(args)) - if args.cmd == "aggregate": - raise SystemExit(command_audit_aggregate(args)) - raise SystemExit(2) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/analysis/public_track_design_audit.py b/projects/memlab/memlab/analysis/public_track_design_audit.py deleted file mode 100644 index 267a036..0000000 --- a/projects/memlab/memlab/analysis/public_track_design_audit.py +++ /dev/null @@ -1,277 +0,0 @@ -from __future__ import annotations - -import argparse -import json -from collections import Counter -from pathlib import Path -from typing import Any - -from memlab.pipeline.control_plane import BranchKey, ControlPlane, normalize_dataset_id - - -def _read_json(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) - - -def _read_jsonl(path: Path) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - with path.open("r", encoding="utf-8") as f: - for line in f: - raw = line.strip() - if not raw: - continue - rows.append(json.loads(raw)) - return rows - - -def _infer_track_goal(track_goal: str, *, track_id: str, experiment_track: str) -> str: - value = str(track_goal or "auto").strip().lower() - if value in {"standard", "memory_sensitive"}: - return value - track_text = f"{track_id}::{experiment_track}".lower() - if "track-m" in track_text or "memory_exposure" in track_text: - return "memory_sensitive" - return "standard" - - -def _safe_hist(counter: Counter[str]) -> dict[str, int]: - return {str(key): int(counter[key]) for key in sorted(counter.keys()) if str(key)} - - -def resolve_design_audit( - *, - base_dir: Path, - dataset_id: str, - track_id: str, - experiment_track: str, - prompt_mode: str, - qa_support_mode: str, - qa_prompt_profile: str, - track_goal: str, - min_memory_needed_ratio: float, -) -> tuple[dict[str, Any], list[str]]: - manifest_path = base_dir / "manifest.json" - tasks_path = base_dir / "tasks_internal.jsonl" - docs_path = base_dir / "docs_internal.jsonl" - report: dict[str, Any] = { - "base_dir": base_dir.as_posix(), - "dataset_id": normalize_dataset_id(dataset_id or base_dir.name), - "track_id": str(track_id or ""), - "experiment_track": str(experiment_track or ""), - "prompt_mode": str(prompt_mode or ""), - "qa_support_mode": str(qa_support_mode or ""), - "qa_prompt_profile": str(qa_prompt_profile or ""), - "track_goal": _infer_track_goal( - track_goal, - track_id=str(track_id or ""), - experiment_track=str(experiment_track or ""), - ), - "checks": {}, - "anomaly_codes": [], - } - anomalies: list[str] = [] - if not manifest_path.exists() or not tasks_path.exists() or not docs_path.exists(): - report["checks"]["required_files_present"] = False - anomalies.append("DATA_COUNT_MISMATCH") - report["anomaly_codes"] = sorted(set(anomalies)) - return report, report["anomaly_codes"] - report["checks"]["required_files_present"] = True - manifest = _read_json(manifest_path) - tasks = _read_jsonl(tasks_path) - docs = _read_jsonl(docs_path) - - memory_needed_true = 0 - memory_needed_false = 0 - metric_hist: Counter[str] = Counter() - task_profile_hist: Counter[str] = Counter() - question_type_hist: Counter[str] = Counter() - for row in tasks: - metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {} - if bool(metadata.get("memory_needed")): - memory_needed_true += 1 - else: - memory_needed_false += 1 - metric_hist[str(metadata.get("answer_metric_hint", "")).strip()] += 1 - task_profile_hist[str(metadata.get("qa_prompt_profile", "")).strip()] += 1 - question_type_hist[str(metadata.get("question_type", metadata.get("task_type", ""))).strip()] += 1 - - total = len(tasks) - ratio = float(memory_needed_true) / float(total) if total else 0.0 - prepared_count = int(manifest.get("sample_count", total) or total) - report["manifest_dataset"] = str(manifest.get("dataset", "")) - report["prepared_sample_count"] = int(prepared_count) - report["task_count"] = int(total) - report["doc_count"] = int(len(docs)) - report["memory_needed_true"] = int(memory_needed_true) - report["memory_needed_false"] = int(memory_needed_false) - report["memory_needed_ratio"] = ratio - report["answer_metric_histogram"] = _safe_hist(metric_hist) - report["task_profile_histogram"] = _safe_hist(task_profile_hist) - report["question_type_histogram"] = _safe_hist(question_type_hist) - report["checks"]["counts_consistent"] = prepared_count == len(tasks) == len(docs) - report["checks"]["memory_needed_ratio_meets_gate"] = ratio >= float(min_memory_needed_ratio) - report["checks"]["qa_support_mode_safe_for_track_goal"] = True - report["checks"]["prompt_mode_safe_for_track_goal"] = True - - goal = str(report["track_goal"]) - if prepared_count != len(tasks) or prepared_count != len(docs): - anomalies.append("DATA_COUNT_MISMATCH") - if goal == "memory_sensitive": - if ratio < float(min_memory_needed_ratio): - anomalies.append("NON_MEMORY_DEPENDENT_DATASET") - if str(prompt_mode or "").strip() != "memory_beneficial": - anomalies.append("TRACK_GOAL_MISMATCH") - report["checks"]["prompt_mode_safe_for_track_goal"] = False - if str(qa_support_mode or "").strip() != "none": - anomalies.append("DIRECT_EVIDENCE_LEAKAGE") - report["checks"]["qa_support_mode_safe_for_track_goal"] = False - report["anomaly_codes"] = sorted(set(anomalies)) - return report, report["anomaly_codes"] - - -def _write_md(path: Path, report: dict[str, Any]) -> None: - lines = [ - "# Public Track Design Audit", - "", - f"- Dataset: `{report.get('dataset_id', '')}`", - f"- Track: `{report.get('track_id', '')}`", - f"- Experiment track: `{report.get('experiment_track', '')}`", - f"- Track goal: `{report.get('track_goal', '')}`", - f"- Prompt mode: `{report.get('prompt_mode', '')}`", - f"- QA support mode: `{report.get('qa_support_mode', '')}`", - f"- QA prompt profile: `{report.get('qa_prompt_profile', '')}`", - f"- Prepared base dir: `{report.get('base_dir', '')}`", - "", - "## Counts", - "", - f"- Prepared sample count: `{report.get('prepared_sample_count', 0)}`", - f"- Task count: `{report.get('task_count', 0)}`", - f"- Doc count: `{report.get('doc_count', 0)}`", - f"- Memory-needed ratio: `{float(report.get('memory_needed_ratio', 0.0)):.3f}`", - "", - "## Histograms", - "", - f"- answer_metric_histogram: `{json.dumps(report.get('answer_metric_histogram', {}), ensure_ascii=False)}`", - f"- task_profile_histogram: `{json.dumps(report.get('task_profile_histogram', {}), ensure_ascii=False)}`", - f"- question_type_histogram: `{json.dumps(report.get('question_type_histogram', {}), ensure_ascii=False)}`", - "", - "## Checks", - "", - ] - checks = report.get("checks", {}) if isinstance(report.get("checks"), dict) else {} - for key in sorted(checks.keys()): - lines.append(f"- {key}: `{checks[key]}`") - lines.extend(["", "## Anomalies", ""]) - anomalies = report.get("anomaly_codes", []) - if anomalies: - for code in anomalies: - lines.append(f"- `{code}`") - else: - lines.append("- `none`") - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--base-dir", type=str, required=True) - ap.add_argument("--dataset-id", type=str, required=True) - ap.add_argument("--track-id", type=str, required=True) - ap.add_argument("--variant-id", type=str, required=True) - ap.add_argument("--repeat-idx", type=int, required=True) - ap.add_argument("--experiment-track", type=str, default="mainline") - ap.add_argument("--prompt-mode", type=str, required=True) - ap.add_argument("--qa-support-mode", type=str, required=True) - ap.add_argument("--qa-prompt-profile", type=str, default="default") - ap.add_argument("--track-goal", type=str, default="auto", choices=("auto", "standard", "memory_sensitive")) - ap.add_argument("--min-memory-needed-ratio", type=float, default=0.5) - ap.add_argument("--artifacts-root", type=str, default="artifacts") - ap.add_argument("--output-json", type=str, default="") - ap.add_argument("--output-md", type=str, default="") - args = ap.parse_args() - - key = BranchKey( - dataset_id=normalize_dataset_id(str(args.dataset_id)), - track_id=str(args.track_id), - variant_id=str(args.variant_id), - repeat_idx=int(args.repeat_idx), - ) - cp = ControlPlane(artifacts_root=args.artifacts_root) - cp.register_branch(key) - - report, anomalies = resolve_design_audit( - base_dir=Path(args.base_dir), - dataset_id=str(args.dataset_id), - track_id=str(args.track_id), - experiment_track=str(args.experiment_track), - prompt_mode=str(args.prompt_mode), - qa_support_mode=str(args.qa_support_mode), - qa_prompt_profile=str(args.qa_prompt_profile), - track_goal=str(args.track_goal), - min_memory_needed_ratio=float(args.min_memory_needed_ratio), - ) - report["branch_key"] = { - "branch_id": key.branch_id, - "dataset_id": key.dataset_id, - "track_id": key.track_id, - "variant_id": key.variant_id, - "repeat_idx": int(key.repeat_idx), - } - report["acceptance_status"] = "accepted" if not anomalies else "invalid" - cp.write_audit_report(key, stage_id="audit_protocol", report=report) - - if str(args.output_json).strip(): - out_json = Path(str(args.output_json)) - out_json.parent.mkdir(parents=True, exist_ok=True) - out_json.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") - if str(args.output_md).strip(): - _write_md(Path(str(args.output_md)), report) - - if anomalies: - reason = ( - "Public track design audit failed: " - + ", ".join(sorted(set(anomalies))) - + ". Fix dataset selection or protocol before smoke." - ) - rerun_plan_id = cp.create_rerun_plan( - key, - rollback_root_stage="audit_protocol", - anomaly_codes=anomalies, - invalidated_run_ids=[], - why=reason, - what="Re-run protocol audit after fixing dataset suitability or track prompt configuration.", - ) - cp.record_anomaly( - key, - run_id=f"audit::{key.branch_id}", - stage_id="audit_protocol", - anomaly_codes=anomalies, - reason=reason, - diagnostic_only=False, - rerun_plan_id=rerun_plan_id, - ) - cp.record_stage_status( - key, - stage_id="audit_protocol", - status="invalid", - run_id="", - note=reason, - anomaly_codes=anomalies, - rollback_root_stage="audit_protocol", - rerun_plan_id=rerun_plan_id, - ) - print(json.dumps(report, ensure_ascii=False)) - raise SystemExit(2) - - cp.record_stage_status( - key, - stage_id="audit_protocol", - status="accepted", - run_id="", - note="Public track design audit passed.", - ) - print(json.dumps(report, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/analysis/public_track_gate.py b/projects/memlab/memlab/analysis/public_track_gate.py deleted file mode 100644 index bb37436..0000000 --- a/projects/memlab/memlab/analysis/public_track_gate.py +++ /dev/null @@ -1,463 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import math -from collections import defaultdict -from pathlib import Path -from typing import Any - -from memlab.pipeline.control_plane import ( - accepted_for_official_reporting, - infer_dataset_id_from_manifest, - infer_track_id_from_manifest, -) - - -def _read_json(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) - - -def _safe_float(value: Any, default: float = 0.0) -> float: - try: - return float(value) - except Exception: - return default - - -def _mean(values: list[float]) -> float: - return sum(values) / max(1, len(values)) - - -def _std(values: list[float]) -> float: - n = len(values) - if n <= 1: - return 0.0 - m = _mean(values) - return math.sqrt(sum((x - m) ** 2 for x in values) / (n - 1)) - - -def _parse_csv_list(raw: str) -> list[str]: - return [token.strip() for token in str(raw).split(",") if token.strip()] - - -def _resolve_baseline_variant(explicit_variant: str, by_variant: dict[str, list[dict[str, Any]]]) -> str: - variant = str(explicit_variant or "").strip() - if variant: - return variant - if "B1" in by_variant: - return "B1" - if "B0" in by_variant: - return "B0" - return "" - - -def _collect_rows( - runs_dir: Path, - *, - workload_id: str, - dataset_id: str, - track_id: str, - model_id: str, -) -> list[dict[str, Any]]: - rows_by_key: dict[tuple[str, int], dict[str, Any]] = {} - updated_ts_by_key: dict[tuple[str, int], float] = {} - for run_dir in runs_dir.iterdir(): - if not run_dir.is_dir(): - continue - manifest_path = run_dir / "manifest.json" - summary_path = run_dir / "summary.json" - if not manifest_path.exists() or not summary_path.exists(): - continue - manifest = _read_json(manifest_path) - summary = _read_json(summary_path) - if str(manifest.get("workload_id", "")) != workload_id: - continue - if dataset_id and infer_dataset_id_from_manifest(manifest) != dataset_id: - continue - if track_id and infer_track_id_from_manifest(manifest) != track_id: - continue - if model_id and str(manifest.get("model_id", "")) != model_id: - continue - if not accepted_for_official_reporting(manifest): - continue - extra = manifest.get("extra") if isinstance(manifest.get("extra"), dict) else {} - row = { - "run_id": str(manifest.get("run_id", run_dir.name)), - "variant": str(manifest.get("variant_id", "")), - "repeat_idx": int(manifest.get("repeat_idx", 0)), - "prompt_mode": str(extra.get("prompt_mode", "")), - "qa_prompt_profile": str(extra.get("qa_prompt_profile", "")), - "qa_support_mode": str(extra.get("qa_support_mode", "")), - "max_evidence_chars": extra.get("max_evidence_chars"), - "context_top_k": extra.get("context_top_k"), - "context_item_max_chars": extra.get("context_item_max_chars"), - "answer_format_mode": str(extra.get("answer_format_mode", "")), - "benchmark_metric": str(summary.get("benchmark_metric", "")), - "benchmark_score_on_total": _safe_float(summary.get("benchmark_score_on_total", 0.0)), - "strict_accuracy_on_total": _safe_float(summary.get("strict_accuracy_on_total", 0.0)), - "partial_accuracy_on_total": _safe_float(summary.get("partial_accuracy_on_total", 0.0)), - "parse_failure_rate_on_total": _safe_float(summary.get("parse_failure_rate_on_total", 0.0)), - "no_answer_rate_on_total": _safe_float(summary.get("no_answer_rate_on_total", 0.0)), - "success_rate": _safe_float(summary.get("success_rate", 0.0)), - } - key = (row["variant"], row["repeat_idx"]) - updated_ts = float(run_dir.stat().st_mtime) - if updated_ts >= updated_ts_by_key.get(key, -1.0): - updated_ts_by_key[key] = updated_ts - rows_by_key[key] = row - rows = list(rows_by_key.values()) - rows.sort(key=lambda item: (str(item["variant"]), int(item["repeat_idx"]))) - return rows - - -def _aggregate_variant(rows: list[dict[str, Any]], variant: str) -> dict[str, Any]: - sub = [row for row in rows if str(row.get("variant", "")) == variant] - metrics = {} - for key in ( - "benchmark_score_on_total", - "strict_accuracy_on_total", - "partial_accuracy_on_total", - "parse_failure_rate_on_total", - "no_answer_rate_on_total", - "success_rate", - ): - values = [_safe_float(row.get(key, 0.0)) for row in sub] - metrics[key] = {"mean": _mean(values), "std": _std(values), "runs": len(values)} - return {"variant": variant, "runs": len(sub), "metrics": metrics} - - -def _build_result( - rows: list[dict[str, Any]], - *, - dataset: str, - track_id: str, - workload_id: str, - stage: str, - expected_variants: list[str], - expected_repeats: list[int], - invalid_rate_threshold: float, - std_threshold: float, - baseline_variant: str = "", - must_beat_baseline_variants: list[str] | None = None, - min_mean_delta: float = 0.0, - min_repeat_delta: float = 0.0, - baseline_lower_bound: float | None = None, - baseline_upper_bound: float | None = None, -) -> dict[str, Any]: - by_variant: dict[str, list[dict[str, Any]]] = defaultdict(list) - prompt_modes = sorted({str(row.get("prompt_mode", "")) for row in rows if str(row.get("prompt_mode", "")).strip()}) - protocol_signatures = sorted( - { - json.dumps( - { - "prompt_mode": row.get("prompt_mode"), - "qa_prompt_profile": row.get("qa_prompt_profile"), - "qa_support_mode": row.get("qa_support_mode"), - "max_evidence_chars": row.get("max_evidence_chars"), - "context_top_k": row.get("context_top_k"), - "context_item_max_chars": row.get("context_item_max_chars"), - "answer_format_mode": row.get("answer_format_mode"), - }, - sort_keys=True, - ensure_ascii=False, - ) - for row in rows - } - ) - for row in rows: - by_variant[str(row.get("variant", ""))].append(row) - - missing_variants = [variant for variant in expected_variants if variant not in by_variant] - missing_repeats: dict[str, list[int]] = {} - invalid_rate_spikes: dict[str, dict[str, float]] = {} - variance_warnings: dict[str, float] = {} - direction_conflicts: dict[str, list[float]] = {} - ordering_reversals: dict[str, dict[str, Any]] = {} - baseline_band_violation: dict[str, Any] = {} - variant_stats = [_aggregate_variant(rows, variant) for variant in expected_variants if variant in by_variant] - - for variant in expected_variants: - sub = by_variant.get(variant, []) - if not sub: - continue - observed = sorted({int(row.get("repeat_idx", 0)) for row in sub}) - expected_missing = [repeat for repeat in expected_repeats if repeat not in observed] - if expected_missing: - missing_repeats[variant] = expected_missing - parse_rate = _mean([_safe_float(row.get("parse_failure_rate_on_total", 0.0)) for row in sub]) - no_answer_rate = _mean([_safe_float(row.get("no_answer_rate_on_total", 0.0)) for row in sub]) - if parse_rate > invalid_rate_threshold or no_answer_rate > invalid_rate_threshold: - invalid_rate_spikes[variant] = { - "parse_failure_rate_on_total_mean": parse_rate, - "no_answer_rate_on_total_mean": no_answer_rate, - } - score_std = _std([_safe_float(row.get("benchmark_score_on_total", 0.0)) for row in sub]) - if len(sub) >= 2 and score_std > std_threshold: - variance_warnings[variant] = score_std - - baseline_variant = _resolve_baseline_variant(baseline_variant, by_variant) - if baseline_variant: - base_by_repeat = { - int(row.get("repeat_idx", 0)): _safe_float(row.get("benchmark_score_on_total", 0.0)) - for row in by_variant.get(baseline_variant, []) - } - base_scores = [_safe_float(row.get("benchmark_score_on_total", 0.0)) for row in by_variant.get(baseline_variant, [])] - baseline_mean = _mean(base_scores) - if baseline_lower_bound is not None and baseline_mean < float(baseline_lower_bound): - baseline_band_violation = { - "variant": baseline_variant, - "mean_score": baseline_mean, - "lower_bound": float(baseline_lower_bound), - "upper_bound": (None if baseline_upper_bound is None else float(baseline_upper_bound)), - } - if baseline_upper_bound is not None and baseline_mean > float(baseline_upper_bound): - baseline_band_violation = { - "variant": baseline_variant, - "mean_score": baseline_mean, - "lower_bound": (None if baseline_lower_bound is None else float(baseline_lower_bound)), - "upper_bound": float(baseline_upper_bound), - } - for variant in ("V1", "F1", "F2"): - deltas: list[float] = [] - for row in by_variant.get(variant, []): - repeat_idx = int(row.get("repeat_idx", 0)) - if repeat_idx not in base_by_repeat: - continue - deltas.append(_safe_float(row.get("benchmark_score_on_total", 0.0)) - base_by_repeat[repeat_idx]) - if len(deltas) >= 2 and any(delta > 0 for delta in deltas) and any(delta < 0 for delta in deltas): - direction_conflicts[variant] = deltas - monitored_variants = [ - variant - for variant in (must_beat_baseline_variants or []) - if variant in by_variant and variant != baseline_variant - ] - base_mean = _mean([_safe_float(row.get("benchmark_score_on_total", 0.0)) for row in by_variant.get(baseline_variant, [])]) - for variant in monitored_variants: - variant_scores = [_safe_float(row.get("benchmark_score_on_total", 0.0)) for row in by_variant.get(variant, [])] - variant_mean = _mean(variant_scores) - paired_deltas: list[float] = [] - for row in by_variant.get(variant, []): - repeat_idx = int(row.get("repeat_idx", 0)) - if repeat_idx not in base_by_repeat: - continue - paired_deltas.append(_safe_float(row.get("benchmark_score_on_total", 0.0)) - base_by_repeat[repeat_idx]) - reasons: list[str] = [] - if variant_mean < (base_mean + float(min_mean_delta)): - reasons.append("mean_delta_below_threshold") - if paired_deltas and any(delta < float(min_repeat_delta) for delta in paired_deltas): - reasons.append("paired_repeat_delta_below_threshold") - if reasons: - ordering_reversals[variant] = { - "baseline_variant": baseline_variant, - "baseline_mean": base_mean, - "variant_mean": variant_mean, - "mean_delta": variant_mean - base_mean, - "paired_repeat_deltas": paired_deltas, - "min_mean_delta": float(min_mean_delta), - "min_repeat_delta": float(min_repeat_delta), - "reasons": reasons, - } - - anomaly_codes: list[str] = [] - if len(prompt_modes) > 1: - anomaly_codes.append("MIXED_TRACK_COMPARISON") - if len(protocol_signatures) > 1: - anomaly_codes.append("PROVIDER_PARAM_DRIFT") - if missing_variants or missing_repeats: - anomaly_codes.append("RAW_RUN_TABLE_MISMATCH") - if invalid_rate_spikes: - anomaly_codes.append("INVALID_RATE_SPIKE") - if direction_conflicts: - anomaly_codes.append("REPEAT_DIRECTION_CONFLICT") - if variance_warnings: - anomaly_codes.append("REPEAT_VARIANCE_ABNORMAL") - if baseline_band_violation: - anomaly_codes.append("BASELINE_OUT_OF_BAND") - if ordering_reversals: - anomaly_codes.append("UNEXPLAINED_ORDERING_REVERSAL") - - hard_block_codes = { - "RAW_RUN_TABLE_MISMATCH", - "BASELINE_OUT_OF_BAND", - "UNEXPLAINED_ORDERING_REVERSAL", - } - if missing_variants or missing_repeats or any(code in hard_block_codes for code in anomaly_codes): - verdict = "invalid" - elif anomaly_codes: - verdict = "diagnostic_only" - else: - verdict = "accepted" - - return { - "dataset": dataset, - "track_id": track_id, - "workload_id": workload_id, - "stage": stage, - "expected_variants": expected_variants, - "expected_repeats": expected_repeats, - "rows": len(rows), - "prompt_modes": prompt_modes, - "protocol_signatures": protocol_signatures, - "baseline_variant": baseline_variant, - "variant_stats": variant_stats, - "missing_variants": missing_variants, - "missing_repeats": missing_repeats, - "invalid_rate_spikes": invalid_rate_spikes, - "variance_warnings": variance_warnings, - "direction_conflicts": direction_conflicts, - "baseline_band_violation": baseline_band_violation, - "ordering_reversals": ordering_reversals, - "anomaly_codes": sorted(set(anomaly_codes)), - "verdict": verdict, - } - - -def _write_md(path: Path, result: dict[str, Any]) -> None: - lines: list[str] = [] - lines.append(f"# {result['dataset']} {str(result['stage']).upper()} Track Gate") - lines.append("") - lines.append(f"- Dataset: `{result['dataset']}`") - lines.append(f"- Track: `{result['track_id']}`") - lines.append(f"- Workload: `{result['workload_id']}`") - lines.append(f"- Verdict: `{result['verdict']}`") - lines.append(f"- Prompt modes observed: `{', '.join(result.get('prompt_modes', []))}`") - lines.append(f"- Protocol signatures observed: `{len(result.get('protocol_signatures', []))}`") - lines.append(f"- Baseline variant: `{result.get('baseline_variant', '')}`") - lines.append("") - lines.append("## Variant Aggregate") - lines.append("") - lines.append("| variant | runs | benchmark_score_on_total mean+/-std | strict_acc_total mean+/-std | partial_acc_total mean+/-std | parse_fail_total mean | no_answer_total mean | success_rate mean |") - lines.append("|---|---:|---:|---:|---:|---:|---:|---:|") - for entry in result.get("variant_stats", []): - metrics = entry.get("metrics", {}) - lines.append( - "| {0} | {1} | {2:.4f}+/-{3:.4f} | {4:.4f}+/-{5:.4f} | {6:.4f}+/-{7:.4f} | {8:.4f} | {9:.4f} | {10:.4f} |".format( - str(entry.get("variant", "")), - int(entry.get("runs", 0)), - _safe_float(((metrics.get("benchmark_score_on_total") or {}).get("mean", 0.0))), - _safe_float(((metrics.get("benchmark_score_on_total") or {}).get("std", 0.0))), - _safe_float(((metrics.get("strict_accuracy_on_total") or {}).get("mean", 0.0))), - _safe_float(((metrics.get("strict_accuracy_on_total") or {}).get("std", 0.0))), - _safe_float(((metrics.get("partial_accuracy_on_total") or {}).get("mean", 0.0))), - _safe_float(((metrics.get("partial_accuracy_on_total") or {}).get("std", 0.0))), - _safe_float(((metrics.get("parse_failure_rate_on_total") or {}).get("mean", 0.0))), - _safe_float(((metrics.get("no_answer_rate_on_total") or {}).get("mean", 0.0))), - _safe_float(((metrics.get("success_rate") or {}).get("mean", 0.0))), - ) - ) - lines.append("") - lines.append("## Gate Findings") - lines.append("") - if result.get("missing_variants"): - lines.append(f"- Missing variants: `{', '.join(result['missing_variants'])}`") - if result.get("missing_repeats"): - for variant, repeats in sorted(result["missing_repeats"].items()): - lines.append(f"- Missing repeats for `{variant}`: `{', '.join(str(x) for x in repeats)}`") - if result.get("invalid_rate_spikes"): - for variant, rates in sorted(result["invalid_rate_spikes"].items()): - lines.append( - "- Invalid-rate spike for `{0}`: parse_fail_total_mean={1:.4f}, no_answer_total_mean={2:.4f}".format( - variant, - _safe_float(rates.get("parse_failure_rate_on_total_mean", 0.0)), - _safe_float(rates.get("no_answer_rate_on_total_mean", 0.0)), - ) - ) - if result.get("variance_warnings"): - for variant, score_std in sorted(result["variance_warnings"].items()): - lines.append(f"- Variance warning for `{variant}`: benchmark_score_on_total std={score_std:.4f}") - if result.get("baseline_band_violation"): - band = result["baseline_band_violation"] - lines.append( - "- Baseline out of band for `{0}`: mean_score={1:.4f}, lower_bound={2}, upper_bound={3}".format( - str(band.get("variant", "")), - _safe_float(band.get("mean_score", 0.0)), - ("n/a" if band.get("lower_bound") is None else f"{_safe_float(band.get('lower_bound', 0.0)):.4f}"), - ("n/a" if band.get("upper_bound") is None else f"{_safe_float(band.get('upper_bound', 0.0)):.4f}"), - ) - ) - if result.get("ordering_reversals"): - for variant, payload in sorted(result["ordering_reversals"].items()): - delta_txt = ", ".join(f"{delta:.4f}" for delta in payload.get("paired_repeat_deltas", [])) - lines.append( - "- Ordering reversal for `{0}` vs `{1}`: mean_delta={2:.4f}, paired_deltas=`{3}`, reasons=`{4}`".format( - variant, - str(payload.get("baseline_variant", "")), - _safe_float(payload.get("mean_delta", 0.0)), - delta_txt, - ", ".join(str(x) for x in payload.get("reasons", [])), - ) - ) - if result.get("direction_conflicts"): - for variant, deltas in sorted(result["direction_conflicts"].items()): - delta_txt = ", ".join(f"{delta:.4f}" for delta in deltas) - lines.append(f"- Direction conflict for `{variant}` vs baseline: `{delta_txt}`") - if result.get("anomaly_codes"): - lines.append(f"- Anomaly codes: `{', '.join(result['anomaly_codes'])}`") - if not result.get("anomaly_codes"): - lines.append("- No structural anomalies detected at this stage.") - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--runs-dir", type=str, default="runs") - ap.add_argument("--workload-id", type=str, required=True) - ap.add_argument("--dataset", type=str, required=True) - ap.add_argument("--track-id", type=str, required=True) - ap.add_argument("--stage", type=str, choices=("smoke", "pilot", "full"), required=True) - ap.add_argument("--expected-variants", type=str, default="B0,B1,V1,F1,F2") - ap.add_argument("--expected-repeats", type=str, default="0") - ap.add_argument("--model-id-filter", type=str, default="") - ap.add_argument("--invalid-rate-threshold", type=float, default=0.15) - ap.add_argument("--score-std-threshold", type=float, default=0.10) - ap.add_argument("--baseline-variant", type=str, default="") - ap.add_argument("--must-beat-baseline-variants", type=str, default="") - ap.add_argument("--min-mean-delta", type=float, default=0.0) - ap.add_argument("--min-repeat-delta", type=float, default=0.0) - ap.add_argument("--baseline-lower-bound", type=float, default=float("nan")) - ap.add_argument("--baseline-upper-bound", type=float, default=float("nan")) - ap.add_argument("--output-md", type=str, required=True) - ap.add_argument("--output-json", type=str, required=True) - args = ap.parse_args() - - expected_variants = _parse_csv_list(args.expected_variants) - expected_repeats = [int(token) for token in _parse_csv_list(args.expected_repeats)] - must_beat_variants = _parse_csv_list(args.must_beat_baseline_variants) - baseline_lower_bound = None if math.isnan(float(args.baseline_lower_bound)) else float(args.baseline_lower_bound) - baseline_upper_bound = None if math.isnan(float(args.baseline_upper_bound)) else float(args.baseline_upper_bound) - rows = _collect_rows( - Path(args.runs_dir), - workload_id=str(args.workload_id), - dataset_id=str(args.dataset), - track_id=str(args.track_id), - model_id=str(args.model_id_filter), - ) - result = _build_result( - rows, - dataset=str(args.dataset), - track_id=str(args.track_id), - workload_id=str(args.workload_id), - stage=str(args.stage), - expected_variants=expected_variants, - expected_repeats=expected_repeats, - invalid_rate_threshold=float(args.invalid_rate_threshold), - std_threshold=float(args.score_std_threshold), - baseline_variant=str(args.baseline_variant or ""), - must_beat_baseline_variants=must_beat_variants, - min_mean_delta=float(args.min_mean_delta), - min_repeat_delta=float(args.min_repeat_delta), - baseline_lower_bound=baseline_lower_bound, - baseline_upper_bound=baseline_upper_bound, - ) - output_json = Path(args.output_json) - output_json.parent.mkdir(parents=True, exist_ok=True) - output_json.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") - _write_md(Path(args.output_md), result) - print(json.dumps(result, ensure_ascii=False)) - raise SystemExit(0 if result.get("verdict") == "accepted" else 2) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/backends/__init__.py b/projects/memlab/memlab/backends/__init__.py deleted file mode 100644 index fd1dc84..0000000 --- a/projects/memlab/memlab/backends/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Backend adapters. - -Import concrete adapters directly to avoid circular imports, e.g.: -`from memlab.backends.redis_store import RedisStore`. -""" - -__all__ = [] diff --git a/projects/memlab/memlab/backends/object_store.py b/projects/memlab/memlab/backends/object_store.py deleted file mode 100644 index c384a19..0000000 --- a/projects/memlab/memlab/backends/object_store.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - - -class LocalObjectStore: - def __init__(self, root_dir: str) -> None: - self.root = Path(root_dir) - - async def ensure_ready(self) -> None: - self.root.mkdir(parents=True, exist_ok=True) - - async def ping(self) -> bool: - try: - await self.ensure_ready() - return self.root.exists() - except Exception: - return False - - @staticmethod - def _safe_chunk(value: str) -> str: - return "".join(ch for ch in value if ch.isalnum() or ch in {"-", "_"}) - - def _blob_path(self, agent_id: str, key: str) -> Path: - safe_agent = self._safe_chunk(agent_id) or "agent" - safe_key = self._safe_chunk(key) or "key" - return self.root / safe_agent / f"{safe_key}.json" - - async def put(self, *, agent_id: str, key: str, text: str, metadata: dict[str, Any] | None = None) -> dict[str, Any]: - await self.ensure_ready() - path = self._blob_path(agent_id, key) - path.parent.mkdir(parents=True, exist_ok=True) - payload = {"text": text, "metadata": metadata or {}} - path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") - return { - "path": str(path.as_posix()), - "bytes": len(text.encode("utf-8")), - } - - async def get(self, pointer: dict[str, Any]) -> str | None: - raw = str(pointer.get("path", "")).strip() - if not raw: - return None - path = Path(raw) - if not path.is_absolute(): - path = self.root / raw - if not path.exists(): - return None - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except Exception: - return None - text = payload.get("text") - return str(text) if isinstance(text, str) else None diff --git a/projects/memlab/memlab/backends/postgres_store.py b/projects/memlab/memlab/backends/postgres_store.py deleted file mode 100644 index 4230a02..0000000 --- a/projects/memlab/memlab/backends/postgres_store.py +++ /dev/null @@ -1,208 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any - -try: - import asyncpg -except Exception: # pragma: no cover - optional dependency guard - asyncpg = None # type: ignore - -from memlab.memory.types import MemoryItem - - -UPSERT_MEMORY_SQL = """ -INSERT INTO memory_items ( - key, agent_id, kind, text_body, bytes, importance, created_at_ms, last_access_ms, access_count, metadata, embedding -) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb, $11::jsonb -) -ON CONFLICT (key, agent_id) DO UPDATE SET - kind = EXCLUDED.kind, - text_body = EXCLUDED.text_body, - bytes = EXCLUDED.bytes, - importance = EXCLUDED.importance, - last_access_ms = EXCLUDED.last_access_ms, - access_count = EXCLUDED.access_count, - metadata = EXCLUDED.metadata, - embedding = EXCLUDED.embedding -""" - - -class PostgresStore: - def __init__(self, pg_uri: str, *, min_pool_size: int = 1, max_pool_size: int = 32) -> None: - self.pg_uri = pg_uri - self.min_pool_size = min_pool_size - self.max_pool_size = max_pool_size - self.pool: asyncpg.Pool | None = None - self.backend = "postgres" - if asyncpg is None: - raise RuntimeError("asyncpg package is required for L2/Postgres operations") - - async def connect(self) -> None: - if self.pool is not None: - return - self.pool = await asyncpg.create_pool( - self.pg_uri, - min_size=self.min_pool_size, - max_size=self.max_pool_size, - command_timeout=30.0, - ssl=False, - ) - - async def close(self) -> None: - if self.pool is not None: - await self.pool.close() - self.pool = None - - async def ping(self) -> bool: - if self.pool is None: - await self.connect() - assert self.pool is not None - async with self.pool.acquire() as conn: - v = await conn.fetchval("SELECT 1") - return v == 1 - - async def put_item(self, item: MemoryItem) -> None: - if self.pool is None: - await self.connect() - assert self.pool is not None - async with self.pool.acquire() as conn: - await conn.execute( - UPSERT_MEMORY_SQL, - item.key, - item.agent_id, - item.kind, - item.text, - item.bytes, - item.importance, - item.created_at_ms, - item.last_access_ms, - item.access_count, - json.dumps(item.metadata or {}, ensure_ascii=False), - None, - ) - - async def put_items_batch(self, items: list[MemoryItem]) -> None: - if not items: - return - if self.pool is None: - await self.connect() - assert self.pool is not None - args = [ - ( - item.key, - item.agent_id, - item.kind, - item.text, - item.bytes, - item.importance, - item.created_at_ms, - item.last_access_ms, - item.access_count, - json.dumps(item.metadata or {}, ensure_ascii=False), - None, - ) - for item in items - ] - async with self.pool.acquire() as conn: - await conn.executemany(UPSERT_MEMORY_SQL, args) - - async def get_recent(self, agent_id: str, limit: int) -> list[MemoryItem]: - if self.pool is None: - await self.connect() - assert self.pool is not None - async with self.pool.acquire() as conn: - rows = await conn.fetch( - """ - SELECT key, agent_id, kind, text_body, bytes, importance, created_at_ms, last_access_ms, - access_count, metadata, embedding - FROM memory_items - WHERE agent_id = $1 - ORDER BY last_access_ms DESC - LIMIT $2 - """, - agent_id, - limit, - ) - return [self._row_to_item(r) for r in rows] - - async def search_text(self, agent_id: str, query: str, limit: int) -> list[MemoryItem]: - if self.pool is None: - await self.connect() - assert self.pool is not None - like = f"%{query}%" - async with self.pool.acquire() as conn: - rows = await conn.fetch( - """ - SELECT key, agent_id, kind, text_body, bytes, importance, created_at_ms, last_access_ms, - access_count, metadata, embedding - FROM memory_items - WHERE agent_id = $1 AND text_body ILIKE $2 - ORDER BY last_access_ms DESC - LIMIT $3 - """, - agent_id, - like, - limit, - ) - return [self._row_to_item(r) for r in rows] - - async def get_by_keys(self, agent_id: str, keys: list[str]) -> list[MemoryItem]: - if not keys: - return [] - deduped_keys = list(dict.fromkeys(keys)) - if self.pool is None: - await self.connect() - assert self.pool is not None - async with self.pool.acquire() as conn: - rows = await conn.fetch( - """ - SELECT key, agent_id, kind, text_body, bytes, importance, created_at_ms, last_access_ms, - access_count, metadata, embedding - FROM memory_items - WHERE agent_id = $1 AND key = ANY($2::text[]) - """, - agent_id, - deduped_keys, - ) - by_key = {r["key"]: self._row_to_item(r) for r in rows} - return [by_key[k] for k in deduped_keys if k in by_key] - - async def write_checkpoint(self, agent_id: str, thread_id: str, ts_ms: int, state_json: dict[str, Any]) -> None: - if self.pool is None: - await self.connect() - assert self.pool is not None - async with self.pool.acquire() as conn: - await conn.execute( - """ - INSERT INTO checkpoints (agent_id, thread_id, ts_ms, state_json) - VALUES ($1, $2, $3, $4::jsonb) - """, - agent_id, - thread_id, - ts_ms, - json.dumps(state_json, ensure_ascii=False), - ) - - @staticmethod - def _row_to_item(r: Any) -> MemoryItem: - metadata = r["metadata"] or {} - if isinstance(metadata, str): - metadata = json.loads(metadata) - embedding = r["embedding"] - if isinstance(embedding, str): - embedding = json.loads(embedding) - return MemoryItem( - key=r["key"], - agent_id=r["agent_id"], - kind=r["kind"], - text=r["text_body"], - bytes=int(r["bytes"]), - importance=float(r["importance"]), - created_at_ms=int(r["created_at_ms"]), - last_access_ms=int(r["last_access_ms"]), - access_count=int(r["access_count"]), - metadata=metadata, - embedding=embedding, - ) diff --git a/projects/memlab/memlab/backends/qdrant_store.py b/projects/memlab/memlab/backends/qdrant_store.py deleted file mode 100644 index 5b4aa69..0000000 --- a/projects/memlab/memlab/backends/qdrant_store.py +++ /dev/null @@ -1,109 +0,0 @@ -from __future__ import annotations - -from typing import Any - -try: - import httpx -except Exception: # pragma: no cover - optional dependency guard - httpx = None # type: ignore - - -class QdrantStore: - def __init__( - self, - qdrant_url: str, - collection: str, - vector_size: int = 64, - shard_number: int = 1, - ) -> None: - if httpx is None: - raise RuntimeError("httpx package is required for Qdrant operations") - self.qdrant_url = qdrant_url.rstrip("/") - self.collection = collection - self.vector_size = vector_size - self.shard_number = max(1, int(shard_number)) - self.client = httpx.AsyncClient(timeout=30.0, trust_env=False) - - async def close(self) -> None: - await self.client.aclose() - - async def ping(self) -> bool: - try: - r = await self.client.get(f"{self.qdrant_url}/collections") - return r.status_code == 200 - except Exception: - return False - - async def ensure_collection(self) -> None: - payload = { - "vectors": { - "size": self.vector_size, - "distance": "Cosine", - }, - "shard_number": self.shard_number, - } - r = await self.client.put(f"{self.qdrant_url}/collections/{self.collection}", json=payload) - if r.status_code == 409: - return - if r.status_code >= 400: - raise RuntimeError(f"Qdrant ensure_collection failed: HTTP {r.status_code} {r.text}") - - async def upsert(self, point_id: str | int, vector: list[float], payload: dict[str, Any]) -> None: - body = { - "points": [ - { - "id": point_id, - "vector": vector, - "payload": payload, - } - ] - } - r = await self.client.put( - f"{self.qdrant_url}/collections/{self.collection}/points", - json=body, - ) - if r.status_code >= 400: - raise RuntimeError(f"Qdrant upsert failed: HTTP {r.status_code} {r.text}") - - async def upsert_many(self, points: list[dict[str, Any]]) -> None: - if not points: - return - body = {"points": points} - r = await self.client.put( - f"{self.qdrant_url}/collections/{self.collection}/points", - json=body, - ) - if r.status_code >= 400: - raise RuntimeError(f"Qdrant upsert_many failed: HTTP {r.status_code} {r.text}") - - async def search( - self, - query_vector: list[float], - *, - limit: int, - agent_id: str | None = None, - ) -> list[dict[str, Any]]: - body: dict[str, Any] = { - "vector": query_vector, - "limit": limit, - "with_payload": True, - } - if agent_id: - body["filter"] = { - "must": [ - { - "key": "agent_id", - "match": {"value": agent_id}, - } - ] - } - r = await self.client.post( - f"{self.qdrant_url}/collections/{self.collection}/points/search", - json=body, - ) - if r.status_code != 200: - raise RuntimeError(f"Qdrant search failed: HTTP {r.status_code} {r.text}") - result = r.json().get("result", []) - if not isinstance(result, list): - raise RuntimeError("Qdrant search response does not contain result list") - return result diff --git a/projects/memlab/memlab/backends/redis_store.py b/projects/memlab/memlab/backends/redis_store.py deleted file mode 100644 index b61425f..0000000 --- a/projects/memlab/memlab/backends/redis_store.py +++ /dev/null @@ -1,94 +0,0 @@ -from __future__ import annotations - -import json - -try: - from redis.asyncio import ConnectionPool, Redis -except Exception: # pragma: no cover - optional dependency guard - ConnectionPool = None # type: ignore - Redis = None # type: ignore - -from memlab.memory.types import MemoryItem - - -class RedisStore: - def __init__(self, redis_url: str, *, max_connections: int = 256) -> None: - if Redis is None or ConnectionPool is None: - raise RuntimeError("redis package is required for L1/Redis operations") - self._pool = ConnectionPool.from_url( - redis_url, - decode_responses=True, - max_connections=max_connections, - ) - self.client = Redis(connection_pool=self._pool) - self._max_cached_text_chars = 1024 - - def _encode_item_payload(self, item: MemoryItem) -> tuple[str, str]: - ikey = self._item_key(item.agent_id, item.key) - payload_dict = dict(item.__dict__) - # L1 cache does not need persisted embeddings; dropping them reduces payload size and latency. - payload_dict["embedding"] = None - text_value = str(payload_dict.get("text", "")) - if len(text_value) > self._max_cached_text_chars: - payload_dict["text"] = text_value[: self._max_cached_text_chars] - meta = dict(payload_dict.get("metadata") or {}) - meta["l1_truncated"] = True - payload_dict["metadata"] = meta - payload = json.dumps(payload_dict, ensure_ascii=False, separators=(",", ":")) - return ikey, payload - - @staticmethod - def _item_key(agent_id: str, key: str) -> str: - return f"mem:{agent_id}:{key}" - - @staticmethod - def _recent_key(agent_id: str) -> str: - return f"mem_recent:{agent_id}" - - async def ping(self) -> bool: - return bool(await self.client.ping()) - - async def put_item(self, item: MemoryItem, ttl_s: int | None = None) -> None: - ikey, payload = self._encode_item_payload(item) - recent_key = self._recent_key(item.agent_id) - async with self.client.pipeline(transaction=False) as pipe: - pipe.set(ikey, payload, ex=ttl_s) - pipe.lpush(recent_key, item.key) - pipe.ltrim(recent_key, 0, 2047) - await pipe.execute() - - async def put_items_batch(self, items: list[MemoryItem], ttl_s: int | None = None) -> None: - if not items: - return - async with self.client.pipeline(transaction=False) as pipe: - for item in items: - ikey, payload = self._encode_item_payload(item) - recent_key = self._recent_key(item.agent_id) - pipe.set(ikey, payload, ex=ttl_s) - pipe.lpush(recent_key, item.key) - pipe.ltrim(recent_key, 0, 2047) - await pipe.execute() - - async def get_item(self, agent_id: str, key: str) -> MemoryItem | None: - raw = await self.client.get(self._item_key(agent_id, key)) - if not raw: - return None - data = json.loads(raw) - return MemoryItem(**data) - - async def get_recent(self, agent_id: str, limit: int) -> list[MemoryItem]: - keys = await self.client.lrange(self._recent_key(agent_id), 0, max(limit - 1, 0)) - out: list[MemoryItem] = [] - if not keys: - return out - redis_keys = [self._item_key(agent_id, key) for key in keys] - raw_items = await self.client.mget(redis_keys) - for raw in raw_items: - if not raw: - continue - out.append(MemoryItem(**json.loads(raw))) - return out - - async def close(self) -> None: - await self.client.aclose() - await self._pool.aclose() diff --git a/projects/memlab/memlab/bench/__init__.py b/projects/memlab/memlab/bench/__init__.py deleted file mode 100644 index ba246f4..0000000 --- a/projects/memlab/memlab/bench/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Microbench runners.""" diff --git a/projects/memlab/memlab/bench/postgres_bench.py b/projects/memlab/memlab/bench/postgres_bench.py deleted file mode 100644 index 08be0af..0000000 --- a/projects/memlab/memlab/bench/postgres_bench.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -import argparse -import os -import shutil -import subprocess -from pathlib import Path - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--pg-uri", type=str, default=os.getenv("PG_URI", "")) - ap.add_argument("--duration-s", type=int, default=300) - ap.add_argument("--clients", type=int, default=64) - ap.add_argument("--threads", type=int, default=16) - ap.add_argument("--out", type=str, default="logs/pgbench.txt") - args = ap.parse_args() - - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - pgbench = shutil.which("pgbench") - if not pgbench: - raise SystemExit("pgbench not found in PATH") - - init = subprocess.run([pgbench, "-i", args.pg_uri], capture_output=True, text=True, check=True) - run = subprocess.run( - [pgbench, "-T", str(args.duration_s), "-c", str(args.clients), "-j", str(args.threads), args.pg_uri], - capture_output=True, - text=True, - check=True, - ) - out.write_text(init.stdout + "\n" + run.stdout, encoding="utf-8") - print(f"Wrote {out}") - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/bench/qdrant_bench.py b/projects/memlab/memlab/bench/qdrant_bench.py deleted file mode 100644 index e8a2aa8..0000000 --- a/projects/memlab/memlab/bench/qdrant_bench.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -import argparse -import asyncio -import random -import time -from pathlib import Path - -from memlab.backends.qdrant_store import QdrantStore -from memlab.metrics import summarize_latency -from memlab.tools.embedding import hash_embedding - - -async def _run(args: argparse.Namespace) -> None: - store = QdrantStore(args.qdrant_url, args.collection, vector_size=args.dim) - await store.ensure_collection() - rng = random.Random(args.seed) - - for i in range(args.points): - text = f"doc_{i} seed_{args.seed}" - emb = hash_embedding(text, dim=args.dim) - await store.upsert( - point_id=f"{args.seed}-{i}", - vector=emb, - payload={"agent_id": "bench", "key": f"bench-{i}", "kind": "bench"}, - ) - - latencies: list[float] = [] - for i in range(args.searches): - q = hash_embedding(f"query_{rng.randint(0, args.points - 1)}", dim=args.dim) - t0 = time.perf_counter() - _ = await store.search(q, limit=args.top_k, agent_id="bench") - latencies.append((time.perf_counter() - t0) * 1000.0) - - summary = summarize_latency(latencies) - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(str(summary), encoding="utf-8") - print(f"Wrote {out}") - await store.close() - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--qdrant-url", type=str, default="http://127.0.0.1:6333") - ap.add_argument("--collection", type=str, default="memlab_docs") - ap.add_argument("--dim", type=int, default=64) - ap.add_argument("--points", type=int, default=20000) - ap.add_argument("--searches", type=int, default=5000) - ap.add_argument("--top-k", type=int, default=8) - ap.add_argument("--seed", type=int, default=42) - ap.add_argument("--out", type=str, default="logs/qdrant_bench.txt") - args = ap.parse_args() - asyncio.run(_run(args)) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/bench/redis_bench.py b/projects/memlab/memlab/bench/redis_bench.py deleted file mode 100644 index f705c35..0000000 --- a/projects/memlab/memlab/bench/redis_bench.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -import argparse -import shutil -import subprocess -from pathlib import Path - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--host", type=str, default="127.0.0.1") - ap.add_argument("--port", type=int, default=6379) - ap.add_argument("--clients", type=int, default=100) - ap.add_argument("--requests", type=int, default=200000) - ap.add_argument("--out", type=str, default="logs/redis_benchmark.csv") - args = ap.parse_args() - - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - - cmd = shutil.which("redis-benchmark") - if not cmd: - raise SystemExit("redis-benchmark not found in PATH") - - run = [ - cmd, - "-h", - args.host, - "-p", - str(args.port), - "-c", - str(args.clients), - "-n", - str(args.requests), - "-t", - "get,set", - "--csv", - ] - proc = subprocess.run(run, capture_output=True, text=True, check=True) - out.write_text(proc.stdout, encoding="utf-8") - print(f"Wrote {out}") - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/checkpoint/__init__.py b/projects/memlab/memlab/checkpoint/__init__.py deleted file mode 100644 index 8db8a5a..0000000 --- a/projects/memlab/memlab/checkpoint/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Checkpoint adapters.""" diff --git a/projects/memlab/memlab/checkpoint/langgraph_pg.py b/projects/memlab/memlab/checkpoint/langgraph_pg.py deleted file mode 100644 index 4e514fa..0000000 --- a/projects/memlab/memlab/checkpoint/langgraph_pg.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import annotations - -import os - - -def get_checkpointer(): - """ - Optional helper for DeerFlow/LangGraph multi-turn persistence. - Returns a configured PostgresSaver instance when dependency exists. - """ - pg_uri = os.environ.get("PG_URI") - if not pg_uri: - raise RuntimeError("PG_URI is required for checkpointer") - try: - from langgraph.checkpoint.postgres import PostgresSaver - except Exception as exc: - raise RuntimeError( - "langgraph-checkpoint-postgres is not installed; install it in the DeerFlow environment" - ) from exc - saver = PostgresSaver.from_conn_string(pg_uri) - saver.setup() - return saver diff --git a/projects/memlab/memlab/config.py b/projects/memlab/memlab/config.py deleted file mode 100644 index c5df24d..0000000 --- a/projects/memlab/memlab/config.py +++ /dev/null @@ -1,131 +0,0 @@ -from __future__ import annotations - -import os -from pathlib import Path -from typing import Literal - -import yaml -from pydantic import BaseModel, Field - - -class ExperimentConfig(BaseModel): - run_id: str = "F1" - variant_id: str = "F1" - seed: int = 42 - repetitions: int = 3 - n_agents: int = 50 - duration_s: int = 300 - llm_backend: Literal["stub", "openai_compatible", "deerflow"] = "stub" - redis_policy: Literal["allkeys-lfu", "allkeys-lru"] = "allkeys-lfu" - summarization_on: bool = True - output_dir: str = "./runs" - logs_dir: str = "./runs" - - -class L0Config(BaseModel): - max_prompt_tokens: int = 8000 - max_memory_injection_tokens: int = 2000 - max_injected_tokens: int = 256 - keep_recent_messages: int = 10 - summarize_trigger_tokens: int = 7000 - summarize_target_tokens: int = 3000 - min_relevance_score: float = 0.2 - - -class L1Config(BaseModel): - enabled: bool = True - ttl_s: int = 300 - max_object_bytes: int = 65536 - reuse_threshold: float = 0.2 - - -class L2Config(BaseModel): - enabled: bool = True - vector_enabled: bool = True - vector_backend: Literal["qdrant"] = "qdrant" - qdrant_shard_number: int = 1 - default_top_k: int = 8 - collection_name: str = "memlab_docs" - write_batch_size: int = 512 - write_flush_interval_ms: int = 50 - min_write_importance: float = 0.7 - max_batch_per_flush: int = 512 - max_flush_ms: int = 50 - drop_queue_depth_threshold: int = 12000 - backpressure_drop_importance_threshold: float = 0.85 - embedding_cache_ttl_ms: int = 120000 - retrieval_cache_ttl_ms: int = 15000 - - -class L3Config(BaseModel): - enabled: bool = False - root_dir: str = "./storage/l3_objects" - pointer_threshold_bytes: int = 65536 - pointer_preview_chars: int = 240 - - -class PolicyConfig(BaseModel): - packing_budget: Literal["strict", "balanced"] = "balanced" - checkpoint_policy: Literal["l2_only", "disabled"] = "l2_only" - direct_evidence_retrieval_bypass: bool = True - - -class MemoryConfig(BaseModel): - enabled_tiers: list[Literal["L0", "L1", "L2", "L3"]] = Field( - default_factory=lambda: ["L0", "L1", "L2", "L3"] - ) - l0: L0Config = Field(default_factory=L0Config) - l1: L1Config = Field(default_factory=L1Config) - l2: L2Config = Field(default_factory=L2Config) - l3: L3Config = Field(default_factory=L3Config) - policy: PolicyConfig = Field(default_factory=PolicyConfig) - - -class ServiceConfig(BaseModel): - redis_url: str = "redis://127.0.0.1:6379/0" - pg_uri: str = Field(default_factory=lambda: os.getenv("PG_URI", "")) - qdrant_url: str = "http://127.0.0.1:6333" - qdrant_collection: str = "memlab_docs" - - -class WorkloadConfig(BaseModel): - corpus_path: str = "./data/docs.jsonl" - tasks_path: str = "./data/tasks.jsonl" - trace_path: str = "./traces/trace_200a.jsonl" - - -class MemlabConfig(BaseModel): - experiment: ExperimentConfig = Field(default_factory=ExperimentConfig) - memory: MemoryConfig = Field(default_factory=MemoryConfig) - services: ServiceConfig = Field(default_factory=ServiceConfig) - workloads: WorkloadConfig = Field(default_factory=WorkloadConfig) - - -def _merge_env_overrides(data: dict) -> dict: - env_map = { - "MEMLAB_OUTPUT_DIR": ("experiment", "output_dir"), - "MEMLAB_LOGS_DIR": ("experiment", "logs_dir"), - "REDIS_URL": ("services", "redis_url"), - "PG_URI": ("services", "pg_uri"), - "QDRANT_URL": ("services", "qdrant_url"), - "QDRANT_COLLECTION": ("services", "qdrant_collection"), - } - for env_key, path in env_map.items(): - v = os.getenv(env_key) - if not v: - continue - target = data - for key in path[:-1]: - target = target.setdefault(key, {}) - target[path[-1]] = v - return data - - -def load_config(path: str | Path | None = None) -> MemlabConfig: - candidate = Path(path or os.getenv("MEMLAB_CONFIG_PATH", "configs/memlab.default.yaml")) - if not candidate.exists(): - return MemlabConfig(**_merge_env_overrides({})) - with candidate.open("r", encoding="utf-8") as f: - raw = yaml.safe_load(f) or {} - merged = _merge_env_overrides(raw) - return MemlabConfig(**merged) diff --git a/projects/memlab/memlab/envcheck.py b/projects/memlab/memlab/envcheck.py deleted file mode 100644 index 51a1cb9..0000000 --- a/projects/memlab/memlab/envcheck.py +++ /dev/null @@ -1,347 +0,0 @@ -from __future__ import annotations - -import json -import os -import socket -import subprocess -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any -from urllib.parse import urlparse - -import httpx - -from memlab.metrics.manifest import resolve_git_commit -from memlab.runtime_paths import deerflow_root - - -INFRA_REQUIRED_VARIANTS = {"V1", "F1", "F2", "A1", "A2"} - - -@dataclass -class EnvCheckResult: - env_valid: bool - env_invalid_reasons: list[str] = field(default_factory=list) - memlab_git_commit: str | None = None - deerflow_git_commit: str | None = None - infra_versions: dict[str, str] = field(default_factory=dict) - docker_image_digests: dict[str, str] = field(default_factory=dict) - - -def _run(cmd: list[str], *, timeout_s: float = 20.0) -> tuple[bool, str]: - try: - out = subprocess.check_output( - cmd, - stderr=subprocess.STDOUT, - text=True, - timeout=timeout_s, - ) - return True, out.strip() - except Exception as exc: - text = "" - if hasattr(exc, "output"): - if isinstance(exc.output, bytes): - text = exc.output.decode("utf-8", errors="ignore").strip() - elif isinstance(exc.output, str): - text = exc.output.strip() - return False, text or str(exc) - - -def _tcp_reachable(host: str, port: int, timeout_s: float = 2.0) -> bool: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(timeout_s) - try: - sock.connect((host, port)) - return True - except Exception: - return False - finally: - sock.close() - - -def _compose_cmd(*args: str) -> list[str]: - return ["docker", "compose", "-f", "infra/docker-compose.yml", *args] - - -def _extract_host_port(url: str) -> tuple[str, int]: - p = urlparse(url) - host = p.hostname or "127.0.0.1" - if p.port: - return host, p.port - return host, 443 if p.scheme == "https" else 80 - - -def _docker_engine_ok() -> bool: - ok, out = _run(["docker", "info", "--format", "{{.ServerVersion}}"], timeout_s=10.0) - if not ok: - return False - txt = out.strip().lower() - if not txt: - return False - if "error" in txt or "request returned" in txt or "cannot connect" in txt: - return False - return True - - -def _service_running(service: str) -> bool: - ok, out = _run(_compose_cmd("ps", "-q", service)) - if not ok or not out.strip(): - return False - cid = out.strip() - ok, state = _run(["docker", "inspect", "--format", "{{.State.Status}}", cid]) - return ok and state.strip() == "running" - - -def _service_healthy(service: str) -> bool: - ok, out = _run(_compose_cmd("ps", "-q", service)) - if not ok or not out.strip(): - return False - cid = out.strip() - ok, health = _run(["docker", "inspect", "--format", "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}", cid]) - if not ok: - return False - status = health.strip().lower() - return status in {"healthy", "none"} - - -def _require_docker_infra(reasons: list[str]) -> None: - if not _docker_engine_ok(): - reasons.append("Docker engine is not running.") - return - for svc in ("redis", "postgres", "qdrant"): - if not _service_running(svc): - reasons.append(f"Container '{svc}' is not running.") - continue - if not _service_healthy(svc): - reasons.append(f"Container '{svc}' is not healthy.") - if not _tcp_reachable("127.0.0.1", 6379): - reasons.append("Redis is unreachable on 127.0.0.1:6379.") - if not _tcp_reachable("127.0.0.1", 5432): - reasons.append("Postgres is unreachable on 127.0.0.1:5432.") - if not _tcp_reachable("127.0.0.1", 6333): - reasons.append("Qdrant is unreachable on 127.0.0.1:6333.") - - -def _probe_infra_versions_and_digests() -> tuple[dict[str, str], dict[str, str], list[str]]: - versions: dict[str, str] = {} - digests: dict[str, str] = {} - errors: list[str] = [] - - ok, redis_info = _run(_compose_cmd("exec", "-T", "redis", "redis-cli", "-h", "127.0.0.1", "-p", "6379", "INFO", "server")) - if ok: - redis_ver = "" - for line in redis_info.splitlines(): - if line.startswith("redis_version:"): - redis_ver = line.split(":", 1)[1].strip() - break - if redis_ver: - versions["redis"] = redis_ver - else: - errors.append("Unable to parse Redis version.") - else: - errors.append(f"Unable to query Redis version: {redis_info}") - - ok, pg_ver = _run(_compose_cmd( - "exec", "-T", "postgres", "sh", "-c", - 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SHOW server_version;"', - )) - if ok and pg_ver: - versions["postgres"] = pg_ver.strip() - else: - errors.append(f"Unable to query Postgres version: {pg_ver}") - - try: - with httpx.Client(timeout=5.0, trust_env=False) as client: - resp = client.get("http://127.0.0.1:6333/") - resp.raise_for_status() - body = resp.json() - versions["qdrant"] = str(body.get("version", "")) - if not versions["qdrant"]: - errors.append("Unable to parse Qdrant version.") - except Exception as exc: - errors.append(f"Unable to query Qdrant version: {exc}") - - for svc in ("redis", "postgres", "qdrant"): - ok, cid = _run(_compose_cmd("ps", "-q", svc)) - if not ok or not cid: - errors.append(f"Unable to resolve container id for '{svc}'.") - continue - ok, image_ref = _run(["docker", "inspect", "--format", "{{.Config.Image}}", cid.strip()]) - if not ok or not image_ref: - errors.append(f"Unable to resolve image reference for '{svc}'.") - continue - ok, digest = _run(["docker", "image", "inspect", image_ref.strip(), "--format", "{{index .RepoDigests 0}}"]) - digests[svc] = digest.strip() if ok and digest else image_ref.strip() - - return versions, digests, errors - - -def _probe_redis_lfu_support(reasons: list[str]) -> None: - ok, out = _run( - _compose_cmd( - "exec", - "-T", - "redis", - "redis-cli", - "-h", - "127.0.0.1", - "-p", - "6379", - "CONFIG", - "SET", - "maxmemory-policy", - "allkeys-lfu", - ) - ) - if not ok or "OK" not in out: - reasons.append("Redis does not support allkeys-lfu; refusing to run comparable variants.") - - -def _check_deerflow( - *, - langgraph_url: str, - gateway_url: str, - reasons: list[str], -) -> None: - lg_host, lg_port = _extract_host_port(langgraph_url) - gw_host, gw_port = _extract_host_port(gateway_url) - if not _tcp_reachable(lg_host, lg_port): - reasons.append(f"DeerFlow LangGraph endpoint is unreachable at {lg_host}:{lg_port}.") - if not _tcp_reachable(gw_host, gw_port): - reasons.append(f"DeerFlow Gateway endpoint is unreachable at {gw_host}:{gw_port}.") - - -def _read_simple_env_file(path: Path) -> dict[str, str]: - values: dict[str, str] = {} - if not path.exists(): - return values - for raw_line in path.read_text(encoding="utf-8").splitlines(): - line = raw_line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - key, value = line.split("=", 1) - values[key.strip()] = value.strip() - return values - - -def _read_local_deerflow_runtime_lock(path: Path) -> dict[str, str]: - if not path.exists(): - return {} - try: - data = json.loads(path.read_text(encoding="utf-8-sig")) - except Exception: - return {} - if not isinstance(data, dict): - return {} - return { - "base_url": str(data.get("base_url", "") or "").strip().rstrip("/"), - "model_id": str(data.get("model_id", "") or "").strip(), - "fallback_model": str(data.get("fallback_model", "") or "").strip(), - "gateway_pid": str(data.get("gateway_pid", "") or "").strip(), - "langgraph_pid": str(data.get("langgraph_pid", "") or "").strip(), - } - - -def _check_local_deerflow_lock_alignment( - *, - langgraph_url: str, - gateway_url: str, - expected_base_url: str, - expected_model_id: str, - reasons: list[str], -) -> None: - lg_host, _ = _extract_host_port(langgraph_url) - gw_host, _ = _extract_host_port(gateway_url) - if {lg_host.lower(), gw_host.lower()} - {"127.0.0.1", "localhost"}: - return - - expected_base = str(expected_base_url or os.getenv("OPENAI_BASE_URL", "") or "").strip().rstrip("/") - expected_model = str(expected_model_id or os.getenv("OPENAI_MODEL", "") or "").strip() - if not expected_base and not expected_model: - return - - runtime_lock = _read_local_deerflow_runtime_lock(deerflow_root() / "logs/runtime.lock.json") - if runtime_lock: - runtime_mismatches: list[str] = [] - runtime_base = runtime_lock.get("base_url", "") - runtime_model = runtime_lock.get("model_id", "") - if expected_base and runtime_base and runtime_base != expected_base: - runtime_mismatches.append(f"base_url={runtime_base} != {expected_base}") - if expected_model and runtime_model and runtime_model != expected_model: - runtime_mismatches.append(f"model_id={runtime_model} != {expected_model}") - if runtime_mismatches: - reasons.append( - "Local DeerFlow runtime lock mismatch in deer-flow/logs/runtime.lock.json: " - + "; ".join(runtime_mismatches) - ) - return - - for env_path in (deerflow_root() / "backend/.env", deerflow_root() / ".env"): - env_map = _read_simple_env_file(env_path) - if not env_map: - continue - file_base = str(env_map.get("OPENAI_BASE_URL", "") or "").strip().rstrip("/") - file_model = str(env_map.get("OPENAI_MODEL", "") or "").strip() - mismatches: list[str] = [] - if expected_base and file_base and file_base != expected_base: - mismatches.append(f"OPENAI_BASE_URL={file_base} != {expected_base}") - if expected_model and file_model and file_model != expected_model: - mismatches.append(f"OPENAI_MODEL={file_model} != {expected_model}") - if mismatches: - reasons.append(f"Local DeerFlow env drift in {env_path.as_posix()}: {'; '.join(mismatches)}") - - -def run_env_checks( - *, - variant_id: str, - require_deerflow: bool, - deerflow_langgraph_url: str, - deerflow_gateway_url: str, - locked_base_url: str | None = None, - locked_model_id: str | None = None, -) -> EnvCheckResult: - reasons: list[str] = [] - memlab_commit = resolve_git_commit(Path.cwd()) - deerflow_commit = resolve_git_commit(deerflow_root()) - if deerflow_commit is None: - pinned = Path("artifacts/deerflow_commit.txt") - if pinned.exists(): - txt = pinned.read_text(encoding="utf-8").strip() - deerflow_commit = txt if txt else None - versions: dict[str, str] = {} - digests: dict[str, str] = {} - infra_reasons: list[str] = [] - _require_docker_infra(infra_reasons) - if not infra_reasons: - versions, digests, infra_errors = _probe_infra_versions_and_digests() - infra_reasons.extend(infra_errors) - _probe_redis_lfu_support(infra_reasons) - reasons.extend(infra_reasons) - if require_deerflow: - _check_deerflow( - langgraph_url=deerflow_langgraph_url, - gateway_url=deerflow_gateway_url, - reasons=reasons, - ) - _check_local_deerflow_lock_alignment( - langgraph_url=deerflow_langgraph_url, - gateway_url=deerflow_gateway_url, - expected_base_url=str(locked_base_url or "").strip().rstrip("/"), - expected_model_id=str(locked_model_id or "").strip(), - reasons=reasons, - ) - - return EnvCheckResult( - env_valid=len(reasons) == 0, - env_invalid_reasons=reasons, - memlab_git_commit=memlab_commit, - deerflow_git_commit=deerflow_commit, - infra_versions=versions, - docker_image_digests=digests, - ) - - -def write_manifest(path: str | Path, manifest: dict[str, Any]) -> None: - p = Path(path) - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(json.dumps(manifest, indent=2), encoding="utf-8") diff --git a/projects/memlab/memlab/event_logger.py b/projects/memlab/memlab/event_logger.py deleted file mode 100644 index 7d67920..0000000 --- a/projects/memlab/memlab/event_logger.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -from dataclasses import asdict -from pathlib import Path -from typing import Any - -from memlab.io import json_dump, jsonl_append -from memlab.types import EventLog - - -class EventLogger: - def __init__(self, run_id: str, event_path: str | Path) -> None: - self.run_id = run_id - self.event_path = Path(event_path) - self.event_path.parent.mkdir(parents=True, exist_ok=True) - - def emit(self, event: EventLog) -> None: - row = asdict(event) - row["extra"] = row.get("extra") or {} - jsonl_append(self.event_path, row) - - def emit_raw(self, row: dict[str, Any]) -> None: - jsonl_append(self.event_path, row) - - @staticmethod - def write_manifest(path: str | Path, manifest: dict[str, Any]) -> None: - json_dump(path, manifest) diff --git a/projects/memlab/memlab/io.py b/projects/memlab/memlab/io.py deleted file mode 100644 index e65c253..0000000 --- a/projects/memlab/memlab/io.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - -import json -import time -from pathlib import Path -from typing import Any - - -def now_ms() -> int: - return int(time.time() * 1000) - - -def ensure_dir(path: str | Path) -> Path: - p = Path(path) - p.mkdir(parents=True, exist_ok=True) - return p - - -def jsonl_append(path: str | Path, row: dict[str, Any]) -> None: - p = Path(path) - p.parent.mkdir(parents=True, exist_ok=True) - with p.open("a", encoding="utf-8") as f: - f.write(json.dumps(row, ensure_ascii=False) + "\n") - - -def json_dump(path: str | Path, obj: dict[str, Any]) -> None: - p = Path(path) - p.parent.mkdir(parents=True, exist_ok=True) - with p.open("w", encoding="utf-8") as f: - json.dump(obj, f, ensure_ascii=False, indent=2) - - -def jsonl_read(path: str | Path) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - with Path(path).open("r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - rows.append(json.loads(line)) - return rows diff --git a/projects/memlab/memlab/memory/__init__.py b/projects/memlab/memlab/memory/__init__.py deleted file mode 100644 index efd0006..0000000 --- a/projects/memlab/memlab/memory/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from .types import ContextBudget, ContextPack, MemoryItem, PlacementDecision - -__all__ = [ - "TieredMemoryManager", - "ContextBudget", - "ContextPack", - "MemoryItem", - "PlacementDecision", -] - - -def __getattr__(name: str): - if name == "TieredMemoryManager": - from .manager import TieredMemoryManager - - return TieredMemoryManager - raise AttributeError(name) diff --git a/projects/memlab/memlab/memory/api.py b/projects/memlab/memlab/memory/api.py deleted file mode 100644 index 3e7deb9..0000000 --- a/projects/memlab/memlab/memory/api.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod - -from .types import ContextBudget, ContextPack, MemoryItem, PlacementDecision - - -class MemoryManager(ABC): - @abstractmethod - async def put(self, item: MemoryItem, *, ttl_s: int | None = None) -> PlacementDecision: - raise NotImplementedError - - @abstractmethod - async def get( - self, - agent_id: str, - query: str, - *, - k: int, - mode: str = "hybrid", - ) -> list[MemoryItem]: - raise NotImplementedError - - @abstractmethod - async def assemble_context( - self, - agent_id: str, - user_query: str, - budget: ContextBudget, - ) -> ContextPack: - raise NotImplementedError - - @abstractmethod - async def checkpoint(self, agent_id: str, thread_id: str, state: dict) -> None: - raise NotImplementedError diff --git a/projects/memlab/memlab/memory/manager.py b/projects/memlab/memlab/memory/manager.py deleted file mode 100644 index 639ecaf..0000000 --- a/projects/memlab/memlab/memory/manager.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import annotations - -from memlab.memory_manager.manager import MultiLevelMemoryManager - - -# Backward-compatible alias for older imports. -TieredMemoryManager = MultiLevelMemoryManager diff --git a/projects/memlab/memlab/memory/types.py b/projects/memlab/memlab/memory/types.py deleted file mode 100644 index 93828df..0000000 --- a/projects/memlab/memlab/memory/types.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Literal, Optional - - -MemoryKind = Literal[ - "message", - "tool_result", - "fact", - "summary", - "checkpoint", - "blob_ref", -] - - -@dataclass -class MemoryItem: - key: str - agent_id: str - kind: MemoryKind - text: str - bytes: int - importance: float - created_at_ms: int - last_access_ms: int - access_count: int - embedding: Optional[list[float]] = None - metadata: dict[str, Any] = field(default_factory=dict) - - -@dataclass -class ContextBudget: - max_prompt_tokens: int - max_memory_injection_tokens: int - keep_recent_messages: int - - -@dataclass -class ContextPack: - messages: list[dict[str, Any]] - injected_memories: list[MemoryItem] - trace: dict[str, Any] - - -@dataclass -class PlacementDecision: - key: str - tiers: list[str] - ttl_s: int | None - reasons: list[str] - admission_reason: str = "" - eviction_reason: str = "" - packed_tokens: int | None = None - pointer_fetch: int = 0 - degrade_mode: str | None = None diff --git a/projects/memlab/memlab/memory_manager/__init__.py b/projects/memlab/memlab/memory_manager/__init__.py deleted file mode 100644 index ef31f3f..0000000 --- a/projects/memlab/memlab/memory_manager/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from .l0 import ContextPackResult, assemble_l0_context, estimate_tokens -from .l1 import L1RedisClient -from .l2 import L2PostgresStore -from .l3 import L3ObjectStore -from .manager import MultiLevelMemoryManager -from .vector import QdrantVectorStore - -__all__ = [ - "ContextPackResult", - "assemble_l0_context", - "estimate_tokens", - "L1RedisClient", - "L2PostgresStore", - "L3ObjectStore", - "QdrantVectorStore", - "MultiLevelMemoryManager", -] diff --git a/projects/memlab/memlab/memory_manager/l0.py b/projects/memlab/memlab/memory_manager/l0.py deleted file mode 100644 index 88e327c..0000000 --- a/projects/memlab/memlab/memory_manager/l0.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -import hashlib -import math -from dataclasses import dataclass -from typing import Any - -from memlab.memory.types import ContextBudget, MemoryItem - - -def estimate_tokens(text: str) -> int: - return max(1, math.ceil(len(text) / 4)) - - -@dataclass -class ContextPackResult: - messages: list[dict[str, Any]] - selected: list[MemoryItem] - dropped_keys: list[str] - injected_tokens: int - injected_item_count: int - total_tokens: int - summarized: bool - - -def _summarize_messages(messages: list[dict[str, Any]], target_tokens: int) -> str: - chunks: list[str] = [] - remain = max(1, target_tokens) - for m in messages: - line = f"{m.get('role', 'unknown')}: {m.get('content', '')}" - cost = estimate_tokens(line) - if cost > remain: - break - chunks.append(line) - remain -= cost - if remain <= 0: - break - return "\n".join(chunks[:64]) if chunks else "No prior context." - - -def _near_duplicate_hash(text: str) -> str: - normalized = " ".join(str(text or "").strip().lower().split()) - # Limit hash input to keep dedupe cheap while still robust against prompt bloat. - return hashlib.sha1(normalized[:512].encode("utf-8")).hexdigest() - - -def assemble_l0_context( - *, - recent_messages: list[MemoryItem], - candidates: list[MemoryItem], - user_query: str, - budget: ContextBudget, - summarization_on: bool, - summarize_trigger_tokens: int, - summarize_target_tokens: int, - min_relevance_score: float, -) -> ContextPackResult: - eligible_recent = [ - m - for m in recent_messages - if not bool((m.metadata or {}).get("exclude_from_recent_context", False)) - ] - keep_n = min(len(eligible_recent), budget.keep_recent_messages) - kept_recent = eligible_recent[-keep_n:] if keep_n > 0 else [] - kept_keys = {m.key for m in kept_recent} - - selected: list[MemoryItem] = [] - dropped: list[str] = [] - injected_tokens = 0 - seen_versions: set[tuple[str, str]] = set() - seen_text_hashes: set[str] = set() - ranked_candidates = sorted( - candidates, - key=lambda item: ( - -float(item.metadata.get("retrieval_score", 0.0) if isinstance(item.metadata.get("retrieval_score"), (int, float)) else 0.0), - str(item.key), - ), - ) - - for item in ranked_candidates: - if item.key in kept_keys: - continue - rel = item.metadata.get("retrieval_score") - score = float(rel) if isinstance(rel, (int, float)) else 1.0 - if score < min_relevance_score: - dropped.append(item.key) - continue - version_raw = item.metadata.get("version", item.created_at_ms) - version = str(version_raw) - dedupe_key = (item.key, version) - if dedupe_key in seen_versions: - dropped.append(item.key) - continue - seen_versions.add(dedupe_key) - text_hash = _near_duplicate_hash(item.text) - if text_hash in seen_text_hashes: - dropped.append(item.key) - continue - seen_text_hashes.add(text_hash) - cost = estimate_tokens(item.text) - if injected_tokens + cost > budget.max_memory_injection_tokens: - dropped.append(item.key) - continue - selected.append(item) - injected_tokens += cost - - messages = [ - { - "role": m.metadata.get("role", "user"), - "content": m.text, - } - for m in kept_recent - ] - used = 0 - injected_item_count = 0 - for item in selected: - cost = estimate_tokens(item.text) - if used + cost > budget.max_memory_injection_tokens: - break - messages.append( - { - "role": "system", - "content": f"[MEMORY:{item.kind}:{item.key}] {item.text}", - } - ) - used += cost - injected_item_count += 1 - injected_tokens = used - messages.append({"role": "user", "content": user_query}) - - total_tokens = sum(estimate_tokens(m["content"]) for m in messages) - summarized = False - if summarization_on and total_tokens > summarize_trigger_tokens and len(messages) > budget.keep_recent_messages: - older = messages[:-budget.keep_recent_messages] - summary_text = _summarize_messages(older, summarize_target_tokens) - messages = [{"role": "system", "content": f"[SUMMARY] {summary_text}"}] + messages[-budget.keep_recent_messages:] - total_tokens = sum(estimate_tokens(m["content"]) for m in messages) - summarized = True - - # Final fail-safe: enforce prompt upper bound by trimming memory injections from the front. - while messages and total_tokens > budget.max_prompt_tokens and len(messages) > 1: - # Keep the final user message. - if messages[0].get("role") == "user" and len(messages) == 1: - break - messages.pop(0) - total_tokens = sum(estimate_tokens(m["content"]) for m in messages) - - return ContextPackResult( - messages=messages, - selected=selected, - dropped_keys=dropped, - injected_tokens=injected_tokens, - injected_item_count=injected_item_count, - total_tokens=total_tokens, - summarized=summarized, - ) diff --git a/projects/memlab/memlab/memory_manager/l1.py b/projects/memlab/memlab/memory_manager/l1.py deleted file mode 100644 index 054e255..0000000 --- a/projects/memlab/memlab/memory_manager/l1.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -from memlab.backends.redis_store import RedisStore -from memlab.memory.types import MemoryItem - - -class L1RedisClient: - def __init__(self, redis_url: str) -> None: - self.store = RedisStore(redis_url) - - async def ping(self) -> bool: - return await self.store.ping() - - async def put(self, item: MemoryItem, ttl_s: int | None) -> None: - await self.store.put_item(item, ttl_s=ttl_s) - - async def put_many(self, items: list[MemoryItem], ttl_s: int | None) -> None: - await self.store.put_items_batch(items, ttl_s=ttl_s) - - async def get_recent(self, agent_id: str, limit: int) -> list[MemoryItem]: - return await self.store.get_recent(agent_id, limit) - - async def get_key(self, agent_id: str, key: str) -> MemoryItem | None: - return await self.store.get_item(agent_id, key) - - async def close(self) -> None: - await self.store.close() diff --git a/projects/memlab/memlab/memory_manager/l2.py b/projects/memlab/memlab/memory_manager/l2.py deleted file mode 100644 index 1cafc1d..0000000 --- a/projects/memlab/memlab/memory_manager/l2.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -from memlab.backends.postgres_store import PostgresStore -from memlab.memory.types import MemoryItem - - -class L2PostgresStore: - def __init__(self, pg_uri: str) -> None: - self.store = PostgresStore(pg_uri) - - async def connect(self) -> None: - await self.store.connect() - - async def ping(self) -> bool: - return await self.store.ping() - - async def put(self, item: MemoryItem) -> None: - await self.store.put_item(item) - - async def put_many(self, items: list[MemoryItem]) -> None: - await self.store.put_items_batch(items) - - async def get_recent(self, agent_id: str, limit: int) -> list[MemoryItem]: - return await self.store.get_recent(agent_id, limit) - - async def search_text(self, agent_id: str, query: str, limit: int) -> list[MemoryItem]: - return await self.store.search_text(agent_id, query, limit) - - async def get_by_keys(self, agent_id: str, keys: list[str]) -> list[MemoryItem]: - return await self.store.get_by_keys(agent_id, keys) - - async def checkpoint(self, agent_id: str, thread_id: str, ts_ms: int, state: dict) -> None: - await self.store.write_checkpoint(agent_id, thread_id, ts_ms, state) - - async def close(self) -> None: - await self.store.close() diff --git a/projects/memlab/memlab/memory_manager/l3.py b/projects/memlab/memlab/memory_manager/l3.py deleted file mode 100644 index ab06ebe..0000000 --- a/projects/memlab/memlab/memory_manager/l3.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from memlab.backends.object_store import LocalObjectStore - - -class L3ObjectStore: - def __init__(self, root_dir: str) -> None: - self.store = LocalObjectStore(root_dir=root_dir) - - async def ensure_ready(self) -> None: - await self.store.ensure_ready() - - async def ping(self) -> bool: - return await self.store.ping() - - async def put(self, *, agent_id: str, key: str, text: str, metadata: dict[str, Any] | None = None) -> dict[str, Any]: - return await self.store.put(agent_id=agent_id, key=key, text=text, metadata=metadata) - - async def get(self, pointer: dict[str, Any]) -> str | None: - return await self.store.get(pointer) diff --git a/projects/memlab/memlab/memory_manager/manager.py b/projects/memlab/memlab/memory_manager/manager.py deleted file mode 100644 index 6e0c56a..0000000 --- a/projects/memlab/memlab/memory_manager/manager.py +++ /dev/null @@ -1,1116 +0,0 @@ -from __future__ import annotations - -import asyncio -import contextlib -import copy -import re -import time -import uuid -from collections import OrderedDict, defaultdict -from dataclasses import asdict -from typing import Any, Callable - -from memlab.config import MemlabConfig -from memlab.io import now_ms -from memlab.memory.api import MemoryManager -from memlab.memory.types import ContextBudget, ContextPack, MemoryItem, PlacementDecision -from memlab.tools.embedding import cosine, hash_embedding - -from .l0 import assemble_l0_context -from .l1 import L1RedisClient -from .l2 import L2PostgresStore -from .l3 import L3ObjectStore -from .vector import QdrantVectorStore - -OpLogger = Callable[[dict[str, Any]], None] -DIRECT_KEY_RE = re.compile(r"^(?:lookup\s+)?(k-\d{6})$", re.IGNORECASE) -RECALL_MARKERS = ("previous", "remember", "earlier", "as we said", "last time", "before") -DIRECT_EVIDENCE_QUERY_MARKERS = ( - "find the secret_code", - "find the secret code", - "document titled", -) - - -def _point_id(agent_id: str, key: str) -> str: - return str(uuid.uuid5(uuid.NAMESPACE_URL, f"memlab:{agent_id}:{key}")) - - -class MultiLevelMemoryManager(MemoryManager): - def __init__(self, config: MemlabConfig, *, op_logger: OpLogger | None = None) -> None: - self.cfg = config - self.op_logger = op_logger - self.l1 = L1RedisClient(config.services.redis_url) if config.memory.l1.enabled else None - self.l2 = L2PostgresStore(config.services.pg_uri) if config.memory.l2.enabled else None - self.vector = ( - QdrantVectorStore( - config.services.qdrant_url, - config.services.qdrant_collection, - vector_size=64, - shard_number=int(getattr(config.memory.l2, "qdrant_shard_number", 1)), - ) - if config.memory.l2.enabled and bool(getattr(config.memory.l2, "vector_enabled", True)) - else None - ) - self.l3 = ( - L3ObjectStore(config.memory.l3.root_dir) - if getattr(config.memory.l3, "enabled", False) - else None - ) - self._recent_messages: dict[str, list[MemoryItem]] = defaultdict(list) - self._l1_write_queue: asyncio.Queue[MemoryItem] | None = ( - asyncio.Queue(maxsize=32768) - if self.cfg.memory.l1.enabled and self.l1 is not None - else None - ) - self._l1_writer_task: asyncio.Task[None] | None = None - self._l1_writer_error: RuntimeError | None = None - self._l1_batch_size = 256 - self._l1_flush_interval_ms = 20 - self._l2_write_queue: asyncio.Queue[MemoryItem] | None = ( - asyncio.Queue(maxsize=16384) - if self.cfg.memory.l2.enabled and self.l2 is not None - else None - ) - self._l2_writer_task: asyncio.Task[None] | None = None - self._l2_writer_error: RuntimeError | None = None - self._l2_batch_size = max(1, int(self.cfg.memory.l2.write_batch_size)) - self._l2_flush_interval_ms = max(1, int(self.cfg.memory.l2.write_flush_interval_ms)) - self._l2_max_batch_per_flush = max(1, int(getattr(self.cfg.memory.l2, "max_batch_per_flush", self._l2_batch_size))) - self._l2_max_flush_ms = max(1, int(getattr(self.cfg.memory.l2, "max_flush_ms", self._l2_flush_interval_ms))) - self._l2_min_importance = float(self.cfg.memory.l2.min_write_importance) - self._l2_write_retry_count = max(1, int(getattr(self.cfg.memory.l2, "write_retry_count", 3))) - self._l2_write_retry_backoff_ms = max(50, int(getattr(self.cfg.memory.l2, "write_retry_backoff_ms", 250))) - self._l2_drop_queue_depth_threshold = max(1, int(getattr(self.cfg.memory.l2, "drop_queue_depth_threshold", 12000))) - self._l2_backpressure_drop_importance_threshold = float( - getattr(self.cfg.memory.l2, "backpressure_drop_importance_threshold", max(self._l2_min_importance, 0.85)) - ) - self._dropped_writes_count = 0 - self._query_embedding_cache: OrderedDict[tuple[str, str], tuple[list[float], int]] = OrderedDict() - self._query_embedding_cache_limit = 8192 - self._embedding_cache_ttl_ms = max(1000, int(getattr(self.cfg.memory.l2, "embedding_cache_ttl_ms", 120000))) - self._retrieval_cache: OrderedDict[tuple[str, str, int, str], tuple[list[MemoryItem], int]] = OrderedDict() - self._retrieval_cache_limit = 8192 - self._retrieval_cache_ttl_ms = max(1000, int(getattr(self.cfg.memory.l2, "retrieval_cache_ttl_ms", 15000))) - self._known_l1_keys: OrderedDict[tuple[str, str], None] = OrderedDict() - self._known_l1_keys_limit = 500000 - self._known_l2_keys: OrderedDict[tuple[str, str], None] = OrderedDict() - self._known_l2_keys_limit = 500000 - self._agents_with_l1_history: set[str] = set() - self._agents_with_l2_history: set[str] = set() - self._close_join_timeout_s = 15.0 - - def _emit_op( - self, - *, - op_name: str, - tier: str, - latency_ms: float, - hit: bool | None, - n_bytes: int | None, - extra: dict[str, Any] | None = None, - ) -> None: - if self.op_logger is None: - return - row = { - "ts_ms": now_ms(), - "op_type": "memory", - "op_name": op_name, - "tier": tier, - "hit": hit, - "latency_ms": float(latency_ms), - "bytes": n_bytes, - "extra": extra or {}, - } - self.op_logger(row) - - def _raise_l2_writer_error(self) -> None: - if self._l2_writer_error is not None: - raise self._l2_writer_error - - def _raise_l1_writer_error(self) -> None: - if self._l1_writer_error is not None: - raise self._l1_writer_error - - def _raise_background_errors(self) -> None: - self._raise_l1_writer_error() - self._raise_l2_writer_error() - - @staticmethod - def _is_transient_l2_write_error(exc: Exception) -> bool: - msg = str(exc or "").lower() - markers = ( - "timeout", - "timed out", - "readtimeout", - "server disconnected", - "connection reset", - "connection aborted", - "temporarily unavailable", - "broken pipe", - ) - return any(marker in msg for marker in markers) - - async def _persist_l1_batch(self, batch: list[MemoryItem], *, flush_interval_ms: float, queue_depth: int) -> None: - if not batch or self.l1 is None: - return - total_bytes = sum(item.bytes for item in batch) - ttl_s = self.cfg.memory.l1.ttl_s if self.cfg.memory.l1.enabled else None - t0 = time.perf_counter() - await self.l1.put_many(batch, ttl_s=ttl_s) - self._emit_op( - op_name="put", - tier="L1", - latency_ms=(time.perf_counter() - t0) * 1000.0, - hit=None, - n_bytes=total_bytes, - extra={ - "batch_size": len(batch), - "batch_size_used": len(batch), - "flush_interval_ms": float(flush_interval_ms), - "queue_depth": int(queue_depth), - }, - ) - - async def _l1_writer_loop(self) -> None: - assert self._l1_write_queue is not None - while True: - try: - item = await self._l1_write_queue.get() - except asyncio.CancelledError: - break - batch = [item] - t_batch_start = time.perf_counter() - cancelled = False - while len(batch) < self._l1_batch_size: - elapsed = (time.perf_counter() - t_batch_start) * 1000.0 - remaining_ms = self._l1_flush_interval_ms - elapsed - if remaining_ms <= 0: - break - try: - nxt = await asyncio.wait_for(self._l1_write_queue.get(), timeout=remaining_ms / 1000.0) - except asyncio.TimeoutError: - break - except asyncio.CancelledError: - cancelled = True - break - batch.append(nxt) - flush_interval_ms = (time.perf_counter() - t_batch_start) * 1000.0 - queue_depth = self._l1_write_queue.qsize() - try: - await self._persist_l1_batch(batch, flush_interval_ms=flush_interval_ms, queue_depth=queue_depth) - except Exception as exc: - self._l1_writer_error = RuntimeError(f"L1 background writer failed: {type(exc).__name__}") - for _ in batch: - self._l1_write_queue.task_done() - while True: - try: - _ = self._l1_write_queue.get_nowait() - self._l1_write_queue.task_done() - except asyncio.QueueEmpty: - break - break - for _ in batch: - self._l1_write_queue.task_done() - if cancelled: - break - - async def _persist_l2_batch(self, batch: list[MemoryItem], *, flush_interval_ms: float, queue_depth: int) -> None: - if not batch: - return - assert self.l2 is not None - - total_bytes = sum(item.bytes for item in batch) - t0 = time.perf_counter() - await self.l2.put_many(batch) - self._emit_op( - op_name="put", - tier="L2", - latency_ms=(time.perf_counter() - t0) * 1000.0, - hit=None, - n_bytes=total_bytes, - extra={ - "batch_size": len(batch), - "batch_size_used": len(batch), - "flush_interval_ms": float(flush_interval_ms), - "queue_depth": int(queue_depth), - "dropped_writes_count": int(self._dropped_writes_count), - }, - ) - - if self.vector is None: - self._emit_op( - op_name="vector_upsert_skip", - tier="L2", - latency_ms=0.0, - hit=False, - n_bytes=total_bytes, - extra={"reason": "vector_unavailable"}, - ) - return - - points = [ - { - "id": _point_id(item.agent_id, item.key), - "vector": item.embedding or hash_embedding(item.text), - "payload": {"agent_id": item.agent_id, "key": item.key, "kind": item.kind}, - } - for item in batch - ] - t1 = time.perf_counter() - await self.vector.upsert_many(points) - self._emit_op( - op_name="vector_upsert", - tier="L2", - latency_ms=(time.perf_counter() - t1) * 1000.0, - hit=None, - n_bytes=total_bytes, - extra={ - "batch_size": len(batch), - "batch_size_used": len(batch), - "flush_interval_ms": float(flush_interval_ms), - "queue_depth": int(queue_depth), - "dropped_writes_count": int(self._dropped_writes_count), - }, - ) - - def _query_embedding(self, agent_id: str, query: str) -> tuple[list[float], bool]: - key = (agent_id, query.strip().lower()) - cached = self._query_embedding_cache.get(key) - now = now_ms() - if cached is not None: - emb, expires_ms = cached - if expires_ms >= now: - self._query_embedding_cache.move_to_end(key) - return emb, True - self._query_embedding_cache.pop(key, None) - emb = hash_embedding(query) - self._query_embedding_cache[key] = (emb, now + self._embedding_cache_ttl_ms) - if len(self._query_embedding_cache) > self._query_embedding_cache_limit: - self._query_embedding_cache.popitem(last=False) - return emb, False - - def _get_retrieval_cache(self, agent_id: str, query: str, k: int, mode: str) -> list[MemoryItem] | None: - key = (agent_id, query.strip().lower(), int(k), str(mode)) - cached = self._retrieval_cache.get(key) - now = now_ms() - if cached is None: - return None - items, expires_ms = cached - if expires_ms < now: - self._retrieval_cache.pop(key, None) - return None - self._retrieval_cache.move_to_end(key) - return [copy.deepcopy(item) for item in items] - - def _set_retrieval_cache(self, agent_id: str, query: str, k: int, mode: str, items: list[MemoryItem]) -> None: - key = (agent_id, query.strip().lower(), int(k), str(mode)) - self._retrieval_cache[key] = ([copy.deepcopy(item) for item in items], now_ms() + self._retrieval_cache_ttl_ms) - self._retrieval_cache.move_to_end(key) - if len(self._retrieval_cache) > self._retrieval_cache_limit: - self._retrieval_cache.popitem(last=False) - - def _has_memory_history(self, agent_id: str) -> bool: - if self._recent_messages.get(agent_id): - return True - if agent_id in self._agents_with_l1_history: - return True - if agent_id in self._agents_with_l2_history: - return True - return False - - def _should_retrieve(self, agent_id: str, query: str) -> tuple[bool, str | None]: - direct_key = self._extract_direct_key(query) - if direct_key: - return True, None - q = query.strip().lower() - if not q: - return False, "empty_query" - if any(marker in q for marker in RECALL_MARKERS): - return True, None - has_history = self._has_memory_history(agent_id) - token_count = len(q.split()) - if not has_history and token_count <= 8 and len(q) <= 80: - return False, "short_query_without_history" - if not has_history: - return False, "no_memory_history" - return True, None - - @staticmethod - def _has_direct_tool_evidence(recent: list[MemoryItem], query: str) -> bool: - q = str(query or "").strip().lower() - if not q: - return False - if not any(marker in q for marker in DIRECT_EVIDENCE_QUERY_MARKERS): - return False - # Use a wider tail window because retrieval shards may occupy the latest slots. - for item in reversed(recent[-256:]): - metadata = item.metadata or {} - if str(metadata.get("role", "")).lower() != "tool": - continue - if bool(metadata.get("retrieval_shard", False)): - continue - # The main evidence body is usually much longer than hint snippets. - if len(str(item.text or "")) < 80: - continue - return True - return False - - @staticmethod - def _finalize_results(items: list[MemoryItem], query: str, query_embedding: list[float], k: int) -> list[MemoryItem]: - scored = sorted(items, key=lambda x: MultiLevelMemoryManager._relevance_score(x, query, query_embedding), reverse=True) - for item in scored: - item.metadata["retrieval_score"] = MultiLevelMemoryManager._relevance_score(item, query, query_embedding) - return scored[:k] - - @staticmethod - def _extract_direct_key(query: str) -> str | None: - m = DIRECT_KEY_RE.match(query.strip()) - if not m: - return None - return m.group(1).lower() - - def _mark_known_l2_key(self, agent_id: str, key: str) -> None: - token = (agent_id, key) - self._known_l2_keys[token] = None - self._known_l2_keys.move_to_end(token) - self._agents_with_l2_history.add(agent_id) - if len(self._known_l2_keys) > self._known_l2_keys_limit: - removed = self._known_l2_keys.popitem(last=False)[0] - removed_agent = removed[0] - if not any(t[0] == removed_agent for t in self._known_l2_keys.keys()): - self._agents_with_l2_history.discard(removed_agent) - - def _mark_known_l1_key(self, agent_id: str, key: str) -> None: - token = (agent_id, key) - self._known_l1_keys[token] = None - self._known_l1_keys.move_to_end(token) - self._agents_with_l1_history.add(agent_id) - if len(self._known_l1_keys) > self._known_l1_keys_limit: - removed = self._known_l1_keys.popitem(last=False)[0] - removed_agent = removed[0] - if not any(t[0] == removed_agent for t in self._known_l1_keys.keys()): - self._agents_with_l1_history.discard(removed_agent) - - async def _l2_writer_loop(self) -> None: - assert self._l2_write_queue is not None - while True: - try: - item = await self._l2_write_queue.get() - except asyncio.CancelledError: - break - batch = [item] - t_batch_start = time.perf_counter() - cancelled = False - target_batch = min(self._l2_batch_size, self._l2_max_batch_per_flush) - while len(batch) < target_batch: - elapsed = (time.perf_counter() - t_batch_start) * 1000.0 - remaining_budget = min(self._l2_flush_interval_ms, self._l2_max_flush_ms) - remaining_ms = remaining_budget - elapsed - if remaining_ms <= 0: - break - try: - nxt = await asyncio.wait_for(self._l2_write_queue.get(), timeout=remaining_ms / 1000.0) - except asyncio.TimeoutError: - break - except asyncio.CancelledError: - cancelled = True - break - batch.append(nxt) - flush_interval_ms = (time.perf_counter() - t_batch_start) * 1000.0 - queue_depth = self._l2_write_queue.qsize() - try: - last_exc: Exception | None = None - persisted = False - for attempt in range(1, self._l2_write_retry_count + 1): - try: - await self._persist_l2_batch(batch, flush_interval_ms=flush_interval_ms, queue_depth=queue_depth) - persisted = True - break - except Exception as exc: - last_exc = exc - if attempt >= self._l2_write_retry_count or not self._is_transient_l2_write_error(exc): - raise - await asyncio.sleep((self._l2_write_retry_backoff_ms / 1000.0) * attempt) - try: - if self.l2 is not None: - await self.l2.connect() - except Exception: - pass - if not persisted and last_exc is not None: - raise last_exc - except Exception as exc: - self._l2_writer_error = RuntimeError(f"L2 background writer failed: {type(exc).__name__}") - for _ in batch: - self._l2_write_queue.task_done() - while True: - try: - _ = self._l2_write_queue.get_nowait() - self._l2_write_queue.task_done() - except asyncio.QueueEmpty: - break - break - for _ in batch: - self._l2_write_queue.task_done() - if cancelled: - break - - async def init(self) -> None: - if self._l1_write_queue is not None and self._l1_writer_task is None: - self._l1_writer_task = asyncio.create_task(self._l1_writer_loop()) - if self.l2 is not None: - t0 = time.perf_counter() - await self.l2.connect() - self._emit_op(op_name="connect", tier="L2", latency_ms=(time.perf_counter() - t0) * 1000.0, hit=None, n_bytes=None) - if self.vector is not None: - try: - t0 = time.perf_counter() - await self.vector.ensure_collection() - self._emit_op(op_name="ensure_collection", tier="L2", latency_ms=(time.perf_counter() - t0) * 1000.0, hit=None, n_bytes=None) - except Exception as exc: - self._emit_op( - op_name="degrade", - tier="L2", - latency_ms=0.0, - hit=False, - n_bytes=None, - extra={"reason": "vector_unavailable", "detail": type(exc).__name__}, - ) - self.vector = None - if self.l3 is not None: - t0 = time.perf_counter() - await self.l3.ensure_ready() - self._emit_op(op_name="ensure_ready", tier="L3", latency_ms=(time.perf_counter() - t0) * 1000.0, hit=None, n_bytes=None) - if self._l2_write_queue is not None and self._l2_writer_task is None: - self._l2_writer_task = asyncio.create_task(self._l2_writer_loop()) - - @staticmethod - def _drain_queue(queue: asyncio.Queue[MemoryItem]) -> int: - drained = 0 - while True: - try: - _ = queue.get_nowait() - queue.task_done() - drained += 1 - except asyncio.QueueEmpty: - break - return drained - - async def close(self) -> None: - close_errors: list[str] = [] - if self._l1_write_queue is not None: - try: - await asyncio.wait_for(self._l1_write_queue.join(), timeout=self._close_join_timeout_s) - except asyncio.TimeoutError: - drained = self._drain_queue(self._l1_write_queue) - close_errors.append( - f"L1 write queue drain timed out after {self._close_join_timeout_s:.1f}s; drained={drained}" - ) - if self._l1_writer_task is not None: - self._l1_writer_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await self._l1_writer_task - self._l1_writer_task = None - if self._l2_write_queue is not None: - try: - await asyncio.wait_for(self._l2_write_queue.join(), timeout=self._close_join_timeout_s) - except asyncio.TimeoutError: - drained = self._drain_queue(self._l2_write_queue) - close_errors.append( - f"L2 write queue drain timed out after {self._close_join_timeout_s:.1f}s; drained={drained}" - ) - if self._l2_writer_task is not None: - self._l2_writer_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await self._l2_writer_task - self._l2_writer_task = None - if self.l1 is not None: - await self.l1.close() - if self.l2 is not None: - await self.l2.close() - if self.vector is not None: - await self.vector.close() - if self._l1_writer_error is not None: - close_errors.append(str(self._l1_writer_error)) - if self._l2_writer_error is not None: - close_errors.append(str(self._l2_writer_error)) - if close_errors: - raise RuntimeError("; ".join(close_errors)) - - async def put(self, item: MemoryItem, *, ttl_s: int | None = None) -> PlacementDecision: - self._raise_background_errors() - tiers: list[str] = [] - reasons: list[str] = [] - degrade_mode: str | None = None - admission_reason = "tiered_ingest" - - if item.embedding is None: - item.embedding = hash_embedding(item.text) - - if self.l3 is not None and int(item.bytes) >= int(self.cfg.memory.l3.pointer_threshold_bytes): - source_kind = item.kind - try: - t_l3 = time.perf_counter() - pointer = await self.l3.put( - agent_id=item.agent_id, - key=item.key, - text=item.text, - metadata={"kind": source_kind, "created_at_ms": int(item.created_at_ms)}, - ) - self._emit_op( - op_name="put", - tier="L3", - latency_ms=(time.perf_counter() - t_l3) * 1000.0, - hit=None, - n_bytes=int(pointer.get("bytes", item.bytes)), - ) - preview_chars = max(80, int(self.cfg.memory.l3.pointer_preview_chars)) - preview = item.text[:preview_chars] - marker = f"[BLOB_REF:{source_kind}:{item.key}] " - pointer_meta = dict(item.metadata or {}) - pointer_meta["l3_pointer"] = pointer - pointer_meta["pointerized"] = True - pointer_meta["source_kind"] = source_kind - item = MemoryItem( - key=item.key, - agent_id=item.agent_id, - kind="blob_ref", - text=marker + preview, - bytes=len((marker + preview).encode("utf-8")), - importance=item.importance, - created_at_ms=item.created_at_ms, - last_access_ms=item.last_access_ms, - access_count=item.access_count, - embedding=item.embedding, - metadata=pointer_meta, - ) - tiers.append("L3") - reasons.append("cold_large_object") - admission_reason = "pointerized_to_l3" - except Exception as exc: - degrade_mode = "l3_unavailable" - self._emit_op( - op_name="put_skip", - tier="L3", - latency_ms=0.0, - hit=False, - n_bytes=item.bytes, - extra={"reason": "l3_put_failed", "detail": type(exc).__name__}, - ) - reasons.append("l3_put_failed_fallback") - - if item.kind in {"message", "fact", "summary", "tool_result", "blob_ref"}: - buf = self._recent_messages[item.agent_id] - buf.append(item) - if len(buf) > 512: - del buf[:-512] - tiers.append("L0") - reasons.append("eligible_for_context") - - if self.cfg.memory.l1.enabled and self.l1 is not None and item.bytes <= self.cfg.memory.l1.max_object_bytes: - if float(item.importance) >= float(self.cfg.memory.l1.reuse_threshold): - if self._l1_write_queue is not None: - t0 = time.perf_counter() - await self._l1_write_queue.put(item) - self._emit_op( - op_name="enqueue_put", - tier="L1", - latency_ms=(time.perf_counter() - t0) * 1000.0, - hit=None, - n_bytes=item.bytes, - extra={"queue_depth": self._l1_write_queue.qsize()}, - ) - else: - t0 = time.perf_counter() - await self.l1.put(item, ttl_s=ttl_s or self.cfg.memory.l1.ttl_s) - self._emit_op( - op_name="put", - tier="L1", - latency_ms=(time.perf_counter() - t0) * 1000.0, - hit=None, - n_bytes=item.bytes, - ) - self._mark_known_l1_key(item.agent_id, item.key) - tiers.append("L1") - reasons.append("hot_cache") - else: - self._emit_op( - op_name="skip_put", - tier="L1", - latency_ms=0.0, - hit=False, - n_bytes=item.bytes, - extra={ - "importance": float(item.importance), - "reuse_threshold": float(self.cfg.memory.l1.reuse_threshold), - }, - ) - reasons.append("below_l1_reuse_threshold") - - if self.cfg.memory.l2.enabled and self.l2 is not None: - if float(item.importance) >= self._l2_min_importance: - if self._l2_write_queue is not None: - queue_depth = self._l2_write_queue.qsize() - if ( - queue_depth >= self._l2_drop_queue_depth_threshold - and float(item.importance) < self._l2_backpressure_drop_importance_threshold - ): - self._dropped_writes_count += 1 - self._emit_op( - op_name="drop_put", - tier="L2", - latency_ms=0.0, - hit=False, - n_bytes=item.bytes, - extra={ - "queue_depth": int(queue_depth), - "drop_queue_depth_threshold": int(self._l2_drop_queue_depth_threshold), - "importance": float(item.importance), - "drop_importance_threshold": float(self._l2_backpressure_drop_importance_threshold), - "dropped_writes_count": int(self._dropped_writes_count), - }, - ) - reasons.append("l2_backpressure_drop") - return PlacementDecision( - key=item.key, - tiers=tiers, - ttl_s=ttl_s, - reasons=reasons, - admission_reason=admission_reason, - degrade_mode=degrade_mode, - ) - t2 = time.perf_counter() - await self._l2_write_queue.put(item) - self._mark_known_l2_key(item.agent_id, item.key) - self._emit_op( - op_name="enqueue_put", - tier="L2", - latency_ms=(time.perf_counter() - t2) * 1000.0, - hit=None, - n_bytes=item.bytes, - extra={ - "queue_depth": self._l2_write_queue.qsize(), - "dropped_writes_count": int(self._dropped_writes_count), - }, - ) - else: - await self._persist_l2_batch([item], flush_interval_ms=0.0, queue_depth=0) - self._mark_known_l2_key(item.agent_id, item.key) - tiers.append("L2") - reasons.append("durable_with_vector_index" if self.vector is not None else "durable_without_vector") - else: - self._emit_op( - op_name="skip_put", - tier="L2", - latency_ms=0.0, - hit=False, - n_bytes=item.bytes, - extra={ - "importance": float(item.importance), - "min_importance": self._l2_min_importance, - }, - ) - reasons.append("below_l2_importance_threshold") - - return PlacementDecision( - key=item.key, - tiers=tiers, - ttl_s=ttl_s, - reasons=reasons, - admission_reason=admission_reason, - degrade_mode=degrade_mode, - ) - - async def get( - self, - agent_id: str, - query: str, - *, - k: int, - mode: str = "hybrid", - telemetry: dict[str, Any] | None = None, - ) -> list[MemoryItem]: - self._raise_background_errors() - if telemetry is not None: - telemetry.setdefault("embedding_cache_hit", None) - telemetry.setdefault("retrieval_cache_hit", None) - telemetry.setdefault("degrade_mode", "none") - telemetry.setdefault("pointer_fetch", 0) - out: list[MemoryItem] = [] - seen: set[str] = set() - degrade_modes: list[str] = [] - direct_key = self._extract_direct_key(query) - if direct_key: - recent = self._recent_messages.get(agent_id, []) - for item in reversed(recent): - if item.key == direct_key: - item.metadata["retrieval_score"] = 1.0 - self._emit_op( - op_name="get_key_recent", - tier="L0", - latency_ms=0.0, - hit=True, - n_bytes=item.bytes, - ) - result = await self._resolve_blob_refs([item], telemetry=telemetry, degrade_modes=degrade_modes) - return result - if telemetry is not None: - cached = self._get_retrieval_cache(agent_id, query, k, mode) - if cached is not None: - telemetry["retrieval_cache_hit"] = True - return cached - telemetry["retrieval_cache_hit"] = False - t_embed = time.perf_counter() - query_embedding, cache_hit = self._query_embedding(agent_id, query) - if telemetry is not None: - telemetry["embedding_cache_hit"] = cache_hit - if self.cfg.memory.l2.enabled: - self._emit_op( - op_name="embed_query", - tier="L2", - latency_ms=(time.perf_counter() - t_embed) * 1000.0, - hit=cache_hit, - n_bytes=None, - ) - best_l1_score = 0.0 - - if self.cfg.memory.l1.enabled and self.l1 is not None and mode in {"hybrid", "l1_first"}: - try: - if direct_key: - if (agent_id, direct_key) in self._known_l1_keys: - t_key = time.perf_counter() - exact = await self.l1.get_key(agent_id, direct_key) - key_lat = (time.perf_counter() - t_key) * 1000.0 - self._emit_op(op_name="get_key", tier="L1", latency_ms=key_lat, hit=exact is not None, n_bytes=None) - if exact is not None: - exact.metadata["retrieval_score"] = 1.0 - result = [exact] - result = await self._resolve_blob_refs(result, telemetry=telemetry, degrade_modes=degrade_modes) - self._set_retrieval_cache(agent_id, query, k, mode, result) - return result - else: - self._emit_op( - op_name="get_key_skip", - tier="L1", - latency_ms=0.0, - hit=False, - n_bytes=None, - extra={"reason": "unknown_direct_key"}, - ) - if not direct_key: - t0 = time.perf_counter() - l1_recent = await self.l1.get_recent(agent_id, limit=min(64, max(8, k * 2))) - lat = (time.perf_counter() - t0) * 1000.0 - for item in l1_recent: - if item.key in seen: - continue - score = self._relevance_score(item, query, query_embedding) - if score > best_l1_score: - best_l1_score = score - if score > 0.05: - item.metadata["retrieval_score"] = score - out.append(item) - seen.add(item.key) - if len(out) >= k: - self._emit_op(op_name="get", tier="L1", latency_ms=lat, hit=True, n_bytes=None) - result = out[:k] - result = await self._resolve_blob_refs(result, telemetry=telemetry, degrade_modes=degrade_modes) - self._set_retrieval_cache(agent_id, query, k, mode, result) - return result - self._emit_op(op_name="get", tier="L1", latency_ms=lat, hit=bool(out), n_bytes=None) - except Exception as exc: - degrade_modes.append("l1_unavailable") - self._emit_op( - op_name="degrade", - tier="L1", - latency_ms=0.0, - hit=False, - n_bytes=None, - extra={"reason": type(exc).__name__}, - ) - - if self.cfg.memory.l2.enabled and self.l2 is not None and mode in {"hybrid", "l2_only", "l1_first"}: - try: - if direct_key: - self._emit_op( - op_name="vector_search_skip", - tier="L2", - latency_ms=0.0, - hit=True, - n_bytes=None, - extra={"reason": "direct_key_query"}, - ) - if (agent_id, direct_key) in self._known_l2_keys: - t1 = time.perf_counter() - pg_items = await self.l2.get_by_keys(agent_id, [direct_key]) - self._emit_op( - op_name="get_by_keys", - tier="L2", - latency_ms=(time.perf_counter() - t1) * 1000.0, - hit=bool(pg_items), - n_bytes=None, - extra={"key_count": 1, "direct_key": True}, - ) - out.extend(pg_items) - else: - self._emit_op( - op_name="get_by_keys_skip", - tier="L2", - latency_ms=0.0, - hit=False, - n_bytes=None, - extra={"reason": "unknown_direct_key"}, - ) - elif self.vector is None: - degrade_modes.append("vector_unavailable") - self._emit_op( - op_name="vector_search_skip", - tier="L2", - latency_ms=0.0, - hit=False, - n_bytes=None, - extra={"reason": "vector_unavailable"}, - ) - elif best_l1_score >= 0.75 and out: - self._emit_op( - op_name="vector_search_skip", - tier="L2", - latency_ms=0.0, - hit=True, - n_bytes=None, - extra={"reason": "high_confidence_l1_hit"}, - ) - result = self._finalize_results(out, query, query_embedding, k) - result = await self._resolve_blob_refs(result, telemetry=telemetry, degrade_modes=degrade_modes) - self._set_retrieval_cache(agent_id, query, k, mode, result) - return result - else: - t0 = time.perf_counter() - q_hits = await self.vector.search(query_embedding, limit=max(k * 2, 16), agent_id=agent_id) # type: ignore[union-attr] - qdrant_ms = (time.perf_counter() - t0) * 1000.0 - self._emit_op( - op_name="vector_search", - tier="L2", - latency_ms=qdrant_ms, - hit=bool(q_hits), - n_bytes=None, - extra={"qdrant_search_time_ms": qdrant_ms}, - ) - q_keys = [] - for hit in q_hits: - payload = hit.get("payload", {}) or {} - key = payload.get("key") - if key and key not in seen: - q_keys.append(key) - seen.add(key) - if q_keys: - deduped_keys = list(dict.fromkeys(q_keys)) - t1 = time.perf_counter() - pg_items = await self.l2.get_by_keys(agent_id, deduped_keys) - self._emit_op( - op_name="get_by_keys", - tier="L2", - latency_ms=(time.perf_counter() - t1) * 1000.0, - hit=bool(pg_items), - n_bytes=None, - extra={"key_count": len(deduped_keys)}, - ) - for item in pg_items: - item.metadata["retrieval_score"] = self._relevance_score(item, query, query_embedding) - out.extend(pg_items) - # For direct-key lookups, avoid expensive broad text scans. - if direct_key: - result = self._finalize_results(out, query, query_embedding, k) - result = await self._resolve_blob_refs(result, telemetry=telemetry, degrade_modes=degrade_modes) - self._set_retrieval_cache(agent_id, query, k, mode, result) - return result - if len(out) < k: - t2 = time.perf_counter() - extra = await self.l2.search_text(agent_id, query=query, limit=(k - len(out))) - self._emit_op(op_name="search_text", tier="L2", latency_ms=(time.perf_counter() - t2) * 1000.0, hit=bool(extra), n_bytes=None) - for item in extra: - if item.key in seen: - continue - item.metadata["retrieval_score"] = self._relevance_score(item, query, query_embedding) - out.append(item) - seen.add(item.key) - except Exception as exc: - degrade_modes.append("l2_unavailable") - self._emit_op( - op_name="degrade", - tier="L2", - latency_ms=0.0, - hit=False, - n_bytes=None, - extra={"reason": type(exc).__name__}, - ) - - result = self._finalize_results(out, query, query_embedding, k) - result = await self._resolve_blob_refs(result, telemetry=telemetry, degrade_modes=degrade_modes) - if telemetry is not None: - telemetry["degrade_mode"] = "|".join(sorted(set(degrade_modes))) if degrade_modes else "none" - self._set_retrieval_cache(agent_id, query, k, mode, result) - return result - - async def assemble_context(self, agent_id: str, user_query: str, budget: ContextBudget) -> ContextPack: - self._raise_background_errors() - recent = list(self._recent_messages.get(agent_id, [])) - if not recent and self.cfg.memory.l2.enabled and self.l2 is not None: - t0 = time.perf_counter() - recent = await self.l2.get_recent(agent_id, limit=max(2 * budget.keep_recent_messages, 20)) - recent = list(reversed(recent)) - self._emit_op(op_name="get_recent", tier="L2", latency_ms=(time.perf_counter() - t0) * 1000.0, hit=bool(recent), n_bytes=None) - - retrieval_k = 8 - if self.cfg.memory.l2.enabled: - retrieval_k = max(1, int(self.cfg.memory.l2.default_top_k)) - retrieve_allowed, retrieve_skip_reason = self._should_retrieve(agent_id, user_query) - if retrieve_allowed and bool(getattr(self.cfg.memory.policy, "direct_evidence_retrieval_bypass", True)): - if self._has_direct_tool_evidence(recent, user_query): - retrieve_allowed = False - retrieve_skip_reason = "direct_tool_evidence_present" - retrieval_telemetry: dict[str, Any] = {} - if retrieve_allowed: - candidates = await self.get(agent_id, user_query, k=retrieval_k, mode="hybrid", telemetry=retrieval_telemetry) - else: - candidates = [] - self._emit_op( - op_name="retrieve_skip", - tier="L2", - latency_ms=0.0, - hit=False, - n_bytes=None, - extra={"reason": retrieve_skip_reason or "heuristic_skip"}, - ) - inject_budget = min( - int(self.cfg.memory.l0.max_memory_injection_tokens), - int(self.cfg.memory.l0.max_injected_tokens), - ) - packed = assemble_l0_context( - recent_messages=recent, - candidates=candidates, - user_query=user_query, - budget=ContextBudget( - max_prompt_tokens=budget.max_prompt_tokens, - max_memory_injection_tokens=max(0, inject_budget), - keep_recent_messages=budget.keep_recent_messages, - ), - summarization_on=self.cfg.experiment.summarization_on, - summarize_trigger_tokens=self.cfg.memory.l0.summarize_trigger_tokens, - summarize_target_tokens=self.cfg.memory.l0.summarize_target_tokens, - min_relevance_score=float(self.cfg.memory.l0.min_relevance_score), - ) - self._emit_op( - op_name="assemble_context", - tier="L0", - latency_ms=0.0, - hit=True, - n_bytes=None, - extra={ - "injected_tokens": packed.injected_tokens, - "injected_items": packed.injected_item_count, - "total_tokens": packed.total_tokens, - "summarized": packed.summarized, - }, - ) - return ContextPack( - messages=packed.messages, - injected_memories=packed.selected, - trace={ - "selected_keys": [i.key for i in packed.selected], - "dropped_keys": packed.dropped_keys, - "summarized": packed.summarized, - "total_tokens": packed.total_tokens, - "packed_tokens": packed.total_tokens, - "injection_tokens": packed.injected_tokens, - "injection_item_count": packed.injected_item_count, - "admission_reason": "retrieval_then_pack", - "eviction_reason": "budget_trim" if packed.dropped_keys else "", - "pointer_fetch": int(retrieval_telemetry.get("pointer_fetch", 0) or 0), - "degrade_mode": retrieval_telemetry.get("degrade_mode", "none"), - "retrieve_skipped_reason": retrieve_skip_reason, - "embedding_cache_hit": retrieval_telemetry.get("embedding_cache_hit"), - "retrieval_cache_hit": retrieval_telemetry.get("retrieval_cache_hit"), - }, - ) - - async def checkpoint(self, agent_id: str, thread_id: str, state: dict) -> None: - self._raise_background_errors() - if str(getattr(self.cfg.memory.policy, "checkpoint_policy", "l2_only")) == "disabled": - self._emit_op( - op_name="checkpoint_skip", - tier="L2", - latency_ms=0.0, - hit=False, - n_bytes=None, - extra={"reason": "policy_disabled"}, - ) - return - if self.l2 is None: - return - t0 = time.perf_counter() - await self.l2.checkpoint(agent_id, thread_id, now_ms(), state) - self._emit_op(op_name="checkpoint", tier="L2", latency_ms=(time.perf_counter() - t0) * 1000.0, hit=None, n_bytes=None) - - async def healthcheck(self) -> dict[str, Any]: - return { - "redis": await self.l1.ping() if self.l1 is not None else True, - "postgres": await self.l2.ping() if self.l2 is not None else True, - "qdrant": await self.vector.ping() if self.vector is not None else True, - "l3_object_store": await self.l3.ping() if self.l3 is not None else True, - "postgres_backend": "postgres" if self.l2 is not None else "none", - "qdrant_mode": "remote" if self.vector is not None else "none", - "l3_mode": "local_object_store" if self.l3 is not None else "none", - } - - async def _resolve_blob_refs( - self, - items: list[MemoryItem], - *, - telemetry: dict[str, Any] | None, - degrade_modes: list[str], - ) -> list[MemoryItem]: - if not items or self.l3 is None: - return items - pointer_fetch = int(telemetry.get("pointer_fetch", 0) or 0) if telemetry is not None else 0 - for item in items: - pointer = (item.metadata or {}).get("l3_pointer") - if item.kind != "blob_ref" or not isinstance(pointer, dict): - continue - try: - t0 = time.perf_counter() - blob_text = await self.l3.get(pointer) - lat = (time.perf_counter() - t0) * 1000.0 - self._emit_op(op_name="get", tier="L3", latency_ms=lat, hit=bool(blob_text), n_bytes=None) - if not blob_text: - continue - pointer_fetch += 1 - preview_chars = max(80, int(self.cfg.memory.l3.pointer_preview_chars)) - preview = blob_text[:preview_chars] - item.metadata["pointer_preview"] = preview - item.text = f"{item.text}\n[POINTER_PREVIEW] {preview}" - item.bytes = len(item.text.encode("utf-8")) - except Exception as exc: - degrade_modes.append("l3_unavailable") - self._emit_op( - op_name="degrade", - tier="L3", - latency_ms=0.0, - hit=False, - n_bytes=None, - extra={"reason": type(exc).__name__}, - ) - if telemetry is not None: - telemetry["pointer_fetch"] = pointer_fetch - return items - - @staticmethod - def _relevance_score(item: MemoryItem, query: str, query_embedding: list[float]) -> float: - emb = item.embedding or hash_embedding(item.text) - sem = max(0.0, cosine(emb, query_embedding)) - lex = 1.0 if any(tok in item.text.lower() for tok in query.lower().split()) else 0.0 - return 0.7 * sem + 0.3 * lex - - @staticmethod - def to_dict(item: MemoryItem) -> dict[str, Any]: - return asdict(item) diff --git a/projects/memlab/memlab/memory_manager/vector.py b/projects/memlab/memlab/memory_manager/vector.py deleted file mode 100644 index d109eea..0000000 --- a/projects/memlab/memlab/memory_manager/vector.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from memlab.backends.qdrant_store import QdrantStore - - -class QdrantVectorStore: - def __init__( - self, - qdrant_url: str, - collection: str, - vector_size: int = 64, - shard_number: int = 1, - ) -> None: - self.store = QdrantStore( - qdrant_url, - collection, - vector_size=vector_size, - shard_number=shard_number, - ) - - async def ensure_collection(self) -> None: - await self.store.ensure_collection() - - async def ping(self) -> bool: - return await self.store.ping() - - async def upsert(self, point_id: str | int, vector: list[float], payload: dict[str, Any]) -> None: - await self.store.upsert(point_id=point_id, vector=vector, payload=payload) - - async def upsert_many(self, points: list[dict[str, Any]]) -> None: - await self.store.upsert_many(points) - - async def search(self, query_vector: list[float], *, limit: int, agent_id: str | None = None) -> list[dict[str, Any]]: - return await self.store.search(query_vector, limit=limit, agent_id=agent_id) - - async def close(self) -> None: - await self.store.close() diff --git a/projects/memlab/memlab/metrics/__init__.py b/projects/memlab/memlab/metrics/__init__.py deleted file mode 100644 index c661e15..0000000 --- a/projects/memlab/memlab/metrics/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from .eventlog import EventLogWriter -from .manifest import build_manifest, resolve_git_commit -from .stats import mean_std_ci95, percentile, summarize_latency -from .summary_builder import aggregate_runs, build_acceptance_result, write_experiment_report - -__all__ = [ - "EventLogWriter", - "build_manifest", - "resolve_git_commit", - "percentile", - "summarize_latency", - "mean_std_ci95", - "aggregate_runs", - "build_acceptance_result", - "write_experiment_report", -] diff --git a/projects/memlab/memlab/metrics/eventlog.py b/projects/memlab/memlab/metrics/eventlog.py deleted file mode 100644 index f611f44..0000000 --- a/projects/memlab/memlab/metrics/eventlog.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - - -class EventLogWriter: - def __init__(self, path: str | Path) -> None: - self.path = Path(path) - self.path.parent.mkdir(parents=True, exist_ok=True) - - def append(self, event: dict[str, Any]) -> None: - with self.path.open("a", encoding="utf-8") as f: - f.write(json.dumps(event, ensure_ascii=False) + "\n") diff --git a/projects/memlab/memlab/metrics/manifest.py b/projects/memlab/memlab/metrics/manifest.py deleted file mode 100644 index e1fc121..0000000 --- a/projects/memlab/memlab/metrics/manifest.py +++ /dev/null @@ -1,126 +0,0 @@ -from __future__ import annotations - -import os -import platform -import subprocess -import ctypes -from pathlib import Path -from typing import Any - -from memlab.pipeline.control_plane import default_manifest_control_fields - - -def resolve_git_commit(repo_path: str | Path) -> str | None: - repo = Path(repo_path) - try: - out = subprocess.check_output( - ["git", "-C", str(repo), "rev-parse", "HEAD"], - stderr=subprocess.DEVNULL, - ) - except Exception: - return None - return out.decode("utf-8").strip() - - -def _get_total_ram_bytes() -> int: - if platform.system().lower().startswith("win"): - class MEMORYSTATUSEX(ctypes.Structure): - _fields_ = [ - ("dwLength", ctypes.c_uint32), - ("dwMemoryLoad", ctypes.c_uint32), - ("ullTotalPhys", ctypes.c_uint64), - ("ullAvailPhys", ctypes.c_uint64), - ("ullTotalPageFile", ctypes.c_uint64), - ("ullAvailPageFile", ctypes.c_uint64), - ("ullTotalVirtual", ctypes.c_uint64), - ("ullAvailVirtual", ctypes.c_uint64), - ("ullAvailExtendedVirtual", ctypes.c_uint64), - ] - - status = MEMORYSTATUSEX() - status.dwLength = ctypes.sizeof(MEMORYSTATUSEX) - if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): # type: ignore[attr-defined] - return int(status.ullTotalPhys) - try: - if hasattr(os, "sysconf"): - pages = int(os.sysconf("SC_PHYS_PAGES")) # type: ignore[attr-defined] - page_size = int(os.sysconf("SC_PAGE_SIZE")) # type: ignore[attr-defined] - return pages * page_size - except Exception: - pass - return 0 - - -def build_manifest( - *, - run_id: str, - variant_id: str, - workload_id: str, - seed: int, - repeat_idx: int, - env_valid: bool, - env_invalid_reasons: list[str], - repro_valid: bool, - repro_invalid_reasons: list[str], - llm_real: bool | None, - llm_invalid_reasons: list[str], - usage_unavailable: bool | None, - usage_unavailable_reasons: list[str], - model_base_url: str | None, - model_id: str | None, - model_fallback_used: bool, - comparable_mode: bool, - comparability_invalid_reasons: list[str], - metrics_schema_version: int, - memlab_git_commit: str | None, - deerflow_git_commit: str | None, - docker_image_digests: dict[str, str], - infra_versions: dict[str, str], - extra: dict[str, Any] | None = None, - track_id: str | None = None, - control: dict[str, Any] | None = None, -) -> dict[str, Any]: - manifest = { - "run_id": run_id, - "variant_id": variant_id, - "workload_id": workload_id, - "seed": int(seed), - "repeat_idx": int(repeat_idx), - "env_valid": bool(env_valid), - "env_invalid_reasons": list(env_invalid_reasons), - "repro_valid": bool(repro_valid), - "repro_invalid_reasons": list(repro_invalid_reasons), - "llm_real": llm_real if llm_real is None else bool(llm_real), - "llm_invalid_reasons": list(llm_invalid_reasons), - "usage_unavailable": usage_unavailable if usage_unavailable is None else bool(usage_unavailable), - "usage_unavailable_reasons": list(usage_unavailable_reasons), - "model_base_url": model_base_url, - "model_id": model_id, - "model_fallback_used": bool(model_fallback_used), - "comparable_mode": bool(comparable_mode), - "comparability_invalid_reasons": list(comparability_invalid_reasons), - "metrics_schema_version": int(metrics_schema_version), - "memlab_git_commit": memlab_git_commit, - "deerflow_git_commit": deerflow_git_commit, - "docker_image_digests": docker_image_digests, - "infra_versions": infra_versions, - "host_info": { - "os": platform.platform(), - "cpu_cores": os.cpu_count() or 0, - "ram_bytes": _get_total_ram_bytes(), - "python_version": platform.python_version(), - }, - "extra": extra or {}, - "track_id": str(track_id or ""), - } - for key, value in default_manifest_control_fields().items(): - manifest.setdefault(key, value) - if control: - manifest.update(control) - if not manifest.get("track_id"): - extra_map = manifest.get("extra") if isinstance(manifest.get("extra"), dict) else {} - prompt_mode = str((extra_map or {}).get("prompt_mode", "") or "").strip() - experiment_track = str((extra_map or {}).get("experiment_track", "") or "").strip() - if experiment_track or prompt_mode: - manifest["track_id"] = f"{experiment_track or 'mainline'}:{prompt_mode or 'memory_neutral'}" - return manifest diff --git a/projects/memlab/memlab/metrics/stats.py b/projects/memlab/memlab/metrics/stats.py deleted file mode 100644 index ecf2c55..0000000 --- a/projects/memlab/memlab/metrics/stats.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import annotations - -from collections.abc import Iterable - -import numpy as np - - -def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - return float(np.percentile(np.asarray(values, dtype=float), p)) - - -def summarize_latency(values: list[float]) -> dict[str, float]: - if not values: - return {"count": 0.0, "mean": 0.0, "p50": 0.0, "p95": 0.0, "p99": 0.0} - arr = np.asarray(values, dtype=float) - return { - "count": float(arr.size), - "mean": float(arr.mean()), - "p50": float(np.percentile(arr, 50)), - "p95": float(np.percentile(arr, 95)), - "p99": float(np.percentile(arr, 99)), - } - - -def mean_std_ci95(values: Iterable[float]) -> dict[str, float]: - arr = np.asarray(list(values), dtype=float) - if arr.size == 0: - return {"mean": 0.0, "std": 0.0, "ci95": 0.0} - mean = float(arr.mean()) - std = float(arr.std(ddof=1)) if arr.size > 1 else 0.0 - ci95 = 1.96 * (std / np.sqrt(arr.size)) if arr.size > 1 else 0.0 - return {"mean": mean, "std": std, "ci95": float(ci95)} diff --git a/projects/memlab/memlab/metrics/summary_builder.py b/projects/memlab/memlab/metrics/summary_builder.py deleted file mode 100644 index b09f0dc..0000000 --- a/projects/memlab/memlab/metrics/summary_builder.py +++ /dev/null @@ -1,689 +0,0 @@ -from __future__ import annotations - -import argparse -import csv -import json -import os -from collections import defaultdict -from pathlib import Path -from typing import Any - -from memlab.pipeline.control_plane import accepted_for_official_reporting, infer_track_id_from_manifest - -from .stats import mean_std_ci95 - - -SUMMARY_FIELDS = [ - "run_id", - "variant_id", - "legacy_variant_alias", - "workload_id", - "track_id", - "prompt_mode", - "tasks_path", - "preflight_passed", - "model_base_url", - "model_id", - "n_sessions", - "repeat_idx", - "seed", - "processing_p50_ms", - "processing_p95_ms", - "processing_p99_ms", - "queue_wait_p50_ms", - "queue_wait_p95_ms", - "queue_wait_p99_ms", - "end_to_end_p50_ms", - "end_to_end_p95_ms", - "end_to_end_p99_ms", - "proc_llm_p50_ms", - "proc_llm_p95_ms", - "proc_llm_p99_ms", - "proc_non_llm_p50_ms", - "proc_non_llm_p95_ms", - "proc_non_llm_p99_ms", - "task_p50_ms", - "task_p95_ms", - "task_p99_ms", - "throughput_tasks_per_min", - "success_rate", - "accuracy", - "strict_accuracy", - "relaxed_accuracy", - "partial_accuracy", - "benchmark_metric", - "benchmark_score", - "strict_accuracy_on_total", - "relaxed_accuracy_on_total", - "partial_accuracy_on_total", - "benchmark_score_on_total", - "strict_correct_total", - "relaxed_correct_total", - "parse_failure_rate", - "no_answer_rate", - "parse_failure_rate_on_total", - "no_answer_rate_on_total", - "parse_failure_tasks", - "no_answer_tasks", - "accuracy_applicable", - "usage_unavailable", - "prompt_tokens_total", - "completion_tokens_total", - "prompt_tokens_per_success", - "total_tokens_per_success", - "retrieval_skip_rate", - "retrieval_engaged_rate", - "memory_injection_nonzero_rate", - "strict_accuracy_retrieval_engaged", - "embedding_cache_hit_rate", - "retrieval_cache_hit_rate", - "l1_hit_rate", - "l2_hit_rate", - "l3_hit_rate", - "memory_ops_per_task", - "db_queries_per_task", - "vector_searches_per_task", - "context_injected_tokens_mean", - "context_injected_items_mean", - "success_tasks", - "failed_tasks", - "error_total", - "error_403_balance", - "error_429_rate_limit", - "error_400_context_length", - "error_tool_call_unsupported", - "error_no_ai_content", - "error_stream_404", - "error_model_not_found", - "retry_count", - "acceptance_status", - "diagnostic_only", - "summary_chain_valid", - "invalid_reason_codes", -] -MIN_METRICS_SCHEMA_VERSION = 3 - - -def _load_json(path: Path) -> dict[str, Any]: - with path.open("r", encoding="utf-8") as f: - return json.load(f) - - -def _mean(values: list[float]) -> float: - if not values: - return 0.0 - return sum(values) / len(values) - - -def _as_float_or_none(value: Any) -> float | None: - if value is None: - return None - if isinstance(value, str): - txt = value.strip() - if not txt: - return None - if txt.lower() == "n/a": - return None - return float(txt) - if isinstance(value, (int, float)): - return float(value) - return None - - -def _mean_optional(values: list[float | None]) -> float | None: - filtered = [v for v in values if v is not None] - if not filtered: - return None - return sum(filtered) / len(filtered) - - -def _is_e2e(workload: str) -> bool: - return workload.upper().startswith("E2E") - - -def _min_e2e_success_rate() -> float: - try: - value = float(os.getenv("MEMLAB_SUMMARY_MIN_E2E_SUCCESS_RATE", "0.60")) - except Exception: - value = 0.60 - return max(0.0, min(1.0, value)) - - -def _iter_valid_run_rows(runs_dir: Path) -> tuple[list[dict[str, Any]], dict[str, dict[str, float]], list[dict[str, Any]]]: - rows_by_key: dict[tuple[str, str, int, int], dict[str, Any]] = {} - op_p95_by_run: dict[str, dict[str, float]] = {} - updated_ts_by_key: dict[tuple[str, str, int, int], float] = {} - excluded_rows: list[dict[str, Any]] = [] - for run_dir in sorted(runs_dir.iterdir()): - if not run_dir.is_dir(): - continue - manifest_path = run_dir / "manifest.json" - summary_path = run_dir / "summary.json" - if not manifest_path.exists() or not summary_path.exists(): - continue - manifest = _load_json(manifest_path) - summary = _load_json(summary_path) - run_id = str(manifest.get("run_id", run_dir.name)) - if not accepted_for_official_reporting(manifest): - excluded_rows.append( - { - "run_id": run_id, - "workload_id": str(manifest.get("workload_id", summary.get("workload_id", ""))), - "variant_id": str(manifest.get("variant_id", summary.get("variant_id", ""))), - "reason": "not_accepted_for_official_reporting", - "acceptance_status": str(manifest.get("acceptance_status", "")), - "diagnostic_only": bool(manifest.get("diagnostic_only", False)), - "summary_chain_valid": manifest.get("summary_chain_valid", None), - } - ) - continue - schema_ver = int(manifest.get("metrics_schema_version", 0)) - if schema_ver < MIN_METRICS_SCHEMA_VERSION: - excluded_rows.append({"run_id": run_id, "reason": "metrics_schema_too_old"}) - continue - if not bool(manifest.get("env_valid", False)): - excluded_rows.append({"run_id": run_id, "reason": "env_invalid"}) - continue - if not bool(manifest.get("repro_valid", False)): - excluded_rows.append({"run_id": run_id, "reason": "repro_invalid"}) - continue - workload = str(manifest.get("workload_id", summary.get("workload_id", ""))) - extra = manifest.get("extra", {}) if isinstance(manifest.get("extra", {}), dict) else {} - if _is_e2e(workload): - if bool(extra.get("debug_print", False)): - excluded_rows.append({"run_id": run_id, "reason": "debug_print_run"}) - continue - tasks_path = str(extra.get("tasks_path", "")).replace("\\", "/") - if tasks_path and not tasks_path.startswith("workloads/e2e/"): - excluded_rows.append({"run_id": run_id, "reason": "tasks_path_not_runtime"}) - continue - if _is_e2e(workload) and not bool(manifest.get("llm_real", False)): - excluded_rows.append({"run_id": run_id, "reason": "llm_not_real"}) - continue - success_rate = float(summary.get("success_rate", 0.0)) - if _is_e2e(workload) and success_rate < _min_e2e_success_rate(): - excluded_rows.append({"run_id": run_id, "reason": "success_rate_below_threshold"}) - continue - latency = summary.get("latency", {}) if isinstance(summary.get("latency"), dict) else {} - queue_wait = latency.get("queue_wait", {}) if isinstance(latency.get("queue_wait"), dict) else {} - processing = latency.get("processing", {}) if isinstance(latency.get("processing"), dict) else {} - proc_llm = latency.get("proc_llm", {}) if isinstance(latency.get("proc_llm"), dict) else {} - proc_non_llm = latency.get("proc_non_llm", {}) if isinstance(latency.get("proc_non_llm"), dict) else {} - end_to_end = latency.get("end_to_end", {}) if isinstance(latency.get("end_to_end"), dict) else {} - if not end_to_end and isinstance(summary.get("task_latency"), dict): - end_to_end = summary.get("task_latency", {}) - accuracy_value = _as_float_or_none(summary.get("accuracy")) - strict_accuracy = _as_float_or_none(summary.get("strict_accuracy")) - relaxed_accuracy = _as_float_or_none(summary.get("relaxed_accuracy")) - benchmark_score = _as_float_or_none(summary.get("benchmark_score")) - strict_accuracy_on_total = _as_float_or_none(summary.get("strict_accuracy_on_total")) - relaxed_accuracy_on_total = _as_float_or_none(summary.get("relaxed_accuracy_on_total")) - benchmark_score_on_total = _as_float_or_none(summary.get("benchmark_score_on_total")) - if strict_accuracy_on_total is None: - strict_accuracy_on_total = strict_accuracy - if relaxed_accuracy_on_total is None: - relaxed_accuracy_on_total = relaxed_accuracy - accuracy_applicable = bool(summary.get("accuracy_applicable", not workload.upper().startswith("SYN"))) - if not accuracy_applicable: - accuracy_value = None - strict_accuracy = None - relaxed_accuracy = None - benchmark_score = None - strict_accuracy_on_total = None - relaxed_accuracy_on_total = None - benchmark_score_on_total = None - tasks_path = str(extra.get("tasks_path", "")).replace("\\", "/") - prompt_mode = str(extra.get("prompt_mode", "memory_neutral") or "memory_neutral") - raw_variant_id = str(manifest.get("variant_id", summary.get("variant_id", ""))) - variant_id = {"A1": "V1", "A2": "F1"}.get(raw_variant_id, raw_variant_id) - legacy_variant_alias = str(extra.get("legacy_variant_alias") or "") - if not legacy_variant_alias and raw_variant_id in {"A1", "A2"}: - legacy_variant_alias = raw_variant_id - track_id = infer_track_id_from_manifest(manifest) - error_breakdown = extra.get("error_breakdown", summary.get("error_breakdown", {})) - by_category = error_breakdown.get("by_category", {}) if isinstance(error_breakdown, dict) else {} - if not isinstance(by_category, dict): - by_category = {} - row = { - "run_id": manifest.get("run_id", run_dir.name), - "variant_id": variant_id, - "legacy_variant_alias": legacy_variant_alias, - "workload_id": workload, - "track_id": track_id, - "prompt_mode": prompt_mode, - "tasks_path": tasks_path, - "preflight_passed": bool(extra.get("preflight_passed", False)), - "model_base_url": manifest.get("model_base_url") or "", - "model_id": manifest.get("model_id") or "", - "n_sessions": int(summary.get("n_sessions", 0)), - "repeat_idx": int(manifest.get("repeat_idx", 0)), - "seed": int(manifest.get("seed", 0)), - "processing_p50_ms": float(processing.get("p50", 0.0)), - "processing_p95_ms": float(processing.get("p95", 0.0)), - "processing_p99_ms": float(processing.get("p99", 0.0)), - "queue_wait_p50_ms": float(queue_wait.get("p50", 0.0)), - "queue_wait_p95_ms": float(queue_wait.get("p95", 0.0)), - "queue_wait_p99_ms": float(queue_wait.get("p99", 0.0)), - "end_to_end_p50_ms": float(end_to_end.get("p50", 0.0)), - "end_to_end_p95_ms": float(end_to_end.get("p95", 0.0)), - "end_to_end_p99_ms": float(end_to_end.get("p99", 0.0)), - "proc_llm_p50_ms": float(proc_llm.get("p50", 0.0)), - "proc_llm_p95_ms": float(proc_llm.get("p95", 0.0)), - "proc_llm_p99_ms": float(proc_llm.get("p99", 0.0)), - "proc_non_llm_p50_ms": float(proc_non_llm.get("p50", 0.0)), - "proc_non_llm_p95_ms": float(proc_non_llm.get("p95", 0.0)), - "proc_non_llm_p99_ms": float(proc_non_llm.get("p99", 0.0)), - "task_p50_ms": float(end_to_end.get("p50", 0.0)), - "task_p95_ms": float(end_to_end.get("p95", 0.0)), - "task_p99_ms": float(end_to_end.get("p99", 0.0)), - "throughput_tasks_per_min": float(summary.get("throughput_tasks_per_min", 0.0)), - "success_rate": success_rate, - "accuracy": accuracy_value, - "strict_accuracy": strict_accuracy, - "relaxed_accuracy": relaxed_accuracy, - "partial_accuracy": _as_float_or_none(summary.get("partial_accuracy")), - "benchmark_metric": str(summary.get("benchmark_metric", "")), - "benchmark_score": benchmark_score, - "strict_accuracy_on_total": strict_accuracy_on_total, - "relaxed_accuracy_on_total": relaxed_accuracy_on_total, - "partial_accuracy_on_total": _as_float_or_none(summary.get("partial_accuracy_on_total")), - "benchmark_score_on_total": benchmark_score_on_total, - "strict_correct_total": int(summary.get("strict_correct_total", 0)), - "relaxed_correct_total": int(summary.get("relaxed_correct_total", 0)), - "parse_failure_rate": float(summary.get("parse_failure_rate", 0.0)), - "no_answer_rate": float(summary.get("no_answer_rate", 0.0)), - "parse_failure_rate_on_total": float(summary.get("parse_failure_rate_on_total", 0.0)), - "no_answer_rate_on_total": float(summary.get("no_answer_rate_on_total", 0.0)), - "parse_failure_tasks": int(summary.get("parse_failure_tasks", 0)), - "no_answer_tasks": int(summary.get("no_answer_tasks", 0)), - "accuracy_applicable": accuracy_applicable, - "usage_unavailable": bool(manifest.get("usage_unavailable", summary.get("usage_unavailable", False))), - "prompt_tokens_total": _as_float_or_none(summary.get("prompt_tokens_total")), - "completion_tokens_total": _as_float_or_none(summary.get("completion_tokens_total")), - "prompt_tokens_per_success": _as_float_or_none(summary.get("prompt_tokens_per_success")), - "total_tokens_per_success": _as_float_or_none(summary.get("total_tokens_per_success")), - "retrieval_skip_rate": float(summary.get("retrieval_skip_rate", 0.0)), - "retrieval_engaged_rate": float(summary.get("retrieval_engaged_rate", 0.0)), - "memory_injection_nonzero_rate": float(summary.get("memory_injection_nonzero_rate", 0.0)), - "strict_accuracy_retrieval_engaged": _as_float_or_none(summary.get("strict_accuracy_retrieval_engaged")), - "embedding_cache_hit_rate": _as_float_or_none(summary.get("embedding_cache_hit_rate")), - "retrieval_cache_hit_rate": _as_float_or_none(summary.get("retrieval_cache_hit_rate")), - "l1_hit_rate": float(summary.get("l1_hit_rate", 0.0)), - "l2_hit_rate": float(summary.get("l2_hit_rate", 0.0)), - "l3_hit_rate": float(summary.get("l3_hit_rate", 0.0)), - "memory_ops_per_task": float(summary.get("memory_ops_per_task", 0.0)), - "db_queries_per_task": float(summary.get("db_queries_per_task", 0.0)), - "vector_searches_per_task": float(summary.get("vector_searches_per_task", 0.0)), - "context_injected_tokens_mean": float(summary.get("context_injected_tokens_mean", 0.0)), - "context_injected_items_mean": float(summary.get("context_injected_items_mean", 0.0)), - "success_tasks": int(summary.get("success_tasks", 0)), - "failed_tasks": int(summary.get("failed_tasks", 0)), - "error_total": int(error_breakdown.get("total_errors", summary.get("failed_tasks", 0)) if isinstance(error_breakdown, dict) else int(summary.get("failed_tasks", 0))), - "error_403_balance": int(by_category.get("provider_403_balance", 0)), - "error_429_rate_limit": int(by_category.get("provider_429_rate_limit", 0)), - "error_400_context_length": int(by_category.get("provider_400_context_length", 0)), - "error_tool_call_unsupported": int(by_category.get("tool_call_unsupported", 0)), - "error_no_ai_content": int(by_category.get("deerflow_no_ai_content", 0)), - "error_stream_404": int(by_category.get("deerflow_stream_404", 0)), - "error_model_not_found": int(by_category.get("model_not_found", 0)), - "retry_count": int(extra.get("retry_count", summary.get("retry_count_total", 0))), - "acceptance_status": str(manifest.get("acceptance_status", "")), - "diagnostic_only": bool(manifest.get("diagnostic_only", False)), - "summary_chain_valid": bool(manifest.get("summary_chain_valid", False)), - "invalid_reason_codes": ",".join([str(x) for x in manifest.get("invalid_reason_codes", [])]), - } - key = ( - str(row["variant_id"]), - str(row["workload_id"]), - int(row["repeat_idx"]), - int(row["seed"]), - ) - updated_ts = float(run_dir.stat().st_mtime) - prev_ts = updated_ts_by_key.get(key, -1.0) - if updated_ts >= prev_ts: - updated_ts_by_key[key] = updated_ts - rows_by_key[key] = row - op_p95 = summary.get("op_p95_ms", {}) or {} - if isinstance(op_p95, dict): - op_p95_by_run[row["run_id"]] = {str(k): float(v) for k, v in op_p95.items()} - rows = sorted( - rows_by_key.values(), - key=lambda r: (str(r["workload_id"]), str(r["variant_id"]), int(r["repeat_idx"]), int(r["seed"])), - ) - return rows, op_p95_by_run, excluded_rows - - -def build_acceptance_result( - rows: list[dict[str, Any]], - *, - min_p95_improve: float = 0.03, - min_prompt_drop: float = 0.03, - max_accuracy_drop: float = 0.03, -) -> dict[str, Any]: - e2e_rows = [ - r - for r in rows - if _is_e2e(str(r.get("workload_id", ""))) - and str(r.get("prompt_mode", "memory_neutral")) == "memory_neutral" - and str(r.get("tasks_path", "")).replace("\\", "/").endswith("tasks.jsonl") - ] - if not e2e_rows: - e2e_rows = [r for r in rows if _is_e2e(str(r.get("workload_id", "")))] - by_variant: dict[str, list[dict[str, Any]]] = defaultdict(list) - for row in e2e_rows: - by_variant[str(row.get("variant_id", ""))].append(row) - - f1 = by_variant.get("F1", []) - b1 = by_variant.get("B1", []) - if not f1 or not b1: - return { - "status": "MISSING", - "reason": "Missing F1 or B1 valid E2E rows", - } - - f1_p95 = _mean([float(r["processing_p95_ms"]) for r in f1]) - b1_p95 = _mean([float(r["processing_p95_ms"]) for r in b1]) - f1_token_subset = [r for r in f1 if not bool(r.get("usage_unavailable"))] - b1_token_subset = [r for r in b1 if not bool(r.get("usage_unavailable"))] - f1_prompt = _mean_optional([_as_float_or_none(r.get("prompt_tokens_per_success")) for r in f1_token_subset]) - b1_prompt = _mean_optional([_as_float_or_none(r.get("prompt_tokens_per_success")) for r in b1_token_subset]) - - def _pick_accuracy(row: dict[str, Any]) -> float | None: - val = _as_float_or_none(row.get("strict_accuracy_on_total")) - if val is not None: - return val - val = _as_float_or_none(row.get("strict_accuracy")) - if val is not None: - return val - return _as_float_or_none(row.get("accuracy")) - - # Use total-task denominator as the primary accuracy lens for memory-effect evaluation. - f1_acc = _mean_optional([_pick_accuracy(r) for r in f1]) - b1_acc = _mean_optional([_pick_accuracy(r) for r in b1]) - if f1_acc is None or b1_acc is None: - return { - "status": "MISSING", - "reason": "Missing E2E accuracy rows for F1 or B1", - } - - p95_improve = ((b1_p95 - f1_p95) / b1_p95) if b1_p95 > 0 else 0.0 - prompt_drop = None - if b1_prompt is not None and f1_prompt is not None and b1_prompt > 0: - prompt_drop = (b1_prompt - f1_prompt) / b1_prompt - acc_drop = b1_acc - f1_acc - - pass_latency = p95_improve >= float(min_p95_improve) - pass_tokens = bool( - prompt_drop is not None - and prompt_drop >= float(min_prompt_drop) - and acc_drop <= float(max_accuracy_drop) - ) - accepted = pass_latency or pass_tokens - return { - "status": "PASS" if accepted else "FAIL", - "latency_metric": "processing_p95_ms", - "accuracy_metric": "strict_accuracy_on_total", - "p95_improve": p95_improve, - "prompt_drop": prompt_drop, - "token_subset_runs_f1": len(f1_token_subset), - "token_subset_runs_b1": len(b1_token_subset), - "accuracy_drop": acc_drop, - "thresholds": { - "min_p95_improve": float(min_p95_improve), - "min_prompt_drop": float(min_prompt_drop), - "max_accuracy_drop": float(max_accuracy_drop), - }, - "pass_latency": pass_latency, - "pass_tokens": pass_tokens, - } - - -def write_experiment_report( - rows: list[dict[str, Any]], - *, - acceptance: dict[str, Any], - op_p95_by_run: dict[str, dict[str, float]], - excluded_rows: list[dict[str, Any]], - output_path: Path, -) -> None: - grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) - for r in rows: - grouped[(str(r["workload_id"]), str(r["variant_id"]))].append(r) - - workloads = sorted({str(r["workload_id"]) for r in rows}) - variants = ["B0", "B1", "V1", "F1", "F2"] - lines: list[str] = [] - lines.append("# Experiment Report") - lines.append("") - usage_unknown_count = sum(1 for r in rows if str(r.get("workload_id", "")).upper().startswith("E2E") and bool(r.get("usage_unavailable"))) - lines.append("## Metric Validity") - lines.append("") - lines.append(f"- Comparable rows in summary.csv: {len(rows)}") - lines.append( - "- E2E comparability requires: " - f"env_valid=true, repro_valid=true, llm_real=true, success_rate>={_min_e2e_success_rate():.2f}." - ) - lines.append(f"- E2E rows with usage unavailable (tokens shown as N/A): {usage_unknown_count}") - lines.append("") - lines.append("## Excluded Runs") - lines.append("") - lines.append(f"- Excluded rows from official summary: {len(excluded_rows)}") - for row in excluded_rows[:20]: - lines.append( - "- `{run_id}`: {reason} (acceptance_status={acceptance_status}, diagnostic_only={diagnostic_only}, summary_chain_valid={summary_chain_valid})".format( - run_id=str(row.get("run_id", "")), - reason=str(row.get("reason", "")), - acceptance_status=str(row.get("acceptance_status", "")), - diagnostic_only=str(row.get("diagnostic_only", "")), - summary_chain_valid=str(row.get("summary_chain_valid", "")), - ) - ) - lines.append("") - - if not rows: - lines.append( - "No comparable runs found " - f"(`env_valid=true`, `repro_valid=true`, `llm_real=true`, and E2E `success_rate>={_min_e2e_success_rate():.2f}`)." - ) - lines.append("") - else: - for workload in workloads: - lines.append(f"## {workload}") - lines.append("") - lines.append("| Variant | proc_whole p95 (ms) | proc_llm p95 (ms) | proc_non_llm p95 (ms) | end-to-end p95 (ms) | throughput (/min) | tokens/success | strict_acc(success) | strict_acc(total) | relaxed_acc(total) | parse_fail_rate(total) | no_answer_rate(total) | retrieval_engaged | injected_nonzero | L1 hit | L2 hit | L3 hit | mem_ops/task |") - lines.append("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|") - for v in variants: - g = grouped.get((workload, v), []) - if not g: - continue - token_values = [_as_float_or_none(x.get("total_tokens_per_success")) for x in g] - strict_acc_values = [_as_float_or_none(x.get("strict_accuracy", x.get("accuracy"))) for x in g] - strict_acc_total_values = [_as_float_or_none(x.get("strict_accuracy_on_total")) for x in g] - relaxed_acc_total_values = [_as_float_or_none(x.get("relaxed_accuracy_on_total")) for x in g] - token_mean = _mean_optional(token_values) - strict_acc_mean = _mean_optional(strict_acc_values) - strict_acc_total_mean = _mean_optional(strict_acc_total_values) - relaxed_acc_total_mean = _mean_optional(relaxed_acc_total_values) - token_txt = "N/A" if token_mean is None else f"{token_mean:.1f}" - strict_acc_txt = "N/A" if strict_acc_mean is None else f"{strict_acc_mean:.4f}" - strict_acc_total_txt = "N/A" if strict_acc_total_mean is None else f"{strict_acc_total_mean:.4f}" - relaxed_acc_total_txt = "N/A" if relaxed_acc_total_mean is None else f"{relaxed_acc_total_mean:.4f}" - lines.append( - "| {0} | {1:.3f} | {2:.3f} | {3:.3f} | {4:.3f} | {5:.2f} | {6} | {7} | {8} | {9} | {10:.2%} | {11:.2%} | {12:.2%} | {13:.2%} | {14:.4f} | {15:.4f} | {16:.4f} | {17:.3f} |".format( - v, - _mean([float(x["processing_p95_ms"]) for x in g]), - _mean([float(x.get("proc_llm_p95_ms", 0.0)) for x in g]), - _mean([float(x.get("proc_non_llm_p95_ms", 0.0)) for x in g]), - _mean([float(x["end_to_end_p95_ms"]) for x in g]), - _mean([float(x.get("throughput_tasks_per_min", 0.0)) for x in g]), - token_txt, - strict_acc_txt, - strict_acc_total_txt, - relaxed_acc_total_txt, - _mean([float(x.get("parse_failure_rate_on_total", 0.0)) for x in g]), - _mean([float(x.get("no_answer_rate_on_total", 0.0)) for x in g]), - _mean([float(x.get("retrieval_engaged_rate", 0.0)) for x in g]), - _mean([float(x.get("memory_injection_nonzero_rate", 0.0)) for x in g]), - _mean([float(x.get("l1_hit_rate", 0.0)) for x in g]), - _mean([float(x.get("l2_hit_rate", 0.0)) for x in g]), - _mean([float(x.get("l3_hit_rate", 0.0)) for x in g]), - _mean([float(x.get("memory_ops_per_task", 0.0)) for x in g]), - ) - ) - lines.append("") - - lines.append("## Acceptance Check (F1 vs B1, E2E)") - lines.append("") - if acceptance.get("status") == "MISSING": - lines.append(f"- Status: MISSING ({acceptance.get('reason', '')})") - else: - lines.append(f"- Status: {acceptance.get('status')}") - thresholds = acceptance.get("thresholds", {}) if isinstance(acceptance.get("thresholds"), dict) else {} - try: - min_p95_improve = float(thresholds.get("min_p95_improve", 0.03)) - except Exception: - min_p95_improve = 0.03 - try: - min_prompt_drop = float(thresholds.get("min_prompt_drop", 0.03)) - except Exception: - min_prompt_drop = 0.03 - try: - max_accuracy_drop = float(thresholds.get("max_accuracy_drop", 0.03)) - except Exception: - max_accuracy_drop = 0.03 - lines.append( - "- acceptance thresholds: " - f"p95_improve>={min_p95_improve:.2%}, " - f"prompt_drop>={min_prompt_drop:.2%}, " - f"accuracy_drop<={max_accuracy_drop:.2%}" - ) - lines.append(f"- primary latency metric: {acceptance.get('latency_metric', 'processing_p95_ms')}") - lines.append(f"- primary accuracy metric: {acceptance.get('accuracy_metric', 'strict_accuracy_on_total')}") - lines.append(f"- processing p95 reduction: {float(acceptance.get('p95_improve', 0.0)):.2%}") - prompt_drop = acceptance.get("prompt_drop") - if prompt_drop is None: - lines.append("- prompt token reduction: N/A (usage unavailable)") - else: - lines.append(f"- prompt token reduction: {float(prompt_drop):.2%}") - lines.append(f"- accuracy drop (B1 - F1): {float(acceptance.get('accuracy_drop', 0.0)):.2%}") - lines.append("") - - if acceptance.get("status") == "FAIL": - lines.append("## Diagnosis") - lines.append("") - f1_runs = [r["run_id"] for r in rows if r.get("variant_id") == "F1" and _is_e2e(str(r.get("workload_id", "")))] - op_buckets: dict[str, list[float]] = defaultdict(list) - for rid in f1_runs: - for op_name, p95 in op_p95_by_run.get(rid, {}).items(): - op_buckets[op_name].append(float(p95)) - if not op_buckets: - lines.append("- No op-level p95 data available for diagnosis.") - else: - ranked = sorted(((k, _mean(v)) for k, v in op_buckets.items()), key=lambda x: x[1], reverse=True) - lines.append("- Highest p95 components in F1:") - for name, value in ranked[:10]: - lines.append(f" - {name}: {value:.3f} ms") - lines.append("") - lines.append("## Ablation Notes") - lines.append("") - f1_rows = [r for r in rows if r.get("variant_id") == "F1" and _is_e2e(str(r.get("workload_id", "")))] - if f1_rows: - lines.append(f"- Mean retrieval skip rate (F1 E2E): {_mean([float(x.get('retrieval_skip_rate', 0.0)) for x in f1_rows]):.2%}.") - emb_hits = _mean_optional([_as_float_or_none(x.get("embedding_cache_hit_rate")) for x in f1_rows]) - ret_hits = _mean_optional([_as_float_or_none(x.get("retrieval_cache_hit_rate")) for x in f1_rows]) - if emb_hits is not None: - lines.append(f"- Mean embedding cache hit rate (F1 E2E): {emb_hits:.2%}.") - if ret_hits is not None: - lines.append(f"- Mean retrieval cache hit rate (F1 E2E): {ret_hits:.2%}.") - lines.append("- Injection controls should reduce context_injected_tokens_mean in F1.") - lines.append("- Retrieval gating/caching should reduce p95 of L2:vector_search and L2:get_by_keys.") - lines.append("") - - output_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def aggregate_runs( - *, - runs_dir: str | Path = "runs", - summary_csv: str | Path = "summary.csv", - stats_json: str | Path = "summary.stats.json", - report_md: str | Path = "experiment_report.md", - report_alias_md: str | Path = "report.md", - min_p95_improve: float = 0.03, - min_prompt_drop: float = 0.03, - max_accuracy_drop: float = 0.03, -) -> dict[str, Any]: - runs_path = Path(runs_dir) - rows, op_p95_by_run, excluded_rows = _iter_valid_run_rows(runs_path) - - summary_path = Path(summary_csv) - summary_path.parent.mkdir(parents=True, exist_ok=True) - with summary_path.open("w", newline="", encoding="utf-8") as f: - writer = csv.DictWriter(f, fieldnames=SUMMARY_FIELDS) - writer.writeheader() - for row in rows: - writer.writerow(row) - - grouped: dict[tuple[str, str], list[float]] = defaultdict(list) - for row in rows: - grouped[(str(row["workload_id"]), str(row["variant_id"]))].append(float(row["processing_p95_ms"])) - stats = { - f"{w}:{v}": mean_std_ci95(vals) - for (w, v), vals in sorted(grouped.items(), key=lambda x: (x[0][0], x[0][1])) - } - stats_path = Path(stats_json) - stats_path.write_text(json.dumps(stats, indent=2), encoding="utf-8") - - acceptance = build_acceptance_result( - rows, - min_p95_improve=float(min_p95_improve), - min_prompt_drop=float(min_prompt_drop), - max_accuracy_drop=float(max_accuracy_drop), - ) - report_path = Path(report_md) - write_experiment_report( - rows, - acceptance=acceptance, - op_p95_by_run=op_p95_by_run, - excluded_rows=excluded_rows, - output_path=report_path, - ) - Path(report_alias_md).write_text(report_path.read_text(encoding="utf-8"), encoding="utf-8") - - return { - "rows": len(rows), - "excluded_rows": len(excluded_rows), - "summary_csv": str(summary_path), - "summary_stats_json": str(stats_path), - "experiment_report": str(report_path), - "report_md": str(Path(report_alias_md)), - "acceptance": acceptance, - } - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--runs-dir", type=str, default="runs") - ap.add_argument("--summary-csv", type=str, default="summary.csv") - ap.add_argument("--stats-json", type=str, default="summary.stats.json") - ap.add_argument("--report-md", type=str, default="experiment_report.md") - ap.add_argument("--report-alias-md", type=str, default="report.md") - ap.add_argument("--min-p95-improve", type=float, default=0.03) - ap.add_argument("--min-prompt-drop", type=float, default=0.03) - ap.add_argument("--max-accuracy-drop", type=float, default=0.03) - args = ap.parse_args() - result = aggregate_runs( - runs_dir=args.runs_dir, - summary_csv=args.summary_csv, - stats_json=args.stats_json, - report_md=args.report_md, - report_alias_md=args.report_alias_md, - min_p95_improve=float(args.min_p95_improve), - min_prompt_drop=float(args.min_prompt_drop), - max_accuracy_drop=float(args.max_accuracy_drop), - ) - print(json.dumps(result, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/microbench.py b/projects/memlab/memlab/microbench.py deleted file mode 100644 index 7ad020c..0000000 --- a/projects/memlab/memlab/microbench.py +++ /dev/null @@ -1,142 +0,0 @@ -from __future__ import annotations - -import argparse -import asyncio -import json -import os -import time -from pathlib import Path - -from memlab.backends.object_store import LocalObjectStore -from memlab.backends.postgres_store import PostgresStore -from memlab.backends.qdrant_store import QdrantStore -from memlab.backends.redis_store import RedisStore -from memlab.metrics.stats import summarize_latency -from memlab.tools.embedding import hash_embedding - - -def _parse_concurrency_levels(raw: str) -> list[int]: - out: list[int] = [] - for part in str(raw or "").split(","): - txt = part.strip() - if not txt: - continue - value = max(1, int(txt)) - if value not in out: - out.append(value) - return out or [1] - - -async def _bench_redis(redis_url: str, n: int, concurrency: int) -> dict[str, float]: - store = RedisStore(redis_url) - lat: list[float] = [0.0 for _ in range(n)] - sem = asyncio.Semaphore(max(1, concurrency)) - - async def _one(i: int) -> None: - async with sem: - t0 = time.perf_counter() - await store.client.set(f"bench:{concurrency}:{i}", f"value:{i}", ex=60) - _ = await store.client.get(f"bench:{concurrency}:{i}") - lat[i] = (time.perf_counter() - t0) * 1000.0 - - await asyncio.gather(*[_one(i) for i in range(n)]) - await store.close() - return summarize_latency(lat) - - -async def _bench_postgres(pg_uri: str, n: int, concurrency: int) -> dict[str, float]: - store = PostgresStore(pg_uri) - await store.connect() - lat: list[float] = [0.0 for _ in range(n)] - sem = asyncio.Semaphore(max(1, concurrency)) - - async def _one(i: int) -> None: - async with sem: - t0 = time.perf_counter() - async with store.pool.acquire() as conn: # type: ignore[union-attr] - _ = await conn.fetchval("SELECT $1::int + 1", i) - lat[i] = (time.perf_counter() - t0) * 1000.0 - - await asyncio.gather(*[_one(i) for i in range(n)]) - await store.close() - return summarize_latency(lat) - - -async def _bench_qdrant(qdrant_url: str, collection: str, n: int, concurrency: int) -> dict[str, float]: - store = QdrantStore(qdrant_url, collection, vector_size=64) - await store.ensure_collection() - for i in range(max(32, n)): - emb = hash_embedding(f"doc:{i}") - await store.upsert(i, emb, {"agent_id": "bench", "key": f"k{i}", "kind": "bench"}) - lat: list[float] = [0.0 for _ in range(n)] - sem = asyncio.Semaphore(max(1, concurrency)) - - async def _one(i: int) -> None: - async with sem: - q = hash_embedding(f"query:{concurrency}:{i}") - t0 = time.perf_counter() - _ = await store.search(q, limit=8, agent_id="bench") - lat[i] = (time.perf_counter() - t0) * 1000.0 - - await asyncio.gather(*[_one(i) for i in range(n)]) - await store.close() - return summarize_latency(lat) - - -async def _bench_l3(root_dir: str, n: int, concurrency: int) -> dict[str, float]: - store = LocalObjectStore(root_dir=root_dir) - await store.ensure_ready() - lat: list[float] = [0.0 for _ in range(n)] - sem = asyncio.Semaphore(max(1, concurrency)) - - async def _one(i: int) -> None: - async with sem: - text = f"payload-{concurrency}-{i}-" + ("x" * 1024) - t0 = time.perf_counter() - pointer = await store.put(agent_id="bench", key=f"obj-{concurrency}-{i}", text=text, metadata={"kind": "bench"}) - _ = await store.get(pointer) - lat[i] = (time.perf_counter() - t0) * 1000.0 - - await asyncio.gather(*[_one(i) for i in range(n)]) - return summarize_latency(lat) - - -async def _main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--redis-url", type=str, default="redis://127.0.0.1:6379/0") - ap.add_argument("--pg-uri", type=str, default=os.getenv("PG_URI", "")) - ap.add_argument("--qdrant-url", type=str, default="http://127.0.0.1:6333") - ap.add_argument("--qdrant-collection", type=str, default="memlab_docs") - ap.add_argument("--l3-root", type=str, default="storage/l3_microbench") - ap.add_argument("--requests", type=int, default=200) - ap.add_argument("--concurrency-levels", type=str, default="1,16,64") - ap.add_argument("--out", type=str, default="logs/microbench.json") - args = ap.parse_args() - - levels = _parse_concurrency_levels(args.concurrency_levels) - result: dict[str, object] = {"concurrency_levels": levels} - redis_by_level: dict[str, dict[str, float]] = {} - postgres_by_level: dict[str, dict[str, float]] = {} - qdrant_by_level: dict[str, dict[str, float]] = {} - l3_by_level: dict[str, dict[str, float]] = {} - for level in levels: - redis_by_level[f"c{level}"] = await _bench_redis(args.redis_url, args.requests, level) - postgres_by_level[f"c{level}"] = await _bench_postgres(args.pg_uri, args.requests, level) - qdrant_by_level[f"c{level}"] = await _bench_qdrant(args.qdrant_url, args.qdrant_collection, args.requests, level) - l3_by_level[f"c{level}"] = await _bench_l3(args.l3_root, args.requests, level) - result["redis"] = redis_by_level - result["postgres"] = postgres_by_level - result["qdrant"] = qdrant_by_level - result["l3_object_store"] = l3_by_level - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(result, indent=2), encoding="utf-8") - print(f"Wrote {out}") - - -def main() -> None: - asyncio.run(_main()) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/pipeline/__init__.py b/projects/memlab/memlab/pipeline/__init__.py deleted file mode 100644 index 32cba1c..0000000 --- a/projects/memlab/memlab/pipeline/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import annotations - -__all__ = [ - "model_profile", -] diff --git a/projects/memlab/memlab/pipeline/control_plane.py b/projects/memlab/memlab/pipeline/control_plane.py deleted file mode 100644 index 54e8257..0000000 --- a/projects/memlab/memlab/pipeline/control_plane.py +++ /dev/null @@ -1,550 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import os -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any - -from memlab.io import now_ms - - -BRANCH_STATUSES = ( - "pending", - "running", - "accepted", - "invalid", - "diagnostic_only", - "blocked", - "superseded", -) - -STAGE_IDS = ( - "audit_data", - "audit_protocol", - "smoke", - "pilot", - "full", - "aggregate", - "promote", -) - -ANOMALY_TO_ROLLBACK_STAGE = { - "DATA_COUNT_MISMATCH": "audit_data", - "FULL_SLICE_MIXED": "audit_data", - "DUPLICATE_OR_MISSING_SAMPLE": "audit_data", - "EMPTY_GOLD_OR_LABEL": "audit_data", - "NON_MEMORY_DEPENDENT_DATASET": "audit_data", - "PROMPT_PROFILE_MISMATCH": "audit_protocol", - "METRIC_BINDING_ERROR": "audit_protocol", - "MODEL_LOCK_DRIFT": "audit_protocol", - "PROVIDER_PARAM_DRIFT": "audit_protocol", - "SESSION_SCOPE_DRIFT": "audit_protocol", - "MIXED_TRACK_COMPARISON": "audit_protocol", - "DIRECT_EVIDENCE_LEAKAGE": "audit_protocol", - "TRACK_GOAL_MISMATCH": "audit_protocol", - "CROSS_SAMPLE_MEMORY_CONTAMINATION": "smoke", - "EVENTLOG_MISSING": "smoke", - "SUMMARY_CHAIN_INCONSISTENT": "smoke", - "INVALID_RATE_SPIKE": "smoke", - "BASELINE_OUT_OF_BAND": "pilot", - "RAW_RUN_TABLE_MISMATCH": "full", - "REPEAT_DIRECTION_CONFLICT": "pilot", - "REPEAT_VARIANCE_ABNORMAL": "pilot", - "UNEXPLAINED_ORDERING_REVERSAL": "pilot", -} - -STRUCTURAL_ANOMALIES = { - "DATA_COUNT_MISMATCH", - "FULL_SLICE_MIXED", - "DUPLICATE_OR_MISSING_SAMPLE", - "EMPTY_GOLD_OR_LABEL", - "NON_MEMORY_DEPENDENT_DATASET", - "PROMPT_PROFILE_MISMATCH", - "METRIC_BINDING_ERROR", - "MODEL_LOCK_DRIFT", - "PROVIDER_PARAM_DRIFT", - "SESSION_SCOPE_DRIFT", - "CROSS_SAMPLE_MEMORY_CONTAMINATION", - "EVENTLOG_MISSING", - "SUMMARY_CHAIN_INCONSISTENT", - "MIXED_TRACK_COMPARISON", - "DIRECT_EVIDENCE_LEAKAGE", - "TRACK_GOAL_MISMATCH", - "RAW_RUN_TABLE_MISMATCH", -} - -DATASET_ALIASES = { - "locomo_preprocess": "locomo", - "longmemeval_24k": "longmemeval", - "squad_v11_dev": "squad_v11", -} - - -@dataclass(frozen=True) -class BranchKey: - dataset_id: str - track_id: str - variant_id: str - repeat_idx: int - - @property - def branch_id(self) -> str: - parts = [self.dataset_id, self.track_id, self.variant_id, f"r{int(self.repeat_idx)}"] - return "::".join([str(p).strip() for p in parts]) - - -@dataclass -class StageStatusRecord: - branch_id: str - dataset_id: str - track_id: str - variant_id: str - repeat_idx: int - stage_id: str - status: str - ts_ms: int - run_id: str = "" - note: str = "" - anomaly_codes: list[str] | None = None - rollback_root_stage: str | None = None - rerun_plan_id: str | None = None - - -def _read_json(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) - - -def _write_json(path: Path, data: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - payload = json.dumps(data, indent=2, ensure_ascii=False) - tmp_path = path.with_name(f".{path.name}.tmp-{os.getpid()}-{now_ms()}") - try: - tmp_path.write_text(payload, encoding="utf-8") - tmp_path.replace(path) - finally: - if tmp_path.exists(): - tmp_path.unlink(missing_ok=True) - - -def _append_jsonl(path: Path, row: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("a", encoding="utf-8", newline="\n") as f: - f.write(json.dumps(row, ensure_ascii=False) + "\n") - - -def _safe_text(path: Path) -> str: - try: - return path.read_text(encoding="utf-8") - except UnicodeDecodeError: - return path.read_bytes().decode("utf-8", errors="ignore") - - -def sha1_text(text: str) -> str: - return hashlib.sha1(text.encode("utf-8")).hexdigest() - - -def sha1_file(path: str | Path) -> str: - p = Path(path) - if not p.exists(): - return "" - h = hashlib.sha1() - with p.open("rb") as f: - while True: - chunk = f.read(1024 * 1024) - if not chunk: - break - h.update(chunk) - return h.hexdigest() - - -def fingerprint_paths(paths: list[str | Path]) -> str: - tokens: list[str] = [] - for raw in paths: - path = Path(raw) - tokens.append(f"{path.as_posix()}::{sha1_file(path)}") - return sha1_text("|".join(tokens)) - - -def normalize_dataset_id(dataset_id: str) -> str: - raw = str(dataset_id or "").strip().lower() - return DATASET_ALIASES.get(raw, raw) - - -def sanitize_path_token(value: str) -> str: - raw = str(value or "").strip() - if not raw: - return "unknown" - clean = "".join(ch if ch not in '<>:"/\\|?*' else "-" for ch in raw) - clean = clean.strip().strip(".") - return clean or "unknown" - - -def infer_dataset_id_from_manifest(manifest: dict[str, Any]) -> str: - extra = manifest.get("extra") or {} - if extra.get("dataset_id"): - return normalize_dataset_id(str(extra.get("dataset_id"))) - tasks_path = Path(str(extra.get("tasks_path", ""))) - if tasks_path.parent.name: - return normalize_dataset_id(tasks_path.parent.name) - workload_id = str(manifest.get("workload_id", "")) - if workload_id.upper().startswith("PUB-"): - return normalize_dataset_id(workload_id.split("-", 2)[1].lower()) - return normalize_dataset_id(str(manifest.get("dataset", "unknown"))) - - -def infer_track_id_from_manifest(manifest: dict[str, Any]) -> str: - if manifest.get("track_id"): - return str(manifest.get("track_id")) - extra = manifest.get("extra") or {} - if extra.get("track_id"): - return str(extra.get("track_id")) - experiment_track = str(extra.get("experiment_track", "mainline") or "mainline").strip() - prompt_mode = str(extra.get("prompt_mode", "memory_neutral") or "memory_neutral").strip() - return f"{experiment_track}:{prompt_mode}" - - -def infer_branch_key_from_manifest(manifest: dict[str, Any]) -> BranchKey: - return BranchKey( - dataset_id=infer_dataset_id_from_manifest(manifest), - track_id=infer_track_id_from_manifest(manifest), - variant_id=str(manifest.get("variant_id", "")).strip(), - repeat_idx=int(manifest.get("repeat_idx", 0)), - ) - - -def accepted_for_official_reporting(manifest: dict[str, Any]) -> bool: - acceptance_status = str(manifest.get("acceptance_status", "") or "").strip().lower() - diagnostic_only = bool(manifest.get("diagnostic_only", False)) - summary_chain_valid = manifest.get("summary_chain_valid", None) - superseded_by = str(manifest.get("superseded_by_run_id", "") or "").strip() - if acceptance_status: - return ( - acceptance_status == "accepted" - and not diagnostic_only - and not superseded_by - and bool(summary_chain_valid) - ) - # Legacy fallback for older runs that predate control-plane metadata. - return ( - bool(manifest.get("env_valid", False)) - and bool(manifest.get("repro_valid", False)) - and bool(manifest.get("llm_real", False)) - ) - - -def merge_control_fields(manifest: dict[str, Any], updates: dict[str, Any]) -> dict[str, Any]: - merged = dict(manifest) - for key, value in updates.items(): - merged[key] = value - return merged - - -class ControlPlane: - def __init__(self, *, artifacts_root: str | Path = "artifacts") -> None: - self.artifacts_root = Path(artifacts_root) - self.ledger_dir = self.artifacts_root / "ledger" - self.audit_dir = self.artifacts_root / "audit" - self.branch_registry_path = self.ledger_dir / "branch_registry.json" - self.invalid_run_ledger_path = self.ledger_dir / "invalid_run_ledger.jsonl" - self.rollback_ledger_path = self.ledger_dir / "rollback_ledger.jsonl" - self.rerun_plan_ledger_path = self.ledger_dir / "rerun_plan_ledger.jsonl" - self.acceptance_status_path = self.ledger_dir / "acceptance_status.jsonl" - self.anomaly_classification_path = self.ledger_dir / "anomaly_classification.jsonl" - self.stage_status_path = self.ledger_dir / "stage_status.jsonl" - - def _load_registry(self) -> dict[str, Any]: - if not self.branch_registry_path.exists(): - return {"branches": {}} - return _read_json(self.branch_registry_path) - - def _write_registry(self, registry: dict[str, Any]) -> None: - _write_json(self.branch_registry_path, registry) - - def register_branch(self, key: BranchKey, *, metadata: dict[str, Any] | None = None) -> dict[str, Any]: - registry = self._load_registry() - branches = registry.setdefault("branches", {}) - record = branches.get(key.branch_id) or { - "branch_id": key.branch_id, - "dataset_id": key.dataset_id, - "track_id": key.track_id, - "variant_id": key.variant_id, - "repeat_idx": int(key.repeat_idx), - "status": "pending", - "created_at_ms": now_ms(), - "updated_at_ms": now_ms(), - "metadata": {}, - "latest_run_id": "", - "latest_stage_id": "", - "latest_rerun_plan_id": "", - "anomaly_count": 0, - } - if metadata: - record["metadata"] = {**record.get("metadata", {}), **metadata} - record["updated_at_ms"] = now_ms() - branches[key.branch_id] = record - self._write_registry(registry) - return record - - def update_branch( - self, - key: BranchKey, - *, - status: str | None = None, - latest_run_id: str | None = None, - latest_stage_id: str | None = None, - latest_rerun_plan_id: str | None = None, - increment_anomaly_count: bool = False, - ) -> None: - registry = self._load_registry() - branches = registry.setdefault("branches", {}) - record = branches.get(key.branch_id) or self.register_branch(key) - if status: - record["status"] = status - if latest_run_id is not None: - record["latest_run_id"] = latest_run_id - if latest_stage_id is not None: - record["latest_stage_id"] = latest_stage_id - if latest_rerun_plan_id is not None: - record["latest_rerun_plan_id"] = latest_rerun_plan_id - if increment_anomaly_count: - record["anomaly_count"] = int(record.get("anomaly_count", 0)) + 1 - record["updated_at_ms"] = now_ms() - branches[key.branch_id] = record - self._write_registry(registry) - - def record_stage_status( - self, - key: BranchKey, - *, - stage_id: str, - status: str, - run_id: str = "", - note: str = "", - anomaly_codes: list[str] | None = None, - rollback_root_stage: str | None = None, - rerun_plan_id: str | None = None, - ) -> None: - record = StageStatusRecord( - branch_id=key.branch_id, - dataset_id=key.dataset_id, - track_id=key.track_id, - variant_id=key.variant_id, - repeat_idx=int(key.repeat_idx), - stage_id=stage_id, - status=status, - ts_ms=now_ms(), - run_id=run_id, - note=note, - anomaly_codes=list(anomaly_codes or []), - rollback_root_stage=rollback_root_stage, - rerun_plan_id=rerun_plan_id, - ) - _append_jsonl(self.stage_status_path, asdict(record)) - _append_jsonl( - self.acceptance_status_path, - { - "branch_id": key.branch_id, - "run_id": run_id, - "stage_id": stage_id, - "status": status, - "ts_ms": record.ts_ms, - "note": note, - }, - ) - self.update_branch( - key, - status=status, - latest_run_id=run_id or None, - latest_stage_id=stage_id, - latest_rerun_plan_id=rerun_plan_id, - increment_anomaly_count=bool(anomaly_codes), - ) - - def audit_report_path(self, key: BranchKey, stage_id: str) -> Path: - return ( - self.audit_dir - / sanitize_path_token(key.dataset_id) - / sanitize_path_token(key.track_id) - / sanitize_path_token(key.variant_id) - / f"r{int(key.repeat_idx)}" - / f"{sanitize_path_token(stage_id)}.json" - ) - - def write_audit_report(self, key: BranchKey, *, stage_id: str, report: dict[str, Any]) -> Path: - path = self.audit_report_path(key, stage_id) - _write_json(path, report) - self.register_branch(key, metadata={"latest_audit_report": path.as_posix()}) - return path - - def annotate_manifest(self, manifest_path: str | Path, **updates: Any) -> dict[str, Any]: - path = Path(manifest_path) - manifest = _read_json(path) - manifest = merge_control_fields(manifest, updates) - _write_json(path, manifest) - return manifest - - def create_rerun_plan( - self, - key: BranchKey, - *, - rollback_root_stage: str, - anomaly_codes: list[str], - invalidated_run_ids: list[str], - why: str, - what: str = "", - downstream_impact: list[str] | None = None, - ) -> str: - rerun_plan_id = f"rerun-{sha1_text(key.branch_id + why + str(now_ms()))[:12]}" - row = { - "rerun_plan_id": rerun_plan_id, - "branch_id": key.branch_id, - "dataset_id": key.dataset_id, - "track_id": key.track_id, - "variant_id": key.variant_id, - "repeat_idx": int(key.repeat_idx), - "rollback_root_stage": rollback_root_stage, - "anomaly_codes": list(sorted(set(anomaly_codes))), - "invalidated_run_ids": list(invalidated_run_ids), - "why": why, - "what": what, - "downstream_impact": list(downstream_impact or []), - "status": "pending", - "created_at_ms": now_ms(), - } - _append_jsonl(self.rerun_plan_ledger_path, row) - self.update_branch(key, latest_rerun_plan_id=rerun_plan_id) - return rerun_plan_id - - def record_anomaly( - self, - key: BranchKey, - *, - run_id: str, - stage_id: str, - anomaly_codes: list[str], - reason: str, - diagnostic_only: bool = False, - rerun_plan_id: str | None = None, - ) -> str: - rollback_root_stage = earliest_rollback_stage(anomaly_codes) - row = { - "branch_id": key.branch_id, - "dataset_id": key.dataset_id, - "track_id": key.track_id, - "variant_id": key.variant_id, - "repeat_idx": int(key.repeat_idx), - "run_id": run_id, - "stage_id": stage_id, - "anomaly_codes": list(sorted(set(anomaly_codes))), - "reason": reason, - "diagnostic_only": bool(diagnostic_only), - "rollback_root_stage": rollback_root_stage, - "rerun_plan_id": rerun_plan_id, - "ts_ms": now_ms(), - } - _append_jsonl(self.anomaly_classification_path, row) - _append_jsonl( - self.rollback_ledger_path, - { - **row, - "action": "rollback", - }, - ) - return rollback_root_stage - - def invalidate_run( - self, - manifest_path: str | Path, - *, - key: BranchKey, - stage_id: str, - anomaly_codes: list[str], - reason: str, - diagnostic_only: bool = False, - supersedes_run_id: str | None = None, - ) -> tuple[dict[str, Any], str, str]: - path = Path(manifest_path) - manifest = _read_json(path) - run_id = str(manifest.get("run_id", path.parent.name)) - rollback_root_stage = earliest_rollback_stage(anomaly_codes) - rerun_plan_id = self.create_rerun_plan( - key, - rollback_root_stage=rollback_root_stage, - anomaly_codes=anomaly_codes, - invalidated_run_ids=[run_id], - why=reason, - what=f"Re-run branch from {rollback_root_stage} after fixing {', '.join(sorted(set(anomaly_codes)))}.", - ) - status = "diagnostic_only" if diagnostic_only else "invalid" - manifest = self.annotate_manifest( - path, - acceptance_status=status, - diagnostic_only=bool(diagnostic_only), - invalid_reason_codes=list(sorted(set(anomaly_codes))), - anomaly_classification=list(sorted(set(anomaly_codes))), - rollback_root_stage=rollback_root_stage, - rerun_plan_id=rerun_plan_id, - supersedes_run_id=supersedes_run_id, - summary_chain_valid=bool(manifest.get("summary_chain_valid", False)), - ) - self.record_anomaly( - key, - run_id=run_id, - stage_id=stage_id, - anomaly_codes=anomaly_codes, - reason=reason, - diagnostic_only=diagnostic_only, - rerun_plan_id=rerun_plan_id, - ) - _append_jsonl( - self.invalid_run_ledger_path, - { - "run_id": run_id, - "branch_id": key.branch_id, - "status": status, - "stage_id": stage_id, - "anomaly_codes": list(sorted(set(anomaly_codes))), - "reason": reason, - "rollback_root_stage": rollback_root_stage, - "rerun_plan_id": rerun_plan_id, - "ts_ms": now_ms(), - }, - ) - self.record_stage_status( - key, - stage_id=stage_id, - status=status, - run_id=run_id, - note=reason, - anomaly_codes=anomaly_codes, - rollback_root_stage=rollback_root_stage, - rerun_plan_id=rerun_plan_id, - ) - return manifest, rollback_root_stage, rerun_plan_id - - -def earliest_rollback_stage(anomaly_codes: list[str]) -> str: - stages = [ANOMALY_TO_ROLLBACK_STAGE.get(code, "audit_protocol") for code in anomaly_codes] - order = {name: idx for idx, name in enumerate(STAGE_IDS)} - return sorted(stages, key=lambda name: order.get(name, 999))[0] if stages else "audit_protocol" - - -def default_manifest_control_fields() -> dict[str, Any]: - return { - "acceptance_status": "pending", - "diagnostic_only": False, - "invalid_reason_codes": [], - "anomaly_classification": [], - "rollback_parent_run_id": None, - "rollback_root_stage": None, - "rerun_plan_id": None, - "supersedes_run_id": None, - "superseded_by_run_id": None, - "input_fingerprint": None, - "protocol_fingerprint": None, - "dataset_fingerprint": None, - "summary_chain_valid": False, - "track_id": "", - } diff --git a/projects/memlab/memlab/pipeline/model_profile.py b/projects/memlab/memlab/pipeline/model_profile.py deleted file mode 100644 index d583d80..0000000 --- a/projects/memlab/memlab/pipeline/model_profile.py +++ /dev/null @@ -1,219 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import os -import re -from pathlib import Path -from typing import Any - -from memlab.io import now_ms - - -MODEL_SIZE_RE = re.compile(r"(\d+(?:\.\d+)?)\s*[bB]\b") - - -def estimate_model_size_b(model_id: str) -> float | None: - match = MODEL_SIZE_RE.search(str(model_id or "")) - if not match: - return None - try: - return float(match.group(1)) - except ValueError: - return None - - -def classify_model_scale(size_b: float | None) -> str: - if size_b is None: - return "unknown" - if size_b <= 8.0: - return "small" - if size_b <= 34.0: - return "medium" - return "large" - - -def runtime_recommendation(scale: str) -> dict[str, Any]: - if scale == "small": - return { - "e2e_inflight": 2, - "worker_processes": 1, - "sessions_per_process": 50, - "context_top_k": 12, - "max_evidence_chars": 3000, - "answer_retry_count": 1, - } - if scale == "medium": - return { - "e2e_inflight": 2, - "worker_processes": 1, - "sessions_per_process": 36, - "context_top_k": 10, - "max_evidence_chars": 2400, - "answer_retry_count": 1, - } - if scale == "large": - return { - "e2e_inflight": 1, - "worker_processes": 1, - "sessions_per_process": 24, - "context_top_k": 8, - "max_evidence_chars": 1800, - "answer_retry_count": 0, - } - # Unknown defaults to conservative medium-like settings. - return { - "e2e_inflight": 1, - "worker_processes": 1, - "sessions_per_process": 30, - "context_top_k": 10, - "max_evidence_chars": 2200, - "answer_retry_count": 1, - } - - -def training_recommendation(scale: str) -> dict[str, Any]: - if scale == "small": - return { - "global_batch_size": 64, - "micro_batch_size": 8, - "gradient_accumulation_steps": 1, - "precision": "bf16", - } - if scale == "medium": - return { - "global_batch_size": 32, - "micro_batch_size": 4, - "gradient_accumulation_steps": 2, - "precision": "bf16", - } - if scale == "large": - return { - "global_batch_size": 8, - "micro_batch_size": 1, - "gradient_accumulation_steps": 8, - "precision": "bf16", - } - return { - "global_batch_size": 16, - "micro_batch_size": 2, - "gradient_accumulation_steps": 4, - "precision": "bf16", - } - - -def _load_registry(path: Path) -> list[dict[str, Any]]: - if not path.exists(): - return [] - payload = json.loads(path.read_text(encoding="utf-8-sig")) - providers = payload.get("providers", []) - out: list[dict[str, Any]] = [] - for provider in providers: - provider_name = str(provider.get("name", "")).strip() - base_url = str(provider.get("base_url", "")).strip().rstrip("/") - for model in provider.get("models", []) or []: - out.append( - { - "provider": provider_name, - "base_url": base_url, - "model_id": str(model.get("id", "")).strip(), - "tags": [str(t) for t in (model.get("tags", []) or [])], - } - ) - return out - - -def _resolve_registry_entry( - entries: list[dict[str, Any]], - *, - model_id: str, - base_url: str, -) -> dict[str, Any] | None: - if not entries: - return None - model_id_norm = str(model_id or "").strip() - base_url_norm = str(base_url or "").strip().rstrip("/") - exact = [ - e - for e in entries - if e.get("model_id") == model_id_norm - and str(e.get("base_url", "")).strip().rstrip("/") == base_url_norm - ] - if exact: - return exact[0] - by_model = [e for e in entries if e.get("model_id") == model_id_norm] - if by_model: - return by_model[0] - return None - - -def build_profile( - *, - model_id: str, - base_url: str, - registry_entries: list[dict[str, Any]], -) -> dict[str, Any]: - size_b = estimate_model_size_b(model_id) - scale = classify_model_scale(size_b) - entry = _resolve_registry_entry(registry_entries, model_id=model_id, base_url=base_url) - provider = entry.get("provider") if entry else "unknown" - tags = entry.get("tags") if entry else [] - profile = { - "generated_at_ms": now_ms(), - "model": { - "provider": provider, - "base_url": str(base_url or "").strip().rstrip("/"), - "model_id": str(model_id or "").strip(), - "estimated_params_b": size_b, - "scale_class": scale, - "tags": tags, - }, - "interface_contract": { - "api_style": "openai_compatible_chat", - "required_env": ["OPENAI_BASE_URL", "OPENAI_API_KEY", "OPENAI_MODEL"], - "request_schema": { - "required_fields": ["model", "messages"], - "optional_fields": ["temperature", "response_format", "max_tokens", "stream"], - }, - "response_schema": { - "required_fields": ["choices"], - "usage_fields": ["prompt_tokens", "completion_tokens"], - }, - }, - "recommended_runtime": runtime_recommendation(scale), - "recommended_training": training_recommendation(scale), - "metric_contract": { - "performance": ["throughput_tasks_per_min", "processing_p95_ms", "end_to_end_p95_ms"], - "cost": ["prompt_tokens_per_success", "total_tokens_per_success", "context_injected_tokens_mean"], - "amplification": ["memory_ops_per_task", "db_queries_per_task", "vector_searches_per_task"], - "reliability": ["success_rate", "retry_count", "no_answer_rate", "parse_failure_rate"], - }, - } - return profile - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--registry", type=str, default="configs/model_registry.json") - ap.add_argument("--model-id", type=str, default="") - ap.add_argument("--base-url", type=str, default="") - ap.add_argument("--out", type=str, default="artifacts/model_runtime_profile.json") - args = ap.parse_args() - - model_id = str(args.model_id or os.getenv("OPENAI_MODEL", "")).strip() - base_url = str(args.base_url or os.getenv("OPENAI_BASE_URL", "")).strip().rstrip("/") - if not model_id: - model_id = "unknown-model" - if not base_url: - base_url = "unknown-base-url" - - registry_entries = _load_registry(Path(args.registry)) - profile = build_profile(model_id=model_id, base_url=base_url, registry_entries=registry_entries) - out_path = Path(args.out) - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(profile, indent=2, ensure_ascii=False), encoding="utf-8") - print(json.dumps({"out": str(out_path), "scale_class": profile["model"]["scale_class"]}, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/runner_core.py b/projects/memlab/memlab/runner_core.py deleted file mode 100644 index 7307a5f..0000000 --- a/projects/memlab/memlab/runner_core.py +++ /dev/null @@ -1,912 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import os -import random -import re -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from memlab.config import MemlabConfig, load_config -from memlab.io import jsonl_read, now_ms -from memlab.metrics.stats import summarize_latency - - -SECRET_RE = re.compile(r"SECRET[_\s-]*CODE\s*[:=]\s*([A-Z0-9_-]+)", re.IGNORECASE) -JSON_SECRET_RE = re.compile(r'"secret_code"\s*:\s*"([A-Z0-9_-]+)"', re.IGNORECASE) -FENCED_JSON_RE = re.compile(r"```(?:json)?\s*(\{.*?\}|\[.*?\])\s*```", re.IGNORECASE | re.DOTALL) -FINAL_ANSWER_RE = re.compile(r"final\s*answer\s*[:：]\s*(.+)", re.IGNORECASE) -LIKELY_CODE_RE = re.compile(r"^[A-Za-z0-9_-]{6,}$") -NON_WORD_RE = re.compile(r"[^a-z0-9]+") -OPTION_LETTER_RE = re.compile(r"\b([A-D])\b", re.IGNORECASE) -LEADING_ARTICLE_RE = re.compile(r"^(?:the|a|an)\s+", re.IGNORECASE) -REFUSAL_HINT_RE = re.compile( - r"(?:\b(can(?:not|'t)|unable|sorry|cannot comply|don't know|not enough info|no answer|refuse|decline)\b|无法|不能|抱歉|不知道|无法回答)", - re.IGNORECASE, -) - - -@dataclass -class CommonArgs: - config: str - variant: str - legacy_variant_alias: str | None - n_sessions: int - llm_max_inflight: int - worker_processes: int - sessions_per_process: int - repeat_idx: int - seed: int - workload_id: str - deerflow_langgraph_url: str - deerflow_gateway_url: str - locked_base_url: str - locked_model_id: str - experiment_track: str - track_id: str - dataset_id: str - prepared_full_or_slice: str - qa_prompt_profile: str - tasks_path: str - docs_path: str - debug_print: bool - debug_max_output_chars: int - allow_fallback_retry: bool - fallback_model: str - prompt_mode: str - qa_support_mode: str - max_evidence_chars: int - answer_retry_count: int - store_raw_output_chars: int - answer_format_mode: str - structured_evidence: bool - context_top_k: int - context_dedupe: bool - context_item_max_chars: int - memory_stress_turns: int - session_scope: str - preflight_only: bool - skip_preflight: bool - assistant_id: str - agent_name: str - - -CANONICAL_VARIANTS = ("B0", "B1", "V1", "F1", "F2") -LEGACY_VARIANT_ALIASES = { - "A1": "V1", - "A2": "F1", -} - - -def normalize_variant(variant: str) -> tuple[str, str | None]: - requested = str(variant or "").upper().strip() - if requested in CANONICAL_VARIANTS: - return requested, None - if requested in LEGACY_VARIANT_ALIASES: - return LEGACY_VARIANT_ALIASES[requested], requested - valid = ", ".join([*CANONICAL_VARIANTS, *LEGACY_VARIANT_ALIASES.keys()]) - raise ValueError(f"Unknown variant: {variant}. Expected one of: {valid}") - - -def apply_variant(cfg: MemlabConfig, variant: str, *, legacy_alias: str | None = None) -> MemlabConfig: - canonical, inferred_alias = normalize_variant(variant) - alias = legacy_alias or inferred_alias - cfg.experiment.variant_id = canonical - if canonical == "B0": - cfg.memory.l1.enabled = False - cfg.memory.l2.enabled = False - cfg.memory.l3.enabled = False - cfg.experiment.summarization_on = False - cfg.memory.l0.max_memory_injection_tokens = 0 - elif canonical == "B1": - cfg.memory.l1.enabled = False - cfg.memory.l2.enabled = False - cfg.memory.l3.enabled = False - cfg.experiment.summarization_on = True - cfg.memory.l0.max_memory_injection_tokens = 0 - elif canonical == "V1": - cfg.memory.l1.enabled = False - cfg.memory.l2.enabled = True - cfg.memory.l3.enabled = False - cfg.experiment.summarization_on = True - elif canonical == "F1": - cfg.memory.l1.enabled = True - cfg.memory.l2.enabled = True - cfg.memory.l3.enabled = False - cfg.experiment.summarization_on = True - elif canonical == "F2": - cfg.memory.l1.enabled = True - cfg.memory.l2.enabled = True - cfg.memory.l3.enabled = True - cfg.experiment.summarization_on = True - if alias == "A2": - # Preserve historical behavior for legacy ablation alias. - cfg.experiment.redis_policy = "allkeys-lru" - return cfg - - -def parse_common_args(default_sessions: int, default_workload_id: str) -> CommonArgs: - ap = argparse.ArgumentParser() - ap.add_argument("--config", type=str, default="configs/memlab.default.yaml") - ap.add_argument("--variant", type=str, required=True, help="B0|B1|V1|F1|F2 (legacy aliases: A1|A2)") - ap.add_argument("--n_sessions", type=int, default=default_sessions) - ap.add_argument("--llm_max_inflight", type=int, default=2) - ap.add_argument("--worker_processes", type=int, default=1) - ap.add_argument("--sessions_per_process", type=int, default=50) - ap.add_argument("--repeat_idx", type=int, default=0) - ap.add_argument("--seed", type=int, default=42) - ap.add_argument("--workload_id", type=str, default=default_workload_id) - ap.add_argument("--deerflow_langgraph_url", type=str, default="http://127.0.0.1:2024") - ap.add_argument("--deerflow_gateway_url", type=str, default="http://127.0.0.1:8001") - ap.add_argument("--locked_base_url", type=str, default=os.getenv("OPENAI_BASE_URL", "")) - ap.add_argument("--locked_model_id", type=str, default=os.getenv("OPENAI_MODEL", "")) - ap.add_argument( - "--experiment_track", - type=str, - default="mainline", - choices=( - "mainline", - "candidate_race", - "emergency_rebaseline", - "memory_exposure", - "focused_validation_v3", - "alignment_diagnosis", - ), - ) - ap.add_argument("--track_id", type=str, default="") - ap.add_argument("--dataset_id", type=str, default="") - ap.add_argument("--prepared_full_or_slice", type=str, default="") - ap.add_argument( - "--qa_prompt_profile", - type=str, - default="default", - choices=("default", "yes_no", "entity_span", "short_answer", "summary", "multiple_choice_letter"), - ) - ap.add_argument("--tasks_path", type=str, default="workloads/e2e/tasks.jsonl") - ap.add_argument("--docs_path", type=str, default="data/docs.jsonl") - ap.add_argument("--debug_print", action="store_true") - ap.add_argument("--debug_max_output_chars", type=int, default=200) - ap.add_argument("--allow_fallback_retry", action="store_true") - ap.add_argument("--fallback_model", type=str, default="") - ap.add_argument( - "--prompt_mode", - type=str, - default="memory_neutral", - choices=("memory_neutral", "memory_beneficial"), - help="memory_neutral injects evidence directly; memory_beneficial relies on memory retrieval.", - ) - ap.add_argument( - "--qa_support_mode", - type=str, - default="standard", - choices=("standard", "none"), - help="Controls whether QA prompts include direct evidence excerpts in addition to memory retrieval context.", - ) - ap.add_argument("--max_evidence_chars", type=int, default=3000) - ap.add_argument("--answer_retry_count", type=int, default=1) - ap.add_argument("--store_raw_output_chars", type=int, default=0) - ap.add_argument( - "--answer_format_mode", - type=str, - default="strict_json", - choices=("strict_json", "final_answer_prefix"), - ) - ap.add_argument("--context_top_k", type=int, default=12) - ap.add_argument("--structured_evidence", action="store_true") - ap.add_argument("--context_dedupe", action="store_true") - ap.add_argument("--context_item_max_chars", type=int, default=700) - ap.add_argument( - "--session_scope", - type=str, - default="worker", - choices=("worker", "task"), - help="worker reuses one memory namespace per worker; task isolates memory namespace per task.", - ) - ap.add_argument( - "--memory_stress_turns", - type=int, - default=0, - help="Inject N distractor turns after evidence write to stress long-history memory retrieval.", - ) - ap.add_argument("--preflight_only", action="store_true") - ap.add_argument("--skip_preflight", action="store_true") - ap.add_argument("--assistant_id", type=str, default="lead_agent") - ap.add_argument("--agent_name", type=str, default="memlab-e2e") - ns = ap.parse_args() - canonical_variant, legacy_alias = normalize_variant(ns.variant) - return CommonArgs( - config=ns.config, - variant=canonical_variant, - legacy_variant_alias=legacy_alias, - n_sessions=int(ns.n_sessions), - llm_max_inflight=int(ns.llm_max_inflight), - worker_processes=int(ns.worker_processes), - sessions_per_process=int(ns.sessions_per_process), - repeat_idx=int(ns.repeat_idx), - seed=int(ns.seed), - workload_id=str(ns.workload_id), - deerflow_langgraph_url=str(ns.deerflow_langgraph_url), - deerflow_gateway_url=str(ns.deerflow_gateway_url), - locked_base_url=str(ns.locked_base_url or "").strip().rstrip("/"), - locked_model_id=str(ns.locked_model_id or "").strip(), - experiment_track=str(ns.experiment_track or "mainline").strip(), - track_id=str(ns.track_id or "").strip(), - dataset_id=str(ns.dataset_id or "").strip(), - prepared_full_or_slice=str(ns.prepared_full_or_slice or "").strip(), - qa_prompt_profile=str(ns.qa_prompt_profile or "default").strip(), - tasks_path=str(ns.tasks_path), - docs_path=str(ns.docs_path), - debug_print=bool(ns.debug_print), - debug_max_output_chars=max(1, int(ns.debug_max_output_chars)), - allow_fallback_retry=bool(ns.allow_fallback_retry), - fallback_model=str(ns.fallback_model or ""), - prompt_mode=str(ns.prompt_mode), - qa_support_mode=str(ns.qa_support_mode or "standard").strip(), - max_evidence_chars=max(256, int(ns.max_evidence_chars)), - answer_retry_count=max(0, int(ns.answer_retry_count)), - store_raw_output_chars=max(0, int(ns.store_raw_output_chars)), - answer_format_mode=str(ns.answer_format_mode), - structured_evidence=bool(ns.structured_evidence), - context_top_k=max(1, int(ns.context_top_k)), - context_dedupe=bool(ns.context_dedupe), - context_item_max_chars=max(120, int(ns.context_item_max_chars)), - memory_stress_turns=max(0, int(ns.memory_stress_turns)), - session_scope=str(ns.session_scope or "worker"), - preflight_only=bool(ns.preflight_only), - skip_preflight=bool(ns.skip_preflight), - assistant_id=str(ns.assistant_id or "lead_agent"), - agent_name=str(ns.agent_name or "memlab-e2e"), - ) - - -def load_runtime_config(common_args: CommonArgs) -> MemlabConfig: - cfg = load_config(common_args.config) - cfg = apply_variant(cfg, common_args.variant, legacy_alias=common_args.legacy_variant_alias) - cfg.experiment.n_agents = common_args.n_sessions - cfg.experiment.seed = common_args.seed - return cfg - - -def load_e2e_tasks(path: str | Path = "workloads/e2e/tasks.jsonl") -> list[dict[str, Any]]: - return jsonl_read(path) - - -def load_docs(path: str | Path = "data/docs.jsonl") -> dict[str, dict[str, Any]]: - docs = jsonl_read(path) - return {str(d["doc_id"]): d for d in docs} - - -def normalize_secret(text: str) -> str: - if not isinstance(text, str): - return "" - return "".join(ch for ch in text.upper() if ch.isalnum()) - - -def normalize_text_relaxed(text: str) -> str: - text = str(text or "").lower().strip() - text = NON_WORD_RE.sub(" ", text) - text = re.sub(r"\s+", " ", text).strip() - return text - - -def normalize_option_letter(text: str) -> str: - raw = str(text or "").strip().upper() - if not raw: - return "" - m = OPTION_LETTER_RE.search(raw) - if m: - return str(m.group(1)).upper() - first = raw[:1] - return first if first in {"A", "B", "C", "D"} else "" - - -def normalize_exact_span(text: str) -> str: - norm = normalize_text_relaxed(text) - norm = LEADING_ARTICLE_RE.sub("", norm).strip() - return norm - - -def resolve_answer_metric_hint(*, task_type: str, answer_metric_hint: str = "") -> str: - if str(task_type or "").strip().lower() == "secret_code": - return "secret_code" - hint = str(answer_metric_hint or "").strip().lower() - return hint or "strict_exact" - - -def compute_benchmark_score( - *, - prediction: str, - answer_key: str, - task_type: str, - answer_metric_hint: str = "", - strict_score: float = 0.0, - relaxed_score: float = 0.0, - partial_score: float = 0.0, -) -> tuple[str, float]: - metric = resolve_answer_metric_hint(task_type=task_type, answer_metric_hint=answer_metric_hint) - pred = str(prediction or "").strip() - gold = str(answer_key or "").strip() - if not pred or not gold: - return metric, 0.0 - - if metric == "secret_code": - score = 1.0 if normalize_secret(pred) == normalize_secret(gold) else 0.0 - elif metric == "exact_option": - score = 1.0 if normalize_option_letter(pred) == normalize_option_letter(gold) else 0.0 - elif metric == "exact_span": - score = 1.0 if normalize_exact_span(pred) == normalize_exact_span(gold) else 0.0 - elif metric == "relaxed_span": - pred_norm = normalize_exact_span(pred) - gold_norm = normalize_exact_span(gold) - score = 1.0 if pred_norm and pred_norm == gold_norm else _token_overlap_f1(pred_norm, gold_norm) - elif metric == "token_f1": - score = _token_overlap_f1(normalize_text_relaxed(pred), normalize_text_relaxed(gold)) - else: - score = strict_score if strict_score > 0.0 else max(relaxed_score, partial_score) - return metric, max(0.0, min(1.0, float(score))) - - -def extract_secret_code(text: str) -> str: - m = SECRET_RE.search(str(text or "")) - if not m: - return "" - return normalize_secret(m.group(1)) - - -def _norm_key(key: Any) -> str: - return str(key).strip().lower().replace("-", "_").replace(" ", "_") - - -def _looks_like_code(text: str) -> bool: - return bool(LIKELY_CODE_RE.match(str(text or "").strip())) - - -def _find_first_field(obj: Any, target_fields: set[str], path: str = "") -> tuple[str, str] | None: - if isinstance(obj, dict): - for key, value in obj.items(): - key_norm = _norm_key(key) - next_path = f"{path}.{key_norm}" if path else key_norm - if key_norm in target_fields and isinstance(value, str): - return str(value), next_path - nested = _find_first_field(value, target_fields, next_path) - if nested: - return nested - return None - if isinstance(obj, list): - for idx, item in enumerate(obj): - next_path = f"{path}[{idx}]" - nested = _find_first_field(item, target_fields, next_path) - if nested: - return nested - return None - return None - - -def _find_wrong_field_candidate(obj: Any, target_fields: set[str], task_type: str) -> str: - if isinstance(obj, dict): - for key, value in obj.items(): - key_norm = _norm_key(key) - if key_norm in target_fields: - continue - if isinstance(value, str): - val = str(value).strip() - if task_type == "secret_code": - if ("answer" in key_norm or "code" in key_norm or "secret" in key_norm) and _looks_like_code(val): - return key_norm - else: - if "answer" in key_norm and val: - return key_norm - nested = _find_wrong_field_candidate(value, target_fields, task_type) - if nested: - return nested - elif isinstance(obj, list): - for item in obj: - nested = _find_wrong_field_candidate(item, target_fields, task_type) - if nested: - return nested - return "" - - -def _json_candidates(text: str) -> list[Any]: - out: list[Any] = [] - try: - out.append(json.loads(text)) - except Exception: - pass - for match in FENCED_JSON_RE.finditer(text): - block = match.group(1) - try: - out.append(json.loads(block)) - except Exception: - continue - for candidate in re.findall(r"\{[^{}]*\}", text): - try: - out.append(json.loads(candidate)) - except Exception: - continue - return out - - -def parse_model_answer( - text: str, - *, - answer_field: str = "secret_code", - task_type: str = "secret_code", -) -> dict[str, Any]: - body = str(text or "") - field = _norm_key(answer_field or ("secret_code" if task_type == "secret_code" else "final_answer")) - target_fields = {field} - if task_type != "secret_code": - target_fields |= {"final_answer", "answer", "result", "output"} - wrong_field_candidate = "" - for obj in _json_candidates(body): - found = _find_first_field(obj, target_fields) - if found: - raw, source_path = found - parsed = normalize_secret(raw) if task_type == "secret_code" else str(raw).strip() - return { - "parsed_answer": parsed, - "parsed_answer_raw": str(raw), - "parse_source": f"json:{source_path}", - "wrong_field_candidate": "", - } - wrong_field_candidate = wrong_field_candidate or _find_wrong_field_candidate(obj, target_fields, task_type) - - if task_type == "secret_code": - m = JSON_SECRET_RE.search(body) - if m: - return { - "parsed_answer": normalize_secret(m.group(1)), - "parsed_answer_raw": str(m.group(1)), - "parse_source": "regex:json_secret", - "wrong_field_candidate": wrong_field_candidate, - } - secret = extract_secret_code(body) - if secret: - return { - "parsed_answer": secret, - "parsed_answer_raw": secret, - "parse_source": "regex:secret_code", - "wrong_field_candidate": wrong_field_candidate, - } - return { - "parsed_answer": "", - "parsed_answer_raw": "", - "parse_source": "none", - "wrong_field_candidate": wrong_field_candidate, - } - - field_pattern = re.compile(rf'"{re.escape(field)}"\s*:\s*"([^"]+)"', re.IGNORECASE) - m_field = field_pattern.search(body) - if m_field: - ans = str(m_field.group(1)) - return { - "parsed_answer": ans.strip(), - "parsed_answer_raw": ans, - "parse_source": "regex:field", - "wrong_field_candidate": wrong_field_candidate, - } - m_final = FINAL_ANSWER_RE.search(body) - if m_final: - ans = str(m_final.group(1)).strip().splitlines()[0].strip() - return { - "parsed_answer": ans, - "parsed_answer_raw": ans, - "parse_source": "regex:final_answer_prefix", - "wrong_field_candidate": wrong_field_candidate, - } - plain = str(body).strip() - if plain: - return { - "parsed_answer": plain.splitlines()[0].strip(), - "parsed_answer_raw": plain.splitlines()[0].strip(), - "parse_source": "fallback:first_line", - "wrong_field_candidate": wrong_field_candidate, - } - return { - "parsed_answer": "", - "parsed_answer_raw": "", - "parse_source": "none", - "wrong_field_candidate": wrong_field_candidate, - } - - -def parse_model_secret(text: str) -> str: - return str(parse_model_answer(text, answer_field="secret_code", task_type="secret_code").get("parsed_answer", "")) - - -def _token_overlap_f1(a: str, b: str) -> float: - aa = [x for x in a.split(" ") if x] - bb = [x for x in b.split(" ") if x] - if not aa or not bb: - return 0.0 - from collections import Counter - - ca = Counter(aa) - cb = Counter(bb) - hit = sum(min(ca[k], cb[k]) for k in set(ca) | set(cb)) - if hit <= 0: - return 0.0 - p = hit / len(aa) - r = hit / len(bb) - return (2.0 * p * r) / (p + r) if (p + r) > 0 else 0.0 - - -def _secret_partial_ratio(pred: str, gold: str) -> float: - if not pred or not gold: - return 0.0 - pred_n = normalize_secret(pred) - gold_n = normalize_secret(gold) - if not pred_n or not gold_n: - return 0.0 - prefix = 0 - for a, b in zip(pred_n, gold_n): - if a != b: - break - prefix += 1 - return prefix / max(1, len(gold_n)) - - -def score_task_answer( - *, - raw_output: str, - parsed_answer: str, - parsed_answer_raw: str, - answer_key: str, - task_type: str, - answer_metric_hint: str = "", - parse_source: str = "", - wrong_field_candidate: str = "", - status: str = "success", -) -> dict[str, Any]: - raw = str(raw_output or "").strip() - parsed = str(parsed_answer or "").strip() - parsed_raw = str(parsed_answer_raw or "").strip() - gold = str(answer_key or "").strip() - strict_score = 0.0 - relaxed_score = 0.0 - partial_score = 0.0 - label = "FACTUALLY_WRONG" - no_answer = False - benchmark_metric = resolve_answer_metric_hint(task_type=task_type, answer_metric_hint=answer_metric_hint) - benchmark_score = 0.0 - - if status != "success": - if not raw or REFUSAL_HINT_RE.search(raw): - no_answer = True - label = "REFUSAL_OR_NOANSWER" - return { - "strict_score": strict_score, - "relaxed_score": relaxed_score, - "partial_score": partial_score, - "label": label, - "parse_failure": False, - "no_answer": no_answer, - "parse_source": parse_source, - "benchmark_metric": benchmark_metric, - "benchmark_score": benchmark_score, - } - - if not raw or raw.lower() in {"{}", "[]", "null"}: - no_answer = True - if not parsed: - if no_answer or REFUSAL_HINT_RE.search(raw): - label = "REFUSAL_OR_NOANSWER" - no_answer = True - elif wrong_field_candidate: - label = "PARSE_WRONG_FIELD" - else: - label = "PARSE_EMPTY" - return { - "strict_score": 0.0, - "relaxed_score": 0.0, - "partial_score": 0.0, - "label": label, - "parse_failure": label in {"PARSE_EMPTY", "PARSE_WRONG_FIELD"}, - "no_answer": no_answer, - "parse_source": parse_source, - "benchmark_metric": benchmark_metric, - "benchmark_score": benchmark_score, - } - - strict_score = 1.0 if parsed_raw == gold else 0.0 - if task_type == "secret_code": - pred_relaxed = normalize_secret(parsed_raw or parsed) - gold_relaxed = normalize_secret(gold) - partial_score = _secret_partial_ratio(parsed_raw or parsed, gold) - else: - pred_relaxed = normalize_text_relaxed(parsed_raw or parsed) - gold_relaxed = normalize_text_relaxed(gold) - partial_score = _token_overlap_f1(pred_relaxed, gold_relaxed) - relaxed_score = 1.0 if pred_relaxed and pred_relaxed == gold_relaxed else 0.0 - benchmark_metric, benchmark_score = compute_benchmark_score( - prediction=(parsed_raw or parsed), - answer_key=gold, - task_type=task_type, - answer_metric_hint=answer_metric_hint, - strict_score=strict_score, - relaxed_score=relaxed_score, - partial_score=partial_score, - ) - - if strict_score >= 1.0: - label = "OK" - elif relaxed_score >= 1.0: - label = "FORMAT_MISMATCH" - elif partial_score >= 0.5: - label = "PARTIAL_MATCH" - else: - label = "FACTUALLY_WRONG" - - return { - "strict_score": strict_score, - "relaxed_score": relaxed_score, - "partial_score": partial_score, - "label": label, - "parse_failure": False, - "no_answer": no_answer, - "parse_source": parse_source, - "benchmark_metric": benchmark_metric, - "benchmark_score": benchmark_score, - } - - -def task_accuracy(predicted_secret: str, answer_key: str) -> float: - return 1.0 if normalize_secret(predicted_secret) == normalize_secret(answer_key) else 0.0 - - -def compute_run_summary( - *, - run_id: str, - variant_id: str, - workload_id: str, - n_sessions: int, - tasks: list[dict[str, Any]], - mem_ops: list[dict[str, Any]], -) -> dict[str, Any]: - successful = [t for t in tasks if t.get("status") == "success"] - failed = [t for t in tasks if t.get("status") != "success"] - prompt_modes = sorted({str(t.get("prompt_mode", "")).strip() for t in tasks if str(t.get("prompt_mode", "")).strip()}) - queue_wait_latencies = [float(t.get("queue_wait_ms", 0.0)) for t in successful if t.get("queue_wait_ms") is not None] - processing_latencies = [float(t.get("proc_whole_ms", t.get("processing_ms", 0.0))) for t in successful if t.get("processing_ms") is not None or t.get("proc_whole_ms") is not None] - proc_llm_latencies = [float(t.get("proc_llm_ms", 0.0)) for t in successful if t.get("proc_llm_ms") is not None] - proc_non_llm_latencies = [float(t.get("proc_non_llm_ms", 0.0)) for t in successful if t.get("proc_non_llm_ms") is not None] - end_to_end_latencies = [float(t.get("end_to_end_ms", t.get("latency_ms", 0.0))) for t in successful] - llm_latencies = [ - float(t.get("llm_latency_ms", t.get("llm_request_latency_ms", 0.0))) - for t in successful - if (t.get("llm_latency_ms") is not None or t.get("llm_request_latency_ms") is not None) - ] - usage_missing = any(t.get("tokens_in") is None or t.get("tokens_out") is None for t in successful) - token_in_total: int | None = None - token_out_total: int | None = None - if successful and not usage_missing: - token_in_total = int(sum(int(t.get("tokens_in", 0)) for t in successful)) - token_out_total = int(sum(int(t.get("tokens_out", 0)) for t in successful)) - injected = [int(t.get("context_injected_tokens", 0)) for t in successful] - injected_items = [int(t.get("context_injected_items", 0)) for t in successful] - is_syn = str(workload_id).upper().startswith("SYN") - accuracy_applicable = not is_syn - total_tasks = len(tasks) - accuracy_values = [float(t.get("accuracy")) for t in successful if t.get("accuracy") is not None] - strict_values = [float(t.get("strict_score")) for t in successful if t.get("strict_score") is not None] - relaxed_values = [float(t.get("relaxed_score")) for t in successful if t.get("relaxed_score") is not None] - partial_values = [float(t.get("partial_score")) for t in successful if t.get("partial_score") is not None] - benchmark_values = [float(t.get("benchmark_score")) for t in successful if t.get("benchmark_score") is not None] - strict_correct_total = int( - sum(1 for t in tasks if float(t.get("strict_score", 0.0) or 0.0) >= 1.0) - ) - relaxed_correct_total = int( - sum(1 for t in tasks if float(t.get("relaxed_score", 0.0) or 0.0) >= 1.0) - ) - partial_score_total = float(sum(float(t.get("partial_score", 0.0) or 0.0) for t in tasks)) - benchmark_score_total = float(sum(float(t.get("benchmark_score", 0.0) or 0.0) for t in tasks)) - benchmark_metrics = sorted( - { - str(t.get("benchmark_metric", "")).strip() - for t in tasks - if str(t.get("benchmark_metric", "")).strip() - } - ) - benchmark_metric = benchmark_metrics[0] if len(benchmark_metrics) == 1 else ("mixed" if benchmark_metrics else "strict_exact") - parse_fail_count = sum(1 for t in successful if bool(t.get("parse_failure", False))) - no_answer_count = sum(1 for t in successful if bool(t.get("no_answer", False))) - parse_fail_count_total = sum(1 for t in tasks if bool(t.get("parse_failure", False))) - no_answer_count_total = sum(1 for t in tasks if bool(t.get("no_answer", False))) - accuracy: float | None = None if not accuracy_applicable else (sum(accuracy_values) / max(1, len(accuracy_values))) - strict_accuracy: float | None = None if not accuracy_applicable else (sum(strict_values) / max(1, len(strict_values))) - relaxed_accuracy: float | None = None if not accuracy_applicable else (sum(relaxed_values) / max(1, len(relaxed_values))) - partial_accuracy: float | None = None if not accuracy_applicable else (sum(partial_values) / max(1, len(partial_values))) - benchmark_score: float | None = None if not accuracy_applicable else (sum(benchmark_values) / max(1, len(benchmark_values))) - strict_accuracy_on_total: float | None = None if not accuracy_applicable else (strict_correct_total / max(1, total_tasks)) - relaxed_accuracy_on_total: float | None = None if not accuracy_applicable else (relaxed_correct_total / max(1, total_tasks)) - partial_accuracy_on_total: float | None = None if not accuracy_applicable else (partial_score_total / max(1, total_tasks)) - benchmark_score_on_total: float | None = None if not accuracy_applicable else (benchmark_score_total / max(1, total_tasks)) - parse_failure_rate = parse_fail_count / max(1, len(successful)) - no_answer_rate = no_answer_count / max(1, len(successful)) - parse_failure_rate_on_total = parse_fail_count_total / max(1, total_tasks) - no_answer_rate_on_total = no_answer_count_total / max(1, total_tasks) - success_count = len(successful) - success_rate = (success_count / total_tasks) if total_tasks > 0 else 0.0 - - l1_hits = [m for m in mem_ops if m.get("tier") == "L1" and m.get("hit") is not None] - l2_hits = [m for m in mem_ops if m.get("tier") == "L2" and m.get("op_name") in {"vector_search", "get_by_keys", "search_text"} and m.get("hit") is not None] - l3_hits = [m for m in mem_ops if m.get("tier") == "L3" and m.get("hit") is not None] - l1_hit_rate = sum(1 for x in l1_hits if x.get("hit")) / max(1, len(l1_hits)) - l2_hit_rate = sum(1 for x in l2_hits if x.get("hit")) / max(1, len(l2_hits)) - l3_hit_rate = sum(1 for x in l3_hits if x.get("hit")) / max(1, len(l3_hits)) - memory_ops_per_task = float(len(mem_ops)) / max(1, len(tasks)) - db_query_ops = [m for m in mem_ops if m.get("tier") == "L2" and m.get("op_name") in {"get_by_keys", "search_text", "get_recent"}] - vector_search_ops = [m for m in mem_ops if m.get("tier") == "L2" and m.get("op_name") == "vector_search"] - db_queries_per_task = float(len(db_query_ops)) / max(1, len(tasks)) - vector_searches_per_task = float(len(vector_search_ops)) / max(1, len(tasks)) - - op_bucket: dict[str, list[float]] = {} - for op in mem_ops: - key = f"{op.get('tier','?')}:{op.get('op_name','?')}" - op_bucket.setdefault(key, []).append(float(op.get("latency_ms", 0.0))) - op_p95 = {k: summarize_latency(v)["p95"] for k, v in sorted(op_bucket.items())} - - prompt_per_success: float | None = None - total_per_success: float | None = None - if token_in_total is not None and token_out_total is not None and success_count > 0: - prompt_per_success = token_in_total / success_count - total_per_success = (token_in_total + token_out_total) / success_count - - retrieval_skipped_count = sum(1 for t in successful if t.get("retrieve_skipped_reason")) - retrieval_skip_rate = retrieval_skipped_count / max(1, len(successful)) - memory_injection_nonzero_tasks = sum(1 for t in successful if int(t.get("context_injected_tokens", 0) or 0) > 0) - memory_injection_nonzero_rate = memory_injection_nonzero_tasks / max(1, len(successful)) - retrieval_engaged_tasks = sum(1 for t in successful if not t.get("retrieve_skipped_reason")) - retrieval_engaged_rate = retrieval_engaged_tasks / max(1, len(successful)) - retrieval_engaged_strict_values = [ - float(t.get("strict_score", 0.0)) - for t in successful - if not t.get("retrieve_skipped_reason") - ] - strict_accuracy_retrieval_engaged: float | None = ( - (sum(retrieval_engaged_strict_values) / len(retrieval_engaged_strict_values)) - if retrieval_engaged_strict_values - else None - ) - embedding_cache_values = [t.get("embedding_cache_hit") for t in successful if t.get("embedding_cache_hit") is not None] - retrieval_cache_values = [t.get("retrieval_cache_hit") for t in successful if t.get("retrieval_cache_hit") is not None] - embedding_cache_hit_rate = ( - sum(1 for v in embedding_cache_values if bool(v)) / len(embedding_cache_values) - if embedding_cache_values - else None - ) - retrieval_cache_hit_rate = ( - sum(1 for v in retrieval_cache_values if bool(v)) / len(retrieval_cache_values) - if retrieval_cache_values - else None - ) - evidence_chars_values = [int(t.get("evidence_chars_used", 0)) for t in successful if t.get("evidence_chars_used") is not None] - - enqueue_ts = [int(t.get("task_enqueued_ts")) for t in tasks if t.get("task_enqueued_ts") is not None] - end_ts = [int(t.get("task_end_ts", t.get("end_ts"))) for t in tasks if t.get("task_end_ts", t.get("end_ts")) is not None] - throughput_tasks_per_min = 0.0 - if enqueue_ts and end_ts: - wall_ms = max(0, max(end_ts) - min(enqueue_ts)) - if wall_ms > 0: - throughput_tasks_per_min = float(success_count) / (wall_ms / 60000.0) - - injected_summary = summarize_latency(injected) if injected else summarize_latency([]) - injected_items_summary = summarize_latency(injected_items) if injected_items else summarize_latency([]) - queue_wait_summary = summarize_latency(queue_wait_latencies) - processing_summary = summarize_latency(processing_latencies) - proc_llm_summary = summarize_latency(proc_llm_latencies) - proc_non_llm_summary = summarize_latency(proc_non_llm_latencies) - end_to_end_summary = summarize_latency(end_to_end_latencies) - - def classify_error(msg: str) -> str: - lower = str(msg or "").lower() - if not lower: - return "none" - if "30001" in lower or ("insufficient" in lower and "balance" in lower): - return "provider_403_balance" - if "429" in lower or "ratelimit" in lower or "tpm" in lower: - return "provider_429_rate_limit" - if "max_seq_len" in lower or "max_prompt_tokens" in lower or "context_length" in lower: - return "provider_400_context_length" - if "input_tokens" in lower and ("exceed" in lower or "larger" in lower): - return "provider_400_context_length" - if "unable to use tool calls" in lower or ("tool calls" in lower and "unable" in lower): - return "tool_call_unsupported" - if "without ai content" in lower: - return "deerflow_no_ai_content" - if "/runs/stream" in lower and "404" in lower: - return "deerflow_stream_404" - if "model not found" in lower or "no chat model could be resolved" in lower: - return "model_not_found" - return "other" - - errors_by_category: dict[str, int] = {} - for row in failed: - cat = classify_error(str(row.get("error", ""))) - errors_by_category[cat] = int(errors_by_category.get(cat, 0)) + 1 - errors_by_category = dict(sorted(errors_by_category.items(), key=lambda kv: (-kv[1], kv[0]))) - - return { - "run_id": run_id, - "variant_id": variant_id, - "workload_id": workload_id, - "n_sessions": int(n_sessions), - "accuracy_applicable": bool(accuracy_applicable), - "success_tasks": int(success_count), - "failed_tasks": int(len(failed)), - "success_rate": float(success_rate), - "parse_failure_tasks": int(parse_fail_count), - "no_answer_tasks": int(no_answer_count), - "throughput_tasks_per_min": float(throughput_tasks_per_min), - "task_latency": end_to_end_summary, - "latency": { - "queue_wait": queue_wait_summary, - "processing": processing_summary, - "proc_llm": proc_llm_summary, - "proc_non_llm": proc_non_llm_summary, - "end_to_end": end_to_end_summary, - }, - "llm_request_latency": summarize_latency(llm_latencies), - "llm_latency": summarize_latency(llm_latencies), - "accuracy": accuracy, - "strict_accuracy": strict_accuracy, - "relaxed_accuracy": relaxed_accuracy, - "partial_accuracy": partial_accuracy, - "benchmark_metric": benchmark_metric, - "benchmark_score": benchmark_score, - "strict_accuracy_on_total": strict_accuracy_on_total, - "relaxed_accuracy_on_total": relaxed_accuracy_on_total, - "partial_accuracy_on_total": partial_accuracy_on_total, - "benchmark_score_on_total": benchmark_score_on_total, - "strict_correct_total": int(strict_correct_total), - "relaxed_correct_total": int(relaxed_correct_total), - "parse_failure_rate": float(parse_failure_rate), - "no_answer_rate": float(no_answer_rate), - "parse_failure_rate_on_total": float(parse_failure_rate_on_total), - "no_answer_rate_on_total": float(no_answer_rate_on_total), - "usage_unavailable": bool(usage_missing), - "prompt_tokens_total": token_in_total, - "completion_tokens_total": token_out_total, - "prompt_tokens_per_success": prompt_per_success, - "total_tokens_per_success": total_per_success, - "l1_hit_rate": float(l1_hit_rate), - "l2_hit_rate": float(l2_hit_rate), - "l3_hit_rate": float(l3_hit_rate), - "memory_ops_per_task": float(memory_ops_per_task), - "db_queries_per_task": float(db_queries_per_task), - "vector_searches_per_task": float(vector_searches_per_task), - "retrieval_skip_rate": float(retrieval_skip_rate), - "retrieval_engaged_rate": float(retrieval_engaged_rate), - "memory_injection_nonzero_rate": float(memory_injection_nonzero_rate), - "strict_accuracy_retrieval_engaged": strict_accuracy_retrieval_engaged, - "embedding_cache_hit_rate": embedding_cache_hit_rate, - "retrieval_cache_hit_rate": retrieval_cache_hit_rate, - "context_injected_tokens_mean": float(sum(injected) / max(1, len(injected))), - "context_injected_items_mean": float(sum(injected_items) / max(1, len(injected_items))), - "evidence_chars_used_mean": float(sum(evidence_chars_values) / max(1, len(evidence_chars_values))), - "prompt_modes": prompt_modes, - "injected_tokens": { - "mean": float(sum(injected) / max(1, len(injected))), - "p95": float(injected_summary.get("p95", 0.0)), - }, - "injected_items": { - "mean": float(sum(injected_items) / max(1, len(injected_items))), - "p95": float(injected_items_summary.get("p95", 0.0)), - }, - "op_p95_ms": op_p95, - "error_breakdown": { - "total_errors": int(len(failed)), - "by_category": errors_by_category, - }, - "generated_at_ms": now_ms(), - } - - -def shuffle_tasks(tasks: list[dict[str, Any]], seed: int) -> list[dict[str, Any]]: - out = list(tasks) - rng = random.Random(seed) - rng.shuffle(out) - return out diff --git a/projects/memlab/memlab/runner_e2e.py b/projects/memlab/memlab/runner_e2e.py deleted file mode 100644 index 21f0571..0000000 --- a/projects/memlab/memlab/runner_e2e.py +++ /dev/null @@ -1,2470 +0,0 @@ -from __future__ import annotations - -import asyncio -import json -import os -import re -import time -from datetime import datetime, timezone -from pathlib import Path -from typing import Any -from urllib.parse import urlparse - -import httpx - -from memlab.runtime_paths import deerflow_root -from memlab.session_logging import current_agent_id, make_operation_logger - -from memlab.envcheck import run_env_checks, write_manifest -from memlab.io import json_dump, now_ms -from memlab.memory.types import ContextBudget, MemoryItem -from memlab.memory_manager.manager import MultiLevelMemoryManager -from memlab.metrics.eventlog import EventLogWriter -from memlab.metrics.manifest import build_manifest -from memlab.runner_core import ( - compute_run_summary, - load_docs, - load_e2e_tasks, - load_runtime_config, - parse_common_args, - parse_model_answer, - score_task_answer, - shuffle_tasks, -) - - -SYSTEM_PROMPT_SECRET = ( - "You are one of many parallel agent sessions. " - "Return STRICT JSON only: {\"task_id\":\"...\",\"secret_code\":\"...\"}." -) -SYSTEM_PROMPT_QA = ( - "You are one of many parallel agent sessions. " - "Return STRICT JSON only: {\"task_id\":\"...\",\"final_answer\":\"...\"}." -) -LOCKED_BASE_URL = "" -LOCKED_MODEL_ID = "" -LOCKED_DEERFLOW_MODEL_NAME = "default-openai-compatible" -DEFAULT_RATE_LIMIT_SAFE_MAX_INFLIGHT = 2 -PRECHECK_CHAT_TIMEOUT_S = 45.0 -PRECHECK_STREAM_TIMEOUT_S = 120.0 -EMBEDDED_WORKER_JSON_PREFIX = "__MEMLAB_JSON__:" - -_EMBEDDED_WORKER_PROC: asyncio.subprocess.Process | None = None -_EMBEDDED_WORKER_LOCK: asyncio.Lock | None = None -_EMBEDDED_WORKER_ERR_HANDLE: Any = None - - -class ProviderBalanceError(RuntimeError): - pass - - -def _safe_error_detail(exc: Exception) -> str: - """Avoid persisting remote bodies, request URLs or exception payloads.""" - if isinstance(exc, httpx.HTTPStatusError): - return f"HTTP {exc.response.status_code}" - return type(exc).__name__ - - -def _utc_now_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _safe_clip(text: str, max_chars: int) -> str: - if max_chars <= 0: - return "" - text = str(text or "") - return text if len(text) <= max_chars else text[:max_chars] - - -def _rate_limit_safe_max_inflight() -> int: - raw = str(os.getenv("MEMLAB_E2E_RATE_LIMIT_SAFE_MAX_INFLIGHT", "") or "").strip() - if not raw: - return DEFAULT_RATE_LIMIT_SAFE_MAX_INFLIGHT - try: - value = int(raw) - except Exception: - return DEFAULT_RATE_LIMIT_SAFE_MAX_INFLIGHT - return max(1, value) - - -def _error_category(msg: str) -> str: - lower = str(msg or "").lower() - if not lower: - return "none" - if "30001" in lower or ("insufficient" in lower and "balance" in lower): - return "provider_403_balance" - if "429" in lower or "ratelimit" in lower or "tpm" in lower: - return "provider_429_rate_limit" - if "max_seq_len" in lower or "max_prompt_tokens" in lower or "context_length" in lower: - return "provider_400_context_length" - if "input_tokens" in lower and ("exceed" in lower or "larger" in lower): - return "provider_400_context_length" - if "unable to use tool calls" in lower or ("tool calls" in lower and "unable" in lower): - return "tool_call_unsupported" - if "without ai content" in lower: - return "deerflow_no_ai_content" - if "/runs/stream" in lower and "404" in lower: - return "deerflow_stream_404" - if "model not found" in lower or "no chat model could be resolved" in lower: - return "model_not_found" - if "403" in lower: - return "provider_403_other" - if "400" in lower: - return "provider_400_other" - return "other" - - -def _is_rate_limit_error(msg: str) -> bool: - return _error_category(msg) == "provider_429_rate_limit" - - -def _is_transient_provider_error(msg: str) -> bool: - lower = str(msg or "").lower() - markers = ( - "readtimeout", - "read timeout", - "connecttimeout", - "connection reset", - "server disconnected without sending a response", - "temporarily unavailable", - "http 500", - "http 502", - "http 503", - "http 504", - "code\":50500", - "unknown error", - ) - return any(marker in lower for marker in markers) - - -def _is_truthy(value: str | None) -> bool: - return str(value or "").strip().lower() in {"1", "true", "yes", "on"} - - -def _build_error_breakdown(task_rows: list[dict[str, Any]]) -> dict[str, Any]: - by_category: dict[str, int] = {} - error_examples: dict[str, str] = {} - total_errors = 0 - for row in task_rows: - if str(row.get("status", "")) != "error": - continue - total_errors += 1 - err = str(row.get("error", "") or "") - cat = _error_category(err) - by_category[cat] = int(by_category.get(cat, 0)) + 1 - if cat not in error_examples and err: - error_examples[cat] = err[:240] - return { - "total_errors": int(total_errors), - "by_category": dict(sorted(by_category.items(), key=lambda kv: (-kv[1], kv[0]))), - "examples": error_examples, - } - - -def _dedupe_lines(lines: list[str]) -> list[str]: - out: list[str] = [] - seen: set[str] = set() - for line in lines: - key = " ".join(str(line).strip().lower().split()) - if not key or key in seen: - continue - seen.add(key) - out.append(str(line)) - return out - - -def _question_focus_text(question: str) -> str: - raw = str(question or "").strip() - if not raw: - return "" - for marker in ("\n\nOptions:", "\nOptions:\n"): - if marker in raw: - raw = raw.split(marker, 1)[0].strip() - break - lines: list[str] = [] - for line in raw.splitlines(): - s = line.strip() - if not s: - continue - if s.lower().startswith("return only "): - continue - lines.append(s) - return " ".join(lines).strip() - - -def _head_tail_clip(text: str, max_chars: int) -> str: - body = str(text or "") - if max_chars <= 0: - return "" - if len(body) <= max_chars: - return body - if max_chars <= 240: - return body[:max_chars] - ellipsis = "\n...\n" - head = max(80, int(max_chars * 0.38)) - tail = max(80, max_chars - head - len(ellipsis)) - return f"{body[:head].strip()}{ellipsis}{body[-tail:].strip()}" - - -def _pack_segments(segments: list[str], *, max_chars: int) -> str: - if max_chars <= 0: - return "" - out: list[str] = [] - used = 0 - for seg in segments: - piece = str(seg or "").strip() - if not piece: - continue - prefix = "\n---\n" if out else "" - remain = max_chars - used - len(prefix) - if remain <= 0: - break - if len(piece) > remain: - if remain < 120: - break - piece = _head_tail_clip(piece, remain) - out.append(prefix + piece) - used += len(prefix) + len(piece) - return "".join(out) - - -def _prepare_context_lines( - packed_messages: list[dict[str, Any]], - *, - top_k: int, - dedupe: bool, - item_max_chars: int, -) -> list[str]: - lines = [f"{m.get('role', 'system')}: {_head_tail_clip(str(m.get('content', '')), int(item_max_chars))}" for m in packed_messages] - if dedupe: - lines = _dedupe_lines(lines) - return lines[: max(1, int(top_k))] - - -def _build_retrieval_shards( - text: str, - *, - shard_chars: int, - max_shards: int, -) -> list[dict[str, Any]]: - body = str(text or "") - if not body: - return [] - shard_chars = max(400, int(shard_chars)) - max_shards = max(1, int(max_shards)) - structured_units = [blk.strip() for blk in re.split(r"\n\s*\n+", body) if blk.strip()] - if len(structured_units) >= 6: - shards: list[dict[str, Any]] = [] - seen_texts: set[str] = set() - cursor = 0 - current_blocks: list[str] = [] - current_start = 0 - current_chars = 0 - for block in structured_units: - block_pos = body.find(block, cursor) - if block_pos < 0: - block_pos = cursor - if not current_blocks: - current_start = block_pos - block_len = len(block) + 2 - if current_blocks and current_chars + block_len > shard_chars: - shard_text = "\n\n".join(current_blocks).strip() - if shard_text: - dedupe_key = " ".join(shard_text[:512].split()) - if dedupe_key not in seen_texts: - seen_texts.add(dedupe_key) - shards.append( - { - "text": shard_text, - "start": int(current_start), - "end": int(current_start + len(shard_text)), - } - ) - current_blocks = [block] - current_start = block_pos - current_chars = block_len - else: - current_blocks.append(block) - current_chars += block_len - cursor = block_pos + len(block) - if current_blocks: - shard_text = "\n\n".join(current_blocks).strip() - if shard_text: - dedupe_key = " ".join(shard_text[:512].split()) - if dedupe_key not in seen_texts: - seen_texts.add(dedupe_key) - shards.append( - { - "text": shard_text, - "start": int(current_start), - "end": int(current_start + len(shard_text)), - } - ) - if shards: - if len(shards) <= max_shards: - return shards - if max_shards == 1: - return [shards[0]] - last_idx = len(shards) - 1 - sampled_idx = { - 0, - last_idx, - *(int(round(last_idx * i / max(1, max_shards - 1))) for i in range(max_shards)), - } - return [shards[i] for i in sorted(sampled_idx)] - if len(body) <= shard_chars: - return [{"text": body, "start": 0, "end": len(body)}] - # Use low-overlap sequential chunks first to avoid shard amplification noise. - starts = list(range(0, len(body), shard_chars)) - if len(starts) > max_shards: - if max_shards == 1: - starts = [0] - else: - last_idx = len(starts) - 1 - sampled_idx = { - 0, - last_idx, - *(int(round(last_idx * i / max(1, max_shards - 1))) for i in range(max_shards)), - } - starts = [starts[i] for i in sorted(sampled_idx)] - - shards: list[dict[str, Any]] = [] - seen_texts: set[str] = set() - for start in starts: - end = min(len(body), start + shard_chars) - shard_text = body[start:end].strip() - if not shard_text: - continue - dedupe_key = " ".join(shard_text[:512].split()) - if dedupe_key in seen_texts: - continue - seen_texts.add(dedupe_key) - shards.append({"text": shard_text, "start": int(start), "end": int(end)}) - return shards - - -def _build_question_anchors( - body: str, - question: str, - *, - max_chunks: int = 4, - chunk_chars: int = 420, -) -> list[str]: - text = str(body or "") - q = str(question or "").lower() - if not text or not q: - return [] - recency_sensitive = bool( - re.search(r"^\s*(where|when)\s+(is|was|are|were)\b", q) - or re.search(r"\b(current|currently|latest|last|now)\b", q) - ) - stop = { - "the", - "and", - "with", - "from", - "that", - "this", - "which", - "what", - "when", - "where", - "who", - "whom", - "why", - "how", - "your", - "into", - "about", - "return", - "only", - "json", - "answer", - } - q_tokens = [t for t in re.findall(r"[a-z0-9_]{3,}", q) if t not in stop] - if not q_tokens: - q_tokens = re.findall(r"[a-z0-9_]{3,}", q) - - def _clip_around_match(seg: str) -> str: - low = seg.lower() - hits: list[int] = [] - for tok in q_tokens: - start = 0 - while True: - idx = low.find(tok, start) - if idx < 0: - break - hits.append(idx) - start = idx + max(1, len(tok)) - if not hits: - return _safe_clip(seg, max(120, int(chunk_chars))) - anchor_pos = max(hits) if recency_sensitive else min(hits) - width = max(120, int(chunk_chars)) - left = max(0, anchor_pos - int(width * 0.35)) - right = min(len(seg), left + width) - left = max(0, right - width) - return seg[left:right].strip() - - structured_blocks = [blk.strip() for blk in re.split(r"\n\s*\n+", text) if blk.strip()] - segments: list[str] = [] - if len(structured_blocks) >= 6: - segments = structured_blocks - else: - for seg in re.split(r"(?<=[.!?])\s+|[\r\n]+", text): - s = seg.strip() - if s: - segments.append(s) - if not segments: - segments = [text] - - scored: list[tuple[float, int, str]] = [] - seg_count = max(1, len(segments)) - for seg_idx, seg in enumerate(segments): - low = seg.lower() - overlap = sum(1 for tok in q_tokens if tok in low) - if overlap <= 0: - continue - token_count = len(re.findall(r"[a-z0-9_]{3,}", low)) - score = float(overlap) + min(1.0, token_count / 18.0) * 0.2 - if len(low) < 64: - score -= 0.4 - if recency_sensitive: - score += 0.35 * ((seg_idx + 1) / seg_count) - scored.append((score, seg_idx, seg)) - scored.sort(key=lambda x: (x[0], x[1]), reverse=True) - - selected: list[tuple[int, str]] = [] - seen: set[str] = set() - for _, seg_idx, seg in scored: - chunk = _clip_around_match(seg) - key = " ".join(chunk.lower().split()) - if key in seen: - continue - seen.add(key) - selected.append((seg_idx, chunk)) - if len(selected) >= max(1, int(max_chunks)): - break - - if selected: - if recency_sensitive: - selected.sort(key=lambda x: x[0], reverse=True) - return [chunk for _, chunk in selected] - - # Fallback: take evenly spaced windows when lexical overlap is weak. - if len(text) <= chunk_chars: - return [text] - starts = list(range(0, len(text), chunk_chars)) - picks: list[int] = [] - if starts: - picks.append(0) - if len(starts) > 1: - picks.append(len(starts) // 2) - if len(starts) > 2: - picks.append(len(starts) - 1) - out: list[str] = [] - for idx in picks: - chunk = text[starts[idx] : min(len(text), starts[idx] + chunk_chars)].strip() - if chunk: - out.append(chunk) - return out[: max(1, int(max_chunks))] - - -def _build_qa_evidence_excerpt( - *, - body: str, - question: str, - max_chars: int, -) -> str: - text = str(body or "") - max_chars = max(240, int(max_chars)) - if len(text) <= max_chars: - return text - - focus = _question_focus_text(question) - anchors = _build_question_anchors( - body=text, - question=focus or question, - max_chunks=min(6, max(2, max_chars // 450)), - chunk_chars=max(220, min(900, max_chars // 3)), - ) - tail_chars = max(200, min(max_chars // 3, 1800)) - head_chars = max(160, min(max_chars // 5, 900)) - segments: list[str] = [] - if focus: - segments.append(f"[QUESTION] {focus}") - for idx, anchor in enumerate(anchors[:6], start=1): - segments.append(f"[EVIDENCE_{idx}] {anchor}") - if anchors: - segments.append(f"[TAIL] {text[-tail_chars:].strip()}") - else: - segments.append(f"[HEAD] {text[:head_chars].strip()}") - segments.append(f"[TAIL] {text[-tail_chars:].strip()}") - packed = _pack_segments(segments, max_chars=max_chars) - return packed if packed else _head_tail_clip(text, max_chars) - - -def _qa_reasoning_hint(*, question: str, qa_prompt_profile: str) -> str: - q = str(question or "").strip().lower() - hints: list[str] = [] - if re.search(r"^\s*(where|when)\s+(is|was|are|were)\b", q) or re.search(r"\b(current|currently|latest|last|now)\b", q): - hints.append("If the evidence contains multiple updates for the same entity or state, choose the latest directly supported state or value.") - if re.search(r"\b(how much more|difference|more expensive|cheaper|compared to|than)\b", q): - hints.append("Compute the requested difference or comparison from the cited values; do not return an intermediate value.") - if re.search(r"^\s*how many\b", q) or re.search(r"\bcount\b", q): - hints.append("Count all directly supported matching items or events before answering.") - if re.search(r"\b(order|first to last|chronological|before|after)\b", q): - hints.append("If multiple events are mentioned, order them chronologically from earliest to latest before answering.") - if re.search(r"\b(recommend|suggest|preference|prefer|would like)\b", q): - hints.append("Base the answer on the user's documented preferences and prior experiences, not on generic suggestions.") - profile = str(qa_prompt_profile or "").strip().lower() - if profile == "entity_span": - hints.append("Return only the bare entity, location, number, or short value without articles or explanation.") - elif profile == "multiple_choice_letter": - hints.append("Reason over the evidence first, then return only one supported option letter.") - elif profile == "summary": - hints.append("Return a concise preference-aware or synthesis answer that preserves the key constraints from the evidence.") - return " ".join(hints).strip() - - -def _resolve_task_qa_prompt_profile(*, task: dict[str, Any], default_profile: str) -> str: - metadata = task.get("metadata") - if isinstance(metadata, dict): - for key in ("qa_prompt_profile", "qa_prompt_profile_override"): - override = str(metadata.get(key, "")).strip().lower() - if override in {"default", "yes_no", "entity_span", "short_answer", "summary", "multiple_choice_letter"}: - return override - profile = str(default_profile or "default").strip().lower() - if profile in {"default", "yes_no", "entity_span", "short_answer", "summary", "multiple_choice_letter"}: - return profile - return "default" - - -def _summarize_prompt( - *, - session_id: str, - question: str, - context_block: str, - body: str, - task_type: str, - answer_field: str, - answer_format_mode: str, - prompt_mode: str, - qa_prompt_profile: str, - qa_support_mode: str, - max_prompt_tokens: int, - max_evidence_chars: int, -) -> tuple[str, int]: - char_budget = max(2500, int(max_prompt_tokens) * 4) - if prompt_mode == "memory_neutral": - max_context_chars = min(1200, max(400, int(char_budget * 0.08))) - evidence_budget = max(0, min(int(max_evidence_chars), int(char_budget * 0.84))) - else: - max_context_chars = max(1000, int(char_budget * 0.45)) - evidence_budget = max(0, min(int(max_evidence_chars), int(char_budget * 0.40))) - context_clip = _safe_clip(context_block, max_context_chars) - if task_type == "secret_code": - evidence_clip = _head_tail_clip(body, evidence_budget) - else: - evidence_clip = _build_qa_evidence_excerpt(body=body, question=question, max_chars=evidence_budget) - if prompt_mode == "memory_beneficial": - if task_type == "secret_code": - evidence_block = ( - "[omitted by prompt_mode=memory_beneficial]\n" - "Use memory retrieval context above to produce the final answer." - ) - evidence_chars_used = 0 - elif str(qa_support_mode or "standard").strip().lower() == "none": - evidence_block = ( - "[direct evidence omitted by qa_support_mode=none]\n" - "Use the memory retrieval context above as the only evidence source. " - "If that context is insufficient, answer conservatively from retrieved memory only." - ) - evidence_chars_used = 0 - else: - support_budget = max(480, min(len(evidence_clip), max(1400, evidence_budget // 2))) - evidence_block = ( - "[supportive direct evidence excerpt]\n" - f"{_safe_clip(evidence_clip, support_budget)}\n" - "Use the memory retrieval context above as the primary recall path, and use this excerpt only as supporting evidence." - ) - evidence_chars_used = min(len(evidence_clip), support_budget) - else: - evidence_block = evidence_clip - evidence_chars_used = len(evidence_clip) - if task_type == "secret_code": - system_prompt = SYSTEM_PROMPT_SECRET - task_instruction = "Return the secret code for this task." - expected_json = '{"task_id":"...","secret_code":"..."}' - else: - system_prompt = SYSTEM_PROMPT_QA - task_instruction = _qa_task_instruction( - qa_prompt_profile=qa_prompt_profile, - answer_field=answer_field, - ) - expected_json = '{"task_id":"...","final_answer":"..."}' - reasoning_hint = "" if task_type == "secret_code" else _qa_reasoning_hint( - question=question, - qa_prompt_profile=qa_prompt_profile, - ) - if answer_format_mode == "final_answer_prefix": - answer_format_note = 'If JSON fails, output one line: "Final Answer: "' - else: - answer_format_note = f"Do not output prose outside JSON. Required schema: {expected_json}" - prompt = ( - f"{system_prompt}\n\n" - f"Session: {session_id}\n" - f"Prompt mode: {prompt_mode}\n" - f"Task type: {task_type}\n" - f"QA prompt profile: {qa_prompt_profile}\n" - f"Question: {question}\n\n" - f"Context:\n{context_clip}\n\n" - f"Evidence:\n{evidence_block}\n\n" - f"Instruction: {task_instruction}\n" - f"{('Reasoning hint: ' + reasoning_hint + chr(10)) if reasoning_hint else ''}" - f"Answer format: {answer_format_note}" - ) - if len(prompt) > char_budget: - prompt = _safe_clip(prompt, char_budget) - return prompt, evidence_chars_used - - -def _extract_text(content: Any) -> str: - if isinstance(content, str): - return content - if isinstance(content, list): - parts: list[str] = [] - for part in content: - if isinstance(part, str): - parts.append(part) - elif isinstance(part, dict) and part.get("type") == "text": - parts.append(str(part.get("text", ""))) - return "\n".join(parts) - if isinstance(content, dict): - return str(content.get("text", "")) - return str(content) - - -def _is_ai_message(msg: dict[str, Any]) -> bool: - mtype = str(msg.get("type", "")).lower() - role = str(msg.get("role", "")).lower() - return mtype == "ai" or role in {"assistant", "ai"} - - -def _extract_usage_from_usage_obj(obj: Any) -> tuple[int, int]: - if not isinstance(obj, dict): - return 0, 0 - prompt_tokens = 0 - completion_tokens = 0 - for key, value in obj.items(): - if not isinstance(value, (int, float)): - continue - lk = str(key).lower() - if lk in {"prompt_tokens", "input_tokens"}: - prompt_tokens = max(prompt_tokens, int(value)) - elif lk in {"completion_tokens", "output_tokens"}: - completion_tokens = max(completion_tokens, int(value)) - return prompt_tokens, completion_tokens - - -def _extract_usage_from_ai_message(msg: dict[str, Any]) -> tuple[int, int]: - prompt_tokens = 0 - completion_tokens = 0 - candidates: list[Any] = [msg] - usage_metadata = msg.get("usage_metadata") - if usage_metadata is not None: - candidates.append(usage_metadata) - response_metadata = msg.get("response_metadata") - if isinstance(response_metadata, dict): - candidates.append(response_metadata) - token_usage = response_metadata.get("token_usage") - if token_usage is not None: - candidates.append(token_usage) - nested_usage = response_metadata.get("usage") - if nested_usage is not None: - candidates.append(nested_usage) - usage = msg.get("usage") - if usage is not None: - candidates.append(usage) - for obj in candidates: - p, c = _extract_usage_from_usage_obj(obj) - prompt_tokens = max(prompt_tokens, p) - completion_tokens = max(completion_tokens, c) - return prompt_tokens, completion_tokens - - -def _extract_ai_message_from_values(data: dict[str, Any]) -> str: - msgs = data.get("messages", []) - if not isinstance(msgs, list): - return "" - for msg in reversed(msgs): - if isinstance(msg, dict) and _is_ai_message(msg): - return _extract_text(msg.get("content", "")) - return "" - - -def _extract_usage_from_values(data: dict[str, Any]) -> tuple[int, int]: - msgs = data.get("messages", []) - if not isinstance(msgs, list): - return 0, 0 - prompt_tokens = 0 - completion_tokens = 0 - for msg in reversed(msgs): - if isinstance(msg, dict) and _is_ai_message(msg): - p, c = _extract_usage_from_ai_message(msg) - prompt_tokens = max(prompt_tokens, p) - completion_tokens = max(completion_tokens, c) - return prompt_tokens, completion_tokens - - -def _is_local_loopback_url(url: str) -> bool: - host = str(urlparse(str(url or "")).hostname or "").strip().lower() - return host in {"127.0.0.1", "localhost"} - - -def _should_use_embedded_deerflow(*, langgraph_url: str, experiment_track: str) -> bool: - return str(experiment_track or "").strip() in { - "candidate_race", - "emergency_rebaseline", - "memory_exposure", - "focused_validation_v3", - "alignment_diagnosis", - } and _is_local_loopback_url(langgraph_url) - - -def _qa_task_instruction(*, qa_prompt_profile: str, answer_field: str) -> str: - profile = str(qa_prompt_profile or "default").strip().lower() - if profile == "multiple_choice_letter": - return ( - "Return only the single correct option letter (for example `A`, `B`, `C`, or `D`) " - f"in `{answer_field}`." - ) - if profile == "yes_no": - return ( - "Answer with only `yes` or `no` when the question is binary. " - f"If the question is not binary, return the shortest exact supported answer in `{answer_field}`." - ) - if profile == "entity_span": - return ( - "Extract only the minimal entity name, short span, or value that directly answers the question, " - f"and put only that text in `{answer_field}`." - ) - if profile == "short_answer": - return ( - "Return the shortest exact answer span or value supported by the evidence. " - f"Do not explain or restate the question; put only the answer in `{answer_field}`." - ) - if profile == "summary": - return ( - "Return a concise synthesis answer that preserves the user's relevant preferences, constraints, or chronology. " - f"When the question is simple, still keep the answer compact and put only the answer in `{answer_field}`." - ) - return ( - "Answer the question with the shortest exact answer span or value supported by the evidence, " - f"and put only that value in `{answer_field}`." - ) - -def _make_embedded_thread_id(seed: str) -> str: - raw = "".join(ch if ch.isalnum() or ch in {"-", "_"} else "-" for ch in str(seed or "")) - collapsed = "-".join(part for part in raw.split("-") if part) - if not collapsed: - collapsed = f"memlab-{now_ms()}" - return collapsed[:96] - -def _embedded_worker_lock() -> asyncio.Lock: - global _EMBEDDED_WORKER_LOCK - if _EMBEDDED_WORKER_LOCK is None: - _EMBEDDED_WORKER_LOCK = asyncio.Lock() - return _EMBEDDED_WORKER_LOCK - - -def _embedded_worker_paths() -> tuple[Path, Path, Path, Path]: - repo_root = Path(__file__).resolve().parents[1] - backend_root = deerflow_root() / "backend" - python_exe = backend_root / ".venv" / "Scripts" / "python.exe" - worker_script = repo_root / "scripts" / "deerflow_embedded_worker.py" - config_path = deerflow_root() / "config.yaml" - deerflow_home = backend_root / ".deer-flow" - return python_exe, worker_script, config_path, deerflow_home - - -async def _shutdown_embedded_worker(force: bool = False) -> None: - global _EMBEDDED_WORKER_PROC, _EMBEDDED_WORKER_ERR_HANDLE - proc = _EMBEDDED_WORKER_PROC - _EMBEDDED_WORKER_PROC = None - if proc is not None: - try: - if not force and proc.returncode is None and proc.stdin is not None and proc.stdout is not None: - try: - proc.stdin.write(b'{"type":"shutdown"}\n') - await proc.stdin.drain() - _ = await asyncio.wait_for(proc.stdout.readline(), timeout=5.0) - except Exception: - pass - if proc.returncode is None: - proc.terminate() - try: - await asyncio.wait_for(proc.wait(), timeout=5.0) - except Exception: - proc.kill() - await proc.wait() - except (ProcessLookupError, AttributeError): - pass - if _EMBEDDED_WORKER_ERR_HANDLE is not None: - try: - _EMBEDDED_WORKER_ERR_HANDLE.close() - except Exception: - pass - _EMBEDDED_WORKER_ERR_HANDLE = None - - -def _shutdown_embedded_worker_sync() -> None: - global _EMBEDDED_WORKER_PROC, _EMBEDDED_WORKER_ERR_HANDLE - proc = _EMBEDDED_WORKER_PROC - _EMBEDDED_WORKER_PROC = None - if proc is not None and proc.returncode is None: - try: - proc.terminate() - except Exception: - try: - proc.kill() - except Exception: - pass - if _EMBEDDED_WORKER_ERR_HANDLE is not None: - try: - _EMBEDDED_WORKER_ERR_HANDLE.close() - except Exception: - pass - _EMBEDDED_WORKER_ERR_HANDLE = None - - -async def _ensure_embedded_worker_started() -> asyncio.subprocess.Process: - global _EMBEDDED_WORKER_PROC, _EMBEDDED_WORKER_ERR_HANDLE - proc = _EMBEDDED_WORKER_PROC - if proc is not None and proc.returncode is None and proc.stdin is not None and proc.stdout is not None: - return proc - - python_exe, worker_script, config_path, deerflow_home = _embedded_worker_paths() - if not python_exe.exists(): - raise FileNotFoundError(f"Embedded DeerFlow Python not found at {python_exe}") - if not worker_script.exists(): - raise FileNotFoundError(f"Embedded DeerFlow worker script not found at {worker_script}") - env = os.environ.copy() - env["DEER_FLOW_CONFIG_PATH"] = str(config_path) - env["DEER_FLOW_HOME"] = str(deerflow_home) - env["PYTHONUTF8"] = "1" - env["PYTHONIOENCODING"] = "utf-8" - log_dir = Path("runs/runtime") - log_dir.mkdir(parents=True, exist_ok=True) - if _EMBEDDED_WORKER_ERR_HANDLE is not None: - try: - _EMBEDDED_WORKER_ERR_HANDLE.close() - except Exception: - pass - _EMBEDDED_WORKER_ERR_HANDLE = open(log_dir / "embedded_worker.err.log", "a", encoding="utf-8") - proc = await asyncio.create_subprocess_exec( - str(python_exe), - "-u", - str(worker_script), - cwd=str(Path(__file__).resolve().parents[1]), - env=env, - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - stderr=_EMBEDDED_WORKER_ERR_HANDLE, - ) - if proc.stdout is None: - await _shutdown_embedded_worker(force=True) - raise RuntimeError("Embedded DeerFlow worker stdout pipe is unavailable.") - try: - deadline = time.perf_counter() + 20.0 - ready_payload: dict[str, Any] | None = None - while time.perf_counter() < deadline: - remaining = max(0.5, deadline - time.perf_counter()) - ready_line = await asyncio.wait_for(proc.stdout.readline(), timeout=remaining) - if not ready_line: - break - ready_text = ready_line.decode("utf-8", errors="ignore").strip() - if not ready_text.startswith(EMBEDDED_WORKER_JSON_PREFIX): - continue - ready_payload = json.loads(ready_text[len(EMBEDDED_WORKER_JSON_PREFIX) :]) - break - except Exception: - await _shutdown_embedded_worker(force=True) - raise - if ready_payload is None: - await _shutdown_embedded_worker(force=True) - raise RuntimeError("Embedded DeerFlow worker exited before signaling readiness.") - if not bool(ready_payload.get("ready", False)): - await _shutdown_embedded_worker(force=True) - raise RuntimeError(f"Embedded DeerFlow worker failed to start: {ready_payload}") - _EMBEDDED_WORKER_PROC = proc - return proc - - -async def _run_deerflow_prompt_embedded_worker( - *, - thread_id: str, - prompt: str, - model_override: str | None, - agent_name: str | None, - request_timeout_s: float, -) -> dict[str, Any]: - lock = _embedded_worker_lock() - for attempt in range(1, 3): - proc = await _ensure_embedded_worker_started() - if proc.stdin is None or proc.stdout is None: - await _shutdown_embedded_worker(force=True) - raise RuntimeError("Embedded DeerFlow worker pipes are unavailable.") - async with lock: - try: - payload = { - "type": "run", - "thread_id": thread_id, - "prompt": prompt, - "model_override": model_override or "", - "agent_name": agent_name or "", - } - proc.stdin.write((json.dumps(payload, ensure_ascii=False) + "\n").encode("utf-8")) - await proc.stdin.drain() - deadline = time.perf_counter() + float(max(5.0, request_timeout_s + 5.0)) - data: dict[str, Any] | None = None - while time.perf_counter() < deadline: - remaining = max(0.5, deadline - time.perf_counter()) - raw_line = await asyncio.wait_for(proc.stdout.readline(), timeout=remaining) - if not raw_line: - raise RuntimeError("Embedded DeerFlow worker closed stdout unexpectedly.") - text = raw_line.decode("utf-8", errors="ignore").strip() - if not text.startswith(EMBEDDED_WORKER_JSON_PREFIX): - continue - data = json.loads(text[len(EMBEDDED_WORKER_JSON_PREFIX) :]) - break - if data is None: - raise RuntimeError("Embedded DeerFlow worker produced no structured response before timeout.") - if not bool(data.get("ok", False)): - raise RuntimeError(str(data.get("error", "embedded worker request failed"))) - return { - "content": str(data.get("content", "") or "{}"), - "prompt_tokens": None, - "completion_tokens": None, - "usage_available": False, - "request_latency_ms": float(data.get("request_latency_ms", 0.0)), - "llm_queue_wait_ms": 0.0, - "model_name": model_override or "", - } - except Exception: - await _shutdown_embedded_worker(force=True) - if attempt >= 2: - raise - raise RuntimeError("Embedded DeerFlow worker exhausted retries.") - - -async def _create_thread(client: httpx.AsyncClient, langgraph_url: str, *, session_id: str, agent_id: str) -> str: - payload = {"metadata": {"session_id": session_id, "agent_id": agent_id}} - r = await client.post(f"{langgraph_url.rstrip('/')}/threads", json=payload) - r.raise_for_status() - body = r.json() - tid = body.get("thread_id") - if not isinstance(tid, str) or not tid: - raise RuntimeError("DeerFlow thread creation returned invalid thread_id") - return tid - - -async def _create_deerflow_thread( - *, - client: httpx.AsyncClient | None, - langgraph_url: str, - session_id: str, - agent_id: str, - use_embedded_local: bool, -) -> str: - if use_embedded_local: - return _make_embedded_thread_id(f"{session_id}-{agent_id}") - if client is None: - raise RuntimeError("HTTP client is required for LangGraph DeerFlow thread creation.") - return await _create_thread(client, langgraph_url, session_id=session_id, agent_id=agent_id) - - -async def _run_deerflow_prompt( - *, - client: httpx.AsyncClient | None, - langgraph_url: str, - thread_id: str, - prompt: str, - llm_sema: asyncio.Semaphore, - model_override: str | None = None, - assistant_id: str = "lead_agent", - agent_name: str | None = None, - use_embedded_local: bool = False, - request_timeout_s: float = 180.0, -) -> dict[str, Any]: - req_t0 = 0.0 - sem_wait_t0 = time.perf_counter() - await llm_sema.acquire() - llm_queue_wait_ms = (time.perf_counter() - sem_wait_t0) * 1000.0 - try: - req_t0 = time.perf_counter() - if use_embedded_local: - result = await _run_deerflow_prompt_embedded_worker( - thread_id=thread_id, - prompt=prompt, - model_override=model_override, - agent_name=agent_name, - request_timeout_s=float(max(1.0, request_timeout_s)), - ) - result["llm_queue_wait_ms"] = llm_queue_wait_ms - return result - - if client is None: - raise RuntimeError("HTTP client is required for LangGraph DeerFlow prompt execution.") - - configurable: dict[str, Any] = { - "thread_id": thread_id, - # DeepSeek-V3.2 with DeerFlow may stream long empty reasoning chunks when - # thinking mode is left on. Force deterministic benchmark mode. - "thinking_enabled": False, - "is_plan_mode": False, - "subagent_enabled": False, - "max_concurrent_subagents": 1, - } - if model_override: - configurable["model_name"] = model_override - configurable["model"] = model_override - if agent_name: - configurable["agent_name"] = agent_name - payload = { - "assistant_id": assistant_id, - "input": { - "messages": [ - { - "role": "user", - "content": prompt, - } - ] - }, - "config": { - "configurable": configurable - }, - "stream_mode": ["values", "messages-tuple"], - } - ai_chunks: list[str] = [] - final_text = "" - prompt_tokens = 0 - completion_tokens = 0 - try: - async with client.stream( - "POST", - f"{langgraph_url.rstrip('/')}/threads/{thread_id}/runs/stream", - json=payload, - timeout=float(max(1.0, request_timeout_s)), - ) as resp: - resp.raise_for_status() - event_name = "" - async for raw_line in resp.aiter_lines(): - line = raw_line.strip() - if not line: - continue - if line.startswith("event:"): - event_name = line[6:].strip() - continue - if not line.startswith("data:"): - continue - data_str = line[5:].strip() - if not data_str or data_str == "[DONE]": - continue - try: - data = json.loads(data_str) - except Exception: - continue - if event_name == "values" and isinstance(data, dict): - p, c = _extract_usage_from_values(data) - prompt_tokens = max(prompt_tokens, p) - completion_tokens = max(completion_tokens, c) - maybe = _extract_ai_message_from_values(data) - if maybe: - final_text = maybe - elif event_name in {"messages", "messages-tuple"}: - node = data - if isinstance(node, list): - if node and isinstance(node[0], dict): - node = node[0] - else: - continue - if isinstance(node, dict): - if "message" in node and isinstance(node["message"], dict): - node = node["message"] - if _is_ai_message(node): - p, c = _extract_usage_from_ai_message(node) - prompt_tokens = max(prompt_tokens, p) - completion_tokens = max(completion_tokens, c) - txt = _extract_text(node.get("content", "")) - if txt: - ai_chunks.append(txt) - elif event_name == "error": - msg = json.dumps(data, ensure_ascii=False) - lower = msg.lower() - if ("insufficient" in lower and "balance" in lower) or "30001" in lower: - raise ProviderBalanceError("provider_balance_rejected") - raise RuntimeError("deerflow_stream_rejected") - elif event_name == "end": - break - except httpx.HTTPStatusError as exc: - msg = str(exc) - lower = msg.lower() - if ("insufficient" in lower and "balance" in lower) or "30001" in lower or "403" in lower: - raise ProviderBalanceError("provider_balance_rejected") from None - raise RuntimeError(f"HTTP {exc.response.status_code}") from None - finally: - llm_sema.release() - req_latency_ms = (time.perf_counter() - req_t0) * 1000.0 - - if not final_text and ai_chunks: - final_text = ai_chunks[-1] - if not final_text: - final_text = "\n".join(ai_chunks).strip() - if not final_text: - final_text = "{}" - - prompt_tokens_out = int(prompt_tokens) if prompt_tokens > 0 else None - completion_tokens_out = int(completion_tokens) if completion_tokens > 0 else None - return { - "content": final_text, - "prompt_tokens": prompt_tokens_out, - "completion_tokens": completion_tokens_out, - "usage_available": (prompt_tokens_out is not None and completion_tokens_out is not None), - "request_latency_ms": req_latency_ms, - "llm_queue_wait_ms": llm_queue_wait_ms, - "model_name": model_override or "", - } - - -async def _run_provider_prompt_direct( - *, - client: httpx.AsyncClient, - base_url: str, - model_id: str, - api_key: str, - prompt: str, - llm_sema: asyncio.Semaphore, - request_timeout_s: float = 180.0, -) -> dict[str, Any]: - req_t0 = 0.0 - sem_wait_t0 = time.perf_counter() - await llm_sema.acquire() - llm_queue_wait_ms = (time.perf_counter() - sem_wait_t0) * 1000.0 - try: - req_t0 = time.perf_counter() - headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} - payload = { - "model": model_id, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": 96, - "temperature": 0.0, - "stream": False, - } - resp = await client.post( - f"{base_url.rstrip('/')}/chat/completions", - headers=headers, - json=payload, - timeout=float(max(1.0, request_timeout_s)), - ) - if resp.status_code != 200: - text = resp.text - lower = text.lower() - if ("insufficient" in lower and "balance" in lower) or "30001" in lower or resp.status_code == 403: - raise ProviderBalanceError(f"HTTP {resp.status_code}: provider_balance_rejected") - raise RuntimeError(f"HTTP {resp.status_code}: provider_request_rejected") - data = resp.json() - choices = data.get("choices", []) - if not isinstance(choices, list) or not choices: - raise RuntimeError("Provider chat response contains no choices.") - message = choices[0].get("message", {}) if isinstance(choices[0], dict) else {} - final_text = _extract_text(message.get("content", "")) if isinstance(message, dict) else "" - if not final_text: - final_text = "{}" - usage = data.get("usage", {}) - prompt_tokens = int(usage.get("prompt_tokens", 0) or 0) - completion_tokens = int(usage.get("completion_tokens", 0) or 0) - finally: - llm_sema.release() - req_latency_ms = (time.perf_counter() - req_t0) * 1000.0 - return { - "content": final_text, - "prompt_tokens": (prompt_tokens if prompt_tokens > 0 else None), - "completion_tokens": (completion_tokens if completion_tokens > 0 else None), - "usage_available": (prompt_tokens > 0 or completion_tokens > 0), - "request_latency_ms": req_latency_ms, - "llm_queue_wait_ms": llm_queue_wait_ms, - "model_name": model_id, - } - - -async def _provider_chat_preflight( - *, - base_url: str, - model_id: str, - api_key: str, -) -> tuple[bool, str]: - url = f"{base_url.rstrip('/')}/chat/completions" - headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} - payload = { - "model": model_id, - "messages": [{"role": "user", "content": "preflight ping"}], - "max_tokens": 16, - "stream": False, - } - try: - async with httpx.AsyncClient(timeout=PRECHECK_CHAT_TIMEOUT_S, trust_env=False) as client: - resp = await client.post(url, json=payload, headers=headers) - if resp.status_code != 200: - return False, f"HTTP {resp.status_code}" - data = resp.json() - choices = data.get("choices") - if not isinstance(choices, list) or not choices: - return False, "Response contains no choices." - return True, "ok" - except Exception as exc: - detail = _safe_error_detail(exc) - return False, detail - - -async def _provider_tool_call_preflight( - *, - base_url: str, - model_id: str, - api_key: str, -) -> tuple[bool, str]: - url = f"{base_url.rstrip('/')}/chat/completions" - headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} - payload = { - "model": model_id, - "messages": [{"role": "user", "content": "Call the ping tool now."}], - "tools": [ - { - "type": "function", - "function": { - "name": "ping", - "description": "Returns pong.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - } - ], - "tool_choice": {"type": "function", "function": {"name": "ping"}}, - "max_tokens": 32, - "stream": False, - } - try: - async with httpx.AsyncClient(timeout=PRECHECK_CHAT_TIMEOUT_S, trust_env=False) as client: - resp = await client.post(url, json=payload, headers=headers) - if resp.status_code != 200: - return False, f"HTTP {resp.status_code}" - data = resp.json() - choices = data.get("choices") - if not isinstance(choices, list) or not choices: - return False, "Response contains no choices." - first = choices[0] if isinstance(choices[0], dict) else {} - msg = first.get("message", {}) if isinstance(first, dict) else {} - if not isinstance(msg, dict): - return False, "Invalid message format." - tool_calls = msg.get("tool_calls") - function_call = msg.get("function_call") - if isinstance(tool_calls, list) and tool_calls: - return True, "ok" - if isinstance(function_call, dict) and function_call.get("name"): - return True, "ok" - return False, "No tool_calls/function_call found in response." - except Exception as exc: - detail = _safe_error_detail(exc) - return False, detail - - -async def _deerflow_stream_preflight( - *, - langgraph_url: str, - model_override: str, - assistant_id: str, - agent_name: str, - use_embedded_local: bool, -) -> tuple[bool, str]: - errors: list[str] = [] - for attempt in range(1, 4): - try: - if use_embedded_local: - client = None - thread_id = await _create_deerflow_thread( - client=None, - langgraph_url=langgraph_url, - session_id=f"preflight-{now_ms()}-{attempt}", - agent_id="agent_preflight", - use_embedded_local=True, - ) - sema = asyncio.Semaphore(1) - _ = await _run_deerflow_prompt( - client=client, - langgraph_url=langgraph_url, - thread_id=thread_id, - prompt='Return strict JSON only: {"ok": true}.', - llm_sema=sema, - model_override=model_override, - assistant_id=assistant_id, - agent_name=agent_name, - use_embedded_local=True, - request_timeout_s=PRECHECK_STREAM_TIMEOUT_S, - ) - else: - async with httpx.AsyncClient(timeout=PRECHECK_STREAM_TIMEOUT_S, trust_env=False) as client: - thread_id = await _create_deerflow_thread( - client=client, - langgraph_url=langgraph_url, - session_id=f"preflight-{now_ms()}-{attempt}", - agent_id="agent_preflight", - use_embedded_local=False, - ) - sema = asyncio.Semaphore(1) - _ = await _run_deerflow_prompt( - client=client, - langgraph_url=langgraph_url, - thread_id=thread_id, - prompt='Return strict JSON only: {"ok": true}.', - llm_sema=sema, - model_override=model_override, - assistant_id=assistant_id, - agent_name=agent_name, - use_embedded_local=False, - request_timeout_s=PRECHECK_STREAM_TIMEOUT_S, - ) - return True, f"ok(attempt={attempt})" - except Exception as exc: - detail = _safe_error_detail(exc) - errors.append(f"attempt={attempt}: {detail}") - await asyncio.sleep(0.6 * attempt) - return False, " | ".join(errors) - - -async def _run_preflight_checks( - *, - model_base_url: str, - model_id: str, - api_key: str, - langgraph_url: str, - locked_base_url: str, - locked_model_id: str, - locked_model_name: str, - llm_max_inflight: int, - prompt_mode: str, - max_evidence_chars: int, - comparable_mode: bool, - assistant_id: str, - agent_name: str, - use_embedded_local: bool, - use_provider_direct: bool, -) -> dict[str, Any]: - report: dict[str, Any] = { - "started_at_utc": _utc_now_iso(), - "checks": {}, - "warnings": [], - "deerflow_transport": ( - "provider_direct" if use_provider_direct else ("embedded_local" if use_embedded_local else "langgraph_http") - ), - } - checks: dict[str, Any] = {} - report["checks"] = checks - if use_embedded_local: - report["warnings"].append("Using embedded local DeerFlow transport for this track.") - if use_provider_direct: - report["warnings"].append("Using provider_direct transport; DeerFlow stream preflight is skipped.") - - def _is_hard_provider_failure(detail: str) -> bool: - txt = str(detail or "").lower() - hard_markers = ( - "401", - "unauthorized", - "invalid api key", - "model not found", - "no chat model could be resolved", - "unable to use tool calls", - ) - return any(marker in txt for marker in hard_markers) - - def _is_timeout_like(detail: str) -> bool: - txt = str(detail or "").lower() - markers = ( - "timed out", - "timeout", - "readtimeout", - "read timeout", - "task was canceled", - "task was cancelled", - "a task was canceled", - "a task was cancelled", - "cancelled", - "canceled", - ) - return any(marker in txt for marker in markers) - - checks["env_vars"] = { - "ok": bool(model_base_url and model_id and api_key), - "base_url_present": bool(model_base_url), - "model_present": bool(model_id), - "api_key_present": bool(api_key), - } - checks["model_lock"] = { - "ok": (not comparable_mode) or (model_base_url == locked_base_url and model_id == locked_model_id), - "expected_base_url": locked_base_url, - "expected_model_id": locked_model_id, - "actual_base_url": model_base_url, - "actual_model_id": model_id, - "comparable_mode": bool(comparable_mode), - } - safe_max_inflight = _rate_limit_safe_max_inflight() - checks["rate_limit_params"] = { - "ok": int(llm_max_inflight) <= safe_max_inflight, - "llm_max_inflight": int(llm_max_inflight), - "recommended_max_inflight": safe_max_inflight, - } - checks["prompt_budget"] = { - "ok": int(max_evidence_chars) >= 256, - "prompt_mode": str(prompt_mode), - "max_evidence_chars": int(max_evidence_chars), - } - - if checks["env_vars"]["ok"]: - (ok_chat, msg_chat), (ok_tool, msg_tool) = await asyncio.gather( - _provider_chat_preflight( - base_url=model_base_url, - model_id=model_id, - api_key=api_key, - ), - _provider_tool_call_preflight( - base_url=model_base_url, - model_id=model_id, - api_key=api_key, - ), - ) - checks["provider_chat"] = {"ok": bool(ok_chat), "detail": msg_chat} - checks["provider_tool_call"] = {"ok": bool(ok_tool), "detail": msg_tool} - else: - checks["provider_chat"] = {"ok": False, "detail": "Skipped due to missing env vars."} - checks["provider_tool_call"] = {"ok": False, "detail": "Skipped due to missing env vars."} - - chat_detail = str(checks.get("provider_chat", {}).get("detail", "")) - tool_detail = str(checks.get("provider_tool_call", {}).get("detail", "")) - provider_checks_ok = bool(checks.get("provider_chat", {}).get("ok", False)) or bool( - checks.get("provider_tool_call", {}).get("ok", False) - ) - skip_deerflow_stream = False - skip_reason = "" - if checks["env_vars"]["ok"] and not provider_checks_ok: - if _is_hard_provider_failure(chat_detail) or _is_hard_provider_failure(tool_detail): - skip_deerflow_stream = True - skip_reason = "Skipped because provider preflight hard-failed for the locked model." - elif _is_timeout_like(chat_detail) and _is_timeout_like(tool_detail): - skip_deerflow_stream = True - skip_reason = "Skipped because provider preflight timed out for the locked model." - - if use_provider_direct: - checks["deerflow_stream"] = { - "ok": True, - "detail": "Skipped because provider_direct transport is enabled.", - "skipped": True, - } - elif skip_deerflow_stream: - checks["deerflow_stream"] = {"ok": False, "detail": skip_reason, "skipped": True} - else: - ok_df, msg_df = await _deerflow_stream_preflight( - langgraph_url=langgraph_url, - model_override=locked_model_name, - assistant_id=assistant_id, - agent_name=agent_name, - use_embedded_local=use_embedded_local, - ) - checks["deerflow_stream"] = {"ok": bool(ok_df), "detail": msg_df} - - if checks["deerflow_stream"]["ok"]: - for check_name in ("provider_chat", "provider_tool_call"): - check = checks.get(check_name, {}) - if bool(check.get("ok", False)): - continue - detail = str(check.get("detail", "")) - if _is_hard_provider_failure(detail): - continue - report["warnings"].append(f"{check_name} degraded to warning: {detail}") - check["ok"] = True - check["degraded"] = True - checks[check_name] = check - - failed: list[str] = [] - for name, detail in checks.items(): - if not bool(detail.get("ok", False)): - failed.append(f"{name}: {detail.get('detail', 'failed')}") - report["failed_checks"] = failed - report["passed"] = len(failed) == 0 - report["finished_at_utc"] = _utc_now_iso() - return report - - -async def _session_worker(**kwargs) -> None: - token = current_agent_id.set(f"agent_{kwargs['session_index'] + 1:04d}") - try: - await _session_worker_body(**kwargs) - finally: - current_agent_id.reset(token) - - -async def _session_worker_body( - *, - queue: asyncio.Queue[dict[str, Any]], - session_index: int, - run_id: str, - docs: dict[str, dict[str, Any]], - memory: MultiLevelMemoryManager, - event_writer: EventLogWriter, - task_rows: list[dict[str, Any]], - mem_rows: list[dict[str, Any]], - langgraph_url: str, - client: httpx.AsyncClient, - llm_sema: asyncio.Semaphore, - cfg, - debug_print: bool, - debug_max_output_chars: int, - stop_event: asyncio.Event, - stop_reasons: list[str], - allow_fallback_retry: bool, - fallback_model: str, - fallback_reasons: list[str], - locked_model_name: str, - prompt_mode: str, - max_evidence_chars: int, - answer_retry_count: int, - store_raw_output_chars: int, - answer_format_mode: str, - structured_evidence: bool, - context_top_k: int, - context_dedupe: bool, - context_item_max_chars: int, - memory_stress_turns: int, - session_scope: str, - assistant_id: str, - agent_name: str, - use_embedded_local: bool, - use_provider_direct: bool, - provider_base_url: str, - provider_model_id: str, - provider_api_key: str, - qa_prompt_profile: str, - qa_support_mode: str, -) -> None: - base_agent_id = f"agent_{session_index + 1:04d}" - base_session_id = f"{run_id}-{base_agent_id}" - agent_id = base_agent_id - session_id = base_session_id - # Hard guardrail to prevent a single task from hanging the entire E2E run. - task_timeout_s = float(max(60.0, float(os.getenv("MEMLAB_E2E_TASK_TIMEOUT_S", "420")))) - thread_timeout_s = float(max(30.0, float(os.getenv("MEMLAB_E2E_THREAD_TIMEOUT_S", "90")))) - stress_turns = max(0, int(memory_stress_turns)) - - while True: - if stop_event.is_set(): - return - try: - task = queue.get_nowait() - except asyncio.QueueEmpty: - return - task_id = str(task["task_id"]) - if str(session_scope or "worker") == "task": - token = re.sub(r"[^A-Za-z0-9_-]+", "-", task_id).strip("-") or f"task-{now_ms()}" - token = token[:48] - agent_id = f"agent_task_{token}" - session_id = f"{run_id}-{agent_id}" - else: - agent_id = base_agent_id - session_id = base_session_id - current_agent_id.set(agent_id) - task_enqueued_ts = int(task.get("task_enqueued_ts", now_ms())) - task_start_ts = now_ms() - status = "success" - error_message = "" - accuracy = 0.0 - strict_score = 0.0 - relaxed_score = 0.0 - partial_score = 0.0 - benchmark_score = 0.0 - benchmark_metric = "strict_exact" - quality_label = "FACTUALLY_WRONG" - parse_failure = False - no_answer = False - parse_source = "" - answer_field = "secret_code" - task_type = str(task.get("task_type", "secret_code") or "secret_code") - answer_metric_hint = str(((task.get("metadata") or {}) if isinstance(task.get("metadata"), dict) else {}).get("answer_metric_hint", "")) - effective_qa_prompt_profile = _resolve_task_qa_prompt_profile(task=task, default_profile=qa_prompt_profile) - tokens_in: int | None = None - tokens_out: int | None = None - injected_tokens = 0 - injected_items = 0 - llm_request_latency_ms: float | None = None - llm_queue_wait_ms = 0.0 - parsed_answer = "" - parsed_answer_raw = "" - raw_output = "" - raw_output_preview = "" - raw_output_stored = "" - retrieve_skipped_reason: str | None = None - embedding_cache_hit: bool | None = None - retrieval_cache_hit: bool | None = None - fallback_retry_used = False - format_retry_count = 0 - evidence_chars_used = 0 - try: - if use_provider_direct: - thread_id = f"provider-direct-{session_id}-{task_id}" - else: - thread_id = await asyncio.wait_for( - _create_deerflow_thread( - client=client, - langgraph_url=langgraph_url, - session_id=f"{session_id}-{task_id}", - agent_id=agent_id, - use_embedded_local=use_embedded_local, - ), - timeout=thread_timeout_s, - ) - doc = docs.get(str(task.get("doc_id", "")), {}) - body = str(doc.get("body", "")) - question = str(task.get("question", "")) - question_focus = _question_focus_text(question) or question - user_text = question_focus - answer_field = str(task.get("answer_field", "secret_code" if task_type == "secret_code" else "final_answer")) - - user_item = MemoryItem( - key=f"user-{task_id}", - agent_id=agent_id, - kind="message", - text=user_text, - bytes=len(user_text.encode("utf-8")), - importance=0.7, - created_at_ms=now_ms(), - last_access_ms=now_ms(), - access_count=1, - metadata={"role": "user"}, - ) - await memory.put(user_item, ttl_s=cfg.memory.l1.ttl_s if cfg.memory.l1.enabled else None) - - if task_type == "secret_code": - evidence_text = _head_tail_clip(body, int(max_evidence_chars)) - else: - evidence_text = _build_qa_evidence_excerpt(body=body, question=question_focus, max_chars=int(max_evidence_chars)) - evidence_metadata: dict[str, Any] = {"role": "tool", "doc_id": task.get("doc_id", "")} - if str(prompt_mode) == "memory_neutral": - # Public standard QA should not duplicate the direct evidence block inside recent-context packing. - evidence_metadata["exclude_from_recent_context"] = True - if stress_turns > 0 and str(prompt_mode) == "memory_beneficial": - # In stress mode, force evidence to be recalled via retrieval rather than direct recent-window carryover. - evidence_metadata["exclude_from_recent_context"] = True - evidence_metadata["memory_stress"] = True - evidence_item = MemoryItem( - key=f"doc-{task.get('doc_id', 'unknown')}-{task_id}", - agent_id=agent_id, - kind="tool_result", - text=evidence_text, - bytes=len(evidence_text.encode("utf-8")), - importance=0.9, - created_at_ms=now_ms(), - last_access_ms=now_ms(), - access_count=1, - metadata=evidence_metadata, - ) - await memory.put(evidence_item, ttl_s=cfg.memory.l1.ttl_s if cfg.memory.l1.enabled else None) - if cfg.memory.l2.enabled and str(prompt_mode) == "memory_beneficial": - anchors = _build_question_anchors( - body=body, - question=question, - max_chunks=4, - chunk_chars=max(240, int(context_item_max_chars)), - ) - for anchor_idx, anchor_text in enumerate(anchors): - if not anchor_text.strip(): - continue - anchor_item = MemoryItem( - key=f"doc-anchor-{task.get('doc_id', 'unknown')}-{task_id}-{anchor_idx:02d}", - agent_id=agent_id, - kind="tool_result", - text=anchor_text, - bytes=len(anchor_text.encode("utf-8")), - importance=0.95, - created_at_ms=now_ms(), - last_access_ms=now_ms(), - access_count=1, - metadata={ - "role": "tool", - "doc_id": task.get("doc_id", ""), - "retrieval_anchor": True, - "exclude_from_recent_context": True, - "anchor_for_question": question[:160], - }, - ) - await memory.put(anchor_item, ttl_s=cfg.memory.l1.ttl_s if cfg.memory.l1.enabled else None) - if cfg.memory.l2.enabled and str(prompt_mode) == "memory_beneficial" and len(body) > int(max_evidence_chars): - shard_chars = max(1600, int(context_item_max_chars) * 3) - for shard_idx, shard in enumerate(_build_retrieval_shards(body, shard_chars=shard_chars, max_shards=8)): - shard_text = str(shard.get("text", "")).strip() - if not shard_text: - continue - shard_item = MemoryItem( - key=f"doc-shard-{task.get('doc_id', 'unknown')}-{task_id}-{shard_idx:02d}", - agent_id=agent_id, - kind="tool_result", - text=shard_text, - bytes=len(shard_text.encode("utf-8")), - importance=0.85, - created_at_ms=now_ms(), - last_access_ms=now_ms(), - access_count=1, - metadata={ - "role": "tool", - "doc_id": task.get("doc_id", ""), - "retrieval_shard": True, - "exclude_from_recent_context": True, - "span_start": int(shard.get("start", 0)), - "span_end": int(shard.get("end", 0)), - }, - ) - await memory.put(shard_item, ttl_s=cfg.memory.l1.ttl_s if cfg.memory.l1.enabled else None) - if bool(structured_evidence) and task_type == "secret_code": - secret_hint = str(doc.get("secret_code", "")).strip() - if not secret_hint: - m_secret = parse_model_answer(body, answer_field="secret_code", task_type="secret_code") - secret_hint = str(m_secret.get("parsed_answer_raw", "")).strip() - if secret_hint: - hint_text = f"SECRET_CODE:{secret_hint}" - hint_item = MemoryItem( - key=f"doc-secret-{task.get('doc_id', 'unknown')}-{task_id}", - agent_id=agent_id, - kind="tool_result", - text=hint_text, - bytes=len(hint_text.encode("utf-8")), - importance=1.0, - created_at_ms=now_ms(), - last_access_ms=now_ms(), - access_count=1, - metadata={"role": "tool", "doc_id": task.get("doc_id", ""), "structured": True}, - ) - await memory.put(hint_item, ttl_s=cfg.memory.l1.ttl_s if cfg.memory.l1.enabled else None) - - packed = await memory.assemble_context( - agent_id=agent_id, - user_query=question_focus, - budget=ContextBudget( - max_prompt_tokens=cfg.memory.l0.max_prompt_tokens, - max_memory_injection_tokens=cfg.memory.l0.max_memory_injection_tokens, - keep_recent_messages=cfg.memory.l0.keep_recent_messages, - ), - ) - if str(prompt_mode) == "memory_neutral": - packed.messages = [{"role": "user", "content": question_focus}] - packed.trace = { - "selected_keys": [], - "dropped_keys": [], - "summarized": False, - "total_tokens": 0, - "packed_tokens": 0, - "injection_tokens": 0, - "injection_item_count": 0, - "admission_reason": "direct_evidence_prompt", - "eviction_reason": "", - "pointer_fetch": 0, - "degrade_mode": "none", - "retrieve_skipped_reason": "prompt_mode_memory_neutral", - "embedding_cache_hit": None, - "retrieval_cache_hit": None, - } - injected_tokens = int(packed.trace.get("injection_tokens", 0)) - injected_items = int(packed.trace.get("injection_item_count", 0)) - retrieve_skipped_reason = packed.trace.get("retrieve_skipped_reason") - embedding_cache_hit = packed.trace.get("embedding_cache_hit") - retrieval_cache_hit = packed.trace.get("retrieval_cache_hit") - context_lines = _prepare_context_lines( - packed.messages, - top_k=int(context_top_k), - dedupe=bool(context_dedupe), - item_max_chars=int(context_item_max_chars), - ) - context_block = "\n".join(context_lines) - prompt, evidence_chars_used = _summarize_prompt( - session_id=session_id, - question=question, - context_block=context_block, - body=body, - task_type=task_type, - answer_field=answer_field, - answer_format_mode=answer_format_mode, - prompt_mode=str(prompt_mode), - qa_prompt_profile=str(effective_qa_prompt_profile), - qa_support_mode=str(qa_support_mode), - max_prompt_tokens=int(cfg.memory.l0.max_prompt_tokens), - max_evidence_chars=int(max_evidence_chars), - ) - - async def _invoke_with_rate_limit_retry(model_override: str, prompt_override: str | None = None) -> dict[str, Any]: - last_exc: Exception | None = None - max_attempts = 6 - for attempt in range(1, max_attempts + 1): - try: - if use_provider_direct: - model_for_direct = str(model_override or provider_model_id).strip() or provider_model_id - return await _run_provider_prompt_direct( - client=client, - base_url=provider_base_url, - model_id=model_for_direct, - api_key=provider_api_key, - prompt=(prompt_override or prompt), - llm_sema=llm_sema, - ) - return await _run_deerflow_prompt( - client=client, - langgraph_url=langgraph_url, - thread_id=thread_id, - prompt=(prompt_override or prompt), - llm_sema=llm_sema, - model_override=model_override, - assistant_id=assistant_id, - agent_name=agent_name, - use_embedded_local=use_embedded_local, - ) - except Exception as exc: - last_exc = exc - if attempt < max_attempts and ( - _is_rate_limit_error(str(exc)) or _is_transient_provider_error(str(exc)) - ): - await asyncio.sleep(float(min(12, 1.5 * attempt))) - continue - break - if last_exc is not None: - raise last_exc - raise RuntimeError("unexpected empty retry state") - - try: - resp = await asyncio.wait_for( - _invoke_with_rate_limit_retry(locked_model_name), - timeout=task_timeout_s, - ) - except ProviderBalanceError as bal_exc: - if allow_fallback_retry and fallback_model: - try: - resp = await asyncio.wait_for( - _invoke_with_rate_limit_retry(fallback_model), - timeout=task_timeout_s, - ) - except ProviderBalanceError as bal_exc2: - reason = f"Provider balance error after fallback retry (task={task_id}): {_safe_error_detail(bal_exc2)}" - if reason not in stop_reasons: - stop_reasons.append(reason) - stop_event.set() - raise - fallback_retry_used = True - reason = f"Fallback retry used model '{fallback_model}' due to provider balance error." - if reason not in fallback_reasons: - fallback_reasons.append(reason) - else: - reason = f"Provider balance error (task={task_id}): {_safe_error_detail(bal_exc)}" - if reason not in stop_reasons: - stop_reasons.append(reason) - stop_event.set() - raise - tokens_in = resp.get("prompt_tokens") - tokens_out = resp.get("completion_tokens") - llm_request_latency_ms = float(resp.get("request_latency_ms", 0.0)) - llm_queue_wait_ms = float(resp.get("llm_queue_wait_ms", 0.0)) - raw_output = str(resp.get("content", "")) - - max_format_retry = int(max(0, answer_retry_count)) - while True: - parsed = parse_model_answer(raw_output, answer_field=answer_field, task_type=task_type) - parsed_answer = str(parsed.get("parsed_answer", "")) - parsed_answer_raw = str(parsed.get("parsed_answer_raw", "")) - score = score_task_answer( - raw_output=raw_output, - parsed_answer=parsed_answer, - parsed_answer_raw=parsed_answer_raw, - answer_key=str(task.get("answer_key", task.get("gold_answer", ""))), - task_type=task_type, - answer_metric_hint=answer_metric_hint, - parse_source=str(parsed.get("parse_source", "")), - wrong_field_candidate=str(parsed.get("wrong_field_candidate", "")), - status=status, - ) - strict_score = float(score.get("strict_score", 0.0)) - relaxed_score = float(score.get("relaxed_score", 0.0)) - partial_score = float(score.get("partial_score", 0.0)) - benchmark_score = float(score.get("benchmark_score", 0.0)) - benchmark_metric = str(score.get("benchmark_metric", "strict_exact")) - quality_label = str(score.get("label", "FACTUALLY_WRONG")) - parse_failure = bool(score.get("parse_failure", False)) - no_answer = bool(score.get("no_answer", False)) - parse_source = str(score.get("parse_source", "")) - accuracy = strict_score - retry_needed = quality_label in {"PARSE_EMPTY", "PARSE_WRONG_FIELD", "REFUSAL_OR_NOANSWER"} - if retry_needed and format_retry_count < max_format_retry: - format_retry_count += 1 - repair_prompt = ( - f"{prompt}\n\n" - "Previous output was invalid.\n" - f"Return STRICT JSON only with field `{answer_field}` and no extra prose." - ) - retry_resp = await asyncio.wait_for( - _invoke_with_rate_limit_retry(locked_model_name, prompt_override=repair_prompt), - timeout=task_timeout_s, - ) - raw_output = str(retry_resp.get("content", "")) - tokens_in = retry_resp.get("prompt_tokens") - tokens_out = retry_resp.get("completion_tokens") - llm_request_latency_ms = float(retry_resp.get("request_latency_ms", 0.0)) - llm_queue_wait_ms = float(retry_resp.get("llm_queue_wait_ms", 0.0)) - continue - break - - result_text = ( - f"task_id={task_id} answer={parsed_answer} strict={strict_score} " - f"relaxed={relaxed_score} benchmark={benchmark_score:.3f} " - f"metric={benchmark_metric} label={quality_label}" - ) - raw_output_preview = raw_output[:debug_max_output_chars] - if int(store_raw_output_chars) > 0: - raw_output_stored = raw_output[: int(store_raw_output_chars)] - if debug_print: - preview = raw_output_preview.replace("\r", " ").replace("\n", " ") - print( - f"[DEBUG][{task_id}] raw='{preview}' parsed='{parsed_answer}' answer='{task.get('answer_key','')}' " - f"tokens_in={tokens_in} tokens_out={tokens_out}", - flush=True, - ) - result_item = MemoryItem( - key=f"result-{task_id}", - agent_id=agent_id, - kind="fact", - text=result_text, - bytes=len(result_text.encode("utf-8")), - importance=1.0 if relaxed_score == 1.0 else 0.3, - created_at_ms=now_ms(), - last_access_ms=now_ms(), - access_count=1, - metadata={ - "role": "assistant", - "strict_score": strict_score, - "relaxed_score": relaxed_score, - "benchmark_score": benchmark_score, - "benchmark_metric": benchmark_metric, - "quality_label": quality_label, - }, - ) - await memory.put(result_item, ttl_s=cfg.memory.l1.ttl_s if cfg.memory.l1.enabled else None) - await memory.checkpoint( - agent_id=agent_id, - thread_id=session_id, - state={ - "task_id": task_id, - "accuracy": accuracy, - "strict_score": strict_score, - "relaxed_score": relaxed_score, - "benchmark_score": benchmark_score, - "benchmark_metric": benchmark_metric, - "tokens_in": tokens_in, - "tokens_out": tokens_out, - }, - ) - except Exception as exc: - status = "error" - detail = _safe_error_detail(exc) - error_message = detail[:320] - task_end_ts = now_ms() - queue_wait_ms = max(0.0, float(task_start_ts - task_enqueued_ts)) - processing_ms = max(0.0, float(task_end_ts - task_start_ts)) - proc_llm_ms = max(0.0, float(llm_request_latency_ms or 0.0)) - proc_non_llm_ms = max(0.0, processing_ms - proc_llm_ms) - end_to_end_ms = max(0.0, float(task_end_ts - task_enqueued_ts)) - row = { - "run_id": run_id, - "op_type": "task", - "op_name": "task_result", - "task_id": task_id, - "agent_id": agent_id, - "session_id": session_id, - "start_ts": task_start_ts, - "end_ts": task_end_ts, - "task_enqueued_ts": task_enqueued_ts, - "task_start_ts": task_start_ts, - "task_end_ts": task_end_ts, - "queue_wait_ms": float(queue_wait_ms), - "processing_ms": float(processing_ms), - "proc_whole_ms": float(processing_ms), - "proc_llm_ms": float(proc_llm_ms), - "proc_non_llm_ms": float(proc_non_llm_ms), - "end_to_end_ms": float(end_to_end_ms), - "status": status, - "accuracy_applicable": True, - "accuracy": float(accuracy), - "strict_score": float(strict_score), - "relaxed_score": float(relaxed_score), - "partial_score": float(partial_score), - "benchmark_score": float(benchmark_score), - "benchmark_metric": benchmark_metric, - "quality_label": quality_label, - "parse_failure": bool(parse_failure), - "no_answer": bool(no_answer), - "parse_source": parse_source, - "answer_metric_hint": answer_metric_hint, - "task_type": task_type, - "answer_field": answer_field, - "qa_prompt_profile": str(effective_qa_prompt_profile), - "qa_support_mode": str(qa_support_mode), - "tokens_in": tokens_in, - "tokens_out": tokens_out, - "latency_ms": float(end_to_end_ms), - "context_injected_tokens": int(injected_tokens), - "context_injected_items": int(injected_items), - "retrieve_skipped_reason": retrieve_skipped_reason, - "embedding_cache_hit": embedding_cache_hit, - "retrieval_cache_hit": retrieval_cache_hit, - "prompt_mode": str(prompt_mode), - "memory_stress_turns": int(stress_turns), - "evidence_chars_used": int(evidence_chars_used), - "llm_queue_wait_ms": float(llm_queue_wait_ms), - "llm_request_latency_ms": llm_request_latency_ms, - "llm_latency_ms": llm_request_latency_ms, - "parsed_answer": parsed_answer, - "parsed_answer_raw": parsed_answer_raw, - "raw_output": raw_output_stored, - "format_retry_count": int(format_retry_count), - "fallback_retry_used": bool(fallback_retry_used), - "error": error_message, - } - if debug_print: - row["raw_output_preview"] = raw_output_preview - task_rows.append(row) - event_writer.append(row) - queue.task_done() - if stop_event.is_set(): - return - - -async def _run() -> int: - args = parse_common_args(default_sessions=50, default_workload_id="E2E-50") - cfg = load_runtime_config(args) - model_base_url = (os.getenv("OPENAI_BASE_URL", "") or "").strip().rstrip("/") - model_id = (os.getenv("OPENAI_MODEL", "") or "").strip() - api_key = (os.getenv("OPENAI_API_KEY", "") or "").strip() - locked_base_url = str(args.locked_base_url or LOCKED_BASE_URL).strip().rstrip("/") - locked_model_id = str(args.locked_model_id or LOCKED_MODEL_ID).strip() - if not model_base_url or not model_id or not api_key: - raise RuntimeError("Set OPENAI_BASE_URL, OPENAI_MODEL and OPENAI_API_KEY before an E2E run") - comparable_mode = not bool(args.allow_fallback_retry) - use_embedded_local = _should_use_embedded_deerflow( - langgraph_url=str(args.deerflow_langgraph_url), - experiment_track=str(args.experiment_track), - ) - use_provider_direct = _is_truthy(os.getenv("MEMLAB_E2E_PROVIDER_DIRECT")) - if (not use_provider_direct) and str(locked_model_id).lower() == "pro/deepseek-ai/deepseek-v3.2": - # Default DeepSeek runs to direct provider transport for stability and throughput. - use_provider_direct = True - runtime_model_name = locked_model_id if use_provider_direct else LOCKED_DEERFLOW_MODEL_NAME - - run_id = f"{args.variant.lower()}-e2e-r{args.repeat_idx}-{now_ms()}" - run_dir = Path("runs") / run_id - run_dir.mkdir(parents=True, exist_ok=True) - - env = run_env_checks( - variant_id=args.variant, - require_deerflow=True, - deerflow_langgraph_url=args.deerflow_langgraph_url, - deerflow_gateway_url=args.deerflow_gateway_url, - locked_base_url=locked_base_url, - locked_model_id=locked_model_id, - ) - manifest = build_manifest( - run_id=run_id, - variant_id=args.variant, - workload_id=args.workload_id, - seed=args.seed, - repeat_idx=args.repeat_idx, - env_valid=env.env_valid, - env_invalid_reasons=env.env_invalid_reasons, - repro_valid=bool(env.memlab_git_commit), - repro_invalid_reasons=[] if env.memlab_git_commit else ["memlab git commit hash is unavailable."], - llm_real=None, - llm_invalid_reasons=[], - usage_unavailable=False, - usage_unavailable_reasons=[], - model_base_url=model_base_url or None, - model_id=model_id or None, - model_fallback_used=False, - comparable_mode=comparable_mode, - comparability_invalid_reasons=[], - metrics_schema_version=5, - memlab_git_commit=env.memlab_git_commit, - deerflow_git_commit=env.deerflow_git_commit, - docker_image_digests=env.docker_image_digests, - infra_versions=env.infra_versions, - track_id=(str(args.track_id).strip() or f"{str(args.experiment_track).strip()}:{str(args.prompt_mode).strip()}"), - extra={ - "runner": "runner_e2e", - "llm_max_inflight": args.llm_max_inflight, - "rate_limit_safe_max_inflight": _rate_limit_safe_max_inflight(), - "experiment_track": args.experiment_track, - "track_id": (str(args.track_id).strip() or f"{str(args.experiment_track).strip()}:{str(args.prompt_mode).strip()}"), - "deerflow_transport": ( - "provider_direct" if use_provider_direct else ("embedded_local" if use_embedded_local else "langgraph_http") - ), - "provider_direct": bool(use_provider_direct), - "dataset_id": str(args.dataset_id or ""), - "prepared_full_or_slice": str(args.prepared_full_or_slice or ""), - "prompt_mode": args.prompt_mode, - "qa_prompt_profile": args.qa_prompt_profile, - "qa_support_mode": args.qa_support_mode, - "session_scope": args.session_scope, - "max_evidence_chars": args.max_evidence_chars, - "assistant_id": args.assistant_id, - "agent_name": args.agent_name, - "worker_processes": args.worker_processes, - "sessions_per_process": args.sessions_per_process, - "legacy_variant_alias": args.legacy_variant_alias, - "deerflow_langgraph_url": args.deerflow_langgraph_url, - "deerflow_gateway_url": args.deerflow_gateway_url, - "tasks_path": args.tasks_path, - "docs_path": args.docs_path, - "debug_print": args.debug_print, - "answer_retry_count": args.answer_retry_count, - "store_raw_output_chars": args.store_raw_output_chars, - "answer_format_mode": args.answer_format_mode, - "structured_evidence": args.structured_evidence, - "context_top_k": args.context_top_k, - "context_dedupe": args.context_dedupe, - "context_item_max_chars": args.context_item_max_chars, - "memory_stress_turns": args.memory_stress_turns, - "allow_fallback_retry": args.allow_fallback_retry, - "fallback_model": (args.fallback_model.strip() or os.getenv("OPENAI_FALLBACK_MODEL", "").strip()), - "model_lock": { - "expected_base_url": locked_base_url, - "expected_model_id": locked_model_id, - "expected_deerflow_model_name": runtime_model_name, - "actual_base_url": model_base_url, - "actual_model_id": model_id, - }, - "preflight_passed": None, - "preflight_report_path": "preflight.json", - "error_breakdown": {}, - }, - ) - write_manifest(run_dir / "manifest.json", manifest) - - if comparable_mode: - if model_base_url != locked_base_url or model_id != locked_model_id: - reason = f"Model lock violation: expected {locked_model_id} on {locked_base_url}" - manifest["env_valid"] = False - manifest.setdefault("env_invalid_reasons", []) - if reason not in manifest["env_invalid_reasons"]: - manifest["env_invalid_reasons"].append(reason) - manifest["repro_valid"] = False - manifest.setdefault("repro_invalid_reasons", []) - if reason not in manifest["repro_invalid_reasons"]: - manifest["repro_invalid_reasons"].append(reason) - manifest.setdefault("comparability_invalid_reasons", []) - if reason not in manifest["comparability_invalid_reasons"]: - manifest["comparability_invalid_reasons"].append(reason) - write_manifest(run_dir / "manifest.json", manifest) - print("STOP: run invalid; refusing fallback.") - print(f"- {reason}") - return 1 - safe_max_inflight = _rate_limit_safe_max_inflight() - if int(args.llm_max_inflight) > safe_max_inflight: - reason = ( - "Rate-limit safety violation: " - f"llm_max_inflight={int(args.llm_max_inflight)} > recommended {safe_max_inflight}." - ) - manifest["env_valid"] = False - manifest.setdefault("env_invalid_reasons", []) - if reason not in manifest["env_invalid_reasons"]: - manifest["env_invalid_reasons"].append(reason) - manifest["repro_valid"] = False - manifest.setdefault("repro_invalid_reasons", []) - if reason not in manifest["repro_invalid_reasons"]: - manifest["repro_invalid_reasons"].append(reason) - manifest.setdefault("comparability_invalid_reasons", []) - if reason not in manifest["comparability_invalid_reasons"]: - manifest["comparability_invalid_reasons"].append(reason) - write_manifest(run_dir / "manifest.json", manifest) - print("STOP: run invalid; refusing fallback.") - print(f"- {reason}") - return 1 - - if not bool(args.skip_preflight): - preflight_report = await _run_preflight_checks( - model_base_url=model_base_url, - model_id=model_id, - api_key=api_key, - langgraph_url=args.deerflow_langgraph_url, - locked_base_url=locked_base_url, - locked_model_id=locked_model_id, - locked_model_name=runtime_model_name, - llm_max_inflight=int(args.llm_max_inflight), - prompt_mode=str(args.prompt_mode), - max_evidence_chars=int(args.max_evidence_chars), - comparable_mode=bool(comparable_mode), - assistant_id=str(args.assistant_id), - agent_name=str(args.agent_name), - use_embedded_local=use_embedded_local, - use_provider_direct=bool(use_provider_direct), - ) - json_dump(run_dir / "preflight.json", preflight_report) - manifest.setdefault("extra", {}) - manifest["extra"]["preflight_passed"] = bool(preflight_report.get("passed", False)) - manifest["extra"]["preflight_report"] = preflight_report - if not bool(preflight_report.get("passed", False)): - manifest["env_valid"] = False - manifest.setdefault("env_invalid_reasons", []) - manifest["repro_valid"] = False - manifest.setdefault("repro_invalid_reasons", []) - for reason in preflight_report.get("failed_checks", []): - reason_text = f"Preflight failed: {reason}" - if reason_text not in manifest["env_invalid_reasons"]: - manifest["env_invalid_reasons"].append(reason_text) - if reason_text not in manifest["repro_invalid_reasons"]: - manifest["repro_invalid_reasons"].append(reason_text) - manifest.setdefault("comparability_invalid_reasons", []) - if reason_text not in manifest["comparability_invalid_reasons"]: - manifest["comparability_invalid_reasons"].append(reason_text) - write_manifest(run_dir / "manifest.json", manifest) - print("STOP: run invalid; refusing fallback.") - for reason in preflight_report.get("failed_checks", []): - print(f"- Preflight failed: {reason}") - return 1 - write_manifest(run_dir / "manifest.json", manifest) - else: - manifest.setdefault("extra", {}) - manifest["extra"]["preflight_passed"] = None - manifest["extra"]["preflight_report"] = None - write_manifest(run_dir / "manifest.json", manifest) - - if bool(args.preflight_only): - print(f"Preflight complete: {run_id}") - print(f"Manifest: {run_dir / 'manifest.json'}") - print(f"Preflight: {run_dir / 'preflight.json'}") - return 0 - - fatal_reasons = list(env.env_invalid_reasons) - if fatal_reasons: - print("STOP: required services missing; refusing fallback.") - for r in fatal_reasons: - print(f"- {r}") - return 1 - - tasks = shuffle_tasks(load_e2e_tasks(args.tasks_path), seed=args.seed + args.repeat_idx) - docs = load_docs(args.docs_path) - - q: asyncio.Queue[dict[str, Any]] = asyncio.Queue() - for task in tasks: - enriched = dict(task) - enriched["task_enqueued_ts"] = now_ms() - await q.put(enriched) - - event_writer = EventLogWriter(run_dir / "eventlog.jsonl") - task_rows: list[dict[str, Any]] = [] - mem_rows: list[dict[str, Any]] = [] - memory = MultiLevelMemoryManager( - cfg, op_logger=make_operation_logger(run_id=run_id, mem_rows=mem_rows, event_writer=event_writer) - ) - await memory.init() - llm_sema = asyncio.Semaphore(max(1, args.llm_max_inflight)) - stop_event = asyncio.Event() - stop_reasons: list[str] = [] - fallback_reasons: list[str] = [] - - capacity = max(1, args.worker_processes * args.sessions_per_process) - if capacity < args.n_sessions: - reason = ( - "Invalid session topology: " - f"worker_processes*sessions_per_process={capacity} is smaller than n_sessions={args.n_sessions}." - ) - manifest["env_valid"] = False - manifest.setdefault("env_invalid_reasons", []) - manifest["env_invalid_reasons"].append(reason) - write_manifest(run_dir / "manifest.json", manifest) - print("STOP: required services missing; refusing fallback.") - print(f"- {reason}") - await memory.close() - return 1 - worker_count = args.n_sessions - async with httpx.AsyncClient(timeout=180.0, trust_env=False) as client: - workers = [ - _session_worker( - queue=q, - session_index=i, - run_id=run_id, - docs=docs, - memory=memory, - event_writer=event_writer, - task_rows=task_rows, - mem_rows=mem_rows, - langgraph_url=args.deerflow_langgraph_url, - client=client, - llm_sema=llm_sema, - cfg=cfg, - debug_print=args.debug_print, - debug_max_output_chars=args.debug_max_output_chars, - stop_event=stop_event, - stop_reasons=stop_reasons, - allow_fallback_retry=args.allow_fallback_retry, - fallback_model=(args.fallback_model.strip() or os.getenv("OPENAI_FALLBACK_MODEL", "").strip()), - fallback_reasons=fallback_reasons, - locked_model_name=runtime_model_name, - prompt_mode=str(args.prompt_mode), - max_evidence_chars=int(args.max_evidence_chars), - answer_retry_count=int(args.answer_retry_count), - store_raw_output_chars=int(args.store_raw_output_chars), - answer_format_mode=str(args.answer_format_mode), - structured_evidence=bool(args.structured_evidence), - context_top_k=int(args.context_top_k), - context_dedupe=bool(args.context_dedupe), - context_item_max_chars=int(args.context_item_max_chars), - memory_stress_turns=int(args.memory_stress_turns), - session_scope=str(args.session_scope), - assistant_id=str(args.assistant_id), - agent_name=str(args.agent_name), - use_embedded_local=use_embedded_local, - use_provider_direct=bool(use_provider_direct), - provider_base_url=locked_base_url, - provider_model_id=locked_model_id, - provider_api_key=api_key, - qa_prompt_profile=str(args.qa_prompt_profile), - qa_support_mode=str(args.qa_support_mode), - ) - for i in range(worker_count) - ] - await asyncio.gather(*workers) - try: - await memory.close() - except Exception as exc: - stop_reasons.append(f"memory_close_failed: {_safe_error_detail(exc)}") - - summary = compute_run_summary( - run_id=run_id, - variant_id=args.variant, - workload_id=args.workload_id, - n_sessions=args.n_sessions, - tasks=task_rows, - mem_ops=mem_rows, - ) - retry_count_total = int(sum(int(r.get("format_retry_count", 0)) for r in task_rows)) - summary["retry_count_total"] = retry_count_total - error_breakdown = _build_error_breakdown(task_rows) - summary["error_breakdown"] = error_breakdown - json_dump(run_dir / "summary.json", summary) - - comparability_reasons: list[str] = [] - llm_post_reasons: list[str] = [] - p50 = float(summary.get("llm_latency", summary.get("llm_request_latency", {})).get("p50", 0.0)) - default_min_success_rate = 0.60 if use_provider_direct else 0.80 - try: - min_success_rate = float(os.getenv("MEMLAB_E2E_MIN_SUCCESS_RATE", str(default_min_success_rate))) - except Exception: - min_success_rate = default_min_success_rate - min_success_rate = max(0.0, min(1.0, float(min_success_rate))) - manifest.setdefault("extra", {}) - manifest["extra"]["comparability_min_success_rate"] = min_success_rate - min_realistic_p50_ms = 120.0 if use_provider_direct else 300.0 - if int(summary.get("success_tasks", 0)) <= 0: - llm_post_reasons.append("E2E run has no successful tasks; refusing to mark comparable.") - if summary.get("success_tasks", 0) > 0 and p50 < min_realistic_p50_ms: - llm_post_reasons.append( - f"E2E LLM request latency suspiciously low (p50={p50:.2f}ms < {min_realistic_p50_ms:.0f}ms)." - ) - success_rate = float(summary.get("success_rate", 0.0)) - if success_rate < min_success_rate: - comparability_reasons.append( - f"E2E success_rate too low for comparability ({success_rate:.3f} < {min_success_rate:.3f})." - ) - usage_unavailable = bool(summary.get("usage_unavailable", False)) - usage_reasons: list[str] = [] - if usage_unavailable: - usage_reasons.append("Provider usage fields are missing for one or more successful tasks.") - - if stop_reasons: - manifest["repro_valid"] = False - manifest["llm_real"] = False - manifest["llm_invalid_reasons"] = stop_reasons - manifest.setdefault("extra", {}) - manifest["extra"]["error_breakdown"] = error_breakdown - manifest["extra"]["retry_count"] = retry_count_total - manifest.setdefault("repro_invalid_reasons", []) - for reason in stop_reasons: - if reason not in manifest["repro_invalid_reasons"]: - manifest["repro_invalid_reasons"].append(reason) - manifest.setdefault("comparability_invalid_reasons", []) - for reason in stop_reasons: - if reason not in manifest["comparability_invalid_reasons"]: - manifest["comparability_invalid_reasons"].append(reason) - manifest["usage_unavailable"] = usage_unavailable - manifest["usage_unavailable_reasons"] = usage_reasons - write_manifest(run_dir / "manifest.json", manifest) - print("STOP: run invalid; refusing fallback.") - for reason in stop_reasons: - print(f"- {reason}") - return 1 - - manifest["llm_real"] = len(llm_post_reasons) == 0 - manifest["llm_invalid_reasons"] = llm_post_reasons - manifest["usage_unavailable"] = usage_unavailable - manifest["usage_unavailable_reasons"] = usage_reasons - manifest.setdefault("extra", {}) - manifest["extra"]["error_breakdown"] = error_breakdown - manifest["extra"]["retry_count"] = retry_count_total - if fallback_reasons: - manifest["repro_valid"] = False - manifest["llm_real"] = False - manifest["model_fallback_used"] = True - manifest.setdefault("repro_invalid_reasons", []) - for reason in fallback_reasons: - if reason not in manifest["repro_invalid_reasons"]: - manifest["repro_invalid_reasons"].append(reason) - manifest.setdefault("comparability_invalid_reasons", []) - for reason in fallback_reasons: - if reason not in manifest["comparability_invalid_reasons"]: - manifest["comparability_invalid_reasons"].append(reason) - manifest["llm_invalid_reasons"] = list(manifest.get("llm_invalid_reasons", [])) + list(fallback_reasons) - if llm_post_reasons or comparability_reasons: - manifest["repro_valid"] = False - manifest.setdefault("repro_invalid_reasons", []) - for reason in llm_post_reasons: - if reason not in manifest["repro_invalid_reasons"]: - manifest["repro_invalid_reasons"].append(reason) - for reason in comparability_reasons: - if reason not in manifest["repro_invalid_reasons"]: - manifest["repro_invalid_reasons"].append(reason) - manifest.setdefault("comparability_invalid_reasons", []) - for reason in llm_post_reasons + comparability_reasons: - if reason not in manifest["comparability_invalid_reasons"]: - manifest["comparability_invalid_reasons"].append(reason) - write_manifest(run_dir / "manifest.json", manifest) - print("STOP: run invalid; refusing fallback.") - for reason in llm_post_reasons + comparability_reasons: - print(f"- {reason}") - return 1 - - manifest["summary_chain_valid"] = bool((run_dir / "eventlog.jsonl").exists() and (run_dir / "summary.json").exists()) - write_manifest(run_dir / "manifest.json", manifest) - print(f"Run complete: {run_id}") - print(f"Manifest: {run_dir / 'manifest.json'}") - print(f"Event log: {run_dir / 'eventlog.jsonl'}") - print(f"Summary: {run_dir / 'summary.json'}") - return 0 - - -def main() -> None: - rc = 1 - try: - rc = asyncio.run(_run()) - finally: - _shutdown_embedded_worker_sync() - if rc != 0: - raise SystemExit(rc) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/runner_stub.py b/projects/memlab/memlab/runner_stub.py deleted file mode 100644 index 21f9ce8..0000000 --- a/projects/memlab/memlab/runner_stub.py +++ /dev/null @@ -1,351 +0,0 @@ -from __future__ import annotations - -import argparse -import asyncio -import random -from pathlib import Path -from typing import Any - -from memlab.config import load_config -from memlab.envcheck import run_env_checks, write_manifest -from memlab.io import json_dump, jsonl_read, now_ms -from memlab.memory.types import ContextBudget, MemoryItem -from memlab.memory_manager.manager import MultiLevelMemoryManager -from memlab.metrics.eventlog import EventLogWriter -from memlab.metrics.manifest import build_manifest -from memlab.runner_core import apply_variant, compute_run_summary -from memlab.runner_core import normalize_variant -from memlab.session_logging import current_agent_id as _CURRENT_AGENT_ID -from memlab.session_logging import make_operation_logger as _make_op_logger - - -def _parse_args() -> argparse.Namespace: - ap = argparse.ArgumentParser() - ap.add_argument("--config", type=str, default="configs/memlab.default.yaml") - ap.add_argument("--variant", type=str, required=True, help="B0|B1|V1|F1|F2 (legacy aliases: A1|A2)") - ap.add_argument("--n_sessions", type=int, default=200) - ap.add_argument("--llm_max_inflight", type=int, default=0) - ap.add_argument("--worker_processes", type=int, default=1) - ap.add_argument("--sessions_per_process", type=int, default=200) - ap.add_argument("--repeat_idx", type=int, default=0) - ap.add_argument("--seed", type=int, default=42) - ap.add_argument("--workload_id", type=str, default="SYN-200") - ap.add_argument("--deerflow_langgraph_url", type=str, default="http://127.0.0.1:2024") - ap.add_argument("--deerflow_gateway_url", type=str, default="http://127.0.0.1:8001") - ap.add_argument("--trace_path", type=str, default="") - ap.add_argument("--ops_per_session", type=int, default=1000) - ap.add_argument("--track_id", type=str, default="stub:default") - return ap.parse_args() - - -def _build_deterministic_stub_tasks(*, n_sessions: int, ops_per_session: int, seed: int) -> list[dict[str, Any]]: - rng = random.Random(seed) - tasks: list[dict[str, Any]] = [] - total = n_sessions * ops_per_session - hotset = max(1000, n_sessions * 20) - for i in range(total): - key_idx = rng.randrange(hotset) - payload_bytes = rng.randint(128, 8192) - tasks.append( - { - "task_id": f"syn_{i:07d}", - "key": f"k-{key_idx:06d}", - "bytes": payload_bytes, - "importance": rng.random(), - } - ) - return tasks - - -def _load_trace_tasks(path: str) -> list[dict[str, Any]]: - rows = jsonl_read(path) - tasks: list[dict[str, Any]] = [] - for i, row in enumerate(rows): - tasks.append( - { - "task_id": f"trace_{i:07d}", - "key": str(row.get("key", f"k-{i:06d}")), - "bytes": int(row.get("bytes", 256)), - "importance": float(row.get("importance", 0.5)), - } - ) - return tasks - - -def _fake_text(key: str, n_bytes: int) -> str: - prefix = f"key={key} " - fill = max(0, n_bytes - len(prefix.encode("utf-8"))) - return prefix + ("x" * fill) - - -async def _session_worker( - *, queue, session_index, run_id, memory, event_writer, task_rows, mem_rows, cfg, -) -> None: - token = _CURRENT_AGENT_ID.set(f"agent_{session_index + 1:04d}") - try: - await _session_worker_body( - queue=queue, session_index=session_index, run_id=run_id, - memory=memory, event_writer=event_writer, task_rows=task_rows, - mem_rows=mem_rows, cfg=cfg, - ) - finally: - _CURRENT_AGENT_ID.reset(token) - - -async def _session_worker_body( - *, - queue: asyncio.Queue[dict[str, Any]], - session_index: int, - run_id: str, - memory: MultiLevelMemoryManager, - event_writer: EventLogWriter, - task_rows: list[dict[str, Any]], - mem_rows: list[dict[str, Any]], - cfg, -) -> None: - agent_id = f"agent_{session_index + 1:04d}" - session_id = f"{run_id}-{agent_id}" - - while True: - try: - task = queue.get_nowait() - except asyncio.QueueEmpty: - return - task_id = str(task["task_id"]) - key = str(task["key"]) - payload_size = int(task.get("bytes", 256)) - task_enqueued_ts = int(task.get("task_enqueued_ts", now_ms())) - task_start_ts = now_ms() - status = "success" - err = "" - accuracy: float | None = None - tokens_in = 0 - tokens_out = 0 - injected_tokens = 0 - injected_items = 0 - retrieve_skipped_reason: str | None = None - embedding_cache_hit: bool | None = None - retrieval_cache_hit: bool | None = None - try: - text = _fake_text(key, payload_size) - item = MemoryItem( - key=key, - agent_id=agent_id, - kind="fact", - text=text, - bytes=payload_size, - importance=float(task.get("importance", 0.5)), - created_at_ms=now_ms(), - last_access_ms=now_ms(), - access_count=1, - metadata={"role": "system", "workload_hint": 1.0}, - ) - await memory.put(item, ttl_s=cfg.memory.l1.ttl_s if cfg.memory.l1.enabled else None) - _ = await memory.get(agent_id=agent_id, query=key, k=1, mode="hybrid") - packed = await memory.assemble_context( - agent_id=agent_id, - user_query=f"lookup {key}", - budget=ContextBudget( - max_prompt_tokens=cfg.memory.l0.max_prompt_tokens, - max_memory_injection_tokens=cfg.memory.l0.max_memory_injection_tokens, - keep_recent_messages=cfg.memory.l0.keep_recent_messages, - ), - ) - injected_tokens = int(packed.trace.get("injection_tokens", 0)) - injected_items = int(packed.trace.get("injection_item_count", 0)) - retrieve_skipped_reason = packed.trace.get("retrieve_skipped_reason") - embedding_cache_hit = packed.trace.get("embedding_cache_hit") - retrieval_cache_hit = packed.trace.get("retrieval_cache_hit") - tokens_in = max(1, int(packed.trace.get("total_tokens", 0))) - tokens_out = 1 - except Exception as exc: - status = "error" - err = type(exc).__name__ - task_end_ts = now_ms() - queue_wait_ms = max(0.0, float(task_start_ts - task_enqueued_ts)) - raw_processing_ms = max(0.0, float(task_end_ts - task_start_ts)) - processing_ms = raw_processing_ms - proc_llm_ms = 0.0 - proc_non_llm_ms = processing_ms - end_to_end_ms = max(0.0, float(task_end_ts - task_enqueued_ts)) - row = { - "run_id": run_id, - "op_type": "task", - "op_name": "task_result", - "task_id": task_id, - "agent_id": agent_id, - "session_id": session_id, - "start_ts": task_start_ts, - "end_ts": task_end_ts, - "task_enqueued_ts": task_enqueued_ts, - "task_start_ts": task_start_ts, - "task_end_ts": task_end_ts, - "queue_wait_ms": float(queue_wait_ms), - "processing_ms": float(processing_ms), - "proc_whole_ms": float(processing_ms), - "proc_llm_ms": float(proc_llm_ms), - "proc_non_llm_ms": float(proc_non_llm_ms), - "end_to_end_ms": float(end_to_end_ms), - "status": status, - "accuracy_applicable": False, - "accuracy": accuracy, - "tokens_in": int(tokens_in), - "tokens_out": int(tokens_out), - "latency_ms": float(end_to_end_ms), - "context_injected_tokens": int(injected_tokens), - "context_injected_items": int(injected_items), - "retrieve_skipped_reason": retrieve_skipped_reason, - "embedding_cache_hit": embedding_cache_hit, - "retrieval_cache_hit": retrieval_cache_hit, - "llm_queue_wait_ms": 0.0, - "llm_latency_ms": None, - "llm_request_latency_ms": None, - "error": err, - } - task_rows.append(row) - event_writer.append(row) - queue.task_done() - - -async def _run() -> int: - args = _parse_args() - variant, legacy_alias = normalize_variant(args.variant) - run_id = f"{variant.lower()}-syn-r{args.repeat_idx}-{now_ms()}" - run_dir = Path("runs") / run_id - run_dir.mkdir(parents=True, exist_ok=True) - - cfg = load_config(args.config) - cfg = apply_variant(cfg, variant, legacy_alias=legacy_alias) - cfg.experiment.seed = int(args.seed) - cfg.experiment.n_agents = int(args.n_sessions) - - env = run_env_checks( - variant_id=variant, - require_deerflow=False, - deerflow_langgraph_url=args.deerflow_langgraph_url, - deerflow_gateway_url=args.deerflow_gateway_url, - ) - manifest = build_manifest( - run_id=run_id, - variant_id=variant, - workload_id=str(args.workload_id), - seed=int(args.seed), - repeat_idx=int(args.repeat_idx), - env_valid=env.env_valid, - env_invalid_reasons=env.env_invalid_reasons, - repro_valid=bool(env.memlab_git_commit), - repro_invalid_reasons=[] if env.memlab_git_commit else ["memlab git commit hash is unavailable."], - llm_real=None, - llm_invalid_reasons=[], - usage_unavailable=None, - usage_unavailable_reasons=[], - model_base_url=None, - model_id=None, - model_fallback_used=False, - comparable_mode=True, - comparability_invalid_reasons=[], - metrics_schema_version=3, - memlab_git_commit=env.memlab_git_commit, - deerflow_git_commit=env.deerflow_git_commit, - docker_image_digests=env.docker_image_digests, - infra_versions=env.infra_versions, - track_id=str(args.track_id or "stub:default"), - extra={ - "runner": "runner_stub", - "track_id": str(args.track_id or "stub:default"), - "worker_processes": int(args.worker_processes), - "sessions_per_process": int(args.sessions_per_process), - "ops_per_session": int(args.ops_per_session), - "trace_path": str(args.trace_path or ""), - "legacy_variant_alias": legacy_alias, - }, - ) - write_manifest(run_dir / "manifest.json", manifest) - - fatal_reasons = list(env.env_invalid_reasons) - if fatal_reasons: - print("STOP: required services missing; refusing fallback.") - for reason in fatal_reasons: - print(f"- {reason}") - return 1 - - if args.trace_path and Path(args.trace_path).exists(): - tasks = _load_trace_tasks(args.trace_path) - else: - tasks = _build_deterministic_stub_tasks( - n_sessions=int(args.n_sessions), - ops_per_session=int(args.ops_per_session), - seed=int(args.seed) + int(args.repeat_idx), - ) - q: asyncio.Queue[dict[str, Any]] = asyncio.Queue() - for task in tasks: - enriched = dict(task) - enriched["task_enqueued_ts"] = now_ms() - await q.put(enriched) - - event_writer = EventLogWriter(run_dir / "eventlog.jsonl") - task_rows: list[dict[str, Any]] = [] - mem_rows: list[dict[str, Any]] = [] - memory = MultiLevelMemoryManager( - cfg, op_logger=_make_op_logger(run_id=run_id, mem_rows=mem_rows, event_writer=event_writer) - ) - await memory.init() - - capacity = max(1, int(args.worker_processes) * int(args.sessions_per_process)) - if capacity < int(args.n_sessions): - reason = ( - "Invalid session topology: " - f"worker_processes*sessions_per_process={capacity} is smaller than n_sessions={int(args.n_sessions)}." - ) - manifest["env_valid"] = False - manifest.setdefault("env_invalid_reasons", []) - manifest["env_invalid_reasons"].append(reason) - write_manifest(run_dir / "manifest.json", manifest) - print("STOP: required services missing; refusing fallback.") - print(f"- {reason}") - await memory.close() - return 1 - worker_count = int(args.n_sessions) - workers = [ - _session_worker( - queue=q, - session_index=i, - run_id=run_id, - memory=memory, - event_writer=event_writer, - task_rows=task_rows, - mem_rows=mem_rows, - cfg=cfg, - ) - for i in range(worker_count) - ] - await asyncio.gather(*workers) - await memory.close() - - summary = compute_run_summary( - run_id=run_id, - variant_id=variant, - workload_id=str(args.workload_id), - n_sessions=int(args.n_sessions), - tasks=task_rows, - mem_ops=mem_rows, - ) - json_dump(run_dir / "summary.json", summary) - manifest["summary_chain_valid"] = bool((run_dir / "eventlog.jsonl").exists() and (run_dir / "summary.json").exists()) - write_manifest(run_dir / "manifest.json", manifest) - - print(f"Run complete: {run_id}") - print(f"Manifest: {run_dir / 'manifest.json'}") - print(f"Event log: {run_dir / 'eventlog.jsonl'}") - print(f"Summary: {run_dir / 'summary.json'}") - return 0 - - -def main() -> None: - rc = asyncio.run(_run()) - if rc != 0: - raise SystemExit(rc) - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/runners/__init__.py b/projects/memlab/memlab/runners/__init__.py deleted file mode 100644 index e627cb1..0000000 --- a/projects/memlab/memlab/runners/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Experiment runners.""" diff --git a/projects/memlab/memlab/runners/common.py b/projects/memlab/memlab/runners/common.py deleted file mode 100644 index 5708c2f..0000000 --- a/projects/memlab/memlab/runners/common.py +++ /dev/null @@ -1,146 +0,0 @@ -from __future__ import annotations - -import argparse -import os -import platform -import random -import re -import subprocess -from dataclasses import asdict -from pathlib import Path -from typing import Any - -from memlab.config import MemlabConfig, load_config -from memlab.io import jsonl_read, now_ms -from memlab.runner_core import normalize_variant -from memlab.types import RunnerConfig - - -SECRET_RE = re.compile(r"SECRET_CODE[:=]\s*([A-Z0-9]+)") - - -def parse_args(default_agents: int = 50) -> argparse.Namespace: - ap = argparse.ArgumentParser() - ap.add_argument("--config", type=str, default="configs/memlab.default.yaml") - ap.add_argument("--variant", type=str, default=None, help="B0|B1|V1|F1|F2 (legacy aliases: A1|A2)") - ap.add_argument("--agents", type=int, default=default_agents) - ap.add_argument("--repeat-idx", type=int, default=0) - return ap.parse_args() - - -def apply_variant(cfg: MemlabConfig, variant: str) -> MemlabConfig: - canonical, legacy_alias = normalize_variant(variant) - cfg.experiment.variant_id = canonical - if canonical == "B0": - cfg.memory.l1.enabled = False - cfg.memory.l2.enabled = False - cfg.memory.l3.enabled = False - cfg.experiment.summarization_on = False - elif canonical == "B1": - cfg.memory.l1.enabled = False - cfg.memory.l2.enabled = False - cfg.memory.l3.enabled = False - cfg.experiment.summarization_on = True - elif canonical == "V1": - cfg.memory.l1.enabled = False - cfg.memory.l2.enabled = True - cfg.memory.l3.enabled = False - cfg.experiment.summarization_on = True - elif canonical == "F1": - cfg.memory.l1.enabled = True - cfg.memory.l2.enabled = True - cfg.memory.l3.enabled = False - cfg.experiment.summarization_on = True - elif canonical == "F2": - cfg.memory.l1.enabled = True - cfg.memory.l2.enabled = True - cfg.memory.l3.enabled = True - cfg.experiment.summarization_on = True - else: - raise ValueError(f"Unknown variant: {variant}") - if legacy_alias == "A2": - cfg.experiment.redis_policy = "allkeys-lru" - return cfg - - -def load_runtime_config(args: argparse.Namespace) -> tuple[MemlabConfig, RunnerConfig]: - cfg = load_config(args.config) - if args.variant: - cfg = apply_variant(cfg, args.variant) - cfg.experiment.n_agents = args.agents - rcfg = RunnerConfig( - variant_id=cfg.experiment.variant_id, - n_agents=cfg.experiment.n_agents, - llm_backend=cfg.experiment.llm_backend, - seed=cfg.experiment.seed, - duration_s=cfg.experiment.duration_s, - redis_policy=cfg.experiment.redis_policy, - summarization_on=cfg.experiment.summarization_on, - repeat_idx=args.repeat_idx, - ) - return cfg, rcfg - - -def load_docs(path: str) -> dict[str, dict[str, Any]]: - rows = jsonl_read(path) - return {r["doc_id"]: r for r in rows} - - -def load_tasks(path: str) -> list[dict[str, Any]]: - return jsonl_read(path) - - -def extract_secret_code(text: str) -> str: - m = SECRET_RE.search(text) - return m.group(1) if m else "" - - -def doc_search(task: dict[str, Any], docs: dict[str, dict[str, Any]], rng: random.Random) -> list[str]: - if task.get("doc_id") in docs: - primary = task["doc_id"] - else: - primary = rng.choice(list(docs.keys())) - # Return primary + a few distractors for realistic retrieval calls. - other = [k for k in docs.keys() if k != primary] - rng.shuffle(other) - return [primary] + other[:4] - - -def task_accuracy(pred_secret: str, answer_key: str) -> float: - return 1.0 if pred_secret == answer_key else 0.0 - - -def get_git_commit() -> str | None: - try: - out = subprocess.check_output(["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL) - except Exception: - return None - return out.decode("utf-8").strip() - - -def build_manifest( - *, - run_id: str, - config_path: str, - runner_config: RunnerConfig, - extra: dict[str, Any] | None = None, -) -> dict[str, Any]: - return { - "run_id": run_id, - "ts_ms": now_ms(), - "config_path": str(Path(config_path).resolve()), - "runner_config": asdict(runner_config), - "platform": { - "system": platform.system(), - "release": platform.release(), - "python": platform.python_version(), - }, - "env": { - "LLM_BACKEND": os.getenv("LLM_BACKEND", ""), - "OPENAI_BASE_URL": os.getenv("OPENAI_BASE_URL", ""), - "OPENAI_MODEL": os.getenv("OPENAI_MODEL", ""), - "MEMLAB_ENABLE_CHECKPOINTER": os.getenv("MEMLAB_ENABLE_CHECKPOINTER", ""), - }, - "git_commit": get_git_commit(), - "extra": extra or {}, - } diff --git a/projects/memlab/memlab/runners/deerflow_adapter.py b/projects/memlab/memlab/runners/deerflow_adapter.py deleted file mode 100644 index 34b5e20..0000000 --- a/projects/memlab/memlab/runners/deerflow_adapter.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -import asyncio -import os -import sys -from dataclasses import dataclass - - -@dataclass -class DeerFlowResponse: - content: str - prompt_tokens: int - completion_tokens: int - - -class DeerFlowAdapter: - def __init__(self) -> None: - backend_path = os.getenv("DEERFLOW_BACKEND_PATH", "").strip() - if not backend_path: - raise RuntimeError("DEERFLOW_BACKEND_PATH must point to DeerFlow backend directory") - if backend_path not in sys.path: - sys.path.append(backend_path) - - from src.client import DeerFlowClient # type: ignore - - checkpointer = None - if os.getenv("MEMLAB_ENABLE_CHECKPOINTER", "1") == "1": - from memlab.checkpoint.langgraph_pg import get_checkpointer - - checkpointer = get_checkpointer() - - self.client = DeerFlowClient(checkpointer=checkpointer) - - async def chat(self, prompt: str, thread_id: str) -> DeerFlowResponse: - def _call() -> str: - resp = self.client.chat(prompt, thread_id=thread_id) - return resp if isinstance(resp, str) else str(resp) - - content = await asyncio.to_thread(_call) - # DeerFlow client does not always expose usage in plain chat return. - return DeerFlowResponse( - content=content, - prompt_tokens=max(1, len(prompt) // 4), - completion_tokens=max(1, len(content) // 4), - ) diff --git a/projects/memlab/memlab/runners/e2e_runner.py b/projects/memlab/memlab/runners/e2e_runner.py deleted file mode 100644 index 5df5e5b..0000000 --- a/projects/memlab/memlab/runners/e2e_runner.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import annotations - -from memlab.runner_e2e import main - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/runners/llm.py b/projects/memlab/memlab/runners/llm.py deleted file mode 100644 index f39f890..0000000 --- a/projects/memlab/memlab/runners/llm.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -import json -import os -import re -from typing import Any - -import httpx - -from .common import extract_secret_code - - -SECRET_JSON_RE = re.compile(r'"secret_code"\s*:\s*"([A-Z0-9]+)"') - - -def _build_stub_json(task_id: str, secret: str) -> str: - return json.dumps({"task_id": task_id, "secret_code": secret}, ensure_ascii=False) - - -async def run_stub_llm(task_id: str, messages: list[dict[str, Any]], evidence_text: str) -> dict[str, Any]: - secret = extract_secret_code(evidence_text) - content = _build_stub_json(task_id, secret) - return { - "content": content, - "prompt_tokens": sum(max(1, len(m.get("content", "")) // 4) for m in messages), - "completion_tokens": max(1, len(content) // 4), - } - - -async def run_openai_compatible(task_id: str, messages: list[dict[str, Any]]) -> dict[str, Any]: - base_url = os.getenv("OPENAI_BASE_URL", "").rstrip("/") - api_key = os.getenv("OPENAI_API_KEY", "") - model = os.getenv("OPENAI_MODEL", "") - if not base_url or not api_key or not model: - raise RuntimeError("OPENAI_BASE_URL, OPENAI_API_KEY, OPENAI_MODEL must be set for openai_compatible mode") - payload = { - "model": model, - "messages": messages, - "temperature": 0, - "response_format": {"type": "json_object"}, - } - headers = {"Authorization": f"Bearer {api_key}"} - try: - async with httpx.AsyncClient(timeout=60.0) as client: - r = await client.post(f"{base_url}/chat/completions", json=payload, headers=headers) - except Exception as exc: - raise RuntimeError(type(exc).__name__) from None - if r.status_code >= 400: - raise RuntimeError(f"HTTP {r.status_code}") - body = r.json() - content = body["choices"][0]["message"]["content"] - usage = body.get("usage", {}) or {} - return { - "content": content, - "prompt_tokens": int(usage.get("prompt_tokens", 0)), - "completion_tokens": int(usage.get("completion_tokens", 0)), - } - - -def parse_model_secret(response_content: str) -> str: - try: - obj = json.loads(response_content) - if isinstance(obj, dict): - sc = obj.get("secret_code") - return sc if isinstance(sc, str) else "" - except Exception: - pass - m = SECRET_JSON_RE.search(response_content) - return m.group(1) if m else "" diff --git a/projects/memlab/memlab/runners/trace_replay.py b/projects/memlab/memlab/runners/trace_replay.py deleted file mode 100644 index 8458c9e..0000000 --- a/projects/memlab/memlab/runners/trace_replay.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import annotations - -from memlab.runner_stub import main - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/runtime_paths.py b/projects/memlab/memlab/runtime_paths.py deleted file mode 100644 index 0f276e1..0000000 --- a/projects/memlab/memlab/runtime_paths.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Resolve the external DeerFlow checkout without copying its source tree.""" -from __future__ import annotations - -import os -from pathlib import Path - - -def deerflow_root() -> Path: - configured = os.getenv("MEMLAB_DEERFLOW_ROOT", "").strip() - return Path(configured or "deer-flow").expanduser().resolve() diff --git a/projects/memlab/memlab/session_logging.py b/projects/memlab/memlab/session_logging.py deleted file mode 100644 index 3c2e573..0000000 --- a/projects/memlab/memlab/session_logging.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Attribute shared-manager operations to the current async session.""" -from contextvars import ContextVar -from typing import Any - -current_agent_id: ContextVar[str | None] = ContextVar("memory_agent_id", default=None) - - -def make_operation_logger(*, run_id, mem_rows, event_writer): - def log_operation(row: dict[str, Any]) -> None: - event = dict(row) - agent_id = event.get("agent_id") or current_agent_id.get() - event["run_id"] = run_id - event["agent_id"] = agent_id - event["session_id"] = f"{run_id}-{agent_id}" if agent_id else None - mem_rows.append(event) - event_writer.append(event) - - return log_operation diff --git a/projects/memlab/memlab/tools/__init__.py b/projects/memlab/memlab/tools/__init__.py deleted file mode 100644 index b08b766..0000000 --- a/projects/memlab/memlab/tools/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Utility tools for local deterministic retrieval/embedding.""" diff --git a/projects/memlab/memlab/tools/doctor.py b/projects/memlab/memlab/tools/doctor.py deleted file mode 100644 index 68d9686..0000000 --- a/projects/memlab/memlab/tools/doctor.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import annotations - -import importlib -import socket - - -def check_import(name: str) -> tuple[str, bool, str]: - try: - importlib.import_module(name) - return name, True, "" - except Exception as exc: - return name, False, str(exc) - - -def check_port(host: str, port: int, timeout: float = 1.0) -> bool: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(timeout) - return s.connect_ex((host, port)) == 0 - - -def main() -> None: - modules = ["yaml", "pydantic", "numpy", "matplotlib", "redis", "asyncpg", "httpx"] - print("== Python dependencies ==") - for m, ok, err in (check_import(x) for x in modules): - status = "OK" if ok else f"MISSING ({err})" - print(f"{m:12} {status}") - - print("\n== Service reachability ==") - for host, port, name in [ - ("127.0.0.1", 6379, "redis"), - ("127.0.0.1", 5432, "postgres"), - ("127.0.0.1", 6333, "qdrant"), - ]: - print(f"{name:12} {'UP' if check_port(host, port) else 'DOWN'} ({host}:{port})") - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/tools/embedding.py b/projects/memlab/memlab/tools/embedding.py deleted file mode 100644 index acf8b16..0000000 --- a/projects/memlab/memlab/tools/embedding.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -import hashlib -import math -import re - - -TOKEN_RE = re.compile(r"[A-Za-z0-9_]+") - - -def hash_embedding(text: str, dim: int = 64) -> list[float]: - """ - Deterministic lightweight embedding for reproducible local experiments. - This is not semantically strong but is stable and cheap for ablation runs. - """ - vec = [0.0] * dim - for token in TOKEN_RE.findall(text.lower()): - h = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest() - idx = int.from_bytes(h, "little") % dim - sign = 1.0 if (h[0] & 1) == 0 else -1.0 - vec[idx] += sign - norm = math.sqrt(sum(v * v for v in vec)) - if norm == 0: - return vec - return [v / norm for v in vec] - - -def cosine(a: list[float], b: list[float]) -> float: - return float(sum(x * y for x, y in zip(a, b))) diff --git a/projects/memlab/memlab/types.py b/projects/memlab/memlab/types.py deleted file mode 100644 index f251bcc..0000000 --- a/projects/memlab/memlab/types.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, Literal - - -Tier = Literal["L0", "L1", "L2", "L3"] - - -@dataclass -class RunnerConfig: - variant_id: str - n_agents: int - llm_backend: Literal["stub", "openai_compatible", "deerflow"] - seed: int - duration_s: int - redis_policy: Literal["allkeys-lfu", "allkeys-lru"] - summarization_on: bool - repeat_idx: int - - -@dataclass -class EventLog: - run_id: str - ts_ms: int - agent_id: str - task_id: str - op: str - tier: str - lat_ms: float - hit: bool | None - bytes: int | None - tokens_in: int | None - tokens_out: int | None - error: str | None - extra: dict[str, Any] | None = None diff --git a/projects/memlab/memlab/workloads/__init__.py b/projects/memlab/memlab/workloads/__init__.py deleted file mode 100644 index 074c1f4..0000000 --- a/projects/memlab/memlab/workloads/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Workload generators.""" diff --git a/projects/memlab/memlab/workloads/gen_corpus.py b/projects/memlab/memlab/workloads/gen_corpus.py deleted file mode 100644 index 9a2010a..0000000 --- a/projects/memlab/memlab/workloads/gen_corpus.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -import argparse -import random -import string -from pathlib import Path - -from memlab.io import jsonl_append - - -def random_secret(rng: random.Random, n: int = 12) -> str: - alphabet = string.ascii_uppercase + string.digits - return "".join(rng.choice(alphabet) for _ in range(n)) - - -def random_body(rng: random.Random, min_bytes: int, max_bytes: int, secret: str) -> str: - target = rng.randint(min_bytes, max_bytes) - words = [] - while len(" ".join(words).encode("utf-8")) < target: - token = "".join(rng.choice(string.ascii_lowercase) for _ in range(rng.randint(4, 11))) - words.append(token) - text = " ".join(words) - return f"{text}\nSECRET_CODE:{secret}\n{text[: min(400, len(text))]}" - - -def make_zipf_sampler(n_docs: int, s: float) -> tuple[list[float], list[int]]: - weights = [1.0 / (i + 1) ** s for i in range(n_docs)] - total = sum(weights) - probs = [w / total for w in weights] - cdf = [] - acc = 0.0 - for p in probs: - acc += p - cdf.append(acc) - return cdf, list(range(n_docs)) - - -def sample_zipf(rng: random.Random, cdf: list[float], items: list[int]) -> int: - x = rng.random() - lo, hi = 0, len(cdf) - 1 - while lo < hi: - mid = (lo + hi) // 2 - if cdf[mid] < x: - lo = mid + 1 - else: - hi = mid - return items[lo] - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--seed", type=int, default=42) - ap.add_argument("--docs", type=int, default=10000) - ap.add_argument("--tasks", type=int, default=1000) - ap.add_argument("--doc-bytes-min", type=int, default=1024) - ap.add_argument("--doc-bytes-max", type=int, default=8192) - ap.add_argument("--zipf-s", type=float, default=1.2) - ap.add_argument("--out", type=str, default="data") - args = ap.parse_args() - - out = Path(args.out) - out.mkdir(parents=True, exist_ok=True) - docs_path = out / "docs.jsonl" - tasks_path = out / "tasks.jsonl" - if docs_path.exists(): - docs_path.unlink() - if tasks_path.exists(): - tasks_path.unlink() - - rng = random.Random(args.seed) - docs: list[dict] = [] - for i in range(args.docs): - doc_id = f"d{i:05d}" - secret = random_secret(rng) - title = f"Document {i:05d}" - body = random_body(rng, args.doc_bytes_min, args.doc_bytes_max, secret) - row = { - "doc_id": doc_id, - "title": title, - "body": body, - "secret_code": secret, - } - docs.append(row) - jsonl_append(docs_path, row) - - cdf, items = make_zipf_sampler(len(docs), args.zipf_s) - for t in range(args.tasks): - idx = sample_zipf(rng, cdf, items) - doc = docs[idx] - task = { - "task_id": f"t{t:05d}", - "doc_id": doc["doc_id"], - "doc_title_hint": doc["title"], - "question": f'Find the SECRET_CODE in the document titled "{doc["title"]}". Return JSON only.', - "answer_key": doc["secret_code"], - } - jsonl_append(tasks_path, task) - - print(f"Wrote {args.docs} docs to {docs_path}") - print(f"Wrote {args.tasks} tasks to {tasks_path}") - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/memlab/workloads/gen_trace.py b/projects/memlab/memlab/workloads/gen_trace.py deleted file mode 100644 index 0e0f56c..0000000 --- a/projects/memlab/memlab/workloads/gen_trace.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import annotations - -import argparse -import random -from pathlib import Path - -from memlab.io import jsonl_append - - -def parse_ratio(ratio: str) -> tuple[int, int]: - left, right = ratio.split(":") - return int(left), int(right) - - -def bounded_lognormal_bytes(rng: random.Random, mean_bytes: int, p99_bytes: int) -> int: - # Approximation tuned for reproducibility, not distribution fitting precision. - sigma = 1.0 - mu = max(1.0, mean_bytes / 3.0) - x = int(rng.lognormvariate(mu=0.0, sigma=sigma) * mu) - return max(64, min(p99_bytes, x)) - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--seed", type=int, default=7) - ap.add_argument("--agents", type=int, default=200) - ap.add_argument("--duration-s", type=int, default=300) - ap.add_argument("--rw-ratio", type=str, default="9:1") - ap.add_argument("--zipf-s", type=float, default=1.3) - ap.add_argument("--mean-bytes", type=int, default=2048) - ap.add_argument("--p99-bytes", type=int, default=131072) - ap.add_argument("--ttl-s", type=int, default=300) - ap.add_argument("--out", type=str, default="traces/trace_200a.jsonl") - args = ap.parse_args() - - rng = random.Random(args.seed) - out_path = Path(args.out) - out_path.parent.mkdir(parents=True, exist_ok=True) - if out_path.exists(): - out_path.unlink() - - read_w, write_w = parse_ratio(args.rw_ratio) - op_weights = [("get", read_w), ("put", write_w)] - - # Build key popularity to emulate hot set access. - n_keys = max(1000, args.agents * 20) - key_weights = [1.0 / ((i + 1) ** args.zipf_s) for i in range(n_keys)] - total = sum(key_weights) - key_probs = [w / total for w in key_weights] - key_cdf = [] - acc = 0.0 - for p in key_probs: - acc += p - key_cdf.append(acc) - - def sample_key() -> str: - x = rng.random() - lo, hi = 0, len(key_cdf) - 1 - while lo < hi: - mid = (lo + hi) // 2 - if key_cdf[mid] < x: - lo = mid + 1 - else: - hi = mid - return f"k-{lo:06d}" - - # Average 20 ops/sec/agent in simulated timeline. - total_ops = args.duration_s * args.agents * 20 - ts_ms = 0 - for _ in range(total_ops): - agent_idx = rng.randrange(args.agents) - agent_id = f"a{agent_idx:04d}" - op = rng.choices([x[0] for x in op_weights], weights=[x[1] for x in op_weights], k=1)[0] - key = sample_key() - b = bounded_lognormal_bytes(rng, args.mean_bytes, args.p99_bytes) - rec = { - "ts_ms": ts_ms, - "agent_id": agent_id, - "op": op, - "key": key, - "bytes": b, - } - if op == "put": - rec["ttl_s"] = args.ttl_s - rec["importance"] = round(rng.random(), 4) - jsonl_append(out_path, rec) - ts_ms += rng.randint(0, 4) - - print(f"Wrote {total_ops} events to {out_path}") - - -if __name__ == "__main__": - main() diff --git a/projects/memlab/pyproject.toml b/projects/memlab/pyproject.toml deleted file mode 100644 index 9b86a08..0000000 --- a/projects/memlab/pyproject.toml +++ /dev/null @@ -1,21 +0,0 @@ -[build-system] -requires = ["setuptools>=68", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "memlab" -version = "0.1.0" -description = "Multi-level agent memory components and experiment bookkeeping." -requires-python = ">=3.12" -dependencies = ["pydantic>=2.8.2,<3", "PyYAML>=6.0.2,<7", "httpx>=0.28.1,<1", "numpy>=2.1.1,<3"] - -[project.optional-dependencies] -dev = ["pytest>=8.3.2,<10", "redis>=5.0.8,<7"] -infra = ["redis>=5.0.8,<7", "asyncpg>=0.30.0,<1"] -benchmarks = ["huggingface_hub>=0.24.6,<2", "pandas>=2.2.2,<4", "pyarrow>=17.0.0"] - -[tool.setuptools.packages.find] -include = ["memlab*", "benchmarks*"] - -[tool.pytest.ini_options] -testpaths = ["tests"] diff --git a/projects/memlab/scripts/_infra_common.ps1 b/projects/memlab/scripts/_infra_common.ps1 deleted file mode 100644 index d54b816..0000000 --- a/projects/memlab/scripts/_infra_common.ps1 +++ /dev/null @@ -1,39 +0,0 @@ -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -function Get-ComposeArgs { - return @("-f", "infra/docker-compose.yml") -} - -function Get-RequiredServices { - return @("redis", "postgres", "qdrant") -} - -function Stop-FailClosed { - param( - [string]$Reason - ) - Write-Error "STOP: required services missing; refusing fallback." - if ($Reason) { - Write-Error $Reason - } - exit 1 -} - -function Invoke-DockerCompose { - param( - [Parameter(Mandatory = $true)] - [string[]]$ComposeArgs - ) - $all = (Get-ComposeArgs) + $ComposeArgs - & docker compose @all -} - -function Test-TcpPort { - param( - [Parameter(Mandatory = $true)] - [int]$Port - ) - $r = Test-NetConnection -ComputerName 127.0.0.1 -Port $Port -WarningAction SilentlyContinue - return [bool]$r.TcpTestSucceeded -} diff --git a/projects/memlab/scripts/check_docker.ps1 b/projects/memlab/scripts/check_docker.ps1 deleted file mode 100644 index d338e79..0000000 --- a/projects/memlab/scripts/check_docker.ps1 +++ /dev/null @@ -1,15 +0,0 @@ -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -. "$PSScriptRoot/_infra_common.ps1" - -try { - $out = (& docker info --format '{{.ServerVersion}}' 2>&1 | Out-String).Trim() -} catch { - Stop-FailClosed -Reason "Docker engine is not running." -} -if ($LASTEXITCODE -ne 0 -or -not $out -or $out.ToLowerInvariant().Contains("error") -or $out.ToLowerInvariant().Contains("request returned")) { - Stop-FailClosed -Reason "Docker engine is not running." -} - -Write-Output "Docker engine is running." diff --git a/projects/memlab/scripts/check_infra.ps1 b/projects/memlab/scripts/check_infra.ps1 deleted file mode 100644 index 6e469ae..0000000 --- a/projects/memlab/scripts/check_infra.ps1 +++ /dev/null @@ -1,93 +0,0 @@ -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -. "$PSScriptRoot/_infra_common.ps1" - -& "$PSScriptRoot/check_docker.ps1" | Out-Null - -$required = Get-RequiredServices -$composeServiceList = Invoke-DockerCompose -ComposeArgs @("ps", "--services", "--filter", "status=running") -$running = @{} -foreach ($svc in $composeServiceList) { - $name = ($svc | Out-String).Trim() - if ($name) { - $running[$name] = $true - } -} - -foreach ($svc in $required) { - if (-not $running.ContainsKey($svc)) { - Stop-FailClosed -Reason "Container '$svc' is not running." - } - $cid = (Invoke-DockerCompose -ComposeArgs @("ps", "-q", $svc) | Out-String).Trim() - if (-not $cid) { - Stop-FailClosed -Reason "Unable to resolve container id for '$svc'." - } - $health = (& docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' $cid 2>$null | Out-String).Trim().ToLowerInvariant() - if ($health -ne "healthy" -and $health -ne "none") { - Stop-FailClosed -Reason "Container '$svc' is not healthy." - } -} - -$ports = @{ - redis = 6379 - postgres = 5432 - qdrant = 6333 -} -foreach ($svc in $required) { - $p = $ports[$svc] - if (-not (Test-TcpPort -Port $p)) { - Stop-FailClosed -Reason "Service '$svc' is unreachable on 127.0.0.1:$p." - } -} - -$redisVersionRaw = Invoke-DockerCompose -ComposeArgs @("exec", "-T", "redis", "redis-cli", "-h", "127.0.0.1", "-p", "6379", "INFO", "server") -$redisVersion = (($redisVersionRaw | Select-String "redis_version:").Line -replace "redis_version:", "").Trim() -if (-not $redisVersion) { - Stop-FailClosed -Reason "Unable to read Redis version from Docker container." -} - -$pgVersion = (Invoke-DockerCompose -ComposeArgs @("exec", "-T", "postgres", "sh", "-c", 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SHOW server_version;"') | Out-String).Trim() -if (-not $pgVersion) { - Stop-FailClosed -Reason "Unable to read Postgres version from Docker container." -} - -try { - $qdrantRoot = Invoke-RestMethod -Uri "http://127.0.0.1:6333/" -Method Get -TimeoutSec 5 - $qdrantVersion = [string]$qdrantRoot.version -} catch { - Stop-FailClosed -Reason "Unable to read Qdrant version from Docker container." -} -if (-not $qdrantVersion) { - Stop-FailClosed -Reason "Qdrant did not return a version." -} - -$probe = (Invoke-DockerCompose -ComposeArgs @("exec", "-T", "redis", "redis-cli", "-h", "127.0.0.1", "-p", "6379", "CONFIG", "SET", "maxmemory-policy", "allkeys-lfu") | Out-String).Trim() -if ($probe -notmatch "OK") { - Stop-FailClosed -Reason "Redis allkeys-lfu probe failed; legacy alias A2 (F1+LRU ablation) would be invalid." -} - -$digests = @{} -foreach ($svc in $required) { - $cid = (Invoke-DockerCompose -ComposeArgs @("ps", "-q", $svc) | Out-String).Trim() - if (-not $cid) { - Stop-FailClosed -Reason "Unable to resolve container id for '$svc'." - } - $imageRef = (& docker inspect --format '{{.Config.Image}}' $cid 2>$null | Out-String).Trim() - if (-not $imageRef) { - Stop-FailClosed -Reason "Unable to resolve image ref for '$svc'." - } - $digest = (& docker image inspect $imageRef --format '{{index .RepoDigests 0}}' 2>$null | Out-String).Trim() - if (-not $digest) { - $digest = $imageRef - } - $digests[$svc] = $digest -} - -Write-Output ("Redis version: {0}" -f $redisVersion) -Write-Output ("Postgres version: {0}" -f $pgVersion) -Write-Output ("Qdrant version: {0}" -f $qdrantVersion) -Write-Output ("Redis image digest: {0}" -f $digests["redis"]) -Write-Output ("Postgres image digest: {0}" -f $digests["postgres"]) -Write-Output ("Qdrant image digest: {0}" -f $digests["qdrant"]) -Write-Output "Infra check passed." diff --git a/projects/memlab/scripts/deerflow_embedded_worker.py b/projects/memlab/scripts/deerflow_embedded_worker.py deleted file mode 100644 index cafa698..0000000 --- a/projects/memlab/scripts/deerflow_embedded_worker.py +++ /dev/null @@ -1,153 +0,0 @@ -from __future__ import annotations - -import json -import os -import sys -import time -from pathlib import Path -from typing import Any - - -REPO_ROOT = Path(__file__).resolve().parents[1] -DEERFLOW_ROOT = Path(os.getenv("MEMLAB_DEERFLOW_ROOT", "") or "deer-flow").expanduser().resolve() -BACKEND_ROOT = DEERFLOW_ROOT / "backend" -CONFIG_PATH = DEERFLOW_ROOT / "config.yaml" -DEERFLOW_HOME = BACKEND_ROOT / ".deer-flow" - -os.environ.setdefault("DEER_FLOW_CONFIG_PATH", str(CONFIG_PATH)) -os.environ.setdefault("DEER_FLOW_HOME", str(DEERFLOW_HOME)) -if str(BACKEND_ROOT) not in sys.path: - sys.path.insert(0, str(BACKEND_ROOT)) - -from langchain_core.messages import AIMessage, HumanMessage -from langchain_core.runnables import RunnableConfig - -from src.agents.lead_agent.agent import make_lead_agent -from src.config.app_config import reset_app_config -import src.config.paths as paths_mod - - -_AGENT_CACHE: dict[tuple[str, str], Any] = {} -JSON_PREFIX = "__MEMLAB_JSON__:" - - -def _emit(payload: dict[str, Any]) -> None: - sys.stdout.write(JSON_PREFIX + json.dumps(payload, ensure_ascii=False) + "\n") - sys.stdout.flush() - - -def _agent_key(model_override: str | None, agent_name: str | None) -> tuple[str, str]: - return (str(model_override or "").strip(), str(agent_name or "").strip()) - - -def _build_config(*, model_override: str | None, agent_name: str | None, thread_id: str) -> RunnableConfig: - configurable: dict[str, Any] = { - "thread_id": thread_id, - "thinking_enabled": False, - "is_plan_mode": False, - "subagent_enabled": False, - } - if model_override: - configurable["model_name"] = model_override - configurable["model"] = model_override - if agent_name: - configurable["agent_name"] = agent_name - return RunnableConfig(configurable=configurable, recursion_limit=100) - - -def _get_agent(*, model_override: str | None, agent_name: str | None): - key = _agent_key(model_override, agent_name) - cached = _AGENT_CACHE.get(key) - if cached is not None: - return cached - config = _build_config(model_override=model_override, agent_name=agent_name, thread_id="worker-bootstrap") - agent = make_lead_agent(config) - _AGENT_CACHE[key] = agent - return agent - - -def _extract_text(content: Any) -> str: - if isinstance(content, str): - return content - if isinstance(content, list): - parts: list[str] = [] - for item in content: - if isinstance(item, str): - parts.append(item) - elif isinstance(item, dict) and item.get("type") == "text": - parts.append(str(item.get("text", ""))) - return "\n".join(parts) - if isinstance(content, dict): - return str(content.get("text", "")) - return str(content) - - -def _run_prompt(*, thread_id: str, prompt: str, model_override: str | None, agent_name: str | None) -> dict[str, Any]: - agent = _get_agent(model_override=model_override, agent_name=agent_name) - config = _build_config(model_override=model_override, agent_name=agent_name, thread_id=thread_id) - state: dict[str, Any] = {"messages": [HumanMessage(content=prompt)]} - context = {"thread_id": thread_id} - seen_ids: set[str] = set() - ai_chunks: list[str] = [] - started = time.perf_counter() - for chunk in agent.stream(state, config=config, context=context, stream_mode="values"): - messages = chunk.get("messages", []) - if not isinstance(messages, list): - continue - for msg in messages: - if not isinstance(msg, AIMessage): - continue - msg_id = getattr(msg, "id", None) - if msg_id and msg_id in seen_ids: - continue - if msg_id: - seen_ids.add(msg_id) - text = _extract_text(getattr(msg, "content", "")) - if text: - ai_chunks.append(text) - return { - "content": ai_chunks[-1] if ai_chunks else "{}", - "request_latency_ms": (time.perf_counter() - started) * 1000.0, - } - - -def main() -> int: - reset_app_config() - if hasattr(paths_mod, "_paths"): - paths_mod._paths = None - _emit( - { - "ready": True, - "pid": os.getpid(), - "config_path": os.environ.get("DEER_FLOW_CONFIG_PATH", ""), - "deerflow_home": os.environ.get("DEER_FLOW_HOME", ""), - } - ) - for raw_line in sys.stdin: - line = str(raw_line or "").strip() - if not line: - continue - try: - request = json.loads(line) - except Exception as exc: - _emit({"ok": False, "error": f"invalid_request_json: {exc}"}) - continue - req_type = str(request.get("type", "run") or "run").strip().lower() - if req_type == "shutdown": - _emit({"ok": True, "shutdown": True}) - return 0 - try: - result = _run_prompt( - thread_id=str(request.get("thread_id", "") or "embedded-thread"), - prompt=str(request.get("prompt", "") or ""), - model_override=str(request.get("model_override", "") or "") or None, - agent_name=str(request.get("agent_name", "") or "") or None, - ) - _emit({"ok": True, **result}) - except Exception as exc: - _emit({"ok": False, "error": f"{type(exc).__name__}: {exc}"}) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/projects/memlab/scripts/down_infra.ps1 b/projects/memlab/scripts/down_infra.ps1 deleted file mode 100644 index 28b1451..0000000 --- a/projects/memlab/scripts/down_infra.ps1 +++ /dev/null @@ -1,7 +0,0 @@ -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -. "$PSScriptRoot/_infra_common.ps1" - -Invoke-DockerCompose -ComposeArgs @("down") -Write-Output "Infra services stopped." diff --git a/projects/memlab/scripts/run_microbench.ps1 b/projects/memlab/scripts/run_microbench.ps1 deleted file mode 100644 index 592f863..0000000 --- a/projects/memlab/scripts/run_microbench.ps1 +++ /dev/null @@ -1,17 +0,0 @@ -param( - [int]$Requests = 200 -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -. "$PSScriptRoot/_infra_common.ps1" - -& "$PSScriptRoot/check_docker.ps1" -& "$PSScriptRoot/check_infra.ps1" - -python -m memlab.microbench --requests $Requests --out logs/microbench.json -if ($LASTEXITCODE -ne 0) { - Stop-FailClosed -Reason "Microbench failed." -} -Write-Output "Microbench complete: logs/microbench.json" diff --git a/projects/memlab/scripts/run_stub.ps1 b/projects/memlab/scripts/run_stub.ps1 deleted file mode 100644 index 2d7e135..0000000 --- a/projects/memlab/scripts/run_stub.ps1 +++ /dev/null @@ -1,45 +0,0 @@ -param( - [ValidateSet("B0", "B1", "V1", "F1", "F2", "A1", "A2")] - [string]$Variant = "B1", - [int]$Sessions = 200, - [int]$Repeat = 0, - [int]$Seed = 42, - [int]$WorkerProcesses = 1, - [int]$SessionsPerProcess = 200, - [int]$OpsPerSession = 50, - [string]$TracePath = "" -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -. "$PSScriptRoot/_infra_common.ps1" - -& "$PSScriptRoot/check_docker.ps1" -& "$PSScriptRoot/check_infra.ps1" - -if ($Variant -eq "A2") { - & "$PSScriptRoot/switch_redis_policy.ps1" -Policy "allkeys-lru" -} else { - & "$PSScriptRoot/switch_redis_policy.ps1" -Policy "allkeys-lfu" -} - -$cmd = @( - "-m", "memlab.runner_stub", - "--variant", $Variant, - "--n_sessions", "$Sessions", - "--repeat_idx", "$Repeat", - "--seed", "$Seed", - "--worker_processes", "$WorkerProcesses", - "--sessions_per_process", "$SessionsPerProcess", - "--ops_per_session", "$OpsPerSession", - "--workload_id", "SYN-200" -) -if ($TracePath) { - $cmd += @("--trace_path", $TracePath) -} - -python @cmd -if ($LASTEXITCODE -ne 0) { - Stop-FailClosed -Reason "Stub runner failed." -} diff --git a/projects/memlab/scripts/switch_redis_policy.ps1 b/projects/memlab/scripts/switch_redis_policy.ps1 deleted file mode 100644 index 7cb05a7..0000000 --- a/projects/memlab/scripts/switch_redis_policy.ps1 +++ /dev/null @@ -1,28 +0,0 @@ -param( - [ValidateSet("allkeys-lfu", "allkeys-lru")] - [string]$Policy = "allkeys-lfu" -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -. "$PSScriptRoot/_infra_common.ps1" - -& "$PSScriptRoot/check_infra.ps1" | Out-Null - -$resp = (Invoke-DockerCompose -ComposeArgs @("exec", "-T", "redis", "redis-cli", "-h", "127.0.0.1", "-p", "6379", "CONFIG", "SET", "maxmemory-policy", $Policy) | Out-String).Trim() -if ($resp -notmatch "OK") { - Stop-FailClosed -Reason "Failed to set Redis maxmemory-policy=$Policy" -} - -$resp2 = (Invoke-DockerCompose -ComposeArgs @("exec", "-T", "redis", "redis-cli", "-h", "127.0.0.1", "-p", "6379", "CONFIG", "SET", "maxmemory", "4gb") | Out-String).Trim() -if ($resp2 -notmatch "OK") { - Stop-FailClosed -Reason "Failed to set Redis maxmemory=4gb." -} - -$verify = (Invoke-DockerCompose -ComposeArgs @("exec", "-T", "redis", "redis-cli", "-h", "127.0.0.1", "-p", "6379", "CONFIG", "GET", "maxmemory-policy") | Out-String).Trim() -if ($verify -notmatch $Policy) { - Stop-FailClosed -Reason "Redis policy verification failed (expected $Policy)." -} - -Write-Output "Applied Redis policy: $Policy" diff --git a/projects/memlab/scripts/up_infra.ps1 b/projects/memlab/scripts/up_infra.ps1 deleted file mode 100644 index feb0fb9..0000000 --- a/projects/memlab/scripts/up_infra.ps1 +++ /dev/null @@ -1,35 +0,0 @@ -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -. "$PSScriptRoot/_infra_common.ps1" - -& "$PSScriptRoot/check_docker.ps1" - -Write-Output "Starting infra services (Redis/Postgres/Qdrant)..." -Invoke-DockerCompose -ComposeArgs @("up", "-d") - -$deadline = (Get-Date).AddMinutes(5) -$required = Get-RequiredServices -while ((Get-Date) -lt $deadline) { - $allHealthy = $true - foreach ($svc in $required) { - $cid = (Invoke-DockerCompose -ComposeArgs @("ps", "-q", $svc) | Out-String).Trim() - if (-not $cid) { - $allHealthy = $false - break - } - $health = (& docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' $cid 2>$null | Out-String).Trim().ToLowerInvariant() - if ($health -ne "healthy" -and $health -ne "none") { - $allHealthy = $false - break - } - } - if ($allHealthy) { - Write-Output "Infra services report healthy state." - & "$PSScriptRoot/check_infra.ps1" - exit 0 - } - Start-Sleep -Seconds 2 -} - -Stop-FailClosed -Reason "Infra services did not become healthy within timeout." diff --git a/projects/memlab/sql/001_schema.sql b/projects/memlab/sql/001_schema.sql deleted file mode 100644 index 254ecdf..0000000 --- a/projects/memlab/sql/001_schema.sql +++ /dev/null @@ -1,28 +0,0 @@ -CREATE TABLE IF NOT EXISTS memory_items ( - key TEXT NOT NULL, - agent_id TEXT NOT NULL, - kind TEXT NOT NULL, - text_body TEXT NOT NULL, - bytes INTEGER NOT NULL, - importance DOUBLE PRECISION NOT NULL, - created_at_ms BIGINT NOT NULL, - last_access_ms BIGINT NOT NULL, - access_count BIGINT NOT NULL DEFAULT 0, - metadata JSONB NOT NULL DEFAULT '{}'::jsonb, - embedding JSONB, - PRIMARY KEY (key, agent_id) -); - -CREATE INDEX IF NOT EXISTS idx_memory_items_agent_time - ON memory_items (agent_id, last_access_ms DESC); - -CREATE TABLE IF NOT EXISTS checkpoints ( - agent_id TEXT NOT NULL, - thread_id TEXT NOT NULL, - ts_ms BIGINT NOT NULL, - state_json JSONB NOT NULL, - PRIMARY KEY (agent_id, thread_id, ts_ms) -); - -CREATE INDEX IF NOT EXISTS idx_checkpoints_latest - ON checkpoints (agent_id, thread_id, ts_ms DESC); diff --git a/projects/memlab/tests/test_benchmark_discriminativeness_gates.py b/projects/memlab/tests/test_benchmark_discriminativeness_gates.py deleted file mode 100644 index 931504b..0000000 --- a/projects/memlab/tests/test_benchmark_discriminativeness_gates.py +++ /dev/null @@ -1,150 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from memlab.analysis.benchmark_discriminativeness_gates import _analyze_workload, _collect_rows - - -def test_collect_rows_honors_track_id_filter(tmp_path: Path) -> None: - runs_dir = tmp_path / "runs" - runs_dir.mkdir() - base_summary = { - "success_rate": 1.0, - "strict_accuracy": 0.5, - "relaxed_accuracy": 0.5, - "benchmark_score_on_total": 0.5, - "benchmark_metric": "exact_option", - "latency": {"processing": {"p95": 10.0}, "end_to_end": {"p95": 12.0}}, - } - for track_id in ("mainline:memory_neutral", "mainline:memory_beneficial"): - run_dir = runs_dir / track_id.replace(":", "-") - run_dir.mkdir() - manifest = { - "run_id": run_dir.name, - "variant_id": "B1", - "workload_id": "PUB-LONGBENCH_V2-SMOKE-MAINLINE", - "repeat_idx": 0, - "seed": 42, - "model_id": "model-x", - "extra": { - "tasks_path": "benchmarks/public_main/longbench_v2/tasks_internal.jsonl", - "experiment_track": "mainline", - "track_id": track_id, - }, - "track_id": track_id, - "env_valid": True, - "repro_valid": True, - "llm_real": True, - } - (run_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") - (run_dir / "summary.json").write_text(json.dumps(base_summary, ensure_ascii=False, indent=2), encoding="utf-8") - - rows = _collect_rows( - runs_dir, - workload_id="PUB-LONGBENCH_V2-SMOKE-MAINLINE", - dataset_filter="longbench_v2", - model_id_filter="model-x", - experiment_track_filter="mainline", - track_id_filter="mainline:memory_beneficial", - ) - assert len(rows) == 1 - assert rows[0]["track_id"] == "mainline:memory_beneficial" - - -def test_collect_rows_excludes_nonaccepted_and_dedupes_latest(tmp_path: Path) -> None: - runs_dir = tmp_path / "runs" - runs_dir.mkdir() - summary_old = { - "success_rate": 1.0, - "strict_accuracy": 0.2, - "relaxed_accuracy": 0.2, - "benchmark_score_on_total": 0.2, - "benchmark_metric": "exact_option", - "latency": {"processing": {"p95": 10.0}, "end_to_end": {"p95": 12.0}}, - } - summary_new = { - "success_rate": 1.0, - "strict_accuracy": 0.8, - "relaxed_accuracy": 0.8, - "benchmark_score_on_total": 0.8, - "benchmark_metric": "exact_option", - "latency": {"processing": {"p95": 10.0}, "end_to_end": {"p95": 12.0}}, - } - accepted_manifest = { - "variant_id": "B1", - "workload_id": "PUB-LONGBENCH_V2-SMOKE-TRACK-M", - "repeat_idx": 0, - "seed": 42, - "model_id": "model-x", - "extra": { - "tasks_path": "benchmarks/public_main/longbench_v2/tasks_internal.jsonl", - "experiment_track": "mainline", - "track_id": "track-m:memory_beneficial", - }, - "track_id": "track-m:memory_beneficial", - "env_valid": True, - "repro_valid": True, - "llm_real": True, - "acceptance_status": "accepted", - "diagnostic_only": False, - "summary_chain_valid": True, - } - invalid_manifest = dict(accepted_manifest) - invalid_manifest["acceptance_status"] = "invalid" - - old_run = runs_dir / "old-accepted" - old_run.mkdir() - (old_run / "manifest.json").write_text(json.dumps({**accepted_manifest, "run_id": old_run.name}, ensure_ascii=False, indent=2), encoding="utf-8") - (old_run / "summary.json").write_text(json.dumps(summary_old, ensure_ascii=False, indent=2), encoding="utf-8") - - invalid_run = runs_dir / "invalid-run" - invalid_run.mkdir() - (invalid_run / "manifest.json").write_text(json.dumps({**invalid_manifest, "run_id": invalid_run.name}, ensure_ascii=False, indent=2), encoding="utf-8") - (invalid_run / "summary.json").write_text(json.dumps(summary_new, ensure_ascii=False, indent=2), encoding="utf-8") - - new_run = runs_dir / "new-accepted" - new_run.mkdir() - (new_run / "manifest.json").write_text(json.dumps({**accepted_manifest, "run_id": new_run.name}, ensure_ascii=False, indent=2), encoding="utf-8") - (new_run / "summary.json").write_text(json.dumps(summary_new, ensure_ascii=False, indent=2), encoding="utf-8") - - rows = _collect_rows( - runs_dir, - workload_id="PUB-LONGBENCH_V2-SMOKE-TRACK-M", - dataset_filter="longbench_v2", - model_id_filter="model-x", - experiment_track_filter="mainline", - track_id_filter="track-m:memory_beneficial", - ) - assert len(rows) == 1 - assert rows[0]["benchmark_score"] == 0.8 - - -def test_analyze_workload_marks_missing_b1_f1_as_insufficient() -> None: - result = _analyze_workload( - dataset="longbench_v2", - workload_id="PUB-LONGBENCH_V2-SMOKE-TRACK-S", - rows=[ - { - "workload_id": "PUB-LONGBENCH_V2-SMOKE-TRACK-S", - "variant": "B0", - "env_valid": True, - "repro_valid": True, - "llm_real": True, - "benchmark_score": 0.25, - "strict_accuracy": 0.25, - "relaxed_accuracy": 0.25, - "no_answer_rate": 0.0, - "success_rate": 1.0, - "parse_failure_rate": 0.0, - "processing_p95_ms": 10.0, - "end_to_end_p95_ms": 12.0, - "benchmark_metric": "exact_option", - } - ], - all_rows=[], - null_workload_id="", - ) - assert result["eligible_for_pilot"] is False - assert result["recommended_disposition"] == "insufficient_comparison_variants" - assert result["benchmark_diff"] is None diff --git a/projects/memlab/tests/test_branch_audit.py b/projects/memlab/tests/test_branch_audit.py deleted file mode 100644 index 6665039..0000000 --- a/projects/memlab/tests/test_branch_audit.py +++ /dev/null @@ -1,214 +0,0 @@ -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -from memlab.analysis.branch_audit import _audit_run_outputs, _resolve_dataset_audit, command_audit_aggregate -from memlab.metrics.summary_builder import aggregate_runs - - -def test_dataset_audit_detects_full_slice_mixed(tmp_path: Path) -> None: - base_dir = tmp_path / "benchmarks" / "public_main" / "babilong" - base_dir.mkdir(parents=True) - raw_path = base_dir / "raw.json" - raw_path.write_text(json.dumps([{"x": 1}, {"x": 2}, {"x": 3}]), encoding="utf-8") - (base_dir / "manifest.json").write_text( - json.dumps( - { - "dataset": "babilong", - "raw_path": str(raw_path), - "sample_count": 2, - }, - ensure_ascii=False, - indent=2, - ), - encoding="utf-8", - ) - tasks = [ - {"task_id": "t1", "doc_id": "d1", "gold_answer": "a"}, - {"task_id": "t2", "doc_id": "d2", "gold_answer": "b"}, - ] - docs = [ - {"doc_id": "d1", "body": "x"}, - {"doc_id": "d2", "body": "y"}, - ] - (base_dir / "tasks_internal.jsonl").write_text("\n".join(json.dumps(x) for x in tasks) + "\n", encoding="utf-8") - (base_dir / "docs_internal.jsonl").write_text("\n".join(json.dumps(x) for x in docs) + "\n", encoding="utf-8") - - report, anomalies = _resolve_dataset_audit(base_dir, max_samples=0) - assert "FULL_SLICE_MIXED" in anomalies - assert report["full_or_slice"] == "full" - - -def test_run_audit_detects_missing_eventlog(tmp_path: Path) -> None: - run_dir = tmp_path / "runs" / "f1-e2e-r0" - run_dir.mkdir(parents=True) - manifest = { - "run_id": "f1-e2e-r0", - "variant_id": "F1", - "workload_id": "PUB-LONGBENCH_V2-FULL-ARCH", - "repeat_idx": 0, - "model_id": "Pro/deepseek-ai/DeepSeek-V3.2", - "model_base_url": "https://example.test/v1", - "extra": { - "tasks_path": "benchmarks/public_main/longbench_v2/tasks_internal.jsonl", - "docs_path": "benchmarks/public_main/longbench_v2/docs_internal.jsonl", - "prompt_mode": "memory_beneficial", - "qa_prompt_profile": "multiple_choice_letter", - "session_scope": "task", - "experiment_track": "mainline", - "track_id": "mainline:memory_beneficial", - "model_lock": { - "expected_base_url": "https://example.test/v1", - "expected_model_id": "Pro/deepseek-ai/DeepSeek-V3.2", - }, - }, - } - summary = { - "run_id": "f1-e2e-r0", - "variant_id": "F1", - "workload_id": "PUB-LONGBENCH_V2-FULL-ARCH", - "parse_failure_rate_on_total": 0.0, - "no_answer_rate_on_total": 0.0, - } - (run_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") - (run_dir / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") - - report, anomalies, diagnostic_only = _audit_run_outputs( - run_dir, - expected_prompt_mode="memory_beneficial", - expected_qa_profile="multiple_choice_letter", - expected_session_scope="task", - expected_model_id="Pro/deepseek-ai/DeepSeek-V3.2", - expected_base_url="https://example.test/v1", - expected_track_id="mainline:memory_beneficial", - ) - assert "EVENTLOG_MISSING" in anomalies - assert not diagnostic_only - assert not report["summary_chain_valid"] - - -def test_aggregate_runs_excludes_nonaccepted_runs(tmp_path: Path) -> None: - runs_dir = tmp_path / "runs" - accepted_dir = runs_dir / "run-accepted" - rejected_dir = runs_dir / "run-rejected" - accepted_dir.mkdir(parents=True) - rejected_dir.mkdir(parents=True) - - base_manifest = { - "variant_id": "B1", - "workload_id": "E2E-UNIT", - "repeat_idx": 0, - "seed": 42, - "metrics_schema_version": 5, - "env_valid": True, - "repro_valid": True, - "llm_real": True, - "track_id": "mainline:memory_neutral", - "summary_chain_valid": True, - "extra": { - "tasks_path": "workloads/e2e/tasks.jsonl", - "prompt_mode": "memory_neutral", - "experiment_track": "mainline", - }, - } - accepted_manifest = { - **base_manifest, - "run_id": "run-accepted", - "acceptance_status": "accepted", - "diagnostic_only": False, - "invalid_reason_codes": [], - } - rejected_manifest = { - **base_manifest, - "run_id": "run-rejected", - "acceptance_status": "invalid", - "diagnostic_only": False, - "invalid_reason_codes": ["MODEL_LOCK_DRIFT"], - } - summary = { - "run_id": "run-accepted", - "variant_id": "B1", - "workload_id": "E2E-UNIT", - "success_rate": 1.0, - "latency": {"processing": {"p95": 10.0}, "end_to_end": {"p95": 11.0}}, - "strict_accuracy_on_total": 0.5, - "relaxed_accuracy_on_total": 0.5, - "benchmark_score_on_total": 0.5, - "accuracy_applicable": True, - "throughput_tasks_per_min": 1.0, - "error_breakdown": {"by_category": {}}, - } - (accepted_dir / "manifest.json").write_text(json.dumps(accepted_manifest, ensure_ascii=False, indent=2), encoding="utf-8") - (accepted_dir / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") - (rejected_dir / "manifest.json").write_text(json.dumps(rejected_manifest, ensure_ascii=False, indent=2), encoding="utf-8") - rejected_summary = dict(summary) - rejected_summary["run_id"] = "run-rejected" - (rejected_dir / "summary.json").write_text(json.dumps(rejected_summary, ensure_ascii=False, indent=2), encoding="utf-8") - - result = aggregate_runs( - runs_dir=runs_dir, - summary_csv=tmp_path / "summary.csv", - stats_json=tmp_path / "summary.stats.json", - report_md=tmp_path / "experiment_report.md", - report_alias_md=tmp_path / "report.md", - ) - assert result["rows"] == 1 - assert result["excluded_rows"] == 1 - csv_text = (tmp_path / "summary.csv").read_text(encoding="utf-8") - assert "run-accepted" in csv_text - assert "run-rejected" not in csv_text - - -def test_aggregate_audit_scopes_to_workload_id(tmp_path: Path) -> None: - runs_dir = tmp_path / "runs" - runs_dir.mkdir(parents=True) - - def write_run(name: str, workload_id: str, prompt_mode: str) -> None: - run_dir = runs_dir / name - run_dir.mkdir() - manifest = { - "run_id": name, - "variant_id": "B0", - "workload_id": workload_id, - "repeat_idx": 0, - "model_id": "Pro/deepseek-ai/DeepSeek-V3.2", - "model_base_url": "https://example.test/v1", - "acceptance_status": "accepted", - "diagnostic_only": False, - "summary_chain_valid": True, - "extra": { - "dataset_id": "longbench_v2", - "track_id": "track-s:memory_neutral", - "prompt_mode": prompt_mode, - "qa_prompt_profile": "multiple_choice_letter", - "session_scope": "task", - "model_lock": { - "expected_base_url": "https://example.test/v1", - "expected_model_id": "Pro/deepseek-ai/DeepSeek-V3.2", - }, - }, - } - summary = { - "run_id": name, - "variant_id": "B0", - "workload_id": workload_id, - "benchmark_score_on_total": 0.5, - } - (run_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") - (run_dir / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") - - write_run("run-smoke", "PUB-LONGBENCH_V2-SMOKE-A", "memory_neutral") - write_run("run-old", "PUB-LONGBENCH_V2-SMOKE-OLD", "memory_beneficial") - - rc = command_audit_aggregate( - argparse.Namespace( - runs_dir=str(runs_dir), - dataset_id="longbench_v2", - track_id="track-s:memory_neutral", - workload_id="PUB-LONGBENCH_V2-SMOKE-A", - artifacts_root=str(tmp_path / "artifacts"), - ) - ) - assert rc == 0 diff --git a/projects/memlab/tests/test_control_plane.py b/projects/memlab/tests/test_control_plane.py deleted file mode 100644 index 0bb8d66..0000000 --- a/projects/memlab/tests/test_control_plane.py +++ /dev/null @@ -1,113 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from memlab.metrics.manifest import build_manifest -from memlab.pipeline.control_plane import ( - BranchKey, - ControlPlane, - accepted_for_official_reporting, - default_manifest_control_fields, - earliest_rollback_stage, - sanitize_path_token, -) - - -def test_build_manifest_includes_control_plane_defaults() -> None: - manifest = build_manifest( - run_id="run-1", - variant_id="F1", - workload_id="PUB-TEST-FULL", - seed=42, - repeat_idx=0, - env_valid=True, - env_invalid_reasons=[], - repro_valid=True, - repro_invalid_reasons=[], - llm_real=True, - llm_invalid_reasons=[], - usage_unavailable=False, - usage_unavailable_reasons=[], - model_base_url="https://example.test/v1", - model_id="model-x", - model_fallback_used=False, - comparable_mode=True, - comparability_invalid_reasons=[], - metrics_schema_version=5, - memlab_git_commit="abc", - deerflow_git_commit="def", - docker_image_digests={}, - infra_versions={}, - extra={"experiment_track": "mainline", "prompt_mode": "memory_beneficial"}, - ) - defaults = default_manifest_control_fields() - for key, value in defaults.items(): - assert key in manifest - if key not in {"track_id"}: - assert manifest[key] == value - assert manifest["track_id"] == "mainline:memory_beneficial" - - -def test_control_plane_invalidate_run_updates_manifest_and_ledgers(tmp_path: Path) -> None: - run_dir = tmp_path / "runs" / "run-1" - run_dir.mkdir(parents=True) - manifest_path = run_dir / "manifest.json" - manifest = build_manifest( - run_id="run-1", - variant_id="F1", - workload_id="PUB-LONGBENCH_V2-FULL", - seed=42, - repeat_idx=0, - env_valid=True, - env_invalid_reasons=[], - repro_valid=True, - repro_invalid_reasons=[], - llm_real=True, - llm_invalid_reasons=[], - usage_unavailable=False, - usage_unavailable_reasons=[], - model_base_url="https://example.test/v1", - model_id="Pro/deepseek-ai/DeepSeek-V3.2", - model_fallback_used=False, - comparable_mode=True, - comparability_invalid_reasons=[], - metrics_schema_version=5, - memlab_git_commit="abc", - deerflow_git_commit="def", - docker_image_digests={}, - infra_versions={}, - extra={ - "tasks_path": "benchmarks/public_main/longbench_v2/tasks_internal.jsonl", - "prompt_mode": "memory_beneficial", - "experiment_track": "mainline", - }, - ) - manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8") - cp = ControlPlane(artifacts_root=tmp_path / "artifacts") - key = BranchKey("longbench_v2", "mainline:memory_beneficial", "F1", 0) - cp.register_branch(key) - cp.invalidate_run( - manifest_path, - key=key, - stage_id="pilot", - anomaly_codes=["REPEAT_DIRECTION_CONFLICT"], - reason="repeat deltas disagree", - diagnostic_only=True, - ) - updated = json.loads(manifest_path.read_text(encoding="utf-8")) - assert updated["acceptance_status"] == "diagnostic_only" - assert updated["rollback_root_stage"] == "pilot" - assert updated["rerun_plan_id"] - assert not accepted_for_official_reporting(updated) - invalid_ledger = (tmp_path / "artifacts" / "ledger" / "invalid_run_ledger.jsonl").read_text(encoding="utf-8") - assert "REPEAT_DIRECTION_CONFLICT" in invalid_ledger - - -def test_earliest_rollback_stage_uses_earliest_upstream_stage() -> None: - assert earliest_rollback_stage(["SUMMARY_CHAIN_INCONSISTENT", "MODEL_LOCK_DRIFT"]) == "audit_protocol" - assert earliest_rollback_stage(["SUMMARY_CHAIN_INCONSISTENT"]) == "smoke" - - -def test_sanitize_path_token_removes_windows_invalid_chars() -> None: - assert sanitize_path_token("mainline:memory_beneficial") == "mainline-memory_beneficial" diff --git a/projects/memlab/tests/test_infra_probe_configuration.py b/projects/memlab/tests/test_infra_probe_configuration.py deleted file mode 100644 index 9382e19..0000000 --- a/projects/memlab/tests/test_infra_probe_configuration.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Check version probing with mocked Docker/HTTP, without starting services.""" -from memlab import envcheck - - -def test_version_probe_uses_container_postgres_configuration(monkeypatch): - commands = [] - - def run(command, **_): - commands.append(command) - if "redis-cli" in command: - return True, "redis_version:8.0.0\n" - if "postgres" in command and "sh" in command: - return True, " 18.3 \n" - if "ps" in command: - return True, "synthetic-container" - if "image" in command: - return True, "synthetic-image@sha256:example" - return True, "synthetic-image" - - class Response: - def raise_for_status(self): - pass - - def json(self): - return {"version": "1.0.0"} - - class Client: - def __init__(self, **_): - pass - - def __enter__(self): - return self - - def __exit__(self, *_): - pass - - def get(self, _): - return Response() - - monkeypatch.setattr(envcheck, "_run", run) - monkeypatch.setattr(envcheck.httpx, "Client", Client) - versions, _, errors = envcheck._probe_infra_versions_and_digests() - assert errors == [] - assert versions["postgres"] == "18.3" - postgres_command = next(cmd for cmd in commands if "postgres" in cmd and "sh" in cmd) - shell_command = postgres_command[-1] - assert '"$POSTGRES_USER"' in shell_command - assert '"$POSTGRES_DB"' in shell_command - assert "POSTGRES_PASSWORD" not in shell_command - assert "PG_URI" not in shell_command diff --git a/projects/memlab/tests/test_memory_manager_policy.py b/projects/memlab/tests/test_memory_manager_policy.py deleted file mode 100644 index 14a1cda..0000000 --- a/projects/memlab/tests/test_memory_manager_policy.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -import asyncio - -from memlab.config import MemlabConfig -from memlab.memory.types import ContextBudget, MemoryItem -from memlab.memory_manager.manager import MultiLevelMemoryManager - - -def _mk_item(*, key: str, role: str, text: str, retrieval_shard: bool = False) -> MemoryItem: - return MemoryItem( - key=key, - agent_id="agent_0001", - kind="tool_result" if role == "tool" else "message", - text=text, - bytes=len(text.encode("utf-8")), - importance=0.9, - created_at_ms=1, - last_access_ms=1, - access_count=1, - metadata={ - "role": role, - "retrieval_shard": retrieval_shard, - }, - ) - - -def test_direct_tool_evidence_detector() -> None: - recent = [ - _mk_item( - key="tool-main", - role="tool", - text="SECRET_CODE: ABC123. " + ("evidence " * 32), - ) - ] - assert MultiLevelMemoryManager._has_direct_tool_evidence( - recent, 'Find the SECRET_CODE in the document titled "Document 00013".' - ) - - -def test_direct_tool_evidence_ignores_shard_only() -> None: - recent = [ - _mk_item( - key="tool-shard", - role="tool", - text="SECRET_CODE: ABC123. " + ("evidence " * 32), - retrieval_shard=True, - ) - ] - assert not MultiLevelMemoryManager._has_direct_tool_evidence( - recent, 'Find the SECRET_CODE in the document titled "Document 00013".' - ) - - -def test_assemble_context_bypasses_retrieval_when_direct_evidence_present() -> None: - cfg = MemlabConfig() - cfg.memory.l1.enabled = False - cfg.memory.l2.enabled = False - cfg.memory.l3.enabled = False - cfg.memory.policy.direct_evidence_retrieval_bypass = True - - manager = MultiLevelMemoryManager(cfg) - manager._recent_messages["agent_0001"] = [ - _mk_item(key="user-1", role="user", text="Task t00013: Find the secret code."), - _mk_item( - key="tool-1", - role="tool", - text="Document body with SECRET_CODE: ABC123 and extra evidence context. " + ("x" * 180), - ), - ] - - pack = asyncio.run( - manager.assemble_context( - "agent_0001", - 'Task t00013: Find the SECRET_CODE in the document titled "Document 00013".', - ContextBudget(max_prompt_tokens=4000, max_memory_injection_tokens=256, keep_recent_messages=10), - ) - ) - - assert pack.trace.get("retrieve_skipped_reason") == "direct_tool_evidence_present" diff --git a/projects/memlab/tests/test_metrics_denominator.py b/projects/memlab/tests/test_metrics_denominator.py deleted file mode 100644 index e4d4915..0000000 --- a/projects/memlab/tests/test_metrics_denominator.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -from memlab.metrics.summary_builder import build_acceptance_result -from memlab.runner_core import compute_run_summary - - -def test_compute_run_summary_emits_total_denominator_accuracy() -> None: - tasks = [ - { - "status": "success", - "strict_score": 1.0, - "relaxed_score": 1.0, - "accuracy": 1.0, - "parse_failure": False, - "no_answer": False, - "queue_wait_ms": 1.0, - "processing_ms": 10.0, - "proc_whole_ms": 10.0, - "proc_llm_ms": 7.0, - "proc_non_llm_ms": 3.0, - "end_to_end_ms": 11.0, - "latency_ms": 11.0, - "task_enqueued_ts": 1000, - "task_end_ts": 1011, - "llm_latency_ms": 7.0, - "context_injected_tokens": 0, - "context_injected_items": 0, - }, - { - "status": "error", - "strict_score": 0.0, - "relaxed_score": 0.0, - "accuracy": 0.0, - "parse_failure": False, - "no_answer": False, - "queue_wait_ms": 2.0, - "processing_ms": 12.0, - "proc_whole_ms": 12.0, - "proc_llm_ms": 8.0, - "proc_non_llm_ms": 4.0, - "end_to_end_ms": 13.0, - "latency_ms": 13.0, - "task_enqueued_ts": 1000, - "task_end_ts": 1013, - "llm_latency_ms": 8.0, - "context_injected_tokens": 0, - "context_injected_items": 0, - }, - ] - summary = compute_run_summary( - run_id="r-test", - variant_id="F1", - workload_id="E2E-UNIT", - n_sessions=2, - tasks=tasks, - mem_ops=[], - ) - assert summary["strict_accuracy"] == 1.0 - assert summary["strict_accuracy_on_total"] == 0.5 - assert summary["strict_correct_total"] == 1 - - -def test_acceptance_prefers_strict_accuracy_on_total() -> None: - rows = [ - { - "variant_id": "B1", - "workload_id": "E2E-UNIT", - "prompt_mode": "memory_neutral", - "tasks_path": "workloads/e2e/tasks.jsonl", - "processing_p95_ms": 1000.0, - "usage_unavailable": True, - "strict_accuracy": 0.20, - "strict_accuracy_on_total": 0.08, - }, - { - "variant_id": "F1", - "workload_id": "E2E-UNIT", - "prompt_mode": "memory_neutral", - "tasks_path": "workloads/e2e/tasks.jsonl", - "processing_p95_ms": 1000.0, - "usage_unavailable": True, - "strict_accuracy": 0.10, - "strict_accuracy_on_total": 0.08, - }, - ] - result = build_acceptance_result(rows, min_p95_improve=0.01, min_prompt_drop=0.01, max_accuracy_drop=0.03) - assert result["accuracy_metric"] == "strict_accuracy_on_total" - assert abs(float(result["accuracy_drop"])) < 1e-9 diff --git a/projects/memlab/tests/test_pipeline_model_profile.py b/projects/memlab/tests/test_pipeline_model_profile.py deleted file mode 100644 index fff7b27..0000000 --- a/projects/memlab/tests/test_pipeline_model_profile.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -from memlab.pipeline.model_profile import ( - build_profile, - classify_model_scale, - estimate_model_size_b, -) - - -def test_estimate_model_size_b() -> None: - assert estimate_model_size_b("Qwen/Qwen3-8B") == 8.0 - assert estimate_model_size_b("SomeModel-70B-Instruct") == 70.0 - assert estimate_model_size_b("no-size-token") is None - - -def test_classify_model_scale() -> None: - assert classify_model_scale(7.0) == "small" - assert classify_model_scale(13.0) == "medium" - assert classify_model_scale(72.0) == "large" - assert classify_model_scale(None) == "unknown" - - -def test_build_profile_runtime_recommendation_uses_scale() -> None: - profile = build_profile( - model_id="Qwen/Qwen3-72B", - base_url="https://api.example.com/v1", - registry_entries=[], - ) - assert profile["model"]["scale_class"] == "large" - rec = profile["recommended_runtime"] - assert rec["e2e_inflight"] == 1 - assert rec["context_top_k"] == 8 - assert "metric_contract" in profile diff --git a/projects/memlab/tests/test_public_qa_packing.py b/projects/memlab/tests/test_public_qa_packing.py deleted file mode 100644 index c3d1de4..0000000 --- a/projects/memlab/tests/test_public_qa_packing.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import annotations - -from benchmarks.public_main.longmemeval.prepare import _resolve_longmemeval_profile -from memlab.runner_core import score_task_answer -from memlab.runner_e2e import ( - _build_qa_evidence_excerpt, - _build_retrieval_shards, - _qa_reasoning_hint, - _question_focus_text, - _resolve_task_qa_prompt_profile, - _summarize_prompt, -) - - -def test_question_focus_text_drops_option_block() -> None: - question = ( - "Which option best describes the author?\n\n" - "Options:\nA. calm\nB. angry\nC. neutral\n\n" - "Return only the option letter (A/B/C/D)." - ) - assert _question_focus_text(question) == "Which option best describes the author?" - - -def test_qa_evidence_excerpt_keeps_tail_signal_for_long_context() -> None: - body = ( - "irrelevant intro " * 200 - + "Sandra moved to the kitchen. " - + "more filler " * 120 - + "Sandra went back to the bathroom." - ) - excerpt = _build_qa_evidence_excerpt( - body=body, - question="Where is Sandra?", - max_chars=480, - ) - assert "bathroom" in excerpt.lower() - - -def test_qa_evidence_excerpt_prefers_latest_state_for_where_question() -> None: - body = ( - ("background " * 40) - + "John moved to the kitchen. " - + ("more filler " * 30) - + "John travelled to the bathroom. " - + ("tail filler " * 20) - ) - excerpt = _build_qa_evidence_excerpt( - body=body, - question="Where is John?", - max_chars=320, - ) - assert "bathroom" in excerpt.lower() - if "kitchen" in excerpt.lower(): - assert excerpt.lower().find("bathroom") < excerpt.lower().find("kitchen") - - -def test_exact_span_benchmark_ignores_leading_article() -> None: - scored = score_task_answer( - raw_output='{"final_answer":"the office"}', - parsed_answer="the office", - parsed_answer_raw="the office", - answer_key="office", - task_type="qa", - answer_metric_hint="exact_span", - ) - assert scored["strict_score"] == 0.0 - assert scored["benchmark_metric"] == "exact_span" - assert scored["benchmark_score"] == 1.0 - - -def test_retrieval_shards_follow_dialogue_boundaries() -> None: - body = ( - "1:00 pm on 1 Jan, 2024\nAlice: I bought a violin.\n\n" - "1:05 pm on 1 Jan, 2024\nBob: Nice instrument.\n\n" - "1:10 pm on 1 Jan, 2024\nAlice: I also bought sheet music.\n\n" - "1:15 pm on 1 Jan, 2024\nBob: Practice every day.\n\n" - "1:20 pm on 1 Jan, 2024\nAlice: I named the violin Aurora.\n\n" - "1:25 pm on 1 Jan, 2024\nBob: Great name.\n" - ) - shards = _build_retrieval_shards(body, shard_chars=120, max_shards=4) - assert shards - assert any("Alice: I named the violin Aurora." in shard["text"] for shard in shards) - - -def test_qa_evidence_excerpt_prefers_substantive_dialogue_blocks_over_greetings() -> None: - body = ( - "1:00 pm\nJohn: Hey Tim!\n\n" - "1:02 pm\nTim: Hey John!\n\n" - "1:10 pm\nJohn: I finally got a signed basketball from the team store.\n\n" - "1:15 pm\nTim: Nice, I also keep my signed basketball in a display case.\n\n" - "1:20 pm\nJohn: That's an awesome collectible.\n" - ) - excerpt = _build_qa_evidence_excerpt( - body=body, - question="What similar sports collectible do Tim and John own?", - max_chars=480, - ) - assert "signed basketball" in excerpt.lower() - - -def test_reasoning_hint_covers_difference_questions() -> None: - hint = _qa_reasoning_hint( - question="For my daily commute, how much more expensive was the taxi ride compared to the train fare?", - qa_prompt_profile="short_answer", - ) - assert "difference or comparison" in hint - - -def test_task_level_qa_profile_override_is_respected() -> None: - profile = _resolve_task_qa_prompt_profile( - task={"metadata": {"qa_prompt_profile": "summary"}}, - default_profile="short_answer", - ) - assert profile == "summary" - - -def test_longmemeval_preference_questions_use_summary_profile() -> None: - profile, metric = _resolve_longmemeval_profile( - "Can you recommend a guide for a synthetic demonstration?", - "single-session-preference", - ) - assert profile == "summary" - assert metric == "token_f1" - - -def test_longmemeval_factoid_questions_stay_short_answer() -> None: - profile, metric = _resolve_longmemeval_profile( - "What color is the example cube?", - "single-session-user", - ) - assert profile == "short_answer" - assert metric == "relaxed_span" - - -def test_memory_beneficial_prompt_can_omit_direct_evidence() -> None: - prompt, evidence_chars_used = _summarize_prompt( - session_id="s1", - question="What did the example agent create?", - context_block="synthetic memory about a blue cube", - body="The example agent created a blue cube.", - task_type="qa", - answer_field="final_answer", - answer_format_mode="strict_json", - prompt_mode="memory_beneficial", - qa_prompt_profile="short_answer", - qa_support_mode="none", - max_prompt_tokens=2048, - max_evidence_chars=600, - ) - assert "[direct evidence omitted by qa_support_mode=none]" in prompt - assert "[supportive direct evidence excerpt]" not in prompt - assert evidence_chars_used == 0 diff --git a/projects/memlab/tests/test_public_track_design_audit.py b/projects/memlab/tests/test_public_track_design_audit.py deleted file mode 100644 index 57ef369..0000000 --- a/projects/memlab/tests/test_public_track_design_audit.py +++ /dev/null @@ -1,118 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -from memlab.analysis.public_track_design_audit import resolve_design_audit - - -def _write_prepared_dir(base_dir: Path, *, dataset: str, tasks: list[dict], docs: list[dict]) -> None: - base_dir.mkdir(parents=True) - (base_dir / "manifest.json").write_text( - json.dumps({"dataset": dataset, "sample_count": len(tasks)}, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - (base_dir / "tasks_internal.jsonl").write_text( - "\n".join(json.dumps(row, ensure_ascii=False) for row in tasks) + "\n", - encoding="utf-8", - ) - (base_dir / "docs_internal.jsonl").write_text( - "\n".join(json.dumps(row, ensure_ascii=False) for row in docs) + "\n", - encoding="utf-8", - ) - - -def test_design_audit_blocks_non_memory_dependent_track_m(tmp_path: Path) -> None: - base_dir = tmp_path / "prepared" / "longbench_v2" - tasks = [ - { - "task_id": "t1", - "doc_id": "d1", - "gold_answer": "A", - "metadata": {"memory_needed": False, "answer_metric_hint": "exact_option"}, - }, - { - "task_id": "t2", - "doc_id": "d2", - "gold_answer": "B", - "metadata": {"memory_needed": False, "answer_metric_hint": "exact_option"}, - }, - ] - docs = [{"doc_id": "d1", "body": "x"}, {"doc_id": "d2", "body": "y"}] - _write_prepared_dir(base_dir, dataset="longbench_v2", tasks=tasks, docs=docs) - - report, anomalies = resolve_design_audit( - base_dir=base_dir, - dataset_id="longbench_v2", - track_id="track-m:memory_beneficial", - experiment_track="memory_exposure", - prompt_mode="memory_beneficial", - qa_support_mode="none", - qa_prompt_profile="multiple_choice_letter", - track_goal="memory_sensitive", - min_memory_needed_ratio=0.5, - ) - assert report["memory_needed_ratio"] == 0.0 - assert "NON_MEMORY_DEPENDENT_DATASET" in anomalies - - -def test_design_audit_blocks_direct_evidence_leak_on_memory_track(tmp_path: Path) -> None: - base_dir = tmp_path / "prepared" / "locomo" - tasks = [ - { - "task_id": "t1", - "doc_id": "d1", - "gold_answer": "a bookmark", - "metadata": {"memory_needed": True, "answer_metric_hint": "token_f1"}, - } - ] - docs = [{"doc_id": "d1", "body": "dialogue"}] - _write_prepared_dir(base_dir, dataset="locomo", tasks=tasks, docs=docs) - - report, anomalies = resolve_design_audit( - base_dir=base_dir, - dataset_id="locomo", - track_id="track-m:memory_beneficial", - experiment_track="memory_exposure", - prompt_mode="memory_beneficial", - qa_support_mode="standard", - qa_prompt_profile="short_answer", - track_goal="memory_sensitive", - min_memory_needed_ratio=0.5, - ) - assert report["memory_needed_ratio"] == 1.0 - assert "DIRECT_EVIDENCE_LEAKAGE" in anomalies - - -def test_design_audit_allows_standard_track_with_mixed_memory_dataset(tmp_path: Path) -> None: - base_dir = tmp_path / "prepared" / "squad_v11" - tasks = [ - { - "task_id": "t1", - "doc_id": "d1", - "gold_answer": "alpha", - "metadata": {"memory_needed": False, "answer_metric_hint": "token_f1"}, - }, - { - "task_id": "t2", - "doc_id": "d2", - "gold_answer": "beta", - "metadata": {"memory_needed": True, "answer_metric_hint": "token_f1"}, - }, - ] - docs = [{"doc_id": "d1", "body": "x"}, {"doc_id": "d2", "body": "y"}] - _write_prepared_dir(base_dir, dataset="squad_v11", tasks=tasks, docs=docs) - - report, anomalies = resolve_design_audit( - base_dir=base_dir, - dataset_id="squad_v11", - track_id="track-s:memory_neutral", - experiment_track="mainline", - prompt_mode="memory_neutral", - qa_support_mode="standard", - qa_prompt_profile="short_answer", - track_goal="standard", - min_memory_needed_ratio=0.5, - ) - assert report["memory_needed_ratio"] == 0.5 - assert anomalies == [] diff --git a/projects/memlab/tests/test_public_track_gate.py b/projects/memlab/tests/test_public_track_gate.py deleted file mode 100644 index 1c16a70..0000000 --- a/projects/memlab/tests/test_public_track_gate.py +++ /dev/null @@ -1,207 +0,0 @@ -from __future__ import annotations - -from memlab.analysis.public_track_gate import _build_result - - -def test_public_track_gate_accepts_complete_consistent_rows() -> None: - rows = [] - for variant, score in { - "B0": 0.30, - "B1": 0.32, - "V1": 0.33, - "F1": 0.35, - "F2": 0.36, - }.items(): - rows.append( - { - "variant": variant, - "repeat_idx": 0, - "prompt_mode": "memory_beneficial", - "benchmark_score_on_total": score, - "strict_accuracy_on_total": score, - "partial_accuracy_on_total": score, - "parse_failure_rate_on_total": 0.0, - "no_answer_rate_on_total": 0.0, - "success_rate": 1.0, - } - ) - result = _build_result( - rows, - dataset="longbench_v2", - track_id="mainline:memory_beneficial", - workload_id="PUB-LONGBENCH_V2-SMOKE-MAINLINE-MEMORY-BENEFICIAL", - stage="smoke", - expected_variants=["B0", "B1", "V1", "F1", "F2"], - expected_repeats=[0], - invalid_rate_threshold=0.15, - std_threshold=0.10, - ) - assert result["verdict"] == "accepted" - assert result["anomaly_codes"] == [] - - -def test_public_track_gate_invalidates_missing_variant() -> None: - rows = [ - { - "variant": "B0", - "repeat_idx": 0, - "prompt_mode": "memory_neutral", - "benchmark_score_on_total": 0.30, - "strict_accuracy_on_total": 0.30, - "partial_accuracy_on_total": 0.30, - "parse_failure_rate_on_total": 0.0, - "no_answer_rate_on_total": 0.0, - "success_rate": 1.0, - }, - { - "variant": "B1", - "repeat_idx": 0, - "prompt_mode": "memory_neutral", - "benchmark_score_on_total": 0.31, - "strict_accuracy_on_total": 0.31, - "partial_accuracy_on_total": 0.31, - "parse_failure_rate_on_total": 0.0, - "no_answer_rate_on_total": 0.0, - "success_rate": 1.0, - }, - ] - result = _build_result( - rows, - dataset="longbench_v2", - track_id="mainline:memory_neutral", - workload_id="PUB-LONGBENCH_V2-SMOKE-MAINLINE-MEMORY-NEUTRAL", - stage="smoke", - expected_variants=["B0", "B1", "V1", "F1", "F2"], - expected_repeats=[0], - invalid_rate_threshold=0.15, - std_threshold=0.10, - ) - assert result["verdict"] == "invalid" - assert "RAW_RUN_TABLE_MISMATCH" in result["anomaly_codes"] - - -def test_public_track_gate_detects_protocol_drift() -> None: - rows = [] - for variant in ("B0", "B1", "V1", "F1", "F2"): - rows.append( - { - "variant": variant, - "repeat_idx": 0, - "prompt_mode": "memory_neutral", - "qa_prompt_profile": "multiple_choice_letter", - "max_evidence_chars": 1600 if variant in {"B0", "B1"} else 18000, - "context_top_k": 10 if variant in {"B0", "B1"} else 12, - "context_item_max_chars": 600 if variant in {"B0", "B1"} else 1800, - "answer_format_mode": "strict_json", - "benchmark_score_on_total": 0.30, - "strict_accuracy_on_total": 0.30, - "partial_accuracy_on_total": 0.30, - "parse_failure_rate_on_total": 0.0, - "no_answer_rate_on_total": 0.0, - "success_rate": 1.0, - } - ) - result = _build_result( - rows, - dataset="longbench_v2", - track_id="track-s:memory_neutral", - workload_id="PUB-LONGBENCH_V2-SMOKE", - stage="smoke", - expected_variants=["B0", "B1", "V1", "F1", "F2"], - expected_repeats=[0], - invalid_rate_threshold=0.15, - std_threshold=0.10, - ) - assert result["verdict"] == "diagnostic_only" - assert "PROVIDER_PARAM_DRIFT" in result["anomaly_codes"] - - -def test_public_track_gate_invalidates_ordering_reversal_against_baseline() -> None: - rows = [] - for repeat_idx, values in enumerate(((0.35, 0.37, 0.34), (0.36, 0.38, 0.33))): - b1_score, v1_score, f1_score = values - rows.extend( - [ - { - "variant": "B1", - "repeat_idx": repeat_idx, - "prompt_mode": "memory_beneficial", - "benchmark_score_on_total": b1_score, - "strict_accuracy_on_total": b1_score, - "partial_accuracy_on_total": b1_score, - "parse_failure_rate_on_total": 0.0, - "no_answer_rate_on_total": 0.0, - "success_rate": 1.0, - }, - { - "variant": "V1", - "repeat_idx": repeat_idx, - "prompt_mode": "memory_beneficial", - "benchmark_score_on_total": v1_score, - "strict_accuracy_on_total": v1_score, - "partial_accuracy_on_total": v1_score, - "parse_failure_rate_on_total": 0.0, - "no_answer_rate_on_total": 0.0, - "success_rate": 1.0, - }, - { - "variant": "F1", - "repeat_idx": repeat_idx, - "prompt_mode": "memory_beneficial", - "benchmark_score_on_total": f1_score, - "strict_accuracy_on_total": f1_score, - "partial_accuracy_on_total": f1_score, - "parse_failure_rate_on_total": 0.0, - "no_answer_rate_on_total": 0.0, - "success_rate": 1.0, - }, - ] - ) - result = _build_result( - rows, - dataset="longbench_v2", - track_id="mainline:memory_beneficial", - workload_id="PUB-LONGBENCH_V2-RIGOR-M", - stage="full", - expected_variants=["B1", "V1", "F1"], - expected_repeats=[0, 1], - invalid_rate_threshold=0.15, - std_threshold=0.10, - baseline_variant="B1", - must_beat_baseline_variants=["V1", "F1"], - ) - assert result["verdict"] == "invalid" - assert "UNEXPLAINED_ORDERING_REVERSAL" in result["anomaly_codes"] - assert "F1" in result["ordering_reversals"] - assert "V1" not in result["ordering_reversals"] - - -def test_public_track_gate_invalidates_baseline_out_of_band() -> None: - rows = [ - { - "variant": "B0", - "repeat_idx": 0, - "prompt_mode": "memory_neutral", - "benchmark_score_on_total": 0.10, - "strict_accuracy_on_total": 0.10, - "partial_accuracy_on_total": 0.10, - "parse_failure_rate_on_total": 0.0, - "no_answer_rate_on_total": 0.0, - "success_rate": 1.0, - } - ] - result = _build_result( - rows, - dataset="longbench_v2", - track_id="mainline:memory_neutral", - workload_id="PUB-LONGBENCH_V2-FULL-MAINLINE-N", - stage="full", - expected_variants=["B0"], - expected_repeats=[0], - invalid_rate_threshold=0.15, - std_threshold=0.10, - baseline_variant="B0", - baseline_lower_bound=0.20, - ) - assert result["verdict"] == "invalid" - assert "BASELINE_OUT_OF_BAND" in result["anomaly_codes"] diff --git a/projects/memlab/tests/test_runtime_error_privacy.py b/projects/memlab/tests/test_runtime_error_privacy.py deleted file mode 100644 index e1817a2..0000000 --- a/projects/memlab/tests/test_runtime_error_privacy.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Remote error payloads must not become persistent diagnostic strings.""" -import asyncio - -import httpx -import pytest - -from memlab import runner_e2e -from memlab.runners import llm - - -SENTINEL = "SYNTHETIC_REMOTE_BODY_MUST_NOT_BE_PERSISTED" - - -class Client: - status = 422 - - def __init__(self, **_): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *_): - pass - - async def post(self, *_, **__): - return httpx.Response(self.status, text=SENTINEL) - - -@pytest.mark.parametrize("status", [403, 422]) -def test_direct_provider_exception_excludes_error_body(status): - client = Client() - client.status = status - with pytest.raises(RuntimeError) as caught: - asyncio.run(runner_e2e._run_provider_prompt_direct( - client=client, base_url="https://example.test/v1", model_id="demo-model", - api_key="placeholder", prompt="synthetic prompt", llm_sema=asyncio.Semaphore(1), - )) - assert SENTINEL not in str(caught.value) - assert str(status) in str(caught.value) - - -@pytest.mark.parametrize("preflight", [ - runner_e2e._provider_chat_preflight, runner_e2e._provider_tool_call_preflight, -]) -def test_preflight_detail_excludes_error_body(monkeypatch, preflight): - monkeypatch.setattr(runner_e2e.httpx, "AsyncClient", Client) - ok, detail = asyncio.run(preflight( - base_url="https://example.test/v1", model_id="demo-model", api_key="placeholder", - )) - assert not ok - assert detail == "HTTP 422" - assert SENTINEL not in detail - - -def test_compatible_adapter_exception_excludes_remote_body(monkeypatch): - monkeypatch.setenv("OPENAI_BASE_URL", "https://example.test/v1") - monkeypatch.setenv("OPENAI_API_KEY", "placeholder") - monkeypatch.setenv("OPENAI_MODEL", "demo-model") - monkeypatch.setattr(llm.httpx, "AsyncClient", Client) - with pytest.raises(RuntimeError) as caught: - asyncio.run(llm.run_openai_compatible("demo", [])) - assert str(caught.value) == "HTTP 422" - - -def test_transport_exception_detail_excludes_payload(monkeypatch): - class FailedClient(Client): - async def post(self, *_, **__): - raise httpx.ConnectError(SENTINEL) - - monkeypatch.setattr(runner_e2e.httpx, "AsyncClient", FailedClient) - ok, detail = asyncio.run(runner_e2e._provider_chat_preflight( - base_url="https://example.test/v1", model_id="demo-model", api_key="placeholder", - )) - assert not ok - assert detail == "ConnectError" - - -def test_memory_failure_events_exclude_exception_payload(): - import json - - from memlab.config import MemlabConfig - from memlab.memory_manager.manager import MultiLevelMemoryManager - - class FailedL1: - async def get_recent(self, *_args, **_kwargs): - raise RuntimeError(SENTINEL) - - async def get_key(self, *_args, **_kwargs): - raise RuntimeError(SENTINEL) - - cfg = MemlabConfig() - cfg.memory.l2.enabled = False - cfg.memory.l3.enabled = False - events = [] - manager = MultiLevelMemoryManager(cfg, op_logger=events.append) - manager.l1 = FailedL1() - asyncio.run(manager.get(agent_id="demo-agent", query="what happened before", k=4)) - assert events - assert SENTINEL not in json.dumps(events) - - -@pytest.mark.parametrize("balance_error", [True, False]) -def test_deerflow_stream_error_excludes_remote_payload(balance_error): - import json - - class Stream: - async def __aenter__(self): - return self - - async def __aexit__(self, *_): - pass - - def raise_for_status(self): - pass - - async def aiter_lines(self): - yield "event: error" - reason = "insufficient balance" if balance_error else "request rejected" - yield "data: " + json.dumps({"error": reason, "echo": SENTINEL}) - - class StreamClient: - def stream(self, *_, **__): - return Stream() - - with pytest.raises(RuntimeError) as caught: - asyncio.run(runner_e2e._run_deerflow_prompt( - client=StreamClient(), langgraph_url="https://example.test", - thread_id="synthetic-thread", prompt="synthetic prompt", - llm_sema=asyncio.Semaphore(1), - )) - expected = "provider_balance_rejected" if balance_error else "deerflow_stream_rejected" - assert str(caught.value) == expected - assert SENTINEL not in str(caught.value) - assert SENTINEL not in runner_e2e._safe_error_detail(caught.value) diff --git a/projects/memlab/tests/test_stub_event_attribution.py b/projects/memlab/tests/test_stub_event_attribution.py deleted file mode 100644 index 02fe1da..0000000 --- a/projects/memlab/tests/test_stub_event_attribution.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Interleave two worker sessions over one fake memory manager.""" -import asyncio -from types import SimpleNamespace - -from memlab.config import MemlabConfig -from memlab.runner_stub import _CURRENT_AGENT_ID, _make_op_logger, _session_worker - - -def test_shared_memory_logger_keeps_interleaved_session_ownership(): - async def run(): - rows, task_rows, written = [], [], [] - writer = SimpleNamespace(append=written.append) - both_started = asyncio.Event() - - class Memory: - def __init__(self): - self.started = 0 - self.op_logger = _make_op_logger(run_id="demo", mem_rows=rows, event_writer=writer) - - async def put(self, item, **_): - self.started += 1 - if self.started == 2: - both_started.set() - await both_started.wait() - await asyncio.sleep(0) - self.op_logger({"op_type": "memory", "expected_agent": item.agent_id}) - - async def get(self, **_): - return [] - - async def assemble_context(self, **_): - return SimpleNamespace(trace={}) - - memory = Memory() - queue = asyncio.Queue() - for i in range(2): - queue.put_nowait({"task_id": f"demo-{i}", "key": f"k-{i:06d}", "bytes": 64}) - cfg = MemlabConfig() - await asyncio.gather(*[ - _session_worker( - queue=queue, session_index=i, run_id="demo", memory=memory, - event_writer=writer, task_rows=task_rows, mem_rows=rows, cfg=cfg, - ) - for i in range(2) - ]) - assert len(rows) == 2 - assert {row["agent_id"] for row in rows} == {"agent_0001", "agent_0002"} - for row in rows: - assert row["agent_id"] == row["expected_agent"] - assert row["session_id"] == f'demo-{row["expected_agent"]}' - assert all(row["status"] == "success" for row in task_rows) - assert _CURRENT_AGENT_ID.get() is None - - asyncio.run(run()) - - -def test_e2e_worker_context_keeps_interleaved_session_ownership(monkeypatch): - from memlab import runner_e2e - - async def run(): - rows = [] - writer = SimpleNamespace(append=lambda _: None) - logger = _make_op_logger(run_id="demo", mem_rows=rows, event_writer=writer) - both_started = asyncio.Event() - started = 0 - - async def body(**kwargs): - nonlocal started - started += 1 - if started == 2: - both_started.set() - await both_started.wait() - await asyncio.sleep(0) - logger({"expected_agent": f"agent_{kwargs['session_index'] + 1:04d}"}) - - monkeypatch.setattr(runner_e2e, "_session_worker_body", body) - await asyncio.gather(*[ - runner_e2e._session_worker(session_index=i) for i in range(2) - ]) - assert len(rows) == 2 - for row in rows: - assert row["agent_id"] == row["expected_agent"] - assert row["session_id"] == f'demo-{row["expected_agent"]}' - assert _CURRENT_AGENT_ID.get() is None - - asyncio.run(run()) diff --git a/projects/memlab/tests/test_system_rebaseline.py b/projects/memlab/tests/test_system_rebaseline.py deleted file mode 100644 index 13627b0..0000000 --- a/projects/memlab/tests/test_system_rebaseline.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import annotations - -import asyncio -from tempfile import TemporaryDirectory - -from memlab.config import MemlabConfig -from memlab.memory.types import MemoryItem -from memlab.memory_manager.manager import MultiLevelMemoryManager -from memlab.runner_core import apply_variant, normalize_variant - - -def test_variant_alias_mapping() -> None: - canonical, alias = normalize_variant("A1") - assert canonical == "V1" - assert alias == "A1" - - cfg = apply_variant(MemlabConfig(), "A2") - assert cfg.experiment.variant_id == "F1" - assert cfg.experiment.redis_policy == "allkeys-lru" - - -def test_pointerization_to_l3() -> None: - async def _run() -> None: - cfg = MemlabConfig() - cfg.memory.l1.enabled = False - cfg.memory.l2.enabled = False - cfg.memory.l3.enabled = True - cfg.memory.l3.pointer_threshold_bytes = 32 - with TemporaryDirectory() as td: - cfg.memory.l3.root_dir = td - mgr = MultiLevelMemoryManager(cfg) - await mgr.init() - try: - item = MemoryItem( - key="k-000001", - agent_id="agent-1", - kind="fact", - text="x" * 256, - bytes=256, - importance=1.0, - created_at_ms=1, - last_access_ms=1, - access_count=1, - metadata={"role": "system"}, - ) - decision = await mgr.put(item) - assert "L3" in decision.tiers - assert decision.admission_reason == "pointerized_to_l3" - recent = mgr._recent_messages["agent-1"][-1] - assert recent.kind == "blob_ref" - assert "l3_pointer" in recent.metadata - finally: - await mgr.close() - - asyncio.run(_run()) - - -def test_degrade_mode_trace_when_l1_fails() -> None: - class _FailingL1: - async def get_recent(self, agent_id: str, limit: int): # noqa: ARG002 - raise RuntimeError("injected_l1_failure") - - async def get_key(self, agent_id: str, key: str): # noqa: ARG002 - raise RuntimeError("injected_l1_failure") - - async def _run() -> None: - cfg = MemlabConfig() - cfg.memory.l1.enabled = True - cfg.memory.l2.enabled = False - cfg.memory.l3.enabled = False - mgr = MultiLevelMemoryManager(cfg) - mgr.l1 = _FailingL1() # type: ignore[assignment] - telemetry: dict[str, object] = {} - rows = await mgr.get(agent_id="agent-1", query="what happened before", k=4, mode="hybrid", telemetry=telemetry) - assert rows == [] - assert telemetry.get("degrade_mode") == "l1_unavailable" - - asyncio.run(_run()) diff --git a/publication-policy.json b/publication-policy.json index fd1e982..5898e3f 100644 --- a/publication-policy.json +++ b/publication-policy.json @@ -1,53 +1,137 @@ { "schema_version": 1, - "purpose": "Publish reviewed source, configuration and documentation without private research data or credentials.", + "purpose": "Maintain only a public project index, documentation, provenance and maintenance checks.", "max_file_bytes": 262144, "max_synthetic_bytes": 65536, - "approved_project_roots": [ - "projects/job-text-classification", - "projects/memlab" - ], + "approved_project_roots": [], "approved_root_files": [ - "README.md", "README.en.md", "CONTRIBUTING.md", "THIRD_PARTY.md", ".gitignore", ".gitattributes", - "pyproject.toml", "requirements.txt", "requirements-dev.txt", - "publication-policy.json", "CITATION.cff", "LICENSE", "NOTICE", ".env.example" - ], - "approved_top_level_directories": ["tools", "tests", "docs", ".github"], - "approved_project_directories": [ - "src", "scripts", "configs", "docs", "tests", "examples", "include", - "memlab", "benchmarks", "sql", "infra" + "README.md", + "CONTRIBUTING.md", + "THIRD_PARTY.md", + ".gitignore", + ".gitattributes", + "publication-policy.json" ], - "approved_project_root_names": [ - "README.md", "README.en.md", "CONTRIBUTING.md", "THIRD_PARTY.md", "pyproject.toml", "requirements.txt", - "requirements-dev.txt", "CMakeLists.txt", "Makefile", ".gitignore", - "CITATION.cff", "LICENSE", "NOTICE", ".env.example" + "approved_top_level_directories": [ + "tools", + "tests", + "docs", + ".github" ], + "approved_project_directories": [], + "approved_project_root_names": [], "approved_source_extensions": [ - ".py", ".md", ".rst", ".toml", ".yaml", ".yml", ".ini", ".cfg", ".conf", - ".ps1", ".sh", ".sql", ".c", ".h", ".cc", ".cpp", ".hpp", ".cmake", ".cff" + ".py", + ".md", + ".rst", + ".toml", + ".yaml", + ".yml", + ".ini", + ".cfg", + ".conf", + ".ps1", + ".sh", + ".sql", + ".c", + ".h", + ".cc", + ".cpp", + ".hpp", + ".cmake", + ".cff" + ], + "synthetic_data_extensions": [ + ".json", + ".jsonl", + ".csv", + ".tsv", + ".txt" ], - "synthetic_data_extensions": [".json", ".jsonl", ".csv", ".tsv", ".txt"], "forbidden_directory_names": [ - "data", "dataset", "datasets", "raw", "private", "outputs", "output", - "runs", "logs", "wandb", "checkpoints", "model", "models", "weights", - "adapters", "artifacts", "results", "rlhf_data", "materials", "third_party", - ".venv", "venv", "node_modules", "__pycache__", ".pytest_cache", - ".mypy_cache", ".ruff_cache", ".cache", "cache" + "data", + "dataset", + "datasets", + "raw", + "private", + "outputs", + "output", + "runs", + "logs", + "wandb", + "checkpoints", + "model", + "models", + "weights", + "adapters", + "artifacts", + "results", + "rlhf_data", + "materials", + "third_party", + ".venv", + "venv", + "node_modules", + "__pycache__", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + ".cache", + "cache" ], "forbidden_extensions": [ - ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", - ".zip", ".rar", ".7z", ".tar", ".gz", ".sqlite", ".sqlite3", ".db", - ".pt", ".pth", ".ckpt", ".bin", ".safetensors", ".npy", ".npz", - ".pkl", ".pickle", ".parquet", ".feather", ".png", ".jpg", ".jpeg", - ".tif", ".tiff", ".bmp", ".webp", ".gif", ".mp4", ".wav", ".pyc" - ], - "allowed_absolute_path_prefixes": ["c:/windows/", "c:/program files/", "c:/program files (x86)/"], - "allowed_email_patterns": ["^[0-9]+\\+[A-Za-z0-9-]+@users\\.noreply\\.github\\.com$", "^[A-Za-z0-9-]+@users\\.noreply\\.github\\.com$"], + ".pdf", + ".doc", + ".docx", + ".xls", + ".xlsx", + ".ppt", + ".pptx", + ".zip", + ".rar", + ".7z", + ".tar", + ".gz", + ".sqlite", + ".sqlite3", + ".db", + ".pt", + ".pth", + ".ckpt", + ".bin", + ".safetensors", + ".npy", + ".npz", + ".pkl", + ".pickle", + ".parquet", + ".feather", + ".png", + ".jpg", + ".jpeg", + ".tif", + ".tiff", + ".bmp", + ".webp", + ".gif", + ".mp4", + ".wav", + ".pyc" + ], + "allowed_absolute_path_prefixes": [ + "c:/windows/", + "c:/program files/", + "c:/program files (x86)/" + ], + "allowed_email_patterns": [ + "^[0-9]+\\+[A-Za-z0-9-]+@users\\.noreply\\.github\\.com$", + "^[A-Za-z0-9-]+@users\\.noreply\\.github\\.com$" + ], "notes": [ "The root .git metadata is skipped; nested Git repositories and every symlink or reparse point are rejected.", "Small data-shaped files are permitted only in examples/synthetic* or tests/fixtures*; configuration JSON is permitted in configs/.", "Synthetic paths are a packaging boundary, not proof of synthetic provenance. Reviewers must confirm that examples were authored from scratch.", "The scanner reports relative file paths, rule names and line numbers, never the matched text.", - "Root Git history is outside this working-tree scan. Start with a clean repository history and review git ls-files before publishing." + "Historical Git objects are retained for traceability; current project source belongs only in canonical standalone repositories." ] } diff --git a/tests/test_publication_policy.py b/tests/test_publication_policy.py index b766fc9..e83bf9d 100644 --- a/tests/test_publication_policy.py +++ b/tests/test_publication_policy.py @@ -22,7 +22,9 @@ CHECKER = importlib.util.module_from_spec(SPEC) sys.modules[SPEC.name] = CHECKER SPEC.loader.exec_module(CHECKER) -POLICY = CHECKER.load_policy(ROOT / "publication-policy.json") +DEPLOYED_POLICY = CHECKER.load_policy(ROOT / "publication-policy.json") +# Authored safety fixtures exercise generic rules independently of the deployed index inventory. +POLICY = dict(DEPLOYED_POLICY, approved_top_level_directories=["tools", "tests", "docs", ".github", "src", "examples", "configs"]) class PublicationPolicyTest(unittest.TestCase): @@ -35,6 +37,17 @@ def make_file(self, root: Path, relative: str, text: str = "# Authored fixture\n def scan(self, root: Path): return CHECKER.scan_repository(root, POLICY) + def test_deployed_index_policy_rejects_project_source(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.make_file(root, "projects/example/source.py") + self.make_file(root, "src/example.py") + self.make_file(root, "pyproject.toml") + findings = CHECKER.scan_repository(root, DEPLOYED_POLICY).findings + self.assertIn(CHECKER.Finding("projects", "DIRECTORY_NOT_ALLOWLISTED"), findings) + self.assertIn(CHECKER.Finding("src", "DIRECTORY_NOT_ALLOWLISTED"), findings) + self.assertIn(CHECKER.Finding("pyproject.toml", "PATH_NOT_ALLOWLISTED"), findings) + def test_secret_report_contains_no_secret_text(self): with tempfile.TemporaryDirectory() as temporary: parent = Path(temporary) @@ -56,9 +69,9 @@ def test_secret_report_contains_no_secret_text(self): def test_private_data_and_manuscript_rejected(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) - self.make_file(root, "projects/memlab/data/observations.csv", "x,y\n1,2\n") - self.make_file(root, "projects/memlab/docs/manuscript.pdf", "authored binary placeholder") - self.make_file(root, "projects/memlab/src/observations.csv", "x,y\n1,2\n") + self.make_file(root, "data/observations.csv", "x,y\n1,2\n") + self.make_file(root, "docs/manuscript.pdf", "authored binary placeholder") + self.make_file(root, "src/observations.csv", "x,y\n1,2\n") rules = {finding.rule for finding in self.scan(root).findings} self.assertIn("FORBIDDEN_DIRECTORY", rules) self.assertIn("FORBIDDEN_FILE_TYPE", rules) @@ -67,10 +80,10 @@ def test_private_data_and_manuscript_rejected(self): def test_small_synthetic_examples_and_fixtures_allowed(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) - self.make_file(root, "projects/job-text-classification/examples/synthetic_jobs.csv", "category,text\nA,authored sample\n") - self.make_file(root, "projects/job-text-classification/src/job_text_classification/examples/synthetic_jobs.jsonl", '{"label": "A", "text": "authored sample"}\n') - self.make_file(root, "projects/memlab/tests/fixtures/tiny.json", '{"sample": [1, 2]}\n') - self.make_file(root, "projects/memlab/configs/demo.json", '{"iterations": 2}\n') + self.make_file(root, "examples/synthetic_jobs.csv", "category,text\nA,authored sample\n") + self.make_file(root, "src/job_text_classification/examples/synthetic_jobs.jsonl", '{"label": "A", "text": "authored sample"}\n') + self.make_file(root, "tests/fixtures/tiny.json", '{"sample": [1, 2]}\n') + self.make_file(root, "configs/demo.json", '{"iterations": 2}\n') self.assertEqual(self.scan(root).findings, []) def test_symlink_is_rejected_without_following_target(self): @@ -98,8 +111,8 @@ def test_nested_git_is_rejected_but_root_metadata_skipped(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.make_file(root, ".git/config", "metadata outside content scan") - self.make_file(root, "projects/memlab/.git", "gitdir: ../elsewhere") - self.assertEqual(self.scan(root).findings, [CHECKER.Finding("projects/memlab/.git", "NESTED_GIT")]) + self.make_file(root, "docs/.git", "gitdir: ../elsewhere") + self.assertEqual(self.scan(root).findings, [CHECKER.Finding("docs/.git", "NESTED_GIT")]) def test_private_identifiers_detected_without_number_false_positives(self): with tempfile.TemporaryDirectory() as temporary: @@ -170,8 +183,8 @@ def test_ignored_runtime_artifacts_do_not_enter_index_scan(self): self.initialize(root) self.make_file(root, ".gitignore", ".venv/\n__pycache__/\n*.egg-info/\n") self.make_file(root, ".venv/ignored.py") - self.make_file(root, "projects/memlab/memlab/__pycache__/ignored.pyc") - self.make_file(root, "projects/memlab/memlab.egg-info/PKG-INFO") + self.make_file(root, "memlab/__pycache__/ignored.pyc") + self.make_file(root, "memlab.egg-info/PKG-INFO") self.git(root, "add", "-A") self.assertEqual(CHECKER.scan_staged(root).findings, []) self.assertTrue(CHECKER.scan_repository(root, POLICY).findings) @@ -182,10 +195,10 @@ def test_symlink_and_submodule_index_entries_rejected_without_materialization(se self.initialize(root) identifier = self.git(root, "hash-object", "-w", "--stdin", payload=b"outside-target").decode().strip() self.git(root, "update-index", "--add", "--cacheinfo", f"120000,{identifier},README.md") - self.git(root, "update-index", "--add", "--cacheinfo", f"160000,{identifier},projects/memlab/external") + self.git(root, "update-index", "--add", "--cacheinfo", f"160000,{identifier},external") findings = CHECKER.scan_staged(root).findings self.assertIn(CHECKER.Finding("README.md", "SYMLINK_INDEX_ENTRY"), findings) - self.assertIn(CHECKER.Finding("projects/memlab/external", "SUBMODULE_INDEX_ENTRY"), findings) + self.assertIn(CHECKER.Finding("external", "SUBMODULE_INDEX_ENTRY"), findings) def test_unmerged_index_is_rejected(self): with tempfile.TemporaryDirectory() as temporary: @@ -213,12 +226,12 @@ def test_unstaged_policy_change_does_not_weaken_index_scan(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) self.initialize(root) - self.make_file(root, "projects/memlab/data/observations.csv", "x,y\n1,2\n") - self.git(root, "add", "projects/memlab/data/observations.csv") + self.make_file(root, "data/observations.csv", "x,y\n1,2\n") + self.git(root, "add", "data/observations.csv") changed = dict(POLICY, forbidden_directory_names=[]) self.make_file(root, "publication-policy.json", json.dumps(changed)) self.assertIn( - CHECKER.Finding("projects/memlab/data", "FORBIDDEN_DIRECTORY"), + CHECKER.Finding("data", "FORBIDDEN_DIRECTORY"), CHECKER.scan_staged(root).findings, ) @@ -226,7 +239,7 @@ def test_unsafe_index_path_validation(self): for name in ("../README.md", "/README.md", "D:" + "/private.py", "a\\b.py", "a/../b.py", "a/./b.py", "a//b.py", "CON.py", "a/name. "): with self.subTest(name=name): self.assertFalse(CHECKER._safe_index_path(name)) - self.assertTrue(CHECKER._safe_index_path("projects/memlab/memlab/config.py")) + self.assertTrue(CHECKER._safe_index_path("tools/check.py")) def test_case_collisions_cannot_hide_a_staged_blob(self): with tempfile.TemporaryDirectory() as temporary: @@ -234,8 +247,8 @@ def test_case_collisions_cannot_hide_a_staged_blob(self): self.initialize(root) first = self.git(root, "hash-object", "-w", "--stdin", payload=b"# first authored blob\n").decode().strip() second = self.git(root, "hash-object", "-w", "--stdin", payload=b"# second authored blob\n").decode().strip() - upper = "projects/memlab/memlab/A.py" - lower = "projects/memlab/memlab/a.py" + upper = "tools/A.py" + lower = "tools/a.py" self.git(root, "-c", "core.ignorecase=false", "update-index", "--add", "--cacheinfo", f"100644,{first},{upper}") self.git(root, "-c", "core.ignorecase=false", "update-index", "--add", "--cacheinfo", f"100644,{second},{lower}") findings = CHECKER.scan_staged(root).findings diff --git a/tools/check_index.py b/tools/check_index.py new file mode 100644 index 0000000..e738c8f --- /dev/null +++ b/tools/check_index.py @@ -0,0 +1,101 @@ +"""Validate the documentation-only inventory and canonical project destinations.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import subprocess +from urllib.parse import unquote, urlsplit + +ROOT = Path(__file__).resolve().parents[1] +EXPECTED = { + '.gitattributes', '.gitignore', '.github/workflows/checks.yml', + 'CONTRIBUTING.md', 'README.md', 'THIRD_PARTY.md', + 'docs/ENGINEERING_NOTES.md', 'docs/PUBLICATION_SCOPE.md', + 'publication-policy.json', 'tests/test_publication_policy.py', + 'tools/check_publication.py', 'tools/check_index.py', +} +CANONICAL = { + 'FracLogGPT-Lite', 'MemLab', 'Recruitment-Text-Classification', 'JSP-UI-Prototype', +} +LINK = re.compile(r'!?\[[^\]\n]*\]\(([^\s)]+)\)') + + +def command(*arguments: str) -> bytes: + result = subprocess.run(arguments, cwd=ROOT, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=60) + if result.returncode: + raise ValueError('Read-only command failed: ' + arguments[0]) + return result.stdout + + +def gh(endpoint: str): + return json.loads(command('gh', 'api', endpoint)) + + +def check(remote: bool = False) -> None: + tracked = set(command('git', 'ls-files', '-z').decode('utf-8').strip('\0').split('\0')) + if tracked != EXPECTED: + raise ValueError(f'Index inventory changed: extra={sorted(tracked - EXPECTED)}, missing={sorted(EXPECTED - tracked)}') + if (ROOT / 'projects').exists(): + raise ValueError('Project source must be maintained in its standalone repository') + policy = json.loads((ROOT / 'publication-policy.json').read_text(encoding='utf-8')) + if policy['approved_project_roots'] or policy['approved_project_directories'] or policy['approved_project_root_names']: + raise ValueError('The deployed publication policy must not allow project subtrees') + readme = (ROOT / 'README.md').read_text(encoding='utf-8') + for name in CANONICAL: + if f'https://github.com/Darssal/{name})' not in readme: + raise ValueError('Missing canonical README entry: ' + name) + + targets: set[tuple[str, str]] = set() + local_count = 0 + for name in sorted(EXPECTED): + if not name.endswith('.md'): + continue + for raw in LINK.findall((ROOT / name).read_text(encoding='utf-8')): + url = urlsplit(raw) + if not url.scheme and not url.netloc: + if not url.path: + continue + target = (ROOT / name).parent / unquote(url.path) + if not target.exists() or not target.resolve().is_relative_to(ROOT.resolve()): + raise ValueError(f'Broken or out-of-root document link in {name}: {raw}') + local_count += 1 + elif url.netloc == 'github.com': + parts = unquote(url.path).strip('/').split('/') + if len(parts) >= 2 and parts[0] == 'Darssal' and parts[1] in CANONICAL: + if len(parts) == 2: + targets.add((parts[1], '')) + elif len(parts) >= 5 and parts[2:4] == ['blob', 'main']: + targets.add((parts[1], '/'.join(parts[4:]))) + else: + raise ValueError(f'Unexpected canonical link in {name}: {raw}') + if '../projects/' in raw: + raise ValueError('Old Collection project link remains') + print(f'Index inventory: {len(tracked)} maintenance/document files; no project source') + print(f'Links: {local_count} local references and {len(targets)} canonical repository/file targets') + if remote: + for name in sorted(CANONICAL): + metadata = gh(f'repos/Darssal/{name}') + if metadata['private'] or metadata['default_branch'] != 'main': + raise ValueError('Expected a public canonical repository with main: ' + name) + tree = gh(f'repos/Darssal/{name}/git/trees/main?recursive=1') + if tree.get('truncated'): + raise ValueError('Canonical tree response was truncated: ' + name) + blobs = {entry['path'] for entry in tree['tree'] if entry['type'] == 'blob'} + for repo, path in targets: + if repo == name and path and path not in blobs: + raise ValueError(f'Canonical linked file is missing: {repo}/{path}') + print(f'Public canonical repository verified: {name} @ {tree["sha"]}') + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--remote', action='store_true', help='Read public GitHub metadata and file trees with gh') + args = parser.parse_args() + check(args.remote) + + +if __name__ == '__main__': + main()