{
  "benchmark": "agent-memory",
  "reproduce": "node scripts/bench/agent-memory.mjs rebuilds the numbers from the per-question results in scripts/bench/agent-memory/. To measure again: the app's `cargo test -p omniget-core --lib assist::memory::bench::locomo_eval -- --ignored` (LOCOMO_K=20 and 80) and `python scripts/bench/agent-memory/hindsight_locomo.py` (default and TOPK=20), both on LoCoMo conv-26 with Claude Haiku through the Claude Code CLI.",
  "metrics": [
    {
      "id": "accuracy_20",
      "unit": "%",
      "better": "higher",
      "from": "k20.accuracy"
    },
    {
      "id": "accuracy_wide",
      "unit": "%",
      "better": "higher",
      "from": "wide.accuracy"
    },
    {
      "id": "disk",
      "unit": "MB",
      "better": "lower",
      "from": "disk_mb"
    }
  ],
  "data": {
    "measured": "2026-09-29",
    "method": "LoCoMo, conversation conv-26: 19 sessions, 419 turns; every second non-adversarial question (76). Claude Haiku 4.5 through the Claude Code CLI 2.1.284, for extraction, answers and the judge. Same answer prompt and judge prompt for both.",
    "machine": "Mac with Apple M5, 24 GB, macOS 26.5",
    "tools": [
      {
        "id": "omniget",
        "version": "OmniGet 0.10.1 with the agent-memory branch (not released yet)",
        "released": "2026-09-21",
        "k20": {
          "accuracy": 67.1,
          "correct": 51,
          "questions": 76,
          "multi_hop": {
            "accuracy": 53.3,
            "correct": 8,
            "questions": 15
          },
          "temporal": {
            "accuracy": 72.2,
            "correct": 13,
            "questions": 18
          },
          "open_domain": {
            "accuracy": 87.5,
            "correct": 7,
            "questions": 8
          },
          "single_hop": {
            "accuracy": 65.7,
            "correct": 23,
            "questions": 35
          }
        },
        "wide": {
          "accuracy": 63.2,
          "correct": 48,
          "questions": 76,
          "multi_hop": {
            "accuracy": 53.3,
            "correct": 8,
            "questions": 15
          },
          "temporal": {
            "accuracy": 66.7,
            "correct": 12,
            "questions": 18
          },
          "open_domain": {
            "accuracy": 100,
            "correct": 8,
            "questions": 8
          },
          "single_hop": {
            "accuracy": 57.1,
            "correct": 20,
            "questions": 35
          },
          "facts_per_answer": 80
        },
        "ingest_s": 710,
        "ingest_note": "4 model calls at a time, 28 calls in total",
        "disk_mb": 316,
        "facts_extracted": 168
      },
      {
        "id": "hindsight",
        "version": "hindsight-all 0.10.1 (embedded server, pg0)",
        "released": "2026-09-21",
        "k20": {
          "accuracy": 64.5,
          "correct": 49,
          "questions": 76,
          "multi_hop": {
            "accuracy": 46.7,
            "correct": 7,
            "questions": 15
          },
          "temporal": {
            "accuracy": 66.7,
            "correct": 12,
            "questions": 18
          },
          "open_domain": {
            "accuracy": 87.5,
            "correct": 7,
            "questions": 8
          },
          "single_hop": {
            "accuracy": 65.7,
            "correct": 23,
            "questions": 35
          }
        },
        "wide": {
          "accuracy": 71.1,
          "correct": 54,
          "questions": 76,
          "multi_hop": {
            "accuracy": 60,
            "correct": 9,
            "questions": 15
          },
          "temporal": {
            "accuracy": 77.8,
            "correct": 14,
            "questions": 18
          },
          "open_domain": {
            "accuracy": 87.5,
            "correct": 7,
            "questions": 8
          },
          "single_hop": {
            "accuracy": 68.6,
            "correct": 24,
            "questions": 35
          },
          "facts_per_answer": 82
        },
        "ingest_s": 1988,
        "ingest_note": "one retain call per session, in sequence",
        "disk_mb": 2390,
        "facts_extracted": null
      }
    ]
  }
}