From 3382c226163a83be64a05e10acd4e37e6603ad65 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 9 Jul 2026 03:30:55 +0000 Subject: [PATCH 01/27] feat(index): add optional Milvus derived index backend Signed-off-by: Cheney Zhang --- config.example.toml | 16 + docs/api.md | 15 +- docs/architecture.md | 26 +- docs/cascade_runbook.md | 16 +- docs/configuration.md | 22 + docs/knowledge.md | 18 +- docs/multimodal.md | 2 +- docs/overview.md | 6 +- docs/reflection.md | 11 +- docs/storage_layout.md | 43 +- src/everos/README.md | 2 +- src/everos/config/__init__.py | 7 +- src/everos/config/default.toml | 18 + src/everos/config/settings.py | 26 + src/everos/core/persistence/memory_root.py | 13 + .../infra/persistence/index/__init__.py | 329 ++++++++ .../infra/persistence/milvus/__init__.py | 49 ++ .../persistence/milvus/milvus_manager.py | 87 ++ src/everos/infra/persistence/milvus/repos.py | 119 +++ .../infra/persistence/milvus/repository.py | 750 ++++++++++++++++++ .../memory/cascade/handlers/agent_case.py | 2 +- .../memory/cascade/handlers/agent_skill.py | 2 +- .../memory/cascade/handlers/atomic_fact.py | 2 +- src/everos/memory/cascade/handlers/episode.py | 2 +- .../memory/cascade/handlers/foresight.py | 2 +- .../cascade/handlers/knowledge_topic.py | 2 +- .../memory/cascade/handlers/user_profile.py | 2 +- src/everos/memory/cascade/registry.py | 2 +- src/everos/memory/get/filters_adapter.py | 19 + src/everos/memory/get/manager.py | 12 +- src/everos/memory/search/filters.py | 215 ++++- src/everos/memory/search/manager.py | 7 +- src/everos/memory/search/recall/agent_case.py | 56 +- .../memory/search/recall/agent_skill.py | 57 +- .../memory/search/recall/atomic_fact.py | 39 +- src/everos/memory/search/recall/episode.py | 36 +- .../memory/search/recall/knowledge_topic.py | 39 +- src/everos/memory/search/recall/profile.py | 2 +- .../memory/strategies/extract_agent_skill.py | 4 +- .../memory/strategies/extract_user_profile.py | 2 +- .../memory/strategies/reflect_episodes.py | 2 +- src/everos/service/get.py | 6 +- tests/unit/test_config/test_settings.py | 27 + .../test_persistence/test_memory_root.py | 3 + .../unit/test_infra/test_milvus/test_repo.py | 127 +++ .../unit/test_memory/test_get/test_manager.py | 4 +- .../test_memory/test_search/test_filters.py | 33 + .../test_memory/test_search/test_manager.py | 16 +- .../test_search/test_recall_episode.py | 68 +- .../test_recall_knowledge_topic.py | 121 ++- uv.lock | 227 +++++- 51 files changed, 2312 insertions(+), 401 deletions(-) create mode 100644 src/everos/infra/persistence/index/__init__.py create mode 100644 src/everos/infra/persistence/milvus/__init__.py create mode 100644 src/everos/infra/persistence/milvus/milvus_manager.py create mode 100644 src/everos/infra/persistence/milvus/repos.py create mode 100644 src/everos/infra/persistence/milvus/repository.py create mode 100644 tests/unit/test_infra/test_milvus/test_repo.py diff --git a/config.example.toml b/config.example.toml index f8043127f..0145dd686 100644 --- a/config.example.toml +++ b/config.example.toml @@ -56,3 +56,19 @@ max_concurrent = 5 # # [lancedb] # read_consistency_seconds = 5.0 + +# ── Optional Milvus derived index ───────────────────── +# Markdown remains the source of truth. Set the index backend to Milvus +# when you want the rebuildable vector/BM25 index to live in Milvus. +# Leave uri empty for Milvus Lite at /.index/milvus/milvus.db. +# +# [index] +# backend = "milvus" +# +# [milvus] +# uri = "" # or "http://localhost:19530" +# token = "" # required for Zilliz Cloud/auth-enabled Milvus +# db_name = "" +# consistency_level = "Session" +# dimension = 1024 # must match your embedding model output +# collection_prefix = "everos" diff --git a/docs/api.md b/docs/api.md index 422b47ffa..d19361f4e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -93,8 +93,9 @@ bare FastAPI `detail`); see [Errors](#errors). `/add` and `/flush` write the markdown file (the source of truth) **synchronously** — when the call returns with `status: "extracted"`, -the new entry exists on disk. The LanceDB vector / BM25 / scalar index -is rebuilt by the in-process **cascade coroutine asynchronously**. +the new entry exists on disk. The configured vector / BM25 / scalar +index backend is rebuilt by the in-process **cascade coroutine +asynchronously**. That means `/search` and `/get` may not see a record immediately after the `/flush` that produced it. Typical sync latency is sub-second, but @@ -371,7 +372,7 @@ A recursive boolean tree of predicates. Used by `/search.filters` and `/get.filters`. The Pydantic envelope only checks the recursive combinator shape; field-level validity (which scalar fields are filterable, which operators apply, value coercion) runs when the -node is compiled to a LanceDB `where` clause server-side. Compile +node is compiled to a backend-specific filter clause server-side. Compile errors surface as `422` with the offending field / operator in `error.message`. @@ -464,12 +465,12 @@ Examples: |---|---| | `"keyword"` | BM25 only — pure lexical match, no embedding cost | | `"vector"` | Dense vector ANN only — semantic recall, no lexical | -| `"hybrid"` *(default)* | Reciprocal-rank fuse of BM25 + vector + optional scalar filter in a single LanceDB query | +| `"hybrid"` *(default)* | Reciprocal-rank fuse of BM25 + vector + optional scalar filter against the configured derived index backend | | `"agentic"` | Iterative cluster-path retrieval driven by a cross-encoder rerank loop; higher quality at higher latency / cost | `"hybrid"` is the default because it balances recall and precision -with one LanceDB roundtrip. `"agentic"` calls the LLM in a loop and -should be reserved for offline or background workflows. +without requiring the agentic loop. `"agentic"` calls the LLM in a loop +and should be reserved for offline or background workflows. ### GetMemoryType @@ -613,7 +614,7 @@ scope. this `(session_id, app_id, project_id)`, or it was already flushed). `/flush` is synchronous with respect to markdown persistence: by the -time the response returns, the new entry is on disk. LanceDB index +time the response returns, the new entry is on disk. Derived index sync is still asynchronous — see [Eventual consistency](#eventual-consistency). diff --git a/docs/architecture.md b/docs/architecture.md index afb5efe30..4aae7826a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,7 +17,7 @@ │ + reflection + strategies + get + events │ ├──────────────────────────────────────────────────────┤ │ infra/persistence (Storage adapters; infra/ may host other adapter types) │ -│ markdown + sqlite + lancedb │ +│ markdown + sqlite + derived index │ └──────────────────────────────────────────────────────┘ Cross-cutting (used by all layers, depends on none): @@ -68,8 +68,8 @@ layers = [ └────────────────────────────────────────────────────────────────┘ ┌──────────────┐ ┌──────────────┐ ┌─────────────────┐ - │ Markdown │ │ SQLite │ │ LanceDB │ - │ (truth) │ │ (state) │ │ (index) │ + │ Markdown │ │ SQLite │ │ Derived index │ + │ (truth) │ │ (state) │ │ LanceDB/Milvus │ ├──────────────┤ ├──────────────┤ ├─────────────────┤ │ entries + │ │ change queue │ │ vector ANN │ │ frontmatter │ │ + state/LSN │ │ BM25 (Tantivy) │ @@ -78,7 +78,7 @@ layers = [ └──────────────┘ └──────────────┘ └─────────────────┘ │ │ │ ▼ ▼ ▼ - memory-root/ .index/sqlite/ .index/lancedb/ + memory-root/ .index/sqlite/ .index// (truth source) (system data) (rebuildable) ``` @@ -101,10 +101,13 @@ External message │ │ ▼ ▼ 4a. SQLite 4b. memory.cascade (async daemon) - audit watches md → diff entries → LanceDB sync + audit watches md → diff entries → index sync ``` -**Key guarantee**: md write is strongly consistent (fsync). LanceDB is eventually consistent. LanceDB unavailability does not block response — changes buffer in the SQLite `md_change_state` queue, replayed on recovery. +**Key guarantee**: md write is strongly consistent (fsync). The derived +index is eventually consistent. Index backend unavailability does not block +response — changes buffer in the SQLite `md_change_state` queue, replayed on +recovery. ## Read path @@ -115,8 +118,8 @@ User query 1. service.search │ ▼ -2. memory.search (hybrid) single LanceDB query = - BM25 + vector ANN + scalar filter +2. memory.search (hybrid) BM25 + vector ANN + scalar filter + through the configured index backend │ ▼ 3. (optional) read md original markdown for context @@ -139,12 +142,13 @@ extract/ ### `memory/cascade/` -Daemon that watches markdown changes and syncs to LanceDB: +Daemon that watches markdown changes and syncs to the configured derived +index backend: - inotify / FSEvents file watcher (cross-platform via `watchdog`) - 500ms debounce - Entry-level diff (added / changed / removed) -- LanceDB single-transaction update (text + vector columns atomic) +- Per-entry index upsert / delete (text + vector columns update together) - LSN-based crash recovery via the SQLite `md_change_state` queue - Handlers for all eight business kinds: episode, atomic_fact, foresight, user_profile, agent_case, agent_skill, knowledge_document, knowledge_topic @@ -225,7 +229,7 @@ holding **only memory extraction algorithms**: everalgo is: - **Stateless** — pure functions, no class hierarchy -- **No I/O** — does not touch md files / LanceDB / SQLite +- **No I/O** — does not touch md files, derived indexes, or SQLite - **No prompts inline** — extractors that accept a prompt-override parameter use the project-supplied value; others use their algo-bundled defaults This boundary lets everalgo be reused across product forms (this open-source build, EverOS Cloud, OpenClaw plugins, etc.). diff --git a/docs/cascade_runbook.md b/docs/cascade_runbook.md index 3e0dcd900..d836672f9 100644 --- a/docs/cascade_runbook.md +++ b/docs/cascade_runbook.md @@ -1,9 +1,13 @@ # Cascade Runbook -The cascade daemon keeps LanceDB in sync with the markdown files under -the memory root. Service / entry points only ever write markdown; the -daemon is the **sole** writer of the LanceDB index. This runbook covers -the recurring operational questions. +The cascade daemon keeps the configured derived index in sync with the +markdown files under the memory root. Service / entry points only ever write +markdown; the daemon is the **sole** writer of the derived index. This runbook +covers the recurring operational questions. + +Sections that mention LanceDB-specific schemas, index cache, file descriptors, +or `lance error` messages apply to the default LanceDB backend. Milvus uses the +same cascade queue with backend-specific collection management. ## What runs where @@ -13,7 +17,7 @@ providers in order: 1. **Metrics** — Prometheus collector. 2. **LLM** — LLM client initialisation. 3. **SQLite** — system DB + schema (`SQLModel.metadata.create_all`). -4. **LanceDB** — async connection + schema verification + FTS indexes. +4. **Derived index** — async connection + schema verification + search indexes. 5. **Cascade** — watcher + scanner + worker, all in-process tasks. 6. **OME** — offline memory engine. @@ -23,7 +27,7 @@ The cascade subsystem itself is three independent loops: |---|---|---| | Watcher | `watchdog` filesystem events (sync thread) | `md_change_state.upsert` per registered kind | | Scanner | Periodic walk (`scan_interval_seconds`, default 30 s) | Same — catches changes the watcher missed | -| Worker | `claim_pending_batch` polling (default 1 s when idle) | Handler dispatch → LanceDB upsert / delete | +| Worker | `claim_pending_batch` polling (default 1 s when idle) | Handler dispatch → index upsert / delete | Every loop talks to the same `md_change_state` sqlite table. The worker's claim mode (`pending → processing → done/failed`) keeps diff --git a/docs/configuration.md b/docs/configuration.md index 98fdf3479..6848d7839 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,6 +99,26 @@ everos init --root /data/everos | `read_consistency_seconds` | float \| null | `null` | Read consistency interval. `null` = no check, `0` = strict, `>0` = eventual. | | `index_cache_size_bytes` | int | `16777216` | Upper bound on LanceDB index cache (16 MB default). | +### `[index]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `backend` | `"lancedb"` \| `"milvus"` | `"lancedb"` | Rebuildable vector/BM25 index backend used by cascade, search, and get. Markdown remains the source of truth; SQLite remains the system state store. | + +### `[milvus]` + +Milvus is optional. Install with `everos[milvus]`, then set +`[index].backend = "milvus"`. + +| Field | Type | Default | Description | +|---|---|---|---| +| `uri` | string \| null | `""` | Empty uses Milvus Lite at `/.index/milvus/milvus.db`. Set to a Milvus server or Zilliz Cloud endpoint to use an external service. | +| `token` | string \| null | `""` | Token for Zilliz Cloud or auth-enabled Milvus. | +| `db_name` | string \| null | `""` | Optional Milvus database name. | +| `consistency_level` | `"Strong"` \| `"Session"` \| `"Bounded"` \| `"Eventually"` | `"Session"` | Milvus consistency level for created collections. | +| `dimension` | int | `1024` | Dense vector dimension. Must match the configured embedding model output. | +| `collection_prefix` | string | `"everos"` | Prefix for EverOS Milvus collections. | + ### `[llm]` | Field | Type | Default | Required | Description | @@ -228,3 +248,5 @@ Examples: | `[llm] api_key = "sk-..."` | `EVEROS_LLM__API_KEY=sk-...` | | `[sqlite] busy_timeout_ms = 10000` | `EVEROS_SQLITE__BUSY_TIMEOUT_MS=10000` | | `[memory] timezone = "Asia/Tokyo"` | `EVEROS_MEMORY__TIMEZONE=Asia/Tokyo` | +| `[index] backend = "milvus"` | `EVEROS_INDEX__BACKEND=milvus` | +| `[milvus] uri = "http://localhost:19530"` | `EVEROS_MILVUS__URI=http://localhost:19530` | diff --git a/docs/knowledge.md b/docs/knowledge.md index d3ad92dd3..a7d7b1770 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -48,8 +48,8 @@ Each level corresponds to a different granularity of API: ## Storage layout Every document is a self-contained directory. Markdown files are the -single source of truth; SQLite and LanceDB are derived indexes built -automatically by the cascade daemon. +single source of truth; SQLite and the configured vector/BM25 index are +derived indexes built automatically by the cascade daemon. ``` ~/.everos///knowledge/ @@ -73,16 +73,16 @@ automatically by the cascade daemon. ### Storage roles ``` -Markdown (source of truth) + SQLite (structured state) + LanceDB (vector + BM25 index) +Markdown (source of truth) + SQLite (structured state) + derived vector/BM25 index ``` | Store | What it holds | Role | |-------|---------------|------| | Markdown | Document metadata, summaries, topic content, original files | Single source of truth; human-readable and editable | | SQLite | Document rows, topic rows (with content), change queue | Structured queries, paginated lists, count aggregation | -| LanceDB | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | +| Derived index | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | -Even if SQLite and LanceDB data is corrupted, as long as the Markdown +Even if SQLite and derived index data is corrupted, as long as the Markdown files are intact, the indexes can be fully rebuilt via the cascade daemon. ### Markdown format @@ -500,7 +500,7 @@ query ─→ embed ─→ keyword (BM25) ─┐ ``` 1. **Embed** — the query is embedded using the configured embedding provider -2. **Recall** — dual-channel retrieval from LanceDB: +2. **Recall** — dual-channel retrieval from the configured derived index: - BM25 channel: keyword matching on `summary_tokens` + `content_tokens` - ANN channel: nearest-neighbor search on the `vector` column - In `hybrid` mode, both channels run in parallel @@ -531,7 +531,7 @@ export EVEROS_KNOWLEDGE__SEARCH__RERANK_N=100 ## Cascade sync The cascade daemon watches the knowledge Markdown directory for file -changes and keeps SQLite + LanceDB in sync. +changes and keeps SQLite + the configured derived index in sync. ``` md file written @@ -539,7 +539,7 @@ md file written → worker picks up from queue (≤1s poll interval) → handler dispatched by file type: index.md → KnowledgeDocumentHandler → SQLite upsert (metadata) - N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + LanceDB upsert + N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + index upsert ``` The topic handler uses a SHA-256 content digest to skip unchanged files — @@ -597,7 +597,7 @@ curl .../documents?app_id=tenant_b&project_id=proj_1 # → { "documents": [], "total": 0 } ``` -Storage paths, SQLite rows, and LanceDB indexes are all scoped by +Storage paths, SQLite rows, and derived index rows are all scoped by `app_id` + `project_id`. ## End-to-end walkthrough diff --git a/docs/multimodal.md b/docs/multimodal.md index eeb181811..5c6b1a6da 100644 --- a/docs/multimodal.md +++ b/docs/multimodal.md @@ -41,7 +41,7 @@ POST /api/v2/memory/add boundary detector → extraction LLM → memory cell (MemCell) │ ▼ - markdown (truth) + SQLite (state) + LanceDB (vector + BM25) + markdown (truth) + SQLite (state) + derived vector/BM25 index │ ▼ retrievable via /search and /get like any text memory diff --git a/docs/overview.md b/docs/overview.md index 07297673d..0bfda5b7d 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -11,7 +11,7 @@ Build an open-source Python memory framework where **AI agents' long-term memory - Local deployment for personal agents or small teams - Conversation, workflow, agent-trace, file-knowledge → structured memory - Hybrid retrieval (BM25 + vector + scalar filter) -- Cascade index sync (md edit → LanceDB sub-second) +- Cascade index sync (md edit → derived index) - Dual-track memory (user-track / agent-track) - Offline memory evolution (Foresight / AtomicFact / Profile / Skill), including Reflection — a consolidation strategy within the OME that @@ -30,7 +30,7 @@ Build an open-source Python memory framework where **AI agents' long-term memory ### 1. Markdown as Source of Truth ``` -delete all LanceDB / SQLite files → can rebuild from md +delete all derived index / SQLite files → can rebuild from md delete any md file → memory is gone ``` @@ -42,7 +42,7 @@ User trust comes from physical visibility — the user can `cat` / `vim` / `grep |---|---|---| | Markdown files | Truth source — entries, frontmatter | Search (grep is degraded fallback only) | | SQLite | Queue, cascade audit log, sensitive data isolation | Vector / full-text | -| LanceDB | Vector ANN + BM25 + scalar filter, single-query hybrid | Be the source of truth (loss = rebuild from md) | +| Derived index | Vector ANN + BM25 + scalar filter | Be the source of truth (loss = rebuild from md) | ### 3. Algorithm-orchestration separation diff --git a/docs/reflection.md b/docs/reflection.md index 63c09f1fd..f3e920bcc 100644 --- a/docs/reflection.md +++ b/docs/reflection.md @@ -147,14 +147,15 @@ narrative. ## Storage layout -Memory uses Markdown as the single source of truth; SQLite and LanceDB are -derived indexes built automatically by the cascade daemon. +Memory uses Markdown as the single source of truth; SQLite and the configured +vector/BM25 index backend are derived indexes built automatically by the +cascade daemon. | Store | What it holds | Role | |---|---|---| | Markdown | Episode bodies, merged narratives, archive markers | Single source of truth; human-readable and editable | | SQLite | Clusters and members, consolidation audit records | Structured state and queries | -| LanceDB | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | +| Derived index | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | The **merged narrative** is written to the Episode daily-log Markdown; its frontmatter marks that it came from a cluster: @@ -186,7 +187,7 @@ deprecated_entries: --- ``` -> Soft-archive, not delete: even if SQLite / LanceDB are corrupted, as long +> Soft-archive, not delete: even if SQLite / the derived index are corrupted, as long > as the Markdown is intact the indexes can be fully rebuilt — and every > consolidation remains traceable back to its original content. @@ -355,5 +356,5 @@ curl -s -X POST "$BASE/memory/search" \ ## See also - [how-memory-works.md](how-memory-works.md) — Episodes and the memory extraction pipeline -- [storage_layout.md](storage_layout.md) — Markdown + SQLite + LanceDB stack +- [storage_layout.md](storage_layout.md) — Markdown + SQLite + derived index stack - [api.md](api.md) — full HTTP API reference diff --git a/docs/storage_layout.md b/docs/storage_layout.md index 860446648..71447905a 100644 --- a/docs/storage_layout.md +++ b/docs/storage_layout.md @@ -3,8 +3,9 @@ How `everos` lays out a memory-root on disk: directory tree, file naming, frontmatter chassis, and entry-id encoding. -The contents are the **source of truth**; SQLite and LanceDB are -derived indexes that can be rebuilt from markdown alone. +The contents are the **source of truth**; SQLite and the configured +vector/BM25 index backend are derived indexes that can be rebuilt from +markdown alone. ## 1. Memory-root tree @@ -49,8 +50,10 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). │ │ ├── ome.db Offline Memory Engine state │ │ ├── ome.aps.db APScheduler jobstore (split to avoid lock contention) │ │ └── ome.db.lock OME single-engine guard (portalocker) -│ └── lancedb/ -│ └── .lance/ one directory per LanceDB table +│ ├── lancedb/ +│ │ └── .lance/ default derived index backend +│ └── milvus/ +│ └── milvus.db Milvus Lite database when configured │ ├── ome.toml user-editable OME strategy overrides (hot-reloaded) └── .tmp/ staging dir for batch / multi-step writes @@ -64,7 +67,7 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py), exposing every path as a property. `MemoryRoot.ensure()` creates the -runtime-required dirs (`.index/{sqlite,lancedb}/`, `.tmp/`); the +runtime-required dirs (`.index/{sqlite,lancedb,milvus}/`, `.tmp/`); the user-visible dirs are *not* pre-created — they appear on first write. Config files (`everos.toml`, `ome.toml`) are created by `everos init`. @@ -168,9 +171,9 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers > **File-level seq, not global**: the same `ep_20260601_00000001` may > appear across two different `user_id`s (each user has its own daily file). > Cross-table joins must therefore key on **`(scope_id, entry_id)`** -> rather than `entry_id` alone — see SQLite/LanceDB tables that follow. +> rather than `entry_id` alone — see the derived index tables that follow. -## 5. SQLite + LanceDB derived indexes +## 5. SQLite + derived indexes ``` .index/ @@ -178,9 +181,10 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers │ └── system.db state / audit / cascade queue + buffer / LSN │ (system tables: md_change_state, memcell, │ unprocessed_buffer, conversation_status, cluster) -└── lancedb/ - └── .lance/ one Arrow table per business kind — the per-kind - rows (text / vector / tokens / metadata) live here +├── lancedb/ +│ └── .lance/ default derived index backend +└── milvus/ + └── milvus.db Milvus Lite database when configured ``` - **SQLite** ([`infra/persistence/sqlite/tables/`](../src/everos/infra/persistence/sqlite/tables/)) @@ -190,13 +194,14 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers per-kind business rows. `reflection_report` is the audit trail for Reflection merges (cluster_id, mode, source_members, merged_entry_id, status). -- **LanceDB** ([`infra/persistence/lancedb/tables/`](../src/everos/infra/persistence/lancedb/tables/)) - holds the per-kind business rows, keyed `_` (so - cross-table joins use `(owner_id, entry_id)`); each table's `Vector(N)` - dimension matches the embedding model output. - -Episode and AtomicFact LanceDB tables carry a `deprecated_by: str | None` -column. When an episode is superseded by a Reflection merge, +- The **derived index backend** holds the per-kind business rows, keyed + `_` (so cross-table joins use `(owner_id, entry_id)`). + LanceDB is the default backend under `.index/lancedb/`; Milvus can be enabled + as the same rebuildable index backend and uses `.index/milvus/milvus.db` by + default for Milvus Lite. + +Episode and AtomicFact index rows carry a `deprecated_by: str | None` column. +When an episode is superseded by a Reflection merge, `deprecated_by` is set to the merged episode's entry_id. Search filters automatically exclude rows where `deprecated_by IS NOT NULL`. @@ -226,5 +231,5 @@ this primitive is **schema-agnostic** — field-level semantics - Code: - [`core/persistence/memory_root.py`](../src/everos/core/persistence/memory_root.py) - [`core/persistence/markdown/`](../src/everos/core/persistence/markdown/) - - [`infra/persistence/{markdown,sqlite,lancedb}/`](../src/everos/infra/persistence/) - - [`memory/cascade/`](../src/everos/memory/cascade/) (md → LanceDB sync) + - [`infra/persistence/{markdown,sqlite,lancedb,milvus,index}/`](../src/everos/infra/persistence/) + - [`memory/cascade/`](../src/everos/memory/cascade/) (md → derived index sync) diff --git a/src/everos/README.md b/src/everos/README.md index 8cc1cef96..2be6caf25 100644 --- a/src/everos/README.md +++ b/src/everos/README.md @@ -10,7 +10,7 @@ everos/ ├── entrypoints/ Presentation: cli + api ├── service/ Application: use case orchestration ├── memory/ Domain: extract + search + cascade + prompt_slots + models -├── infra/ Infrastructure: persistence/{markdown, sqlite, lancedb} +├── infra/ Infrastructure: persistence/{markdown, sqlite, lancedb, milvus, index} ├── component/ Cross-cutting providers: llm / embedding / config / utils ├── core/ Runtime base: observability / lifespan / context └── config/ Data: Settings + default.toml + prompt_slots templates diff --git a/src/everos/config/__init__.py b/src/everos/config/__init__.py index adbd1e354..6ce3d9686 100644 --- a/src/everos/config/__init__.py +++ b/src/everos/config/__init__.py @@ -2,7 +2,8 @@ Public API: from everos.config import ( - Settings, MemorySettings, SqliteSettings, LanceDBSettings, + Settings, MemorySettings, SqliteSettings, IndexSettings, LanceDBSettings, + MilvusSettings, LLMSettings, EmbeddingSettings, RerankSettings, BoundaryDetectionSettings, CascadeSettings, load_settings, resolve_root, @@ -15,9 +16,11 @@ from .settings import BoundaryDetectionSettings as BoundaryDetectionSettings from .settings import CascadeSettings as CascadeSettings from .settings import EmbeddingSettings as EmbeddingSettings +from .settings import IndexSettings as IndexSettings from .settings import LanceDBSettings as LanceDBSettings from .settings import LLMSettings as LLMSettings from .settings import MemorySettings as MemorySettings +from .settings import MilvusSettings as MilvusSettings from .settings import MultimodalSettings as MultimodalSettings from .settings import RerankSettings as RerankSettings from .settings import Settings as Settings @@ -28,9 +31,11 @@ __all__ = [ "BoundaryDetectionSettings", "EmbeddingSettings", + "IndexSettings", "LLMSettings", "LanceDBSettings", "MemorySettings", + "MilvusSettings", "MultimodalSettings", "RerankSettings", "Settings", diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index e2137f84a..d67f0bc99 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -50,6 +50,24 @@ cache_size_kb = 2048 # Uncomment to override: # read_consistency_seconds = 5.0 +[index] +# Rebuildable vector/BM25 index backend. Markdown remains the source of truth +# and SQLite remains the system state store. +# Override via EVEROS_INDEX__BACKEND. +backend = "lancedb" + +[milvus] +# Optional derived index backend. Leave uri empty to use Milvus Lite at +# /.index/milvus/milvus.db. Set uri/token/db_name for Milvus server or +# Zilliz Cloud. +# Override via EVEROS_MILVUS__URI, EVEROS_MILVUS__TOKEN, EVEROS_MILVUS__DB_NAME. +uri = "" +token = "" +db_name = "" +consistency_level = "Session" +dimension = 1024 +collection_prefix = "everos" + [llm] # Provider-agnostic OpenAI-protocol client config. Override via env: # EVEROS_LLM__MODEL, EVEROS_LLM__API_KEY, EVEROS_LLM__BASE_URL diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 50da9947d..5e36b390d 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -375,6 +375,30 @@ class CascadeSettings(BaseModel): optimize_rebuild_interval_seconds: float = 12 * 60 * 60.0 +class IndexSettings(BaseModel): + """Derived-index backend selection.""" + + backend: Literal["lancedb", "milvus"] = "lancedb" + + +class MilvusSettings(BaseModel): + """Remote Milvus Server or Zilliz Cloud connection settings. + + Embedded Milvus Lite is intentionally unsupported. Deployments that select + the Milvus backend must configure an external endpoint and should use a + unique database or collection prefix when sharing a cluster. + """ + + uri: str = "" + token: SecretStr = SecretStr("") + db_name: str = "" + consistency_level: Literal["Strong", "Bounded", "Session", "Eventually"] = ( + "Session" + ) + dimension: int = Field(default=1024, ge=1) + collection_prefix: str = Field(default="everos", min_length=1) + + class KnowledgeSearchSettings(BaseModel): """``[knowledge.search]`` — retrieval tuning for the knowledge module.""" @@ -448,6 +472,8 @@ class Settings(BaseSettings): api: ApiSettings = ApiSettings() sqlite: SqliteSettings = SqliteSettings() lancedb: LanceDBSettings = LanceDBSettings() + index: IndexSettings = IndexSettings() + milvus: MilvusSettings = MilvusSettings() llm: LLMSettings = LLMSettings() embedding: EmbeddingSettings = EmbeddingSettings() rerank: RerankSettings = RerankSettings() diff --git a/src/everos/core/persistence/memory_root.py b/src/everos/core/persistence/memory_root.py index 701ff0012..5ca928c4f 100644 --- a/src/everos/core/persistence/memory_root.py +++ b/src/everos/core/persistence/memory_root.py @@ -11,6 +11,7 @@ .index/ derived indexes (rebuildable from markdown) sqlite/ system.db (+ WAL/SHM), ome.db, ome.aps.db lancedb/ LanceDB tables + milvus/ Milvus Lite database when Milvus is selected .tmp/ atomic-write staging directory .lock single-process lock anchor (created on demand by ``memory_root_lock``) @@ -165,6 +166,16 @@ def lancedb_dir(self) -> Path: """``/.index/lancedb/`` — LanceDB table root.""" return self.index_dir / "lancedb" + @property + def milvus_dir(self) -> Path: + """``/.index/milvus/`` — Milvus Lite derived index root.""" + return self.index_dir / "milvus" + + @property + def milvus_db(self) -> Path: + """``/.index/milvus/milvus.db`` — default Milvus Lite URI.""" + return self.milvus_dir / "milvus.db" + @property def sqlite_dir(self) -> Path: """``/.index/sqlite/`` — SQLite system DB root. @@ -244,10 +255,12 @@ def ensure(self) -> None: /.index/ /.index/sqlite/ /.index/lancedb/ + /.index/milvus/ /.tmp/ """ self.root.mkdir(parents=True, exist_ok=True) self.index_dir.mkdir(parents=True, exist_ok=True) self.sqlite_dir.mkdir(parents=True, exist_ok=True) self.lancedb_dir.mkdir(parents=True, exist_ok=True) + self.milvus_dir.mkdir(parents=True, exist_ok=True) self.tmp_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py new file mode 100644 index 000000000..486900058 --- /dev/null +++ b/src/everos/infra/persistence/index/__init__.py @@ -0,0 +1,329 @@ +"""Backend-neutral derived index persistence facade. + +Markdown is the source of truth and SQLite stores system state. This package +selects the rebuildable vector/BM25 index backend used by cascade, search, and +get. The default backend remains LanceDB; Milvus is opt-in through +``[index] backend = "milvus"``. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from lancedb.query import BooleanQuery, FullTextQuery, MatchQuery + +try: + from lancedb.query import Occur +except ImportError: # pragma: no cover + from lancedb._lancedb import Occur # type: ignore[attr-defined,no-redef] + +from everos.config import load_settings +from everos.infra.persistence import lancedb as _lancedb +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + KnowledgeTopic, + ParentType, + UserProfile, +) + + +def active_backend() -> str: + """Return the configured derived index backend name.""" + return load_settings().index.backend + + +async def startup() -> Any: + """Initialise the configured derived index backend.""" + if active_backend() == "milvus": + milvus = _milvus() + await milvus.get_client() + await milvus.ensure_business_indexes() + return "milvus" + conn = await _lancedb.get_connection() + await _lancedb.verify_business_schemas() + await _lancedb.ensure_business_indexes() + return conn + + +async def shutdown() -> None: + """Dispose the configured derived index backend.""" + if active_backend() == "milvus": + await _milvus().dispose_connection() + else: + await _lancedb.dispose_connection() + + +async def ensure_business_indexes() -> None: + if active_backend() == "milvus": + await _milvus().ensure_business_indexes() + else: + await _lancedb.ensure_business_indexes() + + +async def verify_business_schemas() -> None: + if active_backend() == "milvus": + await _milvus().verify_business_schemas() + else: + await _lancedb.verify_business_schemas() + + +class _LanceIndexRepoAdapter: + """Add backend-neutral recall helpers to an existing LanceDB repo.""" + + def __init__(self, repo: Any, schema: type[Any]) -> None: + self._repo = repo + self.schema = schema + + def __getattr__(self, name: str) -> Any: + return getattr(self._repo, name) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + fields = list(columns or self.schema.BM25_FIELDS) + if not query_terms or not fields: + return [] + table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) + expr = _lance_expr(where) + best: dict[str, dict[str, Any]] = {} + for field in fields: + query = _build_or_query(query_terms, field) + rows = ( + await table.query() + .nearest_to_text(query) + .where(expr) + .limit(limit) + .to_list() + ) + for row in rows: + rid = row.get("id") + if not isinstance(rid, str): + continue + score = float(row.get("_score", 0.0)) + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + shaped = dict(row) + shaped["_score"] = score + best[rid] = shaped + return sorted( + best.values(), key=lambda row: float(row.get("_score", 0.0)), reverse=True + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Any, + *, + limit: int, + ) -> list[dict[str, Any]]: + if not vector: + return [] + table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) + return ( + await table.query() + .nearest_to(list(vector)) + .distance_type("cosine") + .where(_lance_expr(where)) + .limit(limit) + .to_list() + ) + + +class _IndexRepoRouter: + """Route repo calls to the configured derived index backend.""" + + def __init__( + self, lance_repo: Any, milvus_repo_name: str, schema: type[Any] + ) -> None: + self._lance = _LanceIndexRepoAdapter(lance_repo, schema) + self._milvus_repo_name = milvus_repo_name + self.schema = schema + + @property + def table_name(self) -> str: + return self.schema.TABLE_NAME + + def _repo(self) -> Any: + if active_backend() == "milvus": + return getattr(_milvus(), self._milvus_repo_name) + return self._lance + + def __getattr__(self, name: str) -> Any: + return getattr(self._repo(), name) + + async def add(self, records: Sequence[Any]) -> None: + await self._repo().add(records) + + async def upsert(self, records: Sequence[Any], *, by: str = "id") -> None: + await self._repo().upsert(records, by=by) + + async def count(self) -> int: + return await self._repo().count() + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> Any: + return await self._repo().get_by_id(id_value, id_field=id_field) + + async def find_where(self, where: Any, *, limit: int = 100) -> list[Any]: + return await self._repo().find_where(_where_for_backend(where), limit=limit) + + async def find_one_where(self, where: Any) -> Any: + return await self._repo().find_one_where(_where_for_backend(where)) + + async def find_where_paginated( + self, + where: Any, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[Any], int]: + return await self._repo().find_where_paginated( + _where_for_backend(where), + sort_by=sort_by, + descending=descending, + page=page, + page_size=page_size, + max_fetch=max_fetch, + ) + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Any = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + return await self._repo().search( + vector=vector, where=_where_for_backend(where), limit=limit + ) + + async def update(self, updates: dict[str, Any], *, where: Any) -> None: + await self._repo().update(updates, where=_where_for_backend(where)) + + async def delete(self, predicate: Any) -> None: + await self._repo().delete(_where_for_backend(predicate)) + + async def delete_by_md_path(self, md_path: str) -> int: + return await self._repo().delete_by_md_path(md_path) + + async def optimize(self, **kwargs: Any) -> None: + await self._repo().optimize(**kwargs) + + async def rebuild_indexes(self) -> None: + await self._repo().rebuild_indexes() + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + return await self._repo().sparse_search( + query_terms, + _where_for_backend(where), + columns=columns, + limit=limit, + ) + + async def dense_search( + self, vector: Sequence[float], where: Any, *, limit: int + ) -> list[dict[str, Any]]: + return await self._repo().dense_search( + vector, _where_for_backend(where), limit=limit + ) + + +def _where_for_backend(where: Any) -> Any: + if active_backend() == "milvus": + return getattr(where, "milvus", where) + return _lance_expr(where) + + +def _lance_expr(where: Any) -> str: + return getattr(where, "lancedb", where) + + +def _build_or_query(tokens: Sequence[str], column: str) -> FullTextQuery: + clean = [token for token in tokens if token] + if not clean: + return MatchQuery("", column=column) + if len(clean) == 1: + return MatchQuery(clean[0], column=column) + return BooleanQuery( + [(Occur.SHOULD, MatchQuery(token, column=column)) for token in clean] + ) + + +def _milvus() -> Any: + from everos.infra.persistence import milvus + + return milvus + + +episode_repo = _IndexRepoRouter(_lancedb.episode_repo, "episode_repo", Episode) +atomic_fact_repo = _IndexRepoRouter( + _lancedb.atomic_fact_repo, "atomic_fact_repo", AtomicFact +) +foresight_repo = _IndexRepoRouter( + _lancedb.foresight_repo, "foresight_repo", Foresight +) +agent_case_repo = _IndexRepoRouter( + _lancedb.agent_case_repo, "agent_case_repo", AgentCase +) +agent_skill_repo = _IndexRepoRouter( + _lancedb.agent_skill_repo, "agent_skill_repo", AgentSkill +) +user_profile_repo = _IndexRepoRouter( + _lancedb.user_profile_repo, "user_profile_repo", UserProfile +) +knowledge_topic_repo = _IndexRepoRouter( + _lancedb.knowledge_topic_repo, "knowledge_topic_repo", KnowledgeTopic +) + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + +__all__ = [ + "ALL_REPOS", + "AgentCase", + "AgentSkill", + "AtomicFact", + "Episode", + "Foresight", + "KnowledgeTopic", + "ParentType", + "UserProfile", + "active_backend", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "ensure_business_indexes", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "shutdown", + "startup", + "user_profile_repo", + "verify_business_schemas", +] diff --git a/src/everos/infra/persistence/milvus/__init__.py b/src/everos/infra/persistence/milvus/__init__.py new file mode 100644 index 000000000..4396360c8 --- /dev/null +++ b/src/everos/infra/persistence/milvus/__init__.py @@ -0,0 +1,49 @@ +"""Milvus derived index backend. + +This package mirrors the LanceDB business index surface but stores rows in +Milvus collections. It is selected through ``Settings.index.backend`` and is +normally reached through :mod:`everos.infra.persistence.index`. +""" + +from __future__ import annotations + +from .milvus_manager import MilvusSchemaMismatchError as MilvusSchemaMismatchError +from .milvus_manager import dispose_connection as dispose_connection +from .milvus_manager import get_client as get_client +from .repos import ALL_REPOS as ALL_REPOS +from .repos import agent_case_repo as agent_case_repo +from .repos import agent_skill_repo as agent_skill_repo +from .repos import atomic_fact_repo as atomic_fact_repo +from .repos import episode_repo as episode_repo +from .repos import foresight_repo as foresight_repo +from .repos import knowledge_topic_repo as knowledge_topic_repo +from .repos import user_profile_repo as user_profile_repo + + +async def ensure_business_indexes() -> None: + """Create or verify every EverOS Milvus collection.""" + for repo in ALL_REPOS: + await repo.ensure_collection() + + +async def verify_business_schemas() -> None: + """Fail loud if existing Milvus collections drift from EverOS schemas.""" + for repo in ALL_REPOS: + await repo.verify_collection() + + +__all__ = [ + "ALL_REPOS", + "MilvusSchemaMismatchError", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "dispose_connection", + "ensure_business_indexes", + "episode_repo", + "foresight_repo", + "get_client", + "knowledge_topic_repo", + "user_profile_repo", + "verify_business_schemas", +] diff --git a/src/everos/infra/persistence/milvus/milvus_manager.py b/src/everos/infra/persistence/milvus/milvus_manager.py new file mode 100644 index 000000000..602e4a314 --- /dev/null +++ b/src/everos/infra/persistence/milvus/milvus_manager.py @@ -0,0 +1,87 @@ +"""Milvus connection and collection management for the derived index.""" + +from __future__ import annotations + +import re + +from pymilvus import MilvusClient + +from everos.config import MilvusSettings, load_settings +from everos.core.observability.logging import get_logger +from everos.core.persistence import MemoryRoot + +logger = get_logger(__name__) + +_client: MilvusClient | None = None + + +class MilvusSchemaMismatchError(RuntimeError): + """Raised when an existing Milvus collection does not match EverOS.""" + + +def collection_name(table_name: str, settings: MilvusSettings | None = None) -> str: + """Return the configured Milvus collection name for an EverOS table.""" + cfg = settings or load_settings().milvus + prefix = _sanitize_name_part(cfg.collection_prefix) + base = _sanitize_name_part(table_name) + name = f"{prefix}_{base}" if prefix else base + if not re.match(r"^[A-Za-z_]", name): + name = f"_{name}" + return name + + +async def get_client() -> MilvusClient: + """Return the process-wide MilvusClient, creating it lazily.""" + global _client + if _client is None: + settings = load_settings().milvus + uri = _resolve_uri(settings) + token = _secret(settings.token) + db_name = settings.db_name or "" + _client = MilvusClient(uri=uri, token=token, db_name=db_name) + logger.info( + "milvus_connection_opened", + uri=uri, + db_name=db_name or None, + consistency_level=settings.consistency_level, + ) + return _client + + +async def dispose_connection() -> None: + """Close the process-wide Milvus client.""" + global _client + if _client is not None: + _client.close() + _client = None + logger.info("milvus_connection_closed") + + +def _resolve_uri(settings: MilvusSettings) -> str: + if settings.uri: + return settings.uri + memory_root = MemoryRoot.default() + memory_root.milvus_dir.mkdir(parents=True, exist_ok=True) + return str(memory_root.milvus_db) + + +def _secret(value: object | None) -> str: + if value is None: + return "" + getter = getattr(value, "get_secret_value", None) + if callable(getter): + return getter() or "" + return str(value) + + +def _sanitize_name_part(value: str) -> str: + clean = re.sub(r"\W+", "_", value.strip()) + return clean.strip("_") + + +__all__ = [ + "MilvusSchemaMismatchError", + "collection_name", + "dispose_connection", + "get_client", +] diff --git a/src/everos/infra/persistence/milvus/repos.py b/src/everos/infra/persistence/milvus/repos.py new file mode 100644 index 000000000..826bc9267 --- /dev/null +++ b/src/everos/infra/persistence/milvus/repos.py @@ -0,0 +1,119 @@ +"""Milvus repo singletons for EverOS derived index tables.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + KnowledgeTopic, + UserProfile, +) + +from .repository import MilvusRepoBase + + +class _EpisodeRepo(MilvusRepoBase[Episode]): + schema = Episode + + +class _AtomicFactRepo(MilvusRepoBase[AtomicFact]): + schema = AtomicFact + + +class _ForesightRepo(MilvusRepoBase[Foresight]): + schema = Foresight + + +class _AgentCaseRepo(MilvusRepoBase[AgentCase]): + schema = AgentCase + + +class _AgentSkillRepo(MilvusRepoBase[AgentSkill]): + schema = AgentSkill + + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: + rows = await self.find_where( + f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + limit=10_000, + ) + return len(rows) + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[AgentSkill]: + return await self.find_where( + f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + limit=limit, + ) + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[AgentSkill]: + if not query_vector: + raise ValueError( + "query_vector must be non-empty; " + "call find_in_cluster for the scalar fallback" + ) + rows = await self.dense_search( + query_vector, + f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + limit=top_k, + ) + out: list[AgentSkill] = [] + for row in rows: + rid = row.get("id") + if isinstance(rid, str) and (item := await self.get_by_id(rid)) is not None: + out.append(item) + return out + + +class _UserProfileRepo(MilvusRepoBase[UserProfile]): + schema = UserProfile + + +class _KnowledgeTopicRepo(MilvusRepoBase[KnowledgeTopic]): + schema = KnowledgeTopic + + +def _q(value: str) -> str: + return value.replace("\\", "\\\\").replace("'", "\\'") + + +episode_repo = _EpisodeRepo() +atomic_fact_repo = _AtomicFactRepo() +foresight_repo = _ForesightRepo() +agent_case_repo = _AgentCaseRepo() +agent_skill_repo = _AgentSkillRepo() +user_profile_repo = _UserProfileRepo() +knowledge_topic_repo = _KnowledgeTopicRepo() + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + +__all__ = [ + "ALL_REPOS", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "user_profile_repo", +] diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py new file mode 100644 index 000000000..b8828118e --- /dev/null +++ b/src/everos/infra/persistence/milvus/repository.py @@ -0,0 +1,750 @@ +"""Milvus repository implementation for EverOS derived index tables.""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import re +from collections.abc import Sequence +from importlib.metadata import PackageNotFoundError, version +from typing import Any, ClassVar, get_args, get_origin + +from pymilvus import DataType, Function, FunctionType, MilvusClient + +from everos.component.utils.datetime import ( + ensure_utc, + from_iso_format, + from_timestamp, + to_timestamp_ms, +) +from everos.config import load_settings +from everos.core.observability.logging import get_logger +from everos.core.persistence import BaseLanceTable + +from .milvus_manager import ( + MilvusSchemaMismatchError, + collection_name, + get_client, +) + +logger = get_logger(__name__) + +_DUMMY_VECTOR_FIELD = "_everos_dummy_vector" +_SPARSE_SUFFIX = "__sparse" +_MAX_VARCHAR_LENGTH = 65_535 +_ID_MAX_LENGTH = 512 +_ARRAY_MAX_CAPACITY = 256 + + +def _q(value: str) -> str: + """Escape a string for a Milvus single-quoted expression literal.""" + return value.replace("\\", "\\\\").replace("'", "\\'") + + +class MilvusRepoBase[T: BaseLanceTable]: + """Generic Milvus repository for one EverOS derived index collection.""" + + schema: type[T] + _write_locks: ClassVar[dict[str, asyncio.Lock]] = {} + + @property + def table_name(self) -> str: + return self.schema.TABLE_NAME + + @property + def collection_name(self) -> str: + return collection_name(self.table_name) + + @classmethod + def _write_lock(cls, name: str) -> asyncio.Lock: + return cls._write_locks.setdefault(name, asyncio.Lock()) + + @classmethod + def _reset_locks_for_tests(cls) -> None: + cls._write_locks.clear() + + async def ensure_collection(self) -> None: + client = await get_client() + if await _run(client.has_collection, self.collection_name): + await self.verify_collection() + return + + schema = self._build_collection_schema() + index_params = client.prepare_index_params() + if "vector" in self.schema.model_fields: + index_params.add_index( + field_name="vector", + index_type="AUTOINDEX", + metric_type="COSINE", + ) + else: + index_params.add_index( + field_name=_DUMMY_VECTOR_FIELD, + index_type="AUTOINDEX", + metric_type="COSINE", + ) + for field in self.schema.BM25_FIELDS: + index_params.add_index( + field_name=_sparse_field(field), + index_type="AUTOINDEX", + metric_type="BM25", + ) + + settings = load_settings().milvus + await _run( + client.create_collection, + collection_name=self.collection_name, + schema=schema, + index_params=index_params, + consistency_level=settings.consistency_level, + ) + logger.info( + "milvus_collection_created", + table=self.table_name, + collection=self.collection_name, + ) + + async def verify_collection(self) -> None: + client = await get_client() + description = await _run(client.describe_collection, self.collection_name) + actual = {field["name"] for field in description.get("fields", [])} + expected = set(self._stored_field_names()) + missing = expected - actual + if missing: + raise MilvusSchemaMismatchError( + f"Milvus collection {self.collection_name!r} schema drift: " + f"missing={sorted(missing)}. The index is rebuildable from md; " + "drop the collection and restart to rebuild it." + ) + + # ── Create / update ───────────────────────────────────────────── + + async def add(self, records: Sequence[T]) -> None: + if not records: + return + await self.ensure_collection() + client = await get_client() + payload = [self._to_milvus_record(record) for record in records] + async with self._write_lock(self.collection_name): + await _run(client.insert, self.collection_name, payload) + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: + if by != "id": + raise ValueError("MilvusRepoBase only supports upsert by id") + if not records: + return + await self.ensure_collection() + client = await get_client() + payload = [self._to_milvus_record(record) for record in records] + async with self._write_lock(self.collection_name): + await _run(client.upsert, self.collection_name, payload) + + async def update(self, updates: dict[str, Any], *, where: Any) -> None: + await self.ensure_collection() + rows = await self._query_raw(where, limit=10_000, include_vectors=True) + if not rows: + return + patched = [] + for row in rows: + merged = dict(row) + for key, value in updates.items(): + self._write_field_value(merged, key, value) + patched.append(merged) + client = await get_client() + async with self._write_lock(self.collection_name): + await _run(client.upsert, self.collection_name, patched) + + # ── Maintenance ──────────────────────────────────────────────── + + async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None: + """Milvus indexes are managed by the service; no per-write compaction.""" + + async def rebuild_indexes(self) -> None: + """Milvus AUTOINDEX maintenance is service-managed.""" + + # ── Reads ────────────────────────────────────────────────────── + + async def count(self) -> int: + await self.ensure_collection() + client = await get_client() + stats = await _run(client.get_collection_stats, self.collection_name) + row_count = stats.get("row_count", 0) + return int(row_count) + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: + if id_field != "id": + rows = await self.find_where(f"{id_field} = '{_q(id_value)}'", limit=1) + return rows[0] if rows else None + await self.ensure_collection() + client = await get_client() + rows = await _run( + client.get, + self.collection_name, + ids=[id_value], + output_fields=self._output_fields(include_vectors=True), + ) + if not rows: + return None + return self._model_from_milvus(rows[0]) + + async def find_where(self, where: Any, *, limit: int = 100) -> list[T]: + rows = await self._query_raw(where, limit=limit, include_vectors=True) + return [self._model_from_milvus(row) for row in rows] + + async def find_one_where(self, where: Any) -> T | None: + rows = await self.find_where(where, limit=1) + return rows[0] if rows else None + + async def find_where_paginated( + self, + where: Any, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: + raw = await self._query_raw(where, limit=max_fetch, include_vectors=True) + total = len(raw) + if total >= max_fetch: + logger.warning( + "milvus_find_where_paginated_truncated", + table=self.table_name, + total=total, + max_fetch=max_fetch, + ) + rows = [self._model_from_milvus(row) for row in raw] + rows.sort( + key=lambda row: _sort_value(getattr(row, sort_by, None)), + reverse=descending, + ) + offset = (page - 1) * page_size + return rows[offset : offset + page_size], total + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: + return await self.find_where(f"owner_id = '{_q(owner_id)}'", limit=limit) + + async def find_by_md_path(self, md_path: str) -> T | None: + return await self.find_one_where(f"md_path = '{_q(md_path)}'") + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Any = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + if vector is None: + return await self._query_candidate_rows(where, limit=limit) + return await self.dense_search(vector, where, limit=limit) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + if not query_terms: + return [] + await self.ensure_collection() + fields = list(columns or self.schema.BM25_FIELDS) + if not fields: + return [] + client = await get_client() + expr = self._expr(where) + query = " ".join(term for term in query_terms if term) + best: dict[str, dict[str, Any]] = {} + for field in fields: + results = await _run( + client.search, + self.collection_name, + data=[query], + anns_field=_sparse_field(field), + filter=expr, + limit=limit, + output_fields=self._output_fields(include_vectors=False), + search_params={"metric_type": "BM25"}, + ) + for row in _first_result_set(results): + shaped = self._candidate_row_from_search(row) + score = _bm25_score_from_distance(row.get("distance")) + shaped["_score"] = score + rid = shaped.get("id") + if not isinstance(rid, str): + continue + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + best[rid] = shaped + return sorted( + best.values(), key=lambda item: float(item.get("_score", 0.0)), reverse=True + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Any, + *, + limit: int, + ) -> list[dict[str, Any]]: + if not vector or "vector" not in self.schema.model_fields: + return [] + await self.ensure_collection() + client = await get_client() + results = await _run( + client.search, + self.collection_name, + data=[list(vector)], + anns_field="vector", + filter=self._expr(where), + limit=limit, + output_fields=self._output_fields(include_vectors=False), + search_params={"metric_type": "COSINE"}, + ) + return [ + self._candidate_row_from_search(row, normalize_cosine=True) + for row in _first_result_set(results) + ] + + # ── Deletes ──────────────────────────────────────────────────── + + async def delete(self, predicate: Any) -> None: + await self.ensure_collection() + client = await get_client() + async with self._write_lock(self.collection_name): + await _run( + client.delete, + self.collection_name, + filter=self._expr(predicate), + ) + + async def delete_by_md_path(self, md_path: str) -> int: + await self.ensure_collection() + before = await self.find_where(f"md_path = '{_q(md_path)}'", limit=10_000) + if not before: + return 0 + await self.delete(f"md_path = '{_q(md_path)}'") + return len(before) + + # ── Daily-log helpers ────────────────────────────────────────── + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: + return await self.find_one_where( + f"owner_id = '{_q(owner_id)}' AND entry_id = '{_q(entry_id)}' " + f"AND app_id = '{_q(app_id)}' AND project_id = '{_q(project_id)}'" + ) + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: + if not entry_ids: + return [] + quoted = ", ".join(f"'{_q(entry_id)}'" for entry_id in entry_ids) + return await self.find_where( + f"owner_id = '{_q(owner_id)}' AND entry_id IN ({quoted}) " + f"AND app_id = '{_q(app_id)}' AND project_id = '{_q(project_id)}'", + limit=len(entry_ids), + ) + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + f"owner_id = '{_q(owner_id)}' AND session_id = '{_q(session_id)}'", + limit=limit, + ) + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + f"parent_type = '{_q(parent_type)}' AND parent_id = '{_q(parent_id)}'", + limit=limit, + ) + + # ── Field conversion ─────────────────────────────────────────── + + def _build_collection_schema(self): # type: ignore[no-untyped-def] + settings = load_settings().milvus + schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False) + has_vector = False + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + annotation = field.annotation + if name == "id": + schema.add_field( + field_name=name, + datatype=DataType.VARCHAR, + is_primary=True, + max_length=_ID_MAX_LENGTH, + ) + elif name == "vector": + has_vector = True + schema.add_field( + field_name=name, + datatype=DataType.FLOAT_VECTOR, + dim=settings.dimension, + ) + elif _is_datetime(annotation): + schema.add_field( + field_name=_datetime_storage_field(name), + datatype=DataType.INT64, + nullable=_is_optional(annotation), + ) + elif _is_list(annotation): + schema.add_field( + field_name=name, + datatype=DataType.ARRAY, + element_type=DataType.VARCHAR, + max_capacity=_ARRAY_MAX_CAPACITY, + max_length=_ID_MAX_LENGTH, + nullable=_is_optional(annotation), + ) + elif _is_float(annotation): + schema.add_field( + field_name=name, + datatype=DataType.DOUBLE, + nullable=_is_optional(annotation), + ) + elif _is_int(annotation): + schema.add_field( + field_name=name, + datatype=DataType.INT64, + nullable=_is_optional(annotation), + ) + else: + kwargs: dict[str, Any] = { + "field_name": name, + "datatype": DataType.VARCHAR, + "max_length": _MAX_VARCHAR_LENGTH, + } + if name in self.schema.BM25_FIELDS: + kwargs["enable_analyzer"] = True + elif _is_optional(annotation): + kwargs["nullable"] = True + schema.add_field(**kwargs) + + if not has_vector: + schema.add_field( + field_name=_DUMMY_VECTOR_FIELD, + datatype=DataType.FLOAT_VECTOR, + dim=1, + ) + for field in self.schema.BM25_FIELDS: + schema.add_field( + field_name=_sparse_field(field), + datatype=DataType.SPARSE_FLOAT_VECTOR, + ) + schema.add_function( + Function( + name=f"{field}_bm25", + function_type=FunctionType.BM25, + input_field_names=[field], + output_field_names=[_sparse_field(field)], + ) + ) + return schema + + def _stored_field_names(self) -> list[str]: + names: list[str] = [] + has_vector = False + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + if name == "vector": + has_vector = True + names.append(name) + elif _is_datetime(field.annotation): + names.append(_datetime_storage_field(name)) + else: + names.append(name) + if not has_vector: + names.append(_DUMMY_VECTOR_FIELD) + names.extend(_sparse_field(field) for field in self.schema.BM25_FIELDS) + return names + + def _output_fields(self, *, include_vectors: bool) -> list[str]: + fields: list[str] = [] + for name in self._stored_field_names(): + if name.endswith(_SPARSE_SUFFIX) or name == _DUMMY_VECTOR_FIELD: + continue + if not include_vectors and name == "vector": + continue + fields.append(name) + return fields + + def _to_milvus_record(self, record: T) -> dict[str, Any]: + raw = record.model_dump(mode="python") + out: dict[str, Any] = {} + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + value = raw.get(name) + if name in self.schema.BM25_FIELDS and value is None: + value = "" + if _is_datetime(field.annotation): + storage_name = _datetime_storage_field(name) + out[storage_name] = ( + _datetime_to_ms(value) if value is not None else None + ) + elif name == "vector": + out[name] = list(value or []) + elif _is_list(field.annotation): + out[name] = [str(item) for item in (value or [])] + else: + out[name] = value + if "vector" not in self.schema.model_fields: + out[_DUMMY_VECTOR_FIELD] = [0.0] + return out + + def _model_from_milvus(self, row: dict[str, Any]) -> T: + shaped = self._restore_row(row, include_distance=False) + return self.schema.model_validate(shaped) + + def _candidate_row_from_search( + self, row: dict[str, Any], *, normalize_cosine: bool = False + ) -> dict[str, Any]: + shaped = self._restore_row(row.get("entity", {}), include_distance=False) + raw_distance = row.get("distance") + shaped["_distance"] = ( + _cosine_distance_from_milvus(raw_distance) + if normalize_cosine + else raw_distance + ) + return shaped + + def _restore_row( + self, row: dict[str, Any], *, include_distance: bool + ) -> dict[str, Any]: + out: dict[str, Any] = {} + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + if _is_datetime(field.annotation): + storage_name = _datetime_storage_field(name) + value = row.get(storage_name) + if value is None: + out[name] = None + else: + out[name] = from_timestamp(int(value)) + elif name in row: + value = row[name] + if name in self.schema.BM25_FIELDS and value == "" and _is_optional( + field.annotation + ): + out[name] = None + else: + out[name] = value + if include_distance and "distance" in row: + out["_distance"] = row["distance"] + return out + + def _write_field_value( + self, row: dict[str, Any], field_name: str, value: Any + ) -> None: + field = self.schema.model_fields.get(field_name) + if field is None: + row[field_name] = value + elif _is_datetime(field.annotation): + row[_datetime_storage_field(field_name)] = ( + _datetime_to_ms(value) if value is not None else None + ) + else: + row[field_name] = value + + async def _query_raw( + self, where: Any, *, limit: int, include_vectors: bool + ) -> list[dict[str, Any]]: + await self.ensure_collection() + client = await get_client() + return await _run( + client.query, + self.collection_name, + filter=self._expr(where), + output_fields=self._output_fields(include_vectors=include_vectors), + limit=limit, + ) + + async def _query_candidate_rows( + self, where: Any, *, limit: int + ) -> list[dict[str, Any]]: + rows = await self._query_raw(where, limit=limit, include_vectors=False) + return [self._restore_row(row, include_distance=False) for row in rows] + + def _expr(self, where: Any) -> str: + if where is None: + return "" + milvus_expr = getattr(where, "milvus", None) + if isinstance(milvus_expr, str): + return milvus_expr + if not isinstance(where, str): + return str(where) + return _lancedb_expr_to_milvus(where) + + +def _lancedb_expr_to_milvus(expr: str) -> str: + """Best-effort conversion for internal EverOS LanceDB predicates.""" + converted = expr + converted = re.sub(r"\barray_has\s*\(", "array_contains(", converted) + converted = re.sub(r"\bIS\s+NULL\b", "is null", converted, flags=re.IGNORECASE) + converted = re.sub( + r"\b([A-Za-z_][A-Za-z0-9_]*)\s+IN\s+\(([^)]*)\)", + lambda m: f"{m.group(1)} in [{m.group(2)}]", + converted, + flags=re.IGNORECASE, + ) + converted = re.sub(r"(?=])=(?!=)", "==", converted) + converted = re.sub(r"\btimestamp\b", "timestamp_ms", converted) + converted = re.sub(r"\bcreated_at\b", "created_at_ms", converted) + converted = re.sub(r"\bupdated_at\b", "updated_at_ms", converted) + converted = re.sub( + r"TIMESTAMP\s+'([^']*)'", + lambda m: str(_iso_to_ms(m.group(1))), + converted, + ) + return converted + + +def _datetime_storage_field(name: str) -> str: + return f"{name}_ms" + + +def _sparse_field(name: str) -> str: + return f"{name}{_SPARSE_SUFFIX}" + + +def _datetime_to_ms(value: Any) -> int: + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return to_timestamp_ms(aware) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return int(value) + raise TypeError(f"expected datetime or epoch ms, got {type(value).__name__}") + + +def _iso_to_ms(value: str) -> int: + aware = ensure_utc(from_iso_format(value)) + assert aware is not None + return to_timestamp_ms(aware) + + +def _is_optional(annotation: Any) -> bool: + args = get_args(annotation) + return type(None) in args + + +def _non_none_args(annotation: Any) -> tuple[Any, ...]: + return tuple(arg for arg in get_args(annotation) if arg is not type(None)) + + +def _is_datetime(annotation: Any) -> bool: + if annotation is dt.datetime: + return True + return any(arg is dt.datetime for arg in _non_none_args(annotation)) + + +def _is_list(annotation: Any) -> bool: + origin = get_origin(annotation) + if origin is list: + return True + return any(get_origin(arg) is list for arg in _non_none_args(annotation)) + + +def _is_float(annotation: Any) -> bool: + if annotation is float: + return True + return any(arg is float for arg in _non_none_args(annotation)) + + +def _is_int(annotation: Any) -> bool: + if annotation is int: + return True + return any(arg is int for arg in _non_none_args(annotation)) + + +def _sort_value(value: Any) -> Any: + if value is None: + fallback = ensure_utc(dt.datetime.min) + assert fallback is not None + return fallback + return value + + +def _bm25_score_from_distance(distance: Any) -> float: + """Convert the Milvus BM25 search distance into a higher-is-better score.""" + if distance is None: + return 0.0 + return abs(float(distance)) + + +def _cosine_distance_from_milvus(distance: Any) -> float | None: + """Normalize Milvus COSINE results to LanceDB-style distance. + + Milvus server / Zilliz Cloud return cosine similarity through the + ``distance`` field. Milvus Lite 3.0 reports cosine distance instead; keep + this version-gated workaround narrow because the upstream issue is expected + to be fixed after 3.0: + https://github.com/milvus-io/milvus-lite/issues/343 + + Recaller code expects LanceDB's ``1 - similarity`` distance contract. + """ + if distance is None: + return None + value = float(distance) + if not _uses_milvus_lite_3_0_cosine_distance_bug(): + value = 1.0 - value + if value < 0.0: + return 0.0 + if value > 1.0: + return 1.0 + return value + + +def _uses_milvus_lite() -> bool: + uri = load_settings().milvus.uri + if not uri: + return True + return "://" not in uri + + +def _uses_milvus_lite_3_0_cosine_distance_bug() -> bool: + if not _uses_milvus_lite(): + return False + return _milvus_lite_version() in {"3.0", "3.0.0"} + + +def _milvus_lite_version() -> str | None: + try: + return version("milvus-lite") + except PackageNotFoundError: + return None + + +def _first_result_set(results: Any) -> list[dict[str, Any]]: + if not results: + return [] + first = results[0] + return list(first or []) + + +async def _run(func: Any, /, *args: Any, **kwargs: Any) -> Any: + return await asyncio.to_thread(func, *args, **kwargs) + + +__all__ = ["MilvusRepoBase"] diff --git a/src/everos/memory/cascade/handlers/agent_case.py b/src/everos/memory/cascade/handlers/agent_case.py index a6fa9c5b7..2f6bcca20 100644 --- a/src/everos/memory/cascade/handlers/agent_case.py +++ b/src/everos/memory/cascade/handlers/agent_case.py @@ -30,7 +30,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import AgentCase, ParentType, agent_case_repo +from everos.infra.persistence.index import AgentCase, ParentType, agent_case_repo from ._common import require_float, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/agent_skill.py b/src/everos/memory/cascade/handlers/agent_skill.py index 88ae83959..25315d35b 100644 --- a/src/everos/memory/cascade/handlers/agent_skill.py +++ b/src/everos/memory/cascade/handlers/agent_skill.py @@ -42,7 +42,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader -from everos.infra.persistence.lancedb import AgentSkill, agent_skill_repo +from everos.infra.persistence.index import AgentSkill, agent_skill_repo from everos.infra.persistence.markdown import AgentSkillFrontmatter from ..types import HandlerOutcome diff --git a/src/everos/memory/cascade/handlers/atomic_fact.py b/src/everos/memory/cascade/handlers/atomic_fact.py index 6dea35e4e..673423369 100644 --- a/src/everos/memory/cascade/handlers/atomic_fact.py +++ b/src/everos/memory/cascade/handlers/atomic_fact.py @@ -23,7 +23,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import AtomicFact, ParentType, atomic_fact_repo +from everos.infra.persistence.index import AtomicFact, ParentType, atomic_fact_repo from ._common import parse_inline_list, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/episode.py b/src/everos/memory/cascade/handlers/episode.py index 84878d302..57dd6e5ad 100644 --- a/src/everos/memory/cascade/handlers/episode.py +++ b/src/everos/memory/cascade/handlers/episode.py @@ -38,7 +38,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import Episode, ParentType, episode_repo +from everos.infra.persistence.index import Episode, ParentType, episode_repo from ._common import parse_inline_list, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/foresight.py b/src/everos/memory/cascade/handlers/foresight.py index adbefe020..02aeea926 100644 --- a/src/everos/memory/cascade/handlers/foresight.py +++ b/src/everos/memory/cascade/handlers/foresight.py @@ -30,7 +30,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import Foresight, ParentType, foresight_repo +from everos.infra.persistence.index import Foresight, ParentType, foresight_repo from ._common import ( optional_int, diff --git a/src/everos/memory/cascade/handlers/knowledge_topic.py b/src/everos/memory/cascade/handlers/knowledge_topic.py index daa8fbc3c..2535a0572 100644 --- a/src/everos/memory/cascade/handlers/knowledge_topic.py +++ b/src/everos/memory/cascade/handlers/knowledge_topic.py @@ -38,7 +38,7 @@ from everos.component.utils.datetime import get_utc_now from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, ParsedMarkdown -from everos.infra.persistence.lancedb import KnowledgeTopic, knowledge_topic_repo +from everos.infra.persistence.index import KnowledgeTopic, knowledge_topic_repo from everos.infra.persistence.sqlite import ( TopicUpsertPayload, knowledge_topic_sqlite_repo, diff --git a/src/everos/memory/cascade/handlers/user_profile.py b/src/everos/memory/cascade/handlers/user_profile.py index 4842986d0..8d7573521 100644 --- a/src/everos/memory/cascade/handlers/user_profile.py +++ b/src/everos/memory/cascade/handlers/user_profile.py @@ -27,7 +27,7 @@ from typing import Any, ClassVar from everos.core.persistence import MarkdownReader -from everos.infra.persistence.lancedb import UserProfile, user_profile_repo +from everos.infra.persistence.index import UserProfile, user_profile_repo from ..types import HandlerOutcome from ._common import content_sha256 as compute_content_sha256 diff --git a/src/everos/memory/cascade/registry.py b/src/everos/memory/cascade/registry.py index 8537575e6..3054a17a3 100644 --- a/src/everos/memory/cascade/registry.py +++ b/src/everos/memory/cascade/registry.py @@ -18,7 +18,7 @@ from pathlib import PurePosixPath from everos.core.persistence.markdown import BaseFrontmatter -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase, AgentSkill, AtomicFact, diff --git a/src/everos/memory/get/filters_adapter.py b/src/everos/memory/get/filters_adapter.py index 407a06bc7..46b1e6305 100644 --- a/src/everos/memory/get/filters_adapter.py +++ b/src/everos/memory/get/filters_adapter.py @@ -16,6 +16,7 @@ from __future__ import annotations from everos.memory.search import FilterNode, compile_filters +from everos.memory.search.filters import BackendFilters, compile_filters_for_backends def compile_filters_for_get( @@ -38,3 +39,21 @@ def compile_filters_for_get( app_id=app_id, project_id=project_id, ) + + +def compile_filters_for_get_backends( + filters: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str = "default", + project_id: str = "default", +) -> BackendFilters: + """Compile ``/get`` filters for every derived index backend.""" + return compile_filters_for_backends( + filters, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + ) diff --git a/src/everos/memory/get/manager.py b/src/everos/memory/get/manager.py index c42edd7ea..7a9421797 100644 --- a/src/everos/memory/get/manager.py +++ b/src/everos/memory/get/manager.py @@ -12,7 +12,7 @@ Reads only — never writes. Filters are compiled through :func:`compile_filters_for_get` so the column allow-list stays shared with :mod:`memory.search`. Pagination + in-memory sort -runs through :meth:`LanceRepoBase.find_where_paginated`. +runs through the configured derived index repository. """ from __future__ import annotations @@ -34,11 +34,11 @@ GetRequest, GetResponse, ) -from .filters_adapter import compile_filters_for_get +from .filters_adapter import compile_filters_for_get_backends if TYPE_CHECKING: from everos.core.persistence.lancedb import LanceRepoBase - from everos.infra.persistence.lancedb import ( + from everos.infra.persistence.index import ( AgentCase, AgentSkill, Episode, @@ -49,7 +49,7 @@ class GetManager: - """Dispatch ``GetRequest`` to the matching LanceDB-backed repo and + """Dispatch ``GetRequest`` to the matching derived index repo and shape rows into the public DTO.""" def __init__( @@ -70,7 +70,7 @@ def __init__( async def get(self, req: GetRequest) -> GetResponse: request_id = resolve_request_id() descending = req.sort_order == "desc" - where = compile_filters_for_get( + where = compile_filters_for_get_backends( req.filters, owner_id=req.owner_id, owner_type=req.owner_type, @@ -187,7 +187,7 @@ def _shape_agent_skill(row: AgentSkill) -> GetAgentSkillItem: async def _fetch_profile(self, owner_id: str) -> list[GetProfileItem]: """Fetch the owner's single profile row from the ``user_profile`` - LanceDB table (kept in sync with ``users//user.md`` by cascade). + Derived index row (kept in sync with ``users//user.md`` by cascade). Profile is one-row-per-owner KV — there is no pagination / sort / filter surface, so at most one item is returned. Mirrors the diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 2dd9aa804..22df21a47 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -1,4 +1,4 @@ -"""Filters DSL → LanceDB ``where`` string compiler. +"""Filters DSL → derived-index filter compiler. The Filters DSL is intentionally permissive at the JSON layer (so callers can pass whatever they like and get a clean 400 if it is not supported) @@ -23,7 +23,8 @@ kind). Iterate / membership-test only; do not mutate. * :data:`RESERVED_FIELDS` — names rejected inside any ``filters`` block. * :func:`compile_predicate` — render one ``{field: value}`` clause to - SQL. Operator-map and equality-shorthand are both handled. + a backend-specific predicate. Operator-map and equality-shorthand are + both handled. The high-level :func:`compile_filters` remains the entry point for ``/search`` (combinator-aware). @@ -31,10 +32,18 @@ from __future__ import annotations +import dataclasses import datetime as _dt +import json from typing import Any, Final -from everos.component.utils.datetime import from_timestamp, to_iso_format +from everos.component.utils.datetime import ( + ensure_utc, + from_iso_format, + from_timestamp, + to_iso_format, + to_timestamp_ms, +) from everos.core.errors import FilterError as FilterError from .dto import FilterNode @@ -50,6 +59,15 @@ "lte": "<=", "in": "IN", } +_MILVUS_OP_MAP: Final[dict[str, str]] = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", + "in": "in", +} # Field kinds: ``str`` rendered as ``''``; ``ts`` rendered as # ``TIMESTAMP ''`` (DataFusion timestamp literal); ``array_str`` @@ -82,6 +100,17 @@ def __init__(self, column: str, kind: _FieldKind) -> None: ) +@dataclasses.dataclass(frozen=True) +class BackendFilters: + """Compiled filters for every supported derived index backend.""" + + lancedb: str + milvus: str + + def __str__(self) -> str: + return self.lancedb + + # ── Public API ─────────────────────────────────────────────────────────── @@ -102,19 +131,72 @@ def compile_filters( from another — omitting it would let a query bleed across spaces. Both ``/search`` and ``/get`` share this compile path. """ - base = [ - f"owner_id = '{_escape_str(owner_id)}'", - f"owner_type = '{owner_type}'", - f"app_id = '{_escape_str(app_id)}'", - f"project_id = '{_escape_str(project_id)}'", - ] + return _compile_filters_backend( + node, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend="lancedb", + ) + + +def compile_filters_for_backends( + node: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str = "default", + project_id: str = "default", +) -> BackendFilters: + """Compile request filters for every supported derived index backend.""" + return BackendFilters( + lancedb=_compile_filters_backend( + node, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend="lancedb", + ), + milvus=_compile_filters_backend( + node, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend="milvus", + ), + ) + + +def _compile_filters_backend( + node: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str, + project_id: str, + backend: str, +) -> str: + base = _base_clauses( + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend=backend, + ) # Only episode / atomic_fact tables carry the ``deprecated_by`` column # (Reflection V1 marks superseded entries). Agent tables don't have it. if owner_type == "user": - base.append("deprecated_by IS NULL") + base.append( + "deprecated_by IS NULL" + if backend == "lancedb" + else "deprecated_by is null" + ) if node is None: return " AND ".join(base) - compiled = _compile_node(node.model_dump(exclude_none=True)) + compiled = _compile_node(node.model_dump(exclude_none=True), backend=backend) if not compiled: return " AND ".join(base) return " AND ".join([*base, compiled]) @@ -123,7 +205,7 @@ def compile_filters( # ── Internals ──────────────────────────────────────────────────────────── -def _compile_node(raw: dict[str, Any]) -> str: +def _compile_node(raw: dict[str, Any], *, backend: str = "lancedb") -> str: """Walk one DSL node; return the matching SQL fragment (no leading parens). Empty nodes yield ``""`` so :func:`compile_filters` can skip the @@ -133,9 +215,9 @@ def _compile_node(raw: dict[str, Any]) -> str: parts: list[str] = [] if (and_list := raw.pop("AND", None)) is not None: - parts.append(_compile_combinator(and_list, "AND")) + parts.append(_compile_combinator(and_list, "AND", backend=backend)) if (or_list := raw.pop("OR", None)) is not None: - parts.append(_compile_combinator(or_list, "OR")) + parts.append(_compile_combinator(or_list, "OR", backend=backend)) for field, value in raw.items(): if field in RESERVED_FIELDS: @@ -144,7 +226,7 @@ def _compile_node(raw: dict[str, Any]) -> str: ) if field not in ALLOWED_FIELDS: raise FilterError(f"unsupported filter field: {field!r}") - parts.append(compile_predicate(field, value)) + parts.append(compile_predicate(field, value, backend=backend)) # Drop empty fragments coming from empty AND/OR arrays. parts = [p for p in parts if p] @@ -155,7 +237,9 @@ def _compile_node(raw: dict[str, Any]) -> str: return " AND ".join(parts) -def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: +def _compile_combinator( + children: list[dict[str, Any]], op: str, *, backend: str = "lancedb" +) -> str: """Render an ``AND`` / ``OR`` array of child nodes.""" if not isinstance(children, list): raise FilterError(f"{op} expects an array of nodes") @@ -163,7 +247,7 @@ def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: for child in children: if not isinstance(child, dict): raise FilterError(f"{op} children must be objects") - compiled = _compile_node(child) + compiled = _compile_node(child, backend=backend) if compiled: fragments.append(f"({compiled})") if not fragments: @@ -175,7 +259,7 @@ def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: return "(" + glue.join(fragments) + ")" -def compile_predicate(field: str, value: Any) -> str: +def compile_predicate(field: str, value: Any, *, backend: str = "lancedb") -> str: """Render one ``"": `` clause to SQL. Public primitive — :mod:`memory.get` builds a flat (no AND/OR) @@ -195,48 +279,65 @@ def compile_predicate(field: str, value: Any) -> str: if not value: raise FilterError(f"empty operator map for field {field!r}") clauses = [ - _compile_op_clause(spec, field, op, op_val) for op, op_val in value.items() + _compile_op_clause(spec, field, op, op_val, backend=backend) + for op, op_val in value.items() ] if len(clauses) == 1: return clauses[0] return "(" + " AND ".join(clauses) + ")" # Equality shorthand. - return _compile_op_clause(spec, field, "eq", value) + return _compile_op_clause(spec, field, "eq", value, backend=backend) -def _compile_op_clause(spec: _FieldSpec, field: str, op: str, value: Any) -> str: +def _compile_op_clause( + spec: _FieldSpec, field: str, op: str, value: Any, *, backend: str = "lancedb" +) -> str: """Render a single `` `` clause.""" if op not in _OP_MAP: raise FilterError(f"unsupported operator {op!r} on field {field!r}") - sql_op = _OP_MAP[op] + op_map = _MILVUS_OP_MAP if backend == "milvus" else _OP_MAP + sql_op = op_map[op] + column = _column_for_backend(spec, backend) if spec.kind == "array_str": # Only equality / membership make sense on a list column. + fn = "array_contains" if backend == "milvus" else "array_has" if op == "eq": - literal = _escape_str(_require_str(value, field)) - return f"array_has({spec.column}, '{literal}')" + literal = _render_str_literal(_require_str(value, field), backend) + return f"{fn}({column}, {literal})" if op == "in": items = _require_list(value, field) - literals = [f"'{_escape_str(_require_str(v, field))}'" for v in items] - inner = " OR ".join(f"array_has({spec.column}, {lit})" for lit in literals) + literals = [ + _render_str_literal(_require_str(v, field), backend) for v in items + ] + inner = " OR ".join(f"{fn}({column}, {lit})" for lit in literals) return f"({inner})" raise FilterError(f"operator {op!r} is not supported on array field {field!r}") if op == "in": items = _require_list(value, field) - literals = [_render_literal(v, spec.kind, field) for v in items] - return f"{spec.column} IN ({', '.join(literals)})" + literals = [ + _render_literal(v, spec.kind, field, backend=backend) for v in items + ] + if backend == "milvus": + return f"{column} in [{', '.join(literals)}]" + return f"{column} IN ({', '.join(literals)})" - return f"{spec.column} {sql_op} {_render_literal(value, spec.kind, field)}" + literal = _render_literal(value, spec.kind, field, backend=backend) + return f"{column} {sql_op} {literal}" # ── Literal rendering ──────────────────────────────────────────────────── -def _render_literal(value: Any, kind: _FieldKind, field: str) -> str: +def _render_literal( + value: Any, kind: _FieldKind, field: str, *, backend: str = "lancedb" +) -> str: if kind == "str": - return f"'{_escape_str(_require_str(value, field))}'" + return _render_str_literal(_require_str(value, field), backend) if kind == "ts": + if backend == "milvus": + return str(_render_ts_ms(value, field)) return f"TIMESTAMP '{_render_ts(value, field)}'" raise FilterError(f"unsupported field kind {kind!r} for field {field!r}") @@ -257,6 +358,58 @@ def _render_ts(value: Any, field: str) -> str: raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") +def _render_ts_ms(value: Any, field: str) -> int: + if isinstance(value, bool): + raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") + if isinstance(value, (int, float)): + return int(value) + if isinstance(value, str): + if "'" in value: + raise FilterError(f"timestamp string for {field!r} contains a quote") + try: + aware = ensure_utc(from_iso_format(value)) + except (TypeError, ValueError) as exc: + raise FilterError( + f"timestamp value for {field!r} must be ms or ISO string" + ) from exc + assert aware is not None + return to_timestamp_ms(aware) + if isinstance(value, _dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return to_timestamp_ms(aware) + raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") + + +def _render_str_literal(value: str, backend: str) -> str: + if backend == "milvus": + return json.dumps(value, ensure_ascii=False) + return f"'{_escape_str(value)}'" + + +def _base_clauses( + *, + owner_id: str, + owner_type: str, + app_id: str, + project_id: str, + backend: str, +) -> list[str]: + eq = "==" if backend == "milvus" else "=" + return [ + f"owner_id {eq} {_render_str_literal(owner_id, backend)}", + f"owner_type {eq} {_render_str_literal(owner_type, backend)}", + f"app_id {eq} {_render_str_literal(app_id, backend)}", + f"project_id {eq} {_render_str_literal(project_id, backend)}", + ] + + +def _column_for_backend(spec: _FieldSpec, backend: str) -> str: + if backend == "milvus" and spec.kind == "ts": + return f"{spec.column}_ms" + return spec.column + + def _escape_str(value: str) -> str: """Double single quotes — SQL-standard escape for a single-quoted literal.""" return value.replace("'", "''") diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index 1ae5bfd91..7bb15a6b4 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -22,7 +22,8 @@ no query-relevance score we can assign to a fact pulled by parent_id alone, and emitting ``score=0.0`` facts would muddy the contract. -The manager never writes to storage; it only reads LanceDB + markdown. +The manager never writes to storage; it only reads the derived index + +markdown. """ from __future__ import annotations @@ -70,7 +71,7 @@ SearchResponse, UnprocessedMessageDTO, ) -from .filters import compile_filters +from .filters import compile_filters_for_backends from .hierarchy import build_ep_to_fact_parents, heap_expand from .shaper import ( reshape_hybrid_output, @@ -204,7 +205,7 @@ async def search(self, req: SearchRequest) -> SearchResponse: # Compile filters first: a malformed `filters` payload is a user # input error (422) and should surface before the server-side # component guard (500). The two steps are independent. - where = compile_filters( + where = compile_filters_for_backends( req.filters, owner_id=req.owner_id, owner_type=req.owner_type, diff --git a/src/everos/memory/search/recall/agent_case.py b/src/everos/memory/search/recall/agent_case.py index 93a7f9b60..9c17ba45e 100644 --- a/src/everos/memory/search/recall/agent_case.py +++ b/src/everos/memory/search/recall/agent_case.py @@ -13,17 +13,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import AgentCase, get_table +from everos.infra.persistence.index import AgentCase, agent_case_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -50,43 +48,15 @@ async def sparse_recall( per BM25 column (``MatchQuery`` is column-bound), then the two per-column result lists merge by id keeping the max score. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, AgentCase.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AgentCase.TABLE_NAME, AgentCase) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in AgentCase.BM25_FIELDS), + merged_rows = await agent_case_repo.sparse_search( + terms, + where, + columns=AgentCase.BM25_FIELDS, + limit=limit, ) - # Merge by id, keep the max BM25 score across the two columns. - # task_intent hits typically score higher (the retrieval anchor); - # approach hits catch queries that match a step detail. - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - merged_rows = sorted( - best.values(), key=lambda r: float(r.get("_score", 0.0)), reverse=True - )[:limit] return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -97,15 +67,7 @@ async def dense_recall( ) -> list[Candidate]: if not vector: return [] - table = await get_table(AgentCase.TABLE_NAME, AgentCase) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await agent_case_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, diff --git a/src/everos/memory/search/recall/agent_skill.py b/src/everos/memory/search/recall/agent_skill.py index f444e4a38..0c1831c79 100644 --- a/src/everos/memory/search/recall/agent_skill.py +++ b/src/everos/memory/search/recall/agent_skill.py @@ -9,17 +9,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import AgentSkill, get_table +from everos.infra.persistence.index import AgentSkill, agent_skill_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -48,41 +46,15 @@ async def sparse_recall( rationale. One BooleanQuery per BM25 column; merge by id with max score. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, AgentSkill.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in AgentSkill.BM25_FIELDS), + merged_rows = await agent_skill_repo.sparse_search( + terms, + where, + columns=AgentSkill.BM25_FIELDS, + limit=limit, ) - # Merge by id, keep max BM25 score across the two columns. - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - merged_rows = sorted( - best.values(), key=lambda r: float(r.get("_score", 0.0)), reverse=True - )[:limit] return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -93,15 +65,7 @@ async def dense_recall( ) -> list[Candidate]: if not vector: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await agent_skill_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -124,8 +88,7 @@ async def fetch_by_case_ids( """ if not case_ids: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) clause = " OR ".join(f"array_has(source_case_ids, '{_q(c)}')" for c in case_ids) full_where = f"({where}) AND ({clause})" - rows = await table.query().where(full_where).limit(limit).to_list() + rows = await agent_skill_repo.search(where=full_where, limit=limit) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index 69eba22d4..38ec3872a 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -21,11 +21,10 @@ from everalgo.types import Candidate, FactCandidate -from everos.infra.persistence.lancedb import AtomicFact, get_table +from everos.infra.persistence.index import AtomicFact, atomic_fact_repo from .base import ( RecallerDeps, - build_or_query, cosine_score_from_distance, row_to_candidate, ) @@ -49,14 +48,14 @@ async def sparse_recall( self, query: str, where: str, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery (see EpisodeRecaller docstring).""" - bq = build_or_query( - self._deps.tokenizer, query, column=AtomicFact.BM25_FIELDS[0] - ) - if bq is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - rows = ( - await table.query().nearest_to_text(bq).where(where).limit(limit).to_list() + rows = await atomic_fact_repo.sparse_search( + terms, + where, + columns=AtomicFact.BM25_FIELDS, + limit=limit, ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -78,15 +77,7 @@ async def dense_recall( """ if not vector: return [] - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await atomic_fact_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -170,17 +161,11 @@ async def _query_facts_for_parents( clause = f"parent_id IN ({quoted})" full_where = f"({where}) AND ({clause})" limit = per_episode * max(len(parent_to_eps), 1) - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) if query_vector: - return await ( - table.query() - .nearest_to(list(query_vector)) - .distance_type("cosine") - .where(full_where) - .limit(limit) - .to_list() + return await atomic_fact_repo.dense_search( + query_vector, full_where, limit=limit ) - return await table.query().where(full_where).limit(limit).to_list() + return await atomic_fact_repo.search(where=full_where, limit=limit) def _build_parent_to_episode_map( diff --git a/src/everos/memory/search/recall/episode.py b/src/everos/memory/search/recall/episode.py index a2766dcec..1f2c13821 100644 --- a/src/everos/memory/search/recall/episode.py +++ b/src/everos/memory/search/recall/episode.py @@ -7,11 +7,10 @@ from everalgo.types import Candidate -from everos.infra.persistence.lancedb import Episode, get_table +from everos.infra.persistence.index import Episode, active_backend, episode_repo from .base import ( RecallerDeps, - build_or_query, cosine_score_from_distance, row_to_candidate, ) @@ -55,12 +54,14 @@ async def sparse_recall( Mirrors enterprise's ``bool.should + minimum_should_match=1`` ES design. """ - bq = build_or_query(self._deps.tokenizer, query, column=Episode.BM25_FIELDS[0]) - if bq is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query().nearest_to_text(bq).where(where).limit(limit).to_list() + rows = await episode_repo.sparse_search( + terms, + where, + columns=Episode.BM25_FIELDS, + limit=limit, ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -72,16 +73,7 @@ async def dense_recall( ) -> list[Candidate]: if not vector: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query() - .nearest_to(list(vector)) - .column("vector") - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await episode_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -113,6 +105,10 @@ async def dense_recall_subject( """ if not vector: return [] + if active_backend() == "milvus": + return await self.dense_recall(vector, where, limit=limit) + from everos.infra.persistence.lancedb import get_table + table = await get_table(Episode.TABLE_NAME, Episode) rows = ( await table.query() @@ -149,8 +145,7 @@ async def fetch_all_for_owner(self, where: str) -> list[Candidate]: No ``limit`` — the full owner partition is required for cluster membership matching. """ - table = await get_table(Episode.TABLE_NAME, Episode) - rows = await table.query().where(where).to_list() + rows = await episode_repo.search(where=where, limit=10_000) result: list[Candidate] = [] for r in rows: entry_id = r.get("entry_id") @@ -173,8 +168,7 @@ async def fetch_by_entry_ids( """Fetch episodes by entry_id (for facts whose parent_id is an entry_id).""" if not entry_ids: return [] - table = await get_table(Episode.TABLE_NAME, Episode) quoted = ", ".join(f"'{_q(eid)}'" for eid in entry_ids) full_where = f"({where}) AND (entry_id IN ({quoted}))" - rows = await table.query().where(full_where).limit(len(entry_ids)).to_list() + rows = await episode_repo.search(where=full_where, limit=len(entry_ids)) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/knowledge_topic.py b/src/everos/memory/search/recall/knowledge_topic.py index ea484e17a..9cc6a9bfb 100644 --- a/src/everos/memory/search/recall/knowledge_topic.py +++ b/src/everos/memory/search/recall/knowledge_topic.py @@ -12,17 +12,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import KnowledgeTopic, get_table +from everos.infra.persistence.index import KnowledgeTopic, knowledge_topic_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -78,26 +76,15 @@ async def sparse_recall( matching the query in either its summary or its content body is surfaced without double-counting. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, KnowledgeTopic.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in KnowledgeTopic.BM25_FIELDS), + merged_rows = await knowledge_topic_repo.sparse_search( + terms, + where, + columns=KnowledgeTopic.BM25_FIELDS, + limit=limit, ) - merged_rows = _merge_bm25_results(per_column, limit=limit) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -109,15 +96,7 @@ async def dense_recall( """Cosine ANN over the ``summary`` vector (1024-d).""" if not vector: return [] - table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await knowledge_topic_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, diff --git a/src/everos/memory/search/recall/profile.py b/src/everos/memory/search/recall/profile.py index be149b927..640d27091 100644 --- a/src/everos/memory/search/recall/profile.py +++ b/src/everos/memory/search/recall/profile.py @@ -19,7 +19,7 @@ from typing import Any from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import user_profile_repo +from everos.infra.persistence.index import user_profile_repo from ..dto import SearchProfileItem diff --git a/src/everos/memory/strategies/extract_agent_skill.py b/src/everos/memory/strategies/extract_agent_skill.py index 7d4e6bc3b..cb840ea3e 100644 --- a/src/everos/memory/strategies/extract_agent_skill.py +++ b/src/everos/memory/strategies/extract_agent_skill.py @@ -83,10 +83,10 @@ from everos.infra.ome.context import StrategyContext from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.triggers import Immediate -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase as LanceAgentCase, ) -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( agent_case_repo, agent_skill_repo, ) diff --git a/src/everos/memory/strategies/extract_user_profile.py b/src/everos/memory/strategies/extract_user_profile.py index 711eca096..38e7db5d4 100644 --- a/src/everos/memory/strategies/extract_user_profile.py +++ b/src/everos/memory/strategies/extract_user_profile.py @@ -59,7 +59,7 @@ from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.events import BaseEvent from everos.infra.ome.triggers import Immediate -from everos.infra.persistence.lancedb import episode_repo +from everos.infra.persistence.index import episode_repo from everos.infra.persistence.markdown import ( ProfileReader, ProfileWriter, diff --git a/src/everos/memory/strategies/reflect_episodes.py b/src/everos/memory/strategies/reflect_episodes.py index 2d39b8795..44aa0e149 100644 --- a/src/everos/memory/strategies/reflect_episodes.py +++ b/src/everos/memory/strategies/reflect_episodes.py @@ -22,7 +22,7 @@ from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.events import CronTick from everos.infra.ome.triggers import Cron -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( atomic_fact_repo, episode_repo, ) diff --git a/src/everos/service/get.py b/src/everos/service/get.py index 6d5a72bc2..0a83ff746 100644 --- a/src/everos/service/get.py +++ b/src/everos/service/get.py @@ -1,8 +1,8 @@ """Get use case — lazy singleton wiring for ``POST /api/v2/memory/get``. Mirrors :mod:`everos.service.search`: the :class:`GetManager` and its -LanceDB repo singletons are built on first call so the FastAPI module -import order stays decoupled from the lifespan that brings up LanceDB. +derived index repo singletons are built on first call so the FastAPI module +import order stays decoupled from the lifespan that brings up the index backend. ``/get`` is read-only and uses no embedding / LLM / rerank clients — it never blocks on optional components the way ``/search`` does. @@ -11,7 +11,7 @@ from __future__ import annotations from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( agent_case_repo, agent_skill_repo, episode_repo, diff --git a/tests/unit/test_config/test_settings.py b/tests/unit/test_config/test_settings.py index fb4fbb17f..b02d2e78b 100644 --- a/tests/unit/test_config/test_settings.py +++ b/tests/unit/test_config/test_settings.py @@ -120,6 +120,33 @@ def test_embedding_rerank_defaults() -> None: assert s.llm.api_key.get_secret_value() == "" +def test_index_milvus_defaults() -> None: + s = Settings() + assert s.index.backend == "lancedb" + assert s.milvus.uri == "" + assert s.milvus.token.get_secret_value() == "" + assert s.milvus.db_name == "" + assert s.milvus.consistency_level == "Session" + assert s.milvus.dimension == 1024 + assert s.milvus.collection_prefix == "everos" + + +def test_index_milvus_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", "http://localhost:19530") + monkeypatch.setenv("EVEROS_MILVUS__TOKEN", "secret") + monkeypatch.setenv("EVEROS_MILVUS__DB_NAME", "tenant_a") + monkeypatch.setenv("EVEROS_MILVUS__CONSISTENCY_LEVEL", "Strong") + monkeypatch.setenv("EVEROS_MILVUS__DIMENSION", "1536") + s = Settings() + assert s.index.backend == "milvus" + assert s.milvus.uri == "http://localhost:19530" + assert s.milvus.token.get_secret_value() == "secret" + assert s.milvus.db_name == "tenant_a" + assert s.milvus.consistency_level == "Strong" + assert s.milvus.dimension == 1536 + + def test_resolve_root_default(monkeypatch: pytest.MonkeyPatch) -> None: """No --root, no EVEROS_ROOT → ~/.everos.""" monkeypatch.delenv("EVEROS_ROOT", raising=False) diff --git a/tests/unit/test_core/test_persistence/test_memory_root.py b/tests/unit/test_core/test_persistence/test_memory_root.py index 6da6ccd9a..efca98227 100644 --- a/tests/unit/test_core/test_persistence/test_memory_root.py +++ b/tests/unit/test_core/test_persistence/test_memory_root.py @@ -91,6 +91,8 @@ def test_dotfile_paths(tmp_path: Path) -> None: mr = MemoryRoot(tmp_path) assert mr.index_dir == tmp_path / ".index" assert mr.lancedb_dir == tmp_path / ".index" / "lancedb" + assert mr.milvus_dir == tmp_path / ".index" / "milvus" + assert mr.milvus_db == tmp_path / ".index" / "milvus" / "milvus.db" assert mr.sqlite_dir == tmp_path / ".index" / "sqlite" assert mr.system_db == tmp_path / ".index" / "sqlite" / "system.db" assert mr.lock_file == tmp_path / ".lock" @@ -104,6 +106,7 @@ def test_ensure_creates_required_dirs(tmp_path: Path) -> None: assert mr.index_dir.is_dir() assert mr.sqlite_dir.is_dir() assert mr.lancedb_dir.is_dir() + assert mr.milvus_dir.is_dir() assert mr.tmp_dir.is_dir() # User-visible dirs are NOT pre-created. assert not mr.agents_dir().exists() diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py new file mode 100644 index 000000000..901d67034 --- /dev/null +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -0,0 +1,127 @@ +"""Real Milvus Lite coverage for the optional derived index backend.""" + +from __future__ import annotations + +import datetime as dt +import importlib.util +import uuid +from pathlib import Path + +import pytest +import pytest_asyncio + +from everos.config import load_settings + +pytestmark = pytest.mark.skipif( + importlib.util.find_spec("pymilvus") is None, + reason="pymilvus optional extra is not installed", +) + + +def test_milvus_score_normalization_handles_lite_and_cloud( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from everos.infra.persistence.milvus import repository + + monkeypatch.delenv("EVEROS_MILVUS__URI", raising=False) + monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0") + load_settings.cache_clear() + assert repository._cosine_distance_from_milvus(0.25) == pytest.approx(0.25) + assert repository._bm25_score_from_distance(-1.5) == pytest.approx(1.5) + + monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0.0") + assert repository._cosine_distance_from_milvus(0.25) == pytest.approx(0.25) + + monkeypatch.setenv("EVEROS_MILVUS__URI", "https://example.zillizcloud.com") + load_settings.cache_clear() + assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) + assert repository._bm25_score_from_distance(1.5) == pytest.approx(1.5) + + monkeypatch.delenv("EVEROS_MILVUS__URI", raising=False) + monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0.1") + load_settings.cache_clear() + assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) + + +@pytest_asyncio.fixture(autouse=True) +async def _milvus_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", f"test_{uuid.uuid4().hex}") + load_settings.cache_clear() + yield + from everos.infra.persistence.index import shutdown + + await shutdown() + load_settings.cache_clear() + + +async def test_milvus_episode_repo_upsert_search_filter_and_delete() -> None: + from everos.infra.persistence.index import Episode, episode_repo + from everos.memory.search.filters import compile_filters_for_backends + + v_apple = [0.0] * 1024 + v_apple[0] = 1.0 + v_banana = [0.0] * 1024 + v_banana[1] = 1.0 + now = dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + + await episode_repo.upsert( + [ + Episode( + id="u1_ep1", + entry_id="ep1", + owner_id="u1", + owner_type="user", + app_id="default", + project_id="default", + session_id="s1", + timestamp=now, + parent_id="mc1", + sender_ids=["user"], + episode="red apple memory", + episode_tokens="red apple memory", + md_path="default_app/default_project/users/u1/episodes/day.md", + content_sha256="a", + vector=v_apple, + ), + Episode( + id="u1_ep2", + entry_id="ep2", + owner_id="u1", + owner_type="user", + app_id="default", + project_id="default", + session_id="s1", + timestamp=now + dt.timedelta(seconds=1), + parent_id="mc2", + sender_ids=["assistant"], + episode="blue banana memory", + episode_tokens="blue banana memory", + md_path="default_app/default_project/users/u1/episodes/day.md", + content_sha256="b", + vector=v_banana, + ), + ] + ) + + filters = compile_filters_for_backends(None, owner_id="u1", owner_type="user") + rows = await episode_repo.find_where(filters, limit=10) + assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} + assert rows[0].timestamp.utcoffset() == dt.timedelta(0) + + sparse = await episode_repo.sparse_search( + ["apple"], filters, columns=Episode.BM25_FIELDS, limit=5 + ) + assert [row["id"] for row in sparse] == ["u1_ep1"] + assert sparse[0]["_score"] > 0 + + dense = await episode_repo.dense_search(v_apple, filters, limit=5) + assert dense[0]["id"] == "u1_ep1" + assert dense[0]["_distance"] == pytest.approx(0.0) + + deleted = await episode_repo.delete_by_md_path( + "default_app/default_project/users/u1/episodes/day.md" + ) + assert deleted == 2 + assert await episode_repo.count() == 0 diff --git a/tests/unit/test_memory/test_get/test_manager.py b/tests/unit/test_memory/test_get/test_manager.py index 35cec6b79..4cc5e9bfb 100644 --- a/tests/unit/test_memory/test_get/test_manager.py +++ b/tests/unit/test_memory/test_get/test_manager.py @@ -56,7 +56,7 @@ class _StubRepo: async def find_where_paginated( self, - where: str, + where: Any, *, sort_by: str, descending: bool = True, @@ -65,7 +65,7 @@ async def find_where_paginated( max_fetch: int = 20000, ) -> tuple[list[Any], int]: self.last = _CallRecord( - where=where, + where=str(where), sort_by=sort_by, descending=descending, page=page, diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index 808c2a382..a4267d8c8 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -9,6 +9,7 @@ FilterNode, compile_filters, ) +from everos.memory.search.filters import compile_filters_for_backends # ── Base injection ─────────────────────────────────────────────────────── @@ -266,3 +267,35 @@ def test_compile_filters_excludes_deprecated_by_for_user() -> None: def test_compile_filters_omits_deprecated_by_for_agent() -> None: result = compile_filters(None, owner_id="agent_1", owner_type="agent") assert "deprecated_by" not in result + + +# ── Backend-specific rendering ───────────────────────────────────────── + + +def test_compile_filters_for_backends_preserves_lancedb_default() -> None: + filters = compile_filters_for_backends(None, owner_id="u_a", owner_type="user") + assert str(filters) == filters.lancedb + assert "owner_id = 'u_a'" in filters.lancedb + assert "owner_id == \"u_a\"" in filters.milvus + assert "deprecated_by IS NULL" in filters.lancedb + assert "deprecated_by is null" in filters.milvus + + +def test_compile_filters_for_milvus_timestamp_and_array() -> None: + node = FilterNode.model_validate( + {"timestamp": {"gte": 1704067200000}, "sender_id": "u_jason"} + ) + filters = compile_filters_for_backends(node, owner_id="u_a", owner_type="user") + assert "timestamp_ms >= 1704067200000" in filters.milvus + assert 'array_contains(sender_ids, "u_jason")' in filters.milvus + assert "TIMESTAMP '" in filters.lancedb + assert "array_has(sender_ids, 'u_jason')" in filters.lancedb + + +def test_compile_filters_for_milvus_escapes_string_literals() -> None: + node = FilterNode.model_validate({"session_id": "ses's"}) + filters = compile_filters_for_backends(node, owner_id="al'ice", owner_type="user") + assert "owner_id == \"al'ice\"" in filters.milvus + assert "session_id == \"ses's\"" in filters.milvus + assert "owner_id = 'al''ice'" in filters.lancedb + assert "session_id = 'ses''s'" in filters.lancedb diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index 73dd66611..9ee62af13 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -106,25 +106,25 @@ def __init__(self, sparse: list[Candidate], dense: list[Candidate]) -> None: self.last_where: str | None = None async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = where + self.last_where = str(where) return list(self._sparse[:limit]) async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = where + self.last_where = str(where) return list(self._dense[:limit]) async def fetch_by_parent_ids( - self, parent_ids: Sequence[str], where: str + self, parent_ids: Sequence[str], where: Any ) -> list[Candidate]: by_parent = {str(c.metadata.get("parent_id", "")): c for c in self._dense} return [by_parent[p] for p in parent_ids if p in by_parent] async def fetch_by_entry_ids( - self, entry_ids: Sequence[str], where: str + self, entry_ids: Sequence[str], where: Any ) -> list[Candidate]: by_entry = {str(c.metadata.get("entry_id", "")): c for c in self._dense} return [by_entry[e] for e in entry_ids if e in by_entry] @@ -152,7 +152,7 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: async def facts_for_episodes( self, ep_to_parents: Mapping[str, Sequence[str]], - where: str, + where: Any, *, per_episode: int, query_vector: Any = None, @@ -203,7 +203,7 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: return list(self._dense) async def fetch_by_case_ids( - self, case_ids: Sequence[str], where: str, *, limit: int + self, case_ids: Sequence[str], where: Any, *, limit: int ) -> list[Candidate]: return list(self._by_case) diff --git a/tests/unit/test_memory/test_search/test_recall_episode.py b/tests/unit/test_memory/test_search/test_recall_episode.py index 4549322d8..2cde9f368 100644 --- a/tests/unit/test_memory/test_search/test_recall_episode.py +++ b/tests/unit/test_memory/test_search/test_recall_episode.py @@ -58,9 +58,9 @@ async def test_fetch_all_for_owner_returns_entry_id_keyed_candidates( _make_row("ep_2", "mc_2"), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -75,9 +75,9 @@ async def test_fetch_all_for_owner_stores_episode_id_in_metadata( """metadata['episode_id'] carries the real LanceDB episode id for final shaping.""" rows = [_make_row("ep_abc", "mc_xyz")] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -104,9 +104,9 @@ async def test_fetch_all_for_owner_skips_rows_without_entry_id( }, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -131,9 +131,9 @@ async def test_fetch_all_for_owner_merged_episode_uses_entry_id( ), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -156,9 +156,9 @@ async def test_fetch_all_for_owner_mixed_regular_and_merged( ), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -179,14 +179,10 @@ async def test_fetch_by_entry_ids_returns_candidates( entry_id="entry_xyz", ), ] - mock_tbl = MagicMock() - mock_tbl.query.return_value.where.return_value.limit.return_value.to_list = ( - AsyncMock(return_value=rows) - ) with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=mock_tbl, + return_value=rows, ): result = await recaller.fetch_by_entry_ids(["entry_xyz"], "owner_id = 'alice'") @@ -226,9 +222,9 @@ async def test_sparse_recall_as_child_injects_parent_id( {**_make_row("ep_1", "mc_1", entry_id="entry_1"), "_score": 1.0}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.sparse_search", new_callable=AsyncMock, - return_value=_mock_bm25_table(rows), + return_value=rows, ): result = await recaller.sparse_recall_as_child( "hello", "owner_id = 'alice'", limit=10 @@ -256,9 +252,9 @@ async def test_sparse_recall_as_child_falls_back_to_id_when_no_entry_id( "_score": 0.5, } with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.sparse_search", new_callable=AsyncMock, - return_value=_mock_bm25_table([row]), + return_value=[row], ): result = await recaller.sparse_recall_as_child( "hello", "owner_id = 'alice'", limit=10 @@ -286,9 +282,9 @@ async def test_dense_recall_as_child_injects_parent_id( {**_make_row("ep_3", "mc_3", entry_id="entry_3"), "_distance": 0.1}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.dense_search", new_callable=AsyncMock, - return_value=_mock_ann_table(rows), + return_value=rows, ): result = await recaller.dense_recall_as_child( [0.1] * 1024, "owner_id = 'alice'", limit=10 @@ -325,10 +321,16 @@ async def test_dense_recall_subject_returns_subject_vector_source( rows = [ {**_make_row("ep_s1", "mc_s1", entry_id="entry_s1"), "_distance": 0.2}, ] - with patch( - "everos.memory.search.recall.episode.get_table", - new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), + with ( + patch( + "everos.memory.search.recall.episode.active_backend", + return_value="lancedb", + ), + patch( + "everos.infra.persistence.lancedb.get_table", + new_callable=AsyncMock, + return_value=_mock_subject_ann_table(rows), + ), ): result = await recaller.dense_recall_subject( [0.1] * 1024, "owner_id = 'alice'", limit=10 @@ -354,10 +356,16 @@ async def test_dense_recall_subject_as_child_injects_parent_id( rows = [ {**_make_row("ep_s2", "mc_s2", entry_id="entry_s2"), "_distance": 0.15}, ] - with patch( - "everos.memory.search.recall.episode.get_table", - new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), + with ( + patch( + "everos.memory.search.recall.episode.active_backend", + return_value="lancedb", + ), + patch( + "everos.infra.persistence.lancedb.get_table", + new_callable=AsyncMock, + return_value=_mock_subject_ann_table(rows), + ), ): result = await recaller.dense_recall_subject_as_child( [0.1] * 1024, "owner_id = 'alice'", limit=10 diff --git a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py index d40d8a41e..3e98c58b4 100644 --- a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py +++ b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py @@ -1,11 +1,11 @@ """Unit tests for ``KnowledgeTopicRecaller``. -Verifies dual-column BM25 + cosine ANN recall, using ``unittest.mock`` -to patch ``get_table`` so no real LanceDB connection is needed. +Verifies dual-column BM25 delegation + cosine ANN recall, using +``unittest.mock`` to patch the backend-neutral index repo. White-box surfaces touched: - - ``everos.memory.search.recall.knowledge_topic.get_table`` (patched) - - ``KnowledgeTopicRecaller.sparse_recall`` — queries both BM25 columns + - ``everos.memory.search.recall.knowledge_topic.knowledge_topic_repo`` (patched) + - ``KnowledgeTopicRecaller.sparse_recall`` — passes both BM25 columns - ``KnowledgeTopicRecaller.dense_recall`` — cosine ANN with distance→score """ @@ -17,6 +17,7 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.index import KnowledgeTopic from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.knowledge_topic import KnowledgeTopicRecaller @@ -58,41 +59,6 @@ def _make_row( return row -def _mock_bm25_table( - summary_rows: list[dict[str, Any]], - content_rows: list[dict[str, Any]], -) -> MagicMock: - """Build a table mock whose BM25 results differ per column. - - The first ``nearest_to_text`` call (summary_tokens) returns - ``summary_rows``; the second (content_tokens) returns ``content_rows``. - ``asyncio.gather`` fires both concurrently, so we use ``side_effect`` - on the chain rather than recording call order. - """ - summary_chain = MagicMock() - summary_chain.where.return_value.limit.return_value.to_list = AsyncMock( - return_value=summary_rows - ) - - content_chain = MagicMock() - content_chain.where.return_value.limit.return_value.to_list = AsyncMock( - return_value=content_rows - ) - - tbl = MagicMock() - tbl.query.return_value.nearest_to_text.side_effect = [summary_chain, content_chain] - return tbl - - -def _mock_ann_table(rows: list[dict[str, Any]]) -> MagicMock: - """Build a table mock for ANN (dense) queries.""" - tbl = MagicMock() - ann = tbl.query.return_value.nearest_to.return_value - chain = ann.distance_type.return_value.where.return_value.limit.return_value - chain.to_list = AsyncMock(return_value=rows) - return tbl - - @pytest.fixture() def recaller() -> KnowledgeTopicRecaller: return KnowledgeTopicRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) @@ -109,16 +75,20 @@ def recaller() -> KnowledgeTopicRecaller: async def test_sparse_recall_queries_both_columns( recaller: KnowledgeTopicRecaller, ) -> None: - """``nearest_to_text`` must be called once per BM25 column.""" - tbl = _mock_bm25_table( - summary_rows=[_make_row("t1", score=0.9)], - content_rows=[_make_row("t2", score=0.7)], - ) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + """Both BM25 columns must be delegated to the index repo.""" + rows = [_make_row("t1", score=0.9), _make_row("t2", score=0.7)] + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=rows, + ) as mock_sparse: result = await recaller.sparse_recall("topic query", _WHERE, limit=10) - # nearest_to_text called twice (once per column) - assert tbl.query.return_value.nearest_to_text.call_count == 2 + mock_sparse.assert_awaited_once() + assert list(mock_sparse.await_args.args[0]) == ["topic", "query"] + assert mock_sparse.await_args.args[1] == _WHERE + assert mock_sparse.await_args.kwargs["columns"] == KnowledgeTopic.BM25_FIELDS + assert mock_sparse.await_args.kwargs["limit"] == 10 ids = {c.id for c in result} assert ids == {"t1", "t2"} @@ -126,13 +96,14 @@ async def test_sparse_recall_queries_both_columns( async def test_sparse_recall_merges_by_max_score( recaller: KnowledgeTopicRecaller, ) -> None: - """When the same id appears in both columns, keep the higher score.""" + """Scores returned by the repo are preserved on keyword candidates.""" shared_id = "topic_shared" - summary_rows = [_make_row(shared_id, score=0.5)] - content_rows = [_make_row(shared_id, score=0.9)] - tbl = _mock_bm25_table(summary_rows, content_rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=[_make_row(shared_id, score=0.9)], + ): result = await recaller.sparse_recall("overlap", _WHERE, limit=10) assert len(result) == 1 @@ -144,16 +115,16 @@ async def test_sparse_recall_merges_by_max_score( async def test_sparse_recall_returns_sorted_by_score( recaller: KnowledgeTopicRecaller, ) -> None: - """Merged results must be sorted descending by score, truncated to limit.""" - summary_rows = [ - _make_row("a", score=0.3), + """The recaller preserves repo ordering and maps rows to candidates.""" + rows = [ _make_row("b", score=0.8), - ] - content_rows = [ _make_row("c", score=0.6), ] - tbl = _mock_bm25_table(summary_rows, content_rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=rows, + ): result = await recaller.sparse_recall("query", _WHERE, limit=2) assert len(result) == 2 @@ -164,16 +135,19 @@ async def test_sparse_recall_returns_sorted_by_score( async def test_sparse_recall_empty_query_returns_empty( recaller: KnowledgeTopicRecaller, ) -> None: - """Empty tokenisation short-circuits — no LanceDB query is issued.""" + """Empty tokenisation short-circuits — no repo query is issued.""" tok = MagicMock(spec=Tokenizer) tok.tokenize.return_value = [] r = KnowledgeTopicRecaller(RecallerDeps(tokenizer=tok)) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + ) as mock_sparse: result = await r.sparse_recall("", _WHERE, limit=10) assert result == [] - mock_gt.assert_not_called() + mock_sparse.assert_not_called() # --------------------------------------------------------------------------- @@ -189,8 +163,11 @@ async def test_dense_recall_cosine_conversion( _make_row("t1", distance=0.2), _make_row("t2", distance=0.5), ] - tbl = _mock_ann_table(rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + return_value=rows, + ): result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=10) assert len(result) == 2 @@ -203,12 +180,15 @@ async def test_dense_recall_cosine_conversion( async def test_dense_recall_empty_vector_returns_empty( recaller: KnowledgeTopicRecaller, ) -> None: - """Empty vector short-circuits — no LanceDB query is issued.""" - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + """Empty vector short-circuits — no repo query is issued.""" + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + ) as mock_dense: result = await recaller.dense_recall([], _WHERE, limit=10) assert result == [] - mock_gt.assert_not_called() + mock_dense.assert_not_called() async def test_dense_recall_metadata_excludes_noise_columns( @@ -218,8 +198,11 @@ async def test_dense_recall_metadata_excludes_noise_columns( row = _make_row("t1", distance=0.3) row["vector"] = [0.0] * 1024 - tbl = _mock_ann_table([row]) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + return_value=[row], + ): result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=5) assert len(result) == 1 diff --git a/uv.lock b/uv.lock index ec09b1abb..c9c12429c 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,14 @@ version = 1 revision = 3 requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] [[package]] name = "aiosqlite" @@ -99,6 +107,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] +[[package]] +name = "cachetools" +version = "7.1.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/55/af/861ebc2e318a5c3300e3eb63bc4d30f3d70a46d13b360093728ac0705eed/cachetools-7.1.6.tar.gz", hash = "sha256:c7a79e7f30ba9943c1cefd08cc36f006aaae086e017af9166f1d59d6170c47e1", size = 40572, upload-time = "2026-07-23T22:47:53.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/f2/2086ba18a925a73586c4d4e61d25f4a6058e56fd00d77ce8f1d361ab4c9b/cachetools-7.1.6-py3-none-any.whl", hash = "sha256:2c12e255780330af28b91bb7fb96cce4c766f04e38396b9a24510190a5827096", size = 16954, upload-time = "2026-07-23T22:47:52.397Z" }, +] + [[package]] name = "cairocffi" version = "1.7.1" @@ -595,6 +612,9 @@ dependencies = [ ] [package.optional-dependencies] +milvus = [ + { name = "pymilvus", extra = ["milvus-lite"] }, +] multimodal = [ { name = "everalgo-parser", extra = ["svg"] }, ] @@ -641,6 +661,7 @@ requires-dist = [ { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "pydantic", specifier = ">=2.7.1" }, { name = "pydantic-settings", specifier = ">=2.0.0" }, + { name = "pymilvus", extras = ["milvus-lite"], marker = "extra == 'milvus'", specifier = ">=3.0.0" }, { name = "python-multipart", specifier = ">=0.0.7" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "sqlmodel", specifier = ">=0.0.22" }, @@ -651,7 +672,7 @@ requires-dist = [ { name = "watchdog", specifier = ">=4.0.0" }, { name = "watchfiles", specifier = ">=0.21.0" }, ] -provides-extras = ["multimodal", "otel"] +provides-extras = ["multimodal", "otel", "milvus"] [package.metadata.requires-dev] dev = [ @@ -677,6 +698,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] +[[package]] +name = "faiss-cpu" +version = "1.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/b0/48c083d01b7b68c463c1d56507147a9d733f791e1c469a77215a872a9fb5/faiss_cpu-1.14.3-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9369863290a3f0e033757e4c10577b6ef7431f1cede394dabd0a137e4e2ed45", size = 4768290, upload-time = "2026-06-13T02:19:03.427Z" }, + { url = "https://files.pythonhosted.org/packages/ab/34/6b04ef5bae3eada6b5a9457d7875cce041c040d53c890815cbd1e9821c65/faiss_cpu-1.14.3-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:f9d0e84d909194f63f027bbd3c1e35e905e48c9345c2db6e6f24da09a6bc5906", size = 6925734, upload-time = "2026-06-13T02:19:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/b451af4b3c6dd18749ecfb58ccb68503b77e49c9aa4a89d950e9d521e058/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d734cfa9ac90b6a5dfed3a27cb706d05f22824703dafc3969b4e2071877a31c", size = 9661210, upload-time = "2026-06-13T02:19:06.99Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ed/57335bc18c9e18677587bec9bf070c675b29c8e683e13f4def0440731ca0/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8780b526c06e57aad90a8c4655dfba2ff1b3195bd600ff4499752fc45159c9fc", size = 18506292, upload-time = "2026-06-13T02:19:09.621Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d3/c6ca8c44a63b909e78aa8a69e14501c79c89613e62261d58455ea603d710/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b28ba083e8c02f2c9be03783402537fd3f00d27e68799c44bf88931500ee12ec", size = 11238800, upload-time = "2026-06-13T02:19:12.821Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/b405692913a301251749cb175cb3f564ed257fdaa80a22c9a36444d0095d/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cdcb90850cb4b7c27d270839b37bcc0dc8d1fb6a62d1e13053e51ac95061ca25", size = 19237637, upload-time = "2026-06-13T02:19:15.531Z" }, +] + [[package]] name = "fastapi" version = "0.139.0" @@ -850,6 +888,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/8f/774ce522de6a7e70fbeceeaeb6fbe502f5dfb8365728fb3bb4cb23463da8/grimp-3.14-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a424ad14d5deb56721ac24ab939747f72ab3d378d42e7d1f038317d33b052b77", size = 2515157, upload-time = "2025-12-10T17:54:55.874Z" }, ] +[[package]] +name = "grpcio" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/2b/51e32514a4e9b715375c99721aadff0f24164cc2049b8269eda4de82a814/grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930", size = 6303167, upload-time = "2026-07-23T15:19:33.78Z" }, + { url = "https://files.pythonhosted.org/packages/39/33/b5b50fc2c6fbe350e04814047bb2d409feec7b36ef8b170254c050e06bc0/grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da", size = 12160538, upload-time = "2026-07-23T15:19:35.958Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5f/734e72e7b9f79bcf0b2c270b8d3bca0e4ebb97a27a50d06240b145f6d41e/grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16", size = 6869310, upload-time = "2026-07-23T15:19:38.607Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/a1735f215b2a5cd43c38b79eac072ad197e61be9829905b6b29550abd0db/grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf", size = 7613472, upload-time = "2026-07-23T15:19:40.645Z" }, + { url = "https://files.pythonhosted.org/packages/b2/78/c9e81f806ac704b6b145cb01628db398985b1f8dfdc10e23b55fb0902b3d/grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd", size = 7040616, upload-time = "2026-07-23T15:19:42.349Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ba/94cd5af859876049d340480acbb61a959096c84b567f215534faa78d0424/grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c", size = 7570491, upload-time = "2026-07-23T15:19:44.357Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/108d30d5a5c964312ae8b9cb0e8cc5b3c1cc68d8f757cca52b3565534d26/grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5", size = 8605036, upload-time = "2026-07-23T15:19:46.454Z" }, + { url = "https://files.pythonhosted.org/packages/ea/23/3828ae13c3db8233d123ad612747665817b952d8a954f32390230b582336/grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5", size = 7981587, upload-time = "2026-07-23T15:19:48.913Z" }, + { url = "https://files.pythonhosted.org/packages/17/5b/77af31228f55f55a2a5112bb0077ad0a1c4d23dbb0c2853a62475bbdcc14/grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc", size = 4394004, upload-time = "2026-07-23T15:19:50.618Z" }, + { url = "https://files.pythonhosted.org/packages/c0/da/f706e39550e7a3732ce2b9c5926107a93d74a802775b19b642a6df27dc96/grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df", size = 5158525, upload-time = "2026-07-23T15:19:52.246Z" }, + { url = "https://files.pythonhosted.org/packages/56/eb/135daaa713f32d33b8f99b4153b3f8dc3b2a124996ac15581bf9ebdad3c3/grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0", size = 6304480, upload-time = "2026-07-23T15:19:53.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a1/121806ce69f23138dabe06aa595b0e5f1ae051a37e4c1954eed7d692c800/grpcio-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:74fe6f9e8a35c7dbf32255ee154d15e3e5338a81ed39173d079d594d2e544cd1", size = 12154419, upload-time = "2026-07-23T15:19:56.3Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e8/d0389e09cd6b4c4d3089b92967ae4e3ffd64795bd349bf2f85cd6656d3da/grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867", size = 6873200, upload-time = "2026-07-23T15:19:58.701Z" }, + { url = "https://files.pythonhosted.org/packages/f8/51/f464c1d211fa50d5adbabe1b2e519948d99c13757052bfc9ea7afa28e284/grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881", size = 7618811, upload-time = "2026-07-23T15:20:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c0/539fe0832f2dd6500a28f5263071623fb34e8d4867aec632ccf81bd21156/grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f", size = 7042310, upload-time = "2026-07-23T15:20:02.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ca/ccf617d37ffa72567fa8e005ec7090c99da922799be2fb9847c8b21ca18c/grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61", size = 7575412, upload-time = "2026-07-23T15:20:04.712Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b9/fd8d5245f823a8e0fd35d90e20ea3aa4acd47f8d5318fa8df307df52dec6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45", size = 8604248, upload-time = "2026-07-23T15:20:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/14/1e/f37632fc11db72dfa4bba86c3a43e54358e53030df111ecae5e91a733ad6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c", size = 7977458, upload-time = "2026-07-23T15:20:09.109Z" }, + { url = "https://files.pythonhosted.org/packages/93/b6/d70b69ae5c0cfc341b9ba474980e4ed99cbf05c0e4a14e9eee8cb73db0a5/grpcio-1.83.0-cp313-cp313-win32.whl", hash = "sha256:8fe04f1050a59f875601eb55d42b4f66946fe89817f967e34db1462ccd07dadf", size = 4393993, upload-time = "2026-07-23T15:20:11.017Z" }, + { url = "https://files.pythonhosted.org/packages/0f/13/45d4cccb555cf4c476226979bf3d2fd0b0254216f7564c3a053e35117efc/grpcio-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:6e01ecd9d8ef280abe1365138a4dc318f9a5287f4cb1b41d07816f796653f735", size = 5159650, upload-time = "2026-07-23T15:20:12.979Z" }, + { url = "https://files.pythonhosted.org/packages/9c/60/f2cca8147ea213d3e43ae9158d03ad04e020fdf32ff027253e1fe93f921d/grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa", size = 6305607, upload-time = "2026-07-23T15:20:15.353Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ab/d3874931d123a95e83a3ebf8aa04537988fb62425cedb8bf3cefc5ad41b2/grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c", size = 12166617, upload-time = "2026-07-23T15:20:17.435Z" }, + { url = "https://files.pythonhosted.org/packages/92/ff/6f18f9426b69306f4e00a9add3b0ee2748da8aad53836ef80cab0d62d04f/grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b", size = 6880213, upload-time = "2026-07-23T15:20:19.98Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/706d1147c6b93b98f179240c13991fbcc56880eba0c868abb1ad40d8a0a6/grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c", size = 7618335, upload-time = "2026-07-23T15:20:22.161Z" }, + { url = "https://files.pythonhosted.org/packages/74/04/1a8443c889115ec9e213a213e86bc93a71ee9088027e5befa09aaa0edd9d/grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df", size = 7043416, upload-time = "2026-07-23T15:20:24.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/c6/94e0fee5b12bc1da1370185b680988db6f739d19b42d9959db01a7ea50bf/grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b", size = 7583253, upload-time = "2026-07-23T15:20:26.313Z" }, + { url = "https://files.pythonhosted.org/packages/a0/97/de1ccb671fb85575bc5192faedf9ecdbdf5b390d2e6584dcf552bcbd370e/grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404", size = 8605102, upload-time = "2026-07-23T15:20:28.437Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/0e0ec749a7034ffcbaa050e39779872950ead90c22e7e0116be3f28b2b46/grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af", size = 7979826, upload-time = "2026-07-23T15:20:31.182Z" }, + { url = "https://files.pythonhosted.org/packages/83/fa/c3fda157287f64bc65acee6c5aa90c41acf9e0d3a8e69a265eecff6d00a1/grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33", size = 4471765, upload-time = "2026-07-23T15:20:33.195Z" }, + { url = "https://files.pythonhosted.org/packages/a1/00/b1b26431c9d54eee11724fd6e5585473a2ed47fbc1fb95e5204906a642ce/grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9", size = 5298932, upload-time = "2026-07-23T15:20:35.48Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1286,6 +1365,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "milvus-lite" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "faiss-cpu" }, + { name = "grpcio" }, + { name = "numpy" }, + { name = "pyarrow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/58/54c72981958073d365febeae8fed3115c87761f45905f2d4725f506b94d1/milvus_lite-3.1.0.tar.gz", hash = "sha256:8d467ae81173f1ede93723a80f08d85b91988b451cd2b7bbf9b5a7cfb875e04b", size = 636180, upload-time = "2026-07-15T06:40:58.177Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/c2/fb5c2b73618973f943c8064e2d1f076f81731e2fafd5575a6ddaf3deda68/milvus_lite-3.1.0-py3-none-any.whl", hash = "sha256:0d20fffbee3c683eca69c4732a091273123368452159581ed050c933b41a59be", size = 252267, upload-time = "2026-07-15T06:40:52.538Z" }, +] + [[package]] name = "nodeenv" version = "1.10.0" @@ -1446,6 +1540,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, ] +[[package]] +name = "orjson" +version = "3.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515, upload-time = "2026-05-06T15:09:57.265Z" }, + { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409, upload-time = "2026-05-06T15:09:59.063Z" }, + { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106, upload-time = "2026-05-06T15:10:00.798Z" }, + { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864, upload-time = "2026-05-06T15:10:02.15Z" }, + { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213, upload-time = "2026-05-06T15:10:03.515Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994, upload-time = "2026-05-06T15:10:05.083Z" }, + { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744, upload-time = "2026-05-06T15:10:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014, upload-time = "2026-05-06T15:10:08.213Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509, upload-time = "2026-05-06T15:10:09.595Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127, upload-time = "2026-05-06T15:10:11.049Z" }, + { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025, upload-time = "2026-05-06T15:10:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943, upload-time = "2026-05-06T15:10:14.405Z" }, + { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606, upload-time = "2026-05-06T15:10:15.791Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101, upload-time = "2026-05-06T15:10:17.129Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -1455,6 +1602,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + [[package]] name = "parso" version = "0.8.7" @@ -1878,6 +2071,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/97/03635143a12a5d941f545548b00f8ac39d35565321a2effb4154ed267338/pyinstrument-5.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:b6a71f5e7f53c86c9b476b30cf19509463a63581ef17ddbd8680fee37ae509db", size = 128164, upload-time = "2026-01-04T18:38:32.281Z" }, ] +[[package]] +name = "pymilvus" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "grpcio" }, + { name = "orjson" }, + { name = "pandas" }, + { name = "protobuf" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "setuptools" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/c1/01647e61f3a82fd881382746b6dde3401d65b88cd4f75bd059901fb2392b/pymilvus-3.0.0-1-py3-none-any.whl", hash = "sha256:57c8e7c87fbbf579f122b4df893949bc78e50bca2988527864891bd544817b05", size = 344817, upload-time = "2026-05-07T14:57:45.235Z" }, +] + +[package.optional-dependencies] +milvus-lite = [ + { name = "milvus-lite", marker = "sys_platform != 'win32'" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -2183,6 +2399,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, ] +[[package]] +name = "setuptools" +version = "83.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" From c1d5baa9e016c0b05e0b61bc9ce7b4e9b0d6904d Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 9 Jul 2026 10:33:00 +0000 Subject: [PATCH 02/27] fix(milvus): bound fact recall and use exact counts --- .../infra/persistence/milvus/repository.py | 12 ++++++-- .../memory/search/recall/atomic_fact.py | 6 +++- .../unit/test_infra/test_milvus/test_repo.py | 1 + .../test_search/test_recall_atomic_fact.py | 29 +++++++++++++++++++ 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index b8828118e..90019f414 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -167,9 +167,15 @@ async def rebuild_indexes(self) -> None: async def count(self) -> int: await self.ensure_collection() client = await get_client() - stats = await _run(client.get_collection_stats, self.collection_name) - row_count = stats.get("row_count", 0) - return int(row_count) + rows = await _run( + client.query, + self.collection_name, + filter="", + output_fields=["count(*)"], + ) + if not rows: + return 0 + return int(rows[0].get("count(*)", 0)) async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: if id_field != "id": diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index 38ec3872a..f78b3f50a 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -32,6 +32,7 @@ _NOISE_COLUMNS = frozenset( {"vector", "_distance", "_score", "created_at", "updated_at"} ) +_MAX_FACT_RECALL_LIMIT = 1024 class AtomicFactRecaller: @@ -160,7 +161,10 @@ async def _query_facts_for_parents( quoted = ", ".join(f"'{_q(pid)}'" for pid in parent_to_eps) clause = f"parent_id IN ({quoted})" full_where = f"({where}) AND ({clause})" - limit = per_episode * max(len(parent_to_eps), 1) + # Milvus server / Zilliz Cloud reject search topK values above 1024. + # The fact expansion only needs a bounded candidate pool for top-N + # competition, so keep the same cap for every derived-index backend. + limit = min(per_episode * max(len(parent_to_eps), 1), _MAX_FACT_RECALL_LIMIT) if query_vector: return await atomic_fact_repo.dense_search( query_vector, full_where, limit=limit diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index 901d67034..8d265d81f 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -109,6 +109,7 @@ async def test_milvus_episode_repo_upsert_search_filter_and_delete() -> None: rows = await episode_repo.find_where(filters, limit=10) assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} assert rows[0].timestamp.utcoffset() == dt.timedelta(0) + assert await episode_repo.count() == 2 sparse = await episode_repo.sparse_search( ["apple"], filters, columns=Episode.BM25_FIELDS, limit=5 diff --git a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py index f2e80b88e..c3c71d433 100644 --- a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py +++ b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py @@ -29,6 +29,7 @@ atomic_fact_repo, lancedb_manager, ) +from everos.memory.search.recall import atomic_fact as atomic_fact_module from everos.memory.search.recall.atomic_fact import AtomicFactRecaller from everos.memory.search.recall.base import RecallerDeps @@ -267,6 +268,34 @@ async def test_facts_for_episodes_score_zero_without_query_vector() -> None: assert out["alice_ep_a"][0].score == 0.0 +async def test_facts_for_episodes_caps_dense_recall_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Large parent fan-out must not exceed Milvus/Zilliz search topK limits.""" + captured: dict[str, int] = {} + + async def fake_dense_search(vector, where, *, limit): # type: ignore[no-untyped-def] + captured["limit"] = limit + return [] + + monkeypatch.setattr( + atomic_fact_module.atomic_fact_repo, + "dense_search", + fake_dense_search, + ) + ep_to_parents = {f"alice_ep_{i}": [f"parent_{i}"] for i in range(80)} + + out = await _recaller().facts_for_episodes( + ep_to_parents, + "owner_id = 'alice' AND owner_type = 'user'", + per_episode=20, + query_vector=_unit_vector(0), + ) + + assert out == {} + assert captured["limit"] == 1024 + + # ── Dual parent_id (post-1.5 migration) ──────────────────────────────── From e6983075e20017f0bd1585e6ab16ee16ded321db Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Fri, 10 Jul 2026 03:39:14 +0000 Subject: [PATCH 03/27] style: format Milvus index changes --- src/everos/infra/persistence/index/__init__.py | 4 +--- src/everos/infra/persistence/milvus/repository.py | 6 ++++-- src/everos/memory/search/filters.py | 4 +--- tests/unit/test_memory/test_search/test_filters.py | 6 +++--- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py index 486900058..0cc969eb6 100644 --- a/src/everos/infra/persistence/index/__init__.py +++ b/src/everos/infra/persistence/index/__init__.py @@ -278,9 +278,7 @@ def _milvus() -> Any: atomic_fact_repo = _IndexRepoRouter( _lancedb.atomic_fact_repo, "atomic_fact_repo", AtomicFact ) -foresight_repo = _IndexRepoRouter( - _lancedb.foresight_repo, "foresight_repo", Foresight -) +foresight_repo = _IndexRepoRouter(_lancedb.foresight_repo, "foresight_repo", Foresight) agent_case_repo = _IndexRepoRouter( _lancedb.agent_case_repo, "agent_case_repo", AgentCase ) diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index 90019f414..de9143b3d 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -550,8 +550,10 @@ def _restore_row( out[name] = from_timestamp(int(value)) elif name in row: value = row[name] - if name in self.schema.BM25_FIELDS and value == "" and _is_optional( - field.annotation + if ( + name in self.schema.BM25_FIELDS + and value == "" + and _is_optional(field.annotation) ): out[name] = None else: diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 22df21a47..0f417afc0 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -190,9 +190,7 @@ def _compile_filters_backend( # (Reflection V1 marks superseded entries). Agent tables don't have it. if owner_type == "user": base.append( - "deprecated_by IS NULL" - if backend == "lancedb" - else "deprecated_by is null" + "deprecated_by IS NULL" if backend == "lancedb" else "deprecated_by is null" ) if node is None: return " AND ".join(base) diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index a4267d8c8..59e85dd2e 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -276,7 +276,7 @@ def test_compile_filters_for_backends_preserves_lancedb_default() -> None: filters = compile_filters_for_backends(None, owner_id="u_a", owner_type="user") assert str(filters) == filters.lancedb assert "owner_id = 'u_a'" in filters.lancedb - assert "owner_id == \"u_a\"" in filters.milvus + assert 'owner_id == "u_a"' in filters.milvus assert "deprecated_by IS NULL" in filters.lancedb assert "deprecated_by is null" in filters.milvus @@ -295,7 +295,7 @@ def test_compile_filters_for_milvus_timestamp_and_array() -> None: def test_compile_filters_for_milvus_escapes_string_literals() -> None: node = FilterNode.model_validate({"session_id": "ses's"}) filters = compile_filters_for_backends(node, owner_id="al'ice", owner_type="user") - assert "owner_id == \"al'ice\"" in filters.milvus - assert "session_id == \"ses's\"" in filters.milvus + assert 'owner_id == "al\'ice"' in filters.milvus + assert 'session_id == "ses\'s"' in filters.milvus assert "owner_id = 'al''ice'" in filters.lancedb assert "session_id = 'ses''s'" in filters.lancedb From 0d383394931a2dcd3f651dda3e7d8b3ad736d996 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Tue, 18 Aug 2026 12:34:57 +0000 Subject: [PATCH 04/27] refactor(index): require remote Milvus deployments --- .github/workflows/ci.yml | 43 ++ config.example.toml | 12 +- docs/cascade_runbook.md | 13 +- docs/configuration.md | 23 +- docs/storage_layout.md | 10 +- pyproject.toml | 5 + src/everos/config/default.toml | 8 +- src/everos/config/settings.py | 16 +- src/everos/core/persistence/memory_root.py | 13 - .../entrypoints/api/lifespans/lancedb.py | 29 +- .../entrypoints/cli/commands/cascade.py | 47 +- .../infra/persistence/index/__init__.py | 82 ++- .../infra/persistence/index/predicate.py | 45 ++ src/everos/infra/persistence/index/schema.py | 341 +++++++++ .../infra/persistence/lancedb/predicate.py | 83 +++ .../infra/persistence/milvus/__init__.py | 36 + .../persistence/milvus/milvus_manager.py | 30 +- .../infra/persistence/milvus/predicate.py | 100 +++ src/everos/infra/persistence/milvus/repos.py | 61 +- .../infra/persistence/milvus/repository.py | 668 +++++++++--------- src/everos/infra/persistence/predicate.py | 138 ++++ src/everos/memory/cascade/_backfill.py | 8 +- .../cascade/handlers/_daily_log_base.py | 24 +- .../memory/cascade/handlers/agent_skill.py | 15 +- src/everos/memory/get/filters_adapter.py | 7 +- src/everos/memory/reflection/orchestrator.py | 63 +- src/everos/memory/search/filters.py | 438 ++++-------- .../memory/search/recall/agent_skill.py | 25 +- .../memory/search/recall/atomic_fact.py | 24 +- src/everos/memory/search/recall/episode.py | 61 +- src/everos/service/knowledge.py | 11 +- .../test_cascade_cli_integration.py | 2 +- tests/integration/test_milvus_remote.py | 217 ++++++ .../test_reflection_integration.py | 17 + .../test_tiers/test_upgrade_path.py | 18 +- tests/unit/test_config/test_settings.py | 3 - .../test_persistence/test_memory_root.py | 3 - .../unit/test_infra/test_milvus/test_repo.py | 232 +++--- .../test_backfill_subject_null_recovery.py | 12 +- .../test_cascade/test_handler_agent_case.py | 15 +- .../test_cascade/test_handler_agent_skill.py | 30 +- .../test_cascade/test_handler_episode.py | 13 +- .../test_get/test_filters_adapter.py | 26 +- .../unit/test_memory/test_get/test_manager.py | 3 +- .../test_memory/test_search/test_filters.py | 67 +- .../test_memory/test_search/test_manager.py | 5 +- .../test_search/test_recall_agent_skill.py | 3 +- .../test_search/test_recall_atomic_fact.py | 29 +- .../test_search/test_recall_episode.py | 78 +- .../test_search/test_recall_or_semantics.py | 7 +- .../test_service/test_knowledge_search.py | 11 +- uv.lock | 161 ++--- 52 files changed, 2204 insertions(+), 1227 deletions(-) create mode 100644 src/everos/infra/persistence/index/predicate.py create mode 100644 src/everos/infra/persistence/index/schema.py create mode 100644 src/everos/infra/persistence/lancedb/predicate.py create mode 100644 src/everos/infra/persistence/milvus/predicate.py create mode 100644 src/everos/infra/persistence/predicate.py create mode 100644 tests/integration/test_milvus_remote.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b90d55c9f..7d1fdd3b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,49 @@ jobs: - name: Integration tests run: make integration + milvus-integration: + name: Milvus Server integration + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.2.0 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Set up Python + run: uv python install 3.12 + + - name: Install dependencies (frozen) + run: make install-deps + + - name: Start Milvus Server + run: | + curl -fsSL \ + https://github.com/milvus-io/milvus/releases/download/v2.6.22/milvus-standalone-docker-compose.yml \ + -o /tmp/milvus-compose.yml + docker compose -f /tmp/milvus-compose.yml up -d + for attempt in $(seq 1 90); do + if curl -fsS http://127.0.0.1:9091/healthz; then + exit 0 + fi + sleep 2 + done + docker compose -f /tmp/milvus-compose.yml logs + exit 1 + + - name: Milvus derived-index contract + env: + EVEROS_TEST_MILVUS_URI: http://127.0.0.1:19530 + EVEROS_TEST_MILVUS_FULL_STARTUP: "1" + run: uv run pytest tests/integration/test_milvus_remote.py -v + + - name: Stop Milvus Server + if: always() + run: docker compose -f /tmp/milvus-compose.yml down -v + package: name: package build runs-on: ubuntu-latest diff --git a/config.example.toml b/config.example.toml index 0145dd686..00045b2a9 100644 --- a/config.example.toml +++ b/config.example.toml @@ -59,16 +59,16 @@ max_concurrent = 5 # ── Optional Milvus derived index ───────────────────── # Markdown remains the source of truth. Set the index backend to Milvus -# when you want the rebuildable vector/BM25 index to live in Milvus. -# Leave uri empty for Milvus Lite at /.index/milvus/milvus.db. +# when you want the rebuildable vector/BM25 index to live in an external +# Milvus Server or Zilliz Cloud deployment. Embedded Milvus Lite and local +# database paths are not supported. # # [index] # backend = "milvus" # # [milvus] -# uri = "" # or "http://localhost:19530" +# uri = "http://localhost:19530" # required remote endpoint # token = "" # required for Zilliz Cloud/auth-enabled Milvus -# db_name = "" +# db_name = "" # use a dedicated database when available # consistency_level = "Session" -# dimension = 1024 # must match your embedding model output -# collection_prefix = "everos" +# collection_prefix = "everos" # make unique on a shared database diff --git a/docs/cascade_runbook.md b/docs/cascade_runbook.md index d836672f9..338d2a7cf 100644 --- a/docs/cascade_runbook.md +++ b/docs/cascade_runbook.md @@ -127,7 +127,7 @@ parallel with a live `everos server`. ## Rebuild the index: `everos cascade rebuild` -The safe recovery from a drifted or corrupt LanceDB index. It rebuilds +The safe recovery from a drifted or corrupt derived index. It rebuilds the whole index from markdown (the source of truth) in one shot: ```bash @@ -136,15 +136,16 @@ everos cascade rebuild --yes # non-interactive ``` > **Stop the `everos server` first.** Unlike `cascade sync`, rebuild -> **drops and recreates** the LanceDB tables. A running daemon holds +> **drops and recreates** the active backend's tables or collections. A running +> daemon holds > cached table handles that would keep pointing at (and writing to) the > dropped dataset, corrupting the rebuild. This is the one cascade > command that is **not** safe to run alongside a live server. What it does, in order: -1. **Drops** every business LanceDB table (`drop_business_tables`) and - evicts them from the connection cache. +1. **Drops** every business table or collection (`drop_business_tables`) and + evicts it from the process cache. 2. **Recreates** them empty from the current schema + FTS indexes (`ensure_business_indexes`). 3. **Clears** the cascade queue (`md_change_state.reset_all`) so every @@ -164,6 +165,10 @@ Why not a bare `rm`: | `rm -rf .index` | ✅ | ❌ deletes un-extracted messages | | `everos cascade rebuild` | ✅ | ✅ | +For a remote Milvus backend, rebuild acts only on collections whose names use +the configured `collection_prefix`. Use a unique prefix or dedicated database +before running it on a shared Milvus Server or Zilliz Cloud deployment. + ## Recovery paths ### LanceDB schema drift on startup diff --git a/docs/configuration.md b/docs/configuration.md index 6848d7839..0608170e9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -108,16 +108,27 @@ everos init --root /data/everos ### `[milvus]` Milvus is optional. Install with `everos[milvus]`, then set -`[index].backend = "milvus"`. +`[index].backend = "milvus"`. The backend connects only to an external +Milvus Server or Zilliz Cloud endpoint; embedded Milvus Lite and local `.db` +paths are not supported. | Field | Type | Default | Description | |---|---|---|---| -| `uri` | string \| null | `""` | Empty uses Milvus Lite at `/.index/milvus/milvus.db`. Set to a Milvus server or Zilliz Cloud endpoint to use an external service. | -| `token` | string \| null | `""` | Token for Zilliz Cloud or auth-enabled Milvus. | -| `db_name` | string \| null | `""` | Optional Milvus database name. | +| `uri` | string | `""` | Required when the Milvus backend is selected. Must be a remote Milvus Server or Zilliz Cloud endpoint. | +| `token` | string | `""` | Token for Zilliz Cloud or auth-enabled Milvus Server. | +| `db_name` | string | `""` | Optional Milvus database name. Prefer a dedicated database on shared clusters. | | `consistency_level` | `"Strong"` \| `"Session"` \| `"Bounded"` \| `"Eventually"` | `"Session"` | Milvus consistency level for created collections. | -| `dimension` | int | `1024` | Dense vector dimension. Must match the configured embedding model output. | -| `collection_prefix` | string | `"everos"` | Prefix for EverOS Milvus collections. | +| `collection_prefix` | string | `"everos"` | Prefix for EverOS collections. Must start with a letter or underscore and contain only letters, digits, and underscores. Make it unique when deployments share a database. | + +EverOS creates seven collections and supports every dense field in the shared +index schema, including `episode.subject_vector`. The vector dimension is owned +by that schema (currently 1024), rather than duplicated in Milvus config. + +Milvus imposes limits that LanceDB does not: strings are capped at 65,535 +UTF-8 bytes, string arrays at 256 items, and each array item at 512 UTF-8 +bytes. EverOS validates these before a write and reports the exact table and +field instead of relying on a server-side insert error. Ensure the selected +Zilliz Cloud plan permits at least seven collections. ### `[llm]` diff --git a/docs/storage_layout.md b/docs/storage_layout.md index 71447905a..3d90c775d 100644 --- a/docs/storage_layout.md +++ b/docs/storage_layout.md @@ -52,8 +52,6 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). │ │ └── ome.db.lock OME single-engine guard (portalocker) │ ├── lancedb/ │ │ └── .lance/ default derived index backend -│ └── milvus/ -│ └── milvus.db Milvus Lite database when configured │ ├── ome.toml user-editable OME strategy overrides (hot-reloaded) └── .tmp/ staging dir for batch / multi-step writes @@ -67,7 +65,7 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py), exposing every path as a property. `MemoryRoot.ensure()` creates the -runtime-required dirs (`.index/{sqlite,lancedb,milvus}/`, `.tmp/`); the +runtime-required dirs (`.index/{sqlite,lancedb}/`, `.tmp/`); the user-visible dirs are *not* pre-created — they appear on first write. Config files (`everos.toml`, `ome.toml`) are created by `everos init`. @@ -183,8 +181,6 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers │ unprocessed_buffer, conversation_status, cluster) ├── lancedb/ │ └── .lance/ default derived index backend -└── milvus/ - └── milvus.db Milvus Lite database when configured ``` - **SQLite** ([`infra/persistence/sqlite/tables/`](../src/everos/infra/persistence/sqlite/tables/)) @@ -197,8 +193,8 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers - The **derived index backend** holds the per-kind business rows, keyed `_` (so cross-table joins use `(owner_id, entry_id)`). LanceDB is the default backend under `.index/lancedb/`; Milvus can be enabled - as the same rebuildable index backend and uses `.index/milvus/milvus.db` by - default for Milvus Lite. + as the same rebuildable index backend and lives outside the memory root in a + configured Milvus Server or Zilliz Cloud deployment. Episode and AtomicFact index rows carry a `deprecated_by: str | None` column. When an episode is superseded by a Reflection merge, diff --git a/pyproject.toml b/pyproject.toml index 991d9c91b..7006dcd7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ otel = [ "opentelemetry-sdk>=1.27.0", "opentelemetry-exporter-otlp-proto-http>=1.27.0", ] +milvus = ["pymilvus>=3.0.0"] [project.urls] Homepage = "https://evermind.ai" @@ -249,6 +250,7 @@ ignore_imports = [ "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.tables", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.repos", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.lancedb_manager", + "everos.infra.persistence.index -> everos.infra.persistence.lancedb.predicate", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.mds", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.writers", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.readers", @@ -280,4 +282,7 @@ dev = [ # [otel] stack is always present in the dev / CI environment. "opentelemetry-sdk>=1.27.0", "opentelemetry-exporter-otlp-proto-http>=1.27.0", + # Milvus tests must run in every dev/CI environment. The client package + # intentionally excludes the embedded milvus-lite extra. + "pymilvus>=3.0.0", ] diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index d67f0bc99..38d226823 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -57,15 +57,15 @@ cache_size_kb = 2048 backend = "lancedb" [milvus] -# Optional derived index backend. Leave uri empty to use Milvus Lite at -# /.index/milvus/milvus.db. Set uri/token/db_name for Milvus server or -# Zilliz Cloud. +# Optional remote derived index backend for Milvus Server or Zilliz Cloud. +# `uri` is required when [index] backend = "milvus". Embedded Milvus Lite and +# local database paths are not supported. Use a unique db_name or +# collection_prefix when multiple EverOS deployments share a cluster. # Override via EVEROS_MILVUS__URI, EVEROS_MILVUS__TOKEN, EVEROS_MILVUS__DB_NAME. uri = "" token = "" db_name = "" consistency_level = "Session" -dimension = 1024 collection_prefix = "everos" [llm] diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 5e36b390d..1e910ac8e 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -23,6 +23,7 @@ from __future__ import annotations import os +import re from functools import cache from pathlib import Path from typing import Literal @@ -392,12 +393,19 @@ class MilvusSettings(BaseModel): uri: str = "" token: SecretStr = SecretStr("") db_name: str = "" - consistency_level: Literal["Strong", "Bounded", "Session", "Eventually"] = ( - "Session" - ) - dimension: int = Field(default=1024, ge=1) + consistency_level: Literal["Strong", "Bounded", "Session", "Eventually"] = "Session" collection_prefix: str = Field(default="everos", min_length=1) + @field_validator("collection_prefix") + @classmethod + def _validate_collection_prefix(cls, value: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value): + raise ValueError( + "collection_prefix must start with a letter or underscore and " + "contain only letters, digits, and underscores" + ) + return value + class KnowledgeSearchSettings(BaseModel): """``[knowledge.search]`` — retrieval tuning for the knowledge module.""" diff --git a/src/everos/core/persistence/memory_root.py b/src/everos/core/persistence/memory_root.py index 5ca928c4f..701ff0012 100644 --- a/src/everos/core/persistence/memory_root.py +++ b/src/everos/core/persistence/memory_root.py @@ -11,7 +11,6 @@ .index/ derived indexes (rebuildable from markdown) sqlite/ system.db (+ WAL/SHM), ome.db, ome.aps.db lancedb/ LanceDB tables - milvus/ Milvus Lite database when Milvus is selected .tmp/ atomic-write staging directory .lock single-process lock anchor (created on demand by ``memory_root_lock``) @@ -166,16 +165,6 @@ def lancedb_dir(self) -> Path: """``/.index/lancedb/`` — LanceDB table root.""" return self.index_dir / "lancedb" - @property - def milvus_dir(self) -> Path: - """``/.index/milvus/`` — Milvus Lite derived index root.""" - return self.index_dir / "milvus" - - @property - def milvus_db(self) -> Path: - """``/.index/milvus/milvus.db`` — default Milvus Lite URI.""" - return self.milvus_dir / "milvus.db" - @property def sqlite_dir(self) -> Path: """``/.index/sqlite/`` — SQLite system DB root. @@ -255,12 +244,10 @@ def ensure(self) -> None: /.index/ /.index/sqlite/ /.index/lancedb/ - /.index/milvus/ /.tmp/ """ self.root.mkdir(parents=True, exist_ok=True) self.index_dir.mkdir(parents=True, exist_ok=True) self.sqlite_dir.mkdir(parents=True, exist_ok=True) self.lancedb_dir.mkdir(parents=True, exist_ok=True) - self.milvus_dir.mkdir(parents=True, exist_ok=True) self.tmp_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/everos/entrypoints/api/lifespans/lancedb.py b/src/everos/entrypoints/api/lifespans/lancedb.py index d20e05b63..56994f565 100644 --- a/src/everos/entrypoints/api/lifespans/lancedb.py +++ b/src/everos/entrypoints/api/lifespans/lancedb.py @@ -1,4 +1,4 @@ -"""LanceDB lifespan provider (HTTP API entrypoint). +"""Derived-index lifespan provider (HTTP API entrypoint). Startup: Open the connection via ``get_connection`` (lazy, idempotent). @@ -31,13 +31,18 @@ from everos.core.lifespan import LifespanProvider from everos.core.observability.logging import get_logger +from everos.infra.persistence.index import ( + active_backend, +) +from everos.infra.persistence.index import ( + shutdown as shutdown_index, +) +from everos.infra.persistence.index import ( + startup as startup_index, +) from everos.infra.persistence.lancedb import ( BUSINESS_SCHEMAS_WITH_VECTOR, - dispose_connection, - ensure_business_indexes, - get_connection, get_table, - verify_business_schemas, ) logger = get_logger(__name__) @@ -83,7 +88,7 @@ async def _log_unbackfilled_hint() -> None: class LanceDBLifespanProvider(LifespanProvider): - """Manage the LanceDB connection + table cache for the app lifecycle. + """Manage the configured derived index for the app lifecycle. Startup runs four steps: @@ -100,12 +105,12 @@ def __init__(self, order: int = 11) -> None: super().__init__(name="lancedb", order=order) async def startup(self, app: FastAPI) -> Any: - conn = await get_connection() - await verify_business_schemas() - await ensure_business_indexes() - await _log_unbackfilled_hint() - logger.info("lancedb_ready", uri=conn.uri) + backend = active_backend() + conn = await startup_index() + if backend == "lancedb": + await _log_unbackfilled_hint() + logger.info("derived_index_ready", backend=backend) return conn async def shutdown(self, app: FastAPI) -> None: - await dispose_connection() + await shutdown_index() diff --git a/src/everos/entrypoints/cli/commands/cascade.py b/src/everos/entrypoints/cli/commands/cascade.py index 8710f947b..d39f46944 100644 --- a/src/everos/entrypoints/cli/commands/cascade.py +++ b/src/everos/entrypoints/cli/commands/cascade.py @@ -15,7 +15,7 @@ vectors, build clusters, extract skills. See :func:`everos.entrypoints.cli.commands._backfill_cmd.run_backfill` for the phase orchestration. -- ``cascade rebuild`` — drop every business LanceDB table and re-index +- ``cascade rebuild`` — drop every business derived-index table and re-index all md from scratch. Recovery for a drifted / corrupt index; safe because md is the source of truth and un-extracted buffered messages are preserved. Skips the schema-verify guard (which the drift would @@ -46,13 +46,16 @@ from everos.core.persistence import MemoryRoot from everos.entrypoints.cli._log_setup import configure_cli_logging from everos.entrypoints.cli.commands._backfill_cmd import run_backfill -from everos.infra.persistence.lancedb import ( - dispose_connection, +from everos.infra.persistence.index import ( + active_backend, + connect, drop_business_tables, ensure_business_indexes, - get_connection, verify_business_schemas, ) +from everos.infra.persistence.index import ( + shutdown as shutdown_index, +) from everos.infra.persistence.sqlite import ( dispose_engine, get_engine, @@ -68,7 +71,7 @@ app = typer.Typer( name="cascade", - help="Inspect and operate the md → LanceDB sync queue", + help="Inspect and operate the markdown → derived-index sync queue", no_args_is_help=True, ) @@ -138,7 +141,7 @@ def _apply_verbose_logging(verbose: bool | None) -> None: async def _runtime( # type: ignore[no-untyped-def] *, verify: bool = True, ensure: bool = True ): - """Stand up sqlite + lancedb the same way the API lifespan would. + """Stand up SQLite + the configured index the way the API lifespan would. The CLI uses the same lazy, process-wide singletons the API lifespan does. They are **per-process**: a running daemon has its own @@ -163,15 +166,21 @@ async def _runtime( # type: ignore[no-untyped-def] engine = get_engine() async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) - await get_connection() - if verify: - await verify_business_schemas() - if ensure: - await ensure_business_indexes() + await connect() + if active_backend() == "milvus": + if ensure: + await ensure_business_indexes() + if verify: + await verify_business_schemas() + else: + if verify: + await verify_business_schemas() + if ensure: + await ensure_business_indexes() try: yield finally: - await dispose_connection() + await shutdown_index() await dispose_engine() @@ -451,7 +460,7 @@ def rebuild( typer.Option("--yes", "-y", help="Skip the confirmation prompt."), ] = False, ) -> None: - """Rebuild the LanceDB index from markdown (recover from schema drift). + """Rebuild the configured derived index from markdown. **Stop the ``everos server`` first** — this is the one cascade command that is not safe alongside a live daemon. It drops and recreates the @@ -459,13 +468,13 @@ def rebuild( the dropped dataset; the command refuses to start while a server holds the OME lock. - Drops every business LanceDB table and re-indexes all md from + Drops every business table or collection and re-indexes all markdown from scratch. Markdown is the source of truth, so no memory content is lost, and this is the safe recovery from a drifted / corrupt index (e.g. the ``verify_business_schemas`` startup failure): - - unlike ``rm -rf ~/.everos/.index/lancedb``, it re-populates - already-indexed entries (that command leaves the cascade queue + - unlike deleting ``~/.everos/.index/lancedb`` on the default backend, it + re-populates already-indexed entries (that command leaves the cascade queue marked ``done``, so nothing re-indexes and the index comes back empty); - unlike ``rm -rf ~/.everos/.index``, it preserves SQLite state that @@ -476,7 +485,7 @@ def rebuild( typer.echo( "error: a server (or another exclusive CLI phase) is running on " "this memory root.\n" - " cascade rebuild drops and recreates the LanceDB tables; a live " + " cascade rebuild drops and recreates the derived index; a live " "daemon holds cached\n" " table handles and would keep writing to the dropped dataset. " "Stop `everos server`\n" @@ -486,7 +495,7 @@ def rebuild( raise typer.Exit(code=3) if not yes: typer.confirm( - "Drop all LanceDB business tables and re-index from markdown? " + "Drop all derived-index business data and re-index from markdown? " "(requires the server to be stopped)", abort=True, ) @@ -507,7 +516,7 @@ async def _run() -> None: typer.echo(f"reset {cleared} cascade queue row(s)") dropped = await drop_business_tables() typer.echo( - f"dropped {len(dropped)} LanceDB table(s): " + f"dropped {len(dropped)} derived-index table(s): " f"{', '.join(dropped) or '(none)'}" ) # Recreate the tables (current schema) + FTS indexes. diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py index 0cc969eb6..dfe2bcab3 100644 --- a/src/everos/infra/persistence/index/__init__.py +++ b/src/everos/infra/persistence/index/__init__.py @@ -30,6 +30,23 @@ ParentType, UserProfile, ) +from everos.infra.persistence.lancedb.predicate import render_predicate as _render_lance + +from .predicate import ( + Predicate, + all_of, + any_of, + contains, + eq, + gt, + gte, + is_null, + lt, + lte, + ne, + one_of, +) +from .schema import schema_for def active_backend() -> str: @@ -50,6 +67,13 @@ async def startup() -> Any: return conn +async def connect() -> Any: + """Open the active backend without creating or verifying indexes.""" + if active_backend() == "milvus": + return await _milvus().get_client() + return await _lancedb.get_connection() + + async def shutdown() -> None: """Dispose the configured derived index backend.""" if active_backend() == "milvus": @@ -72,12 +96,20 @@ async def verify_business_schemas() -> None: await _lancedb.verify_business_schemas() +async def drop_business_tables() -> list[str]: + """Drop all business indexes for the active backend.""" + if active_backend() == "milvus": + return await _milvus().drop_business_tables() + return await _lancedb.drop_business_tables() + + class _LanceIndexRepoAdapter: """Add backend-neutral recall helpers to an existing LanceDB repo.""" def __init__(self, repo: Any, schema: type[Any]) -> None: self._repo = repo self.schema = schema + self.index_schema = schema_for(schema) def __getattr__(self, name: str) -> Any: return getattr(self._repo, name) @@ -90,7 +122,7 @@ async def sparse_search( columns: Sequence[str] | None = None, limit: int, ) -> list[dict[str, Any]]: - fields = list(columns or self.schema.BM25_FIELDS) + fields = list(columns or self.index_schema.bm25_fields) if not query_terms or not fields: return [] table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) @@ -125,6 +157,7 @@ async def dense_search( where: Any, *, limit: int, + vector_field: str = "vector", ) -> list[dict[str, Any]]: if not vector: return [] @@ -132,6 +165,7 @@ async def dense_search( return ( await table.query() .nearest_to(list(vector)) + .column(vector_field) .distance_type("cosine") .where(_lance_expr(where)) .limit(limit) @@ -148,10 +182,11 @@ def __init__( self._lance = _LanceIndexRepoAdapter(lance_repo, schema) self._milvus_repo_name = milvus_repo_name self.schema = schema + self.index_schema = schema_for(schema) @property def table_name(self) -> str: - return self.schema.TABLE_NAME + return self.index_schema.table_name def _repo(self) -> Any: if active_backend() == "milvus": @@ -240,21 +275,40 @@ async def sparse_search( ) async def dense_search( - self, vector: Sequence[float], where: Any, *, limit: int + self, + vector: Sequence[float], + where: Any, + *, + limit: int, + vector_field: str = "vector", ) -> list[dict[str, Any]]: return await self._repo().dense_search( - vector, _where_for_backend(where), limit=limit + vector, + _where_for_backend(where), + limit=limit, + vector_field=vector_field, ) def _where_for_backend(where: Any) -> Any: + if where is None: + return None if active_backend() == "milvus" else "" + if not isinstance(where, Predicate): + raise TypeError( + "derived-index repository predicates must use the neutral " + f"Predicate AST, got {type(where).__name__}" + ) if active_backend() == "milvus": - return getattr(where, "milvus", where) - return _lance_expr(where) + return where + return _render_lance(where) def _lance_expr(where: Any) -> str: - return getattr(where, "lancedb", where) + if isinstance(where, Predicate): + return _render_lance(where) + if isinstance(where, str): + return where + raise TypeError(f"unsupported LanceDB predicate: {type(where).__name__}") def _build_or_query(tokens: Sequence[str], column: str) -> FullTextQuery: @@ -311,15 +365,29 @@ def _milvus() -> Any: "Foresight", "KnowledgeTopic", "ParentType", + "Predicate", "UserProfile", "active_backend", "agent_case_repo", "agent_skill_repo", + "all_of", + "any_of", "atomic_fact_repo", + "connect", + "contains", + "drop_business_tables", "ensure_business_indexes", "episode_repo", + "eq", "foresight_repo", + "gt", + "gte", + "is_null", "knowledge_topic_repo", + "lt", + "lte", + "ne", + "one_of", "shutdown", "startup", "user_profile_repo", diff --git a/src/everos/infra/persistence/index/predicate.py b/src/everos/infra/persistence/index/predicate.py new file mode 100644 index 000000000..4892356eb --- /dev/null +++ b/src/everos/infra/persistence/index/predicate.py @@ -0,0 +1,45 @@ +"""Compatibility re-exports for the shared derived-index predicate AST.""" + +from everos.infra.persistence.predicate import All as All +from everos.infra.persistence.predicate import AnyOf as AnyOf +from everos.infra.persistence.predicate import Comparison as Comparison +from everos.infra.persistence.predicate import Contains as Contains +from everos.infra.persistence.predicate import In as In +from everos.infra.persistence.predicate import IsNull as IsNull +from everos.infra.persistence.predicate import Predicate as Predicate +from everos.infra.persistence.predicate import Scalar as Scalar +from everos.infra.persistence.predicate import all_of as all_of +from everos.infra.persistence.predicate import any_of as any_of +from everos.infra.persistence.predicate import compare as compare +from everos.infra.persistence.predicate import contains as contains +from everos.infra.persistence.predicate import eq as eq +from everos.infra.persistence.predicate import gt as gt +from everos.infra.persistence.predicate import gte as gte +from everos.infra.persistence.predicate import is_null as is_null +from everos.infra.persistence.predicate import lt as lt +from everos.infra.persistence.predicate import lte as lte +from everos.infra.persistence.predicate import ne as ne +from everos.infra.persistence.predicate import one_of as one_of + +__all__ = [ + "All", + "AnyOf", + "Comparison", + "Contains", + "In", + "IsNull", + "Predicate", + "Scalar", + "all_of", + "any_of", + "compare", + "contains", + "eq", + "gt", + "gte", + "is_null", + "lt", + "lte", + "ne", + "one_of", +] diff --git a/src/everos/infra/persistence/index/schema.py b/src/everos/infra/persistence/index/schema.py new file mode 100644 index 000000000..dd9294baa --- /dev/null +++ b/src/everos/infra/persistence/index/schema.py @@ -0,0 +1,341 @@ +"""Backend-neutral schemas for rebuildable business indexes. + +Each table has one explicit field description consumed by every backend +adapter. The model-field parity check fails immediately when a domain field is +added without a storage decision, avoiding fallback coercions and silently +divergent schemas. +""" + +from __future__ import annotations + +import datetime as dt +from dataclasses import dataclass +from enum import StrEnum +from functools import cache +from typing import Any, get_args, get_origin + +_DEFAULT_STRING_LENGTH = 65_535 +_ID_LENGTH = 512 +_ARRAY_CAPACITY = 256 +_VECTOR_DIMENSION = 1024 + + +class IndexFieldKind(StrEnum): + STRING = "string" + STRING_ARRAY = "string_array" + FLOAT = "float" + INTEGER = "integer" + DATETIME = "datetime" + DENSE_VECTOR = "dense_vector" + + +@dataclass(frozen=True) +class IndexField: + name: str + kind: IndexFieldKind + nullable: bool = False + primary: bool = False + max_length: int | None = None + max_capacity: int | None = None + dimension: int | None = None + + +@dataclass(frozen=True) +class IndexSchema: + table_name: str + model: type[Any] + fields: tuple[IndexField, ...] + bm25_fields: tuple[str, ...] + + def field(self, name: str) -> IndexField: + for field in self.fields: + if field.name == name: + return field + raise KeyError(name) + + @property + def vector_fields(self) -> tuple[IndexField, ...]: + return tuple( + field for field in self.fields if field.kind is IndexFieldKind.DENSE_VECTOR + ) + + @property + def datetime_fields(self) -> frozenset[str]: + return frozenset( + field.name for field in self.fields if field.kind is IndexFieldKind.DATETIME + ) + + +def _s( + name: str, + *, + nullable: bool = False, + primary: bool = False, + max_length: int = _DEFAULT_STRING_LENGTH, +) -> IndexField: + return IndexField( + name, + IndexFieldKind.STRING, + nullable=nullable, + primary=primary, + max_length=max_length, + ) + + +def _id(name: str = "id") -> IndexField: + return _s(name, primary=True, max_length=_ID_LENGTH) + + +def _a(name: str, *, nullable: bool = False) -> IndexField: + return IndexField( + name, + IndexFieldKind.STRING_ARRAY, + nullable=nullable, + max_length=_ID_LENGTH, + max_capacity=_ARRAY_CAPACITY, + ) + + +def _f(name: str, *, nullable: bool = False) -> IndexField: + return IndexField(name, IndexFieldKind.FLOAT, nullable=nullable) + + +def _i(name: str, *, nullable: bool = False) -> IndexField: + return IndexField(name, IndexFieldKind.INTEGER, nullable=nullable) + + +def _d(name: str, *, nullable: bool = False) -> IndexField: + return IndexField(name, IndexFieldKind.DATETIME, nullable=nullable) + + +def _v(name: str, *, nullable: bool = True) -> IndexField: + return IndexField( + name, + IndexFieldKind.DENSE_VECTOR, + nullable=nullable, + dimension=_VECTOR_DIMENSION, + ) + + +_FIELDS: dict[str, tuple[IndexField, ...]] = { + "episode": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id", nullable=True), + _d("timestamp"), + _s("parent_type"), + _s("parent_id"), + _a("sender_ids"), + _s("subject", nullable=True), + _s("summary", nullable=True), + _s("episode"), + _s("episode_tokens"), + _s("md_path"), + _s("content_sha256"), + _s("deprecated_by", nullable=True), + _v("vector"), + _v("subject_vector"), + _d("created_at"), + _d("updated_at"), + ), + "atomic_fact": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id", nullable=True), + _d("timestamp"), + _s("parent_type"), + _s("parent_id"), + _a("sender_ids"), + _s("fact"), + _s("fact_tokens"), + _s("md_path"), + _s("content_sha256"), + _s("deprecated_by", nullable=True), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "foresight": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id", nullable=True), + _d("timestamp"), + _d("start_time", nullable=True), + _d("end_time", nullable=True), + _i("duration_days", nullable=True), + _s("parent_type"), + _s("parent_id"), + _a("sender_ids"), + _s("foresight"), + _s("foresight_tokens"), + _s("evidence", nullable=True), + _s("evidence_tokens", nullable=True), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "agent_case": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id"), + _d("timestamp"), + _s("parent_type"), + _s("parent_id"), + _f("quality_score"), + _s("task_intent"), + _s("task_intent_tokens"), + _s("approach"), + _s("approach_tokens"), + _s("key_insight", nullable=True), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "agent_skill": ( + _id(), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("name"), + _s("description"), + _s("description_tokens"), + _s("content"), + _s("content_tokens"), + _f("confidence"), + _f("maturity_score"), + _a("source_case_ids"), + _s("cluster_id", nullable=True), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "user_profile": ( + _id(), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("summary"), + _s("explicit_info_json"), + _s("implicit_traits_json"), + _i("profile_timestamp_ms"), + _s("md_path"), + _s("content_sha256"), + _d("created_at"), + _d("updated_at"), + ), + "knowledge_topic": ( + _id(), + _s("doc_id"), + _s("category_id"), + _s("app_id"), + _s("project_id"), + _s("topic_name"), + _s("topic_path"), + _i("depth"), + _s("parent_node_id"), + _s("summary"), + _s("summary_tokens"), + _s("content_tokens"), + _a("content_labels"), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), +} + + +@cache +def schema_for(model: type[Any]) -> IndexSchema: + """Return and validate the explicit neutral schema for a model.""" + table_name = model.TABLE_NAME + try: + fields = _FIELDS[table_name] + except KeyError as exc: + raise ValueError(f"no derived-index schema for {table_name!r}") from exc + + declared = {field.name for field in fields} + actual = set(model.model_fields) + if declared != actual: + raise ValueError( + f"derived-index schema drift for {table_name!r}: " + f"missing={sorted(actual - declared)}, stale={sorted(declared - actual)}" + ) + for field in fields: + _validate_model_field(table_name, field, model.model_fields[field.name]) + bm25_fields = tuple(model.BM25_FIELDS) + unknown_bm25 = set(bm25_fields) - declared + if unknown_bm25: + raise ValueError( + f"derived-index schema {table_name!r} has unknown BM25 fields: " + f"{sorted(unknown_bm25)}" + ) + return IndexSchema(table_name, model, fields, bm25_fields) + + +def _validate_model_field(table_name: str, field: IndexField, model_field: Any) -> None: + annotation = model_field.annotation + args = get_args(annotation) + optional = type(None) in args + candidates = ( + tuple(arg for arg in args if arg is not type(None)) + if optional + else (annotation,) + ) + if optional != field.nullable: + raise ValueError( + f"derived-index schema {table_name}.{field.name} nullable drift: " + f"model={optional}, schema={field.nullable}" + ) + + valid = False + if field.kind is IndexFieldKind.STRING: + valid = candidates == (str,) + elif field.kind is IndexFieldKind.STRING_ARRAY: + valid = len(candidates) == 1 and ( + get_origin(candidates[0]) is list and get_args(candidates[0]) == (str,) + ) + elif field.kind is IndexFieldKind.FLOAT: + valid = candidates == (float,) + elif field.kind is IndexFieldKind.INTEGER: + valid = candidates == (int,) + elif field.kind is IndexFieldKind.DATETIME: + valid = candidates == (dt.datetime,) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + dimension = getattr(candidates[0], "dim", None) if candidates else None + if callable(dimension): + dimension = dimension() + valid = len(candidates) == 1 and dimension == field.dimension + if not valid: + raise ValueError( + f"derived-index schema {table_name}.{field.name} type drift: " + f"model={annotation!r}, schema={field.kind.value}" + ) + + +__all__ = ["IndexField", "IndexFieldKind", "IndexSchema", "schema_for"] diff --git a/src/everos/infra/persistence/lancedb/predicate.py b/src/everos/infra/persistence/lancedb/predicate.py new file mode 100644 index 000000000..8a429d57e --- /dev/null +++ b/src/everos/infra/persistence/lancedb/predicate.py @@ -0,0 +1,83 @@ +"""Render backend-neutral predicates as LanceDB DataFusion expressions.""" + +from __future__ import annotations + +import datetime as dt +import re +from typing import Final + +from everos.component.utils.datetime import ensure_utc, to_iso_format +from everos.infra.persistence.predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, +) + +_FIELD_NAME: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_OPERATORS: Final[dict[str, str]] = { + "eq": "=", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def render_predicate(predicate: Predicate | None) -> str: + """Render a predicate with LanceDB escaping and timestamp literals.""" + if predicate is None: + return "" + if isinstance(predicate, Comparison): + return ( + f"{_field(predicate.field)} {_OPERATORS[predicate.operator]} " + f"{_literal(predicate.value)}" + ) + if isinstance(predicate, In): + values = ", ".join(_literal(value) for value in predicate.values) + return f"{_field(predicate.field)} IN ({values})" + if isinstance(predicate, Contains): + return f"array_has({_field(predicate.field)}, {_literal(predicate.value)})" + if isinstance(predicate, IsNull): + return f"{_field(predicate.field)} IS NULL" + if isinstance(predicate, All): + return _render_group(predicate.children, "AND") + if isinstance(predicate, AnyOf): + return _render_group(predicate.children, "OR") + raise TypeError(f"unsupported predicate: {type(predicate).__name__}") + + +def _render_group(children: tuple[Predicate, ...], operator: str) -> str: + rendered = [render_predicate(child) for child in children] + rendered = [item for item in rendered if item] + if not rendered: + return "" + if len(rendered) == 1: + return rendered[0] + return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" + + +def _field(value: str) -> str: + if not _FIELD_NAME.fullmatch(value): + raise ValueError(f"invalid predicate field: {value!r}") + return value + + +def _literal(value: Scalar) -> str: + if isinstance(value, str): + return f"'{value.replace(chr(39), chr(39) * 2)}'" + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return f"TIMESTAMP '{to_iso_format(aware)}'" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + return str(value) + + +__all__ = ["render_predicate"] diff --git a/src/everos/infra/persistence/milvus/__init__.py b/src/everos/infra/persistence/milvus/__init__.py index 4396360c8..910f8e64b 100644 --- a/src/everos/infra/persistence/milvus/__init__.py +++ b/src/everos/infra/persistence/milvus/__init__.py @@ -7,6 +7,11 @@ from __future__ import annotations +import asyncio + +from everos.core.observability.logging import get_logger + +from .milvus_manager import MilvusConfigurationError as MilvusConfigurationError from .milvus_manager import MilvusSchemaMismatchError as MilvusSchemaMismatchError from .milvus_manager import dispose_connection as dispose_connection from .milvus_manager import get_client as get_client @@ -18,6 +23,10 @@ from .repos import foresight_repo as foresight_repo from .repos import knowledge_topic_repo as knowledge_topic_repo from .repos import user_profile_repo as user_profile_repo +from .repository import MilvusRepoBase as MilvusRepoBase +from .repository import MilvusValueLimitError as MilvusValueLimitError + +logger = get_logger(__name__) async def ensure_business_indexes() -> None: @@ -32,13 +41,40 @@ async def verify_business_schemas() -> None: await repo.verify_collection() +async def drop_business_tables() -> list[str]: + """Drop every configured Milvus collection and return their names.""" + client = await get_client() + dropped: list[str] = [] + for repo in ALL_REPOS: + name = repo.collection_name + if await asyncio.to_thread(client.has_collection, name): + try: + await asyncio.to_thread(client.drop_collection, name) + except Exception: + # Zilliz Serverless can complete the drop server-side while + # its gateway returns DEADLINE_EXCEEDED. Confirm state before + # turning an already-successful rebuild/cleanup into failure. + if await asyncio.to_thread(client.has_collection, name): + raise + logger.warning( + "milvus_collection_drop_confirmed_after_client_error", + collection=name, + ) + dropped.append(name) + MilvusRepoBase._reset_collection_cache() + return dropped + + __all__ = [ "ALL_REPOS", + "MilvusConfigurationError", "MilvusSchemaMismatchError", + "MilvusValueLimitError", "agent_case_repo", "agent_skill_repo", "atomic_fact_repo", "dispose_connection", + "drop_business_tables", "ensure_business_indexes", "episode_repo", "foresight_repo", diff --git a/src/everos/infra/persistence/milvus/milvus_manager.py b/src/everos/infra/persistence/milvus/milvus_manager.py index 602e4a314..05af230b4 100644 --- a/src/everos/infra/persistence/milvus/milvus_manager.py +++ b/src/everos/infra/persistence/milvus/milvus_manager.py @@ -7,8 +7,8 @@ from pymilvus import MilvusClient from everos.config import MilvusSettings, load_settings +from everos.core.errors import ConfigurationError from everos.core.observability.logging import get_logger -from everos.core.persistence import MemoryRoot logger = get_logger(__name__) @@ -19,6 +19,10 @@ class MilvusSchemaMismatchError(RuntimeError): """Raised when an existing Milvus collection does not match EverOS.""" +class MilvusConfigurationError(ConfigurationError): + """Raised when the remote Milvus profile is incomplete or invalid.""" + + def collection_name(table_name: str, settings: MilvusSettings | None = None) -> str: """Return the configured Milvus collection name for an EverOS table.""" cfg = settings or load_settings().milvus @@ -55,14 +59,27 @@ async def dispose_connection() -> None: _client.close() _client = None logger.info("milvus_connection_closed") + from .repository import MilvusRepoBase + + MilvusRepoBase._reset_collection_cache() def _resolve_uri(settings: MilvusSettings) -> str: - if settings.uri: - return settings.uri - memory_root = MemoryRoot.default() - memory_root.milvus_dir.mkdir(parents=True, exist_ok=True) - return str(memory_root.milvus_db) + uri = settings.uri.strip() + if not uri: + raise MilvusConfigurationError( + "[index] backend = 'milvus' requires EVEROS_MILVUS__URI (or " + "[milvus] uri) pointing to Milvus Server or Zilliz Cloud; " + "embedded Milvus Lite is not supported" + ) + scheme, separator, _rest = uri.partition("://") + if not separator or scheme.lower() not in {"http", "https"}: + raise MilvusConfigurationError( + "[milvus] uri must be a remote http(s) endpoint for Milvus Server " + "or Zilliz Cloud, not a local database path; embedded Milvus Lite " + "is not supported" + ) + return uri def _secret(value: object | None) -> str: @@ -80,6 +97,7 @@ def _sanitize_name_part(value: str) -> str: __all__ = [ + "MilvusConfigurationError", "MilvusSchemaMismatchError", "collection_name", "dispose_connection", diff --git a/src/everos/infra/persistence/milvus/predicate.py b/src/everos/infra/persistence/milvus/predicate.py new file mode 100644 index 000000000..37f14b289 --- /dev/null +++ b/src/everos/infra/persistence/milvus/predicate.py @@ -0,0 +1,100 @@ +"""Render backend-neutral predicates as Milvus filter expressions.""" + +from __future__ import annotations + +import datetime as dt +import json +import re +from collections.abc import Collection +from typing import Final + +from everos.component.utils.datetime import ensure_utc, to_timestamp_ms +from everos.infra.persistence.predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, +) + +_FIELD_NAME: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_OPERATORS: Final[dict[str, str]] = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def render_predicate( + predicate: Predicate | None, + *, + datetime_fields: Collection[str] = (), +) -> str: + """Render a predicate using Milvus operators and physical field names.""" + if predicate is None: + return "" + if isinstance(predicate, Comparison): + return ( + f"{_field(predicate.field, datetime_fields)} " + f"{_OPERATORS[predicate.operator]} {_literal(predicate.value)}" + ) + if isinstance(predicate, In): + values = ", ".join(_literal(value) for value in predicate.values) + return f"{_field(predicate.field, datetime_fields)} in [{values}]" + if isinstance(predicate, Contains): + return ( + f"array_contains({_field(predicate.field, datetime_fields)}, " + f"{_literal(predicate.value)})" + ) + if isinstance(predicate, IsNull): + return f"{_field(predicate.field, datetime_fields)} is null" + if isinstance(predicate, All): + return _render_group(predicate.children, "and", datetime_fields) + if isinstance(predicate, AnyOf): + return _render_group(predicate.children, "or", datetime_fields) + raise TypeError(f"unsupported predicate: {type(predicate).__name__}") + + +def _render_group( + children: tuple[Predicate, ...], + operator: str, + datetime_fields: Collection[str], +) -> str: + rendered = [ + render_predicate(child, datetime_fields=datetime_fields) for child in children + ] + rendered = [item for item in rendered if item] + if not rendered: + return "" + if len(rendered) == 1: + return rendered[0] + return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" + + +def _field(value: str, datetime_fields: Collection[str]) -> str: + if not _FIELD_NAME.fullmatch(value): + raise ValueError(f"invalid predicate field: {value!r}") + if value in datetime_fields: + return f"{value}_ms" + return value + + +def _literal(value: Scalar) -> str: + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False) + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return str(to_timestamp_ms(aware)) + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +__all__ = ["render_predicate"] diff --git a/src/everos/infra/persistence/milvus/repos.py b/src/everos/infra/persistence/milvus/repos.py index 826bc9267..9319a9d9e 100644 --- a/src/everos/infra/persistence/milvus/repos.py +++ b/src/everos/infra/persistence/milvus/repos.py @@ -3,7 +3,10 @@ from __future__ import annotations from collections.abc import Sequence +from typing import Any +from everos.component.utils.datetime import from_timestamp +from everos.infra.persistence.index.predicate import all_of, eq, gt, is_null from everos.infra.persistence.lancedb import ( AgentCase, AgentSkill, @@ -20,6 +23,50 @@ class _EpisodeRepo(MilvusRepoBase[Episode]): schema = Episode + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: + return await self._count_where( + all_of( + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), + is_null("deprecated_by"), + eq("parent_type", parent_type) if parent_type is not None else None, + ) + ) + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[Episode] | list[dict[str, Any]]: + predicate = all_of( + eq("owner_id", owner_id), + gt("timestamp", from_timestamp(after_ts)), + eq("parent_type", parent_type), + eq("app_id", app_id), + eq("project_id", project_id), + is_null("deprecated_by"), + ) + rows = await self.find_where(predicate, limit=limit or 20_000) + rows.sort(key=lambda row: row.timestamp) + if columns is None: + return rows + projection = list(dict.fromkeys([*columns, "timestamp"])) + return [{name: getattr(row, name) for name in projection} for row in rows] + class _AtomicFactRepo(MilvusRepoBase[AtomicFact]): schema = AtomicFact @@ -37,17 +84,15 @@ class _AgentSkillRepo(MilvusRepoBase[AgentSkill]): schema = AgentSkill async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: - rows = await self.find_where( - f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", - limit=10_000, + return await self._count_where( + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)) ) - return len(rows) async def find_in_cluster( self, *, owner_id: str, cluster_id: str, limit: int ) -> list[AgentSkill]: return await self.find_where( - f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)), limit=limit, ) @@ -66,7 +111,7 @@ async def find_topk_relevant_in_cluster( ) rows = await self.dense_search( query_vector, - f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)), limit=top_k, ) out: list[AgentSkill] = [] @@ -85,10 +130,6 @@ class _KnowledgeTopicRepo(MilvusRepoBase[KnowledgeTopic]): schema = KnowledgeTopic -def _q(value: str) -> str: - return value.replace("\\", "\\\\").replace("'", "\\'") - - episode_repo = _EpisodeRepo() atomic_fact_repo = _AtomicFactRepo() foresight_repo = _ForesightRepo() diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index de9143b3d..44eaa11f5 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -1,55 +1,62 @@ -"""Milvus repository implementation for EverOS derived index tables.""" +"""Milvus repository for EverOS rebuildable derived indexes.""" from __future__ import annotations import asyncio import datetime as dt -import re from collections.abc import Sequence -from importlib.metadata import PackageNotFoundError, version -from typing import Any, ClassVar, get_args, get_origin +from typing import Any, ClassVar +from pydantic import BaseModel from pymilvus import DataType, Function, FunctionType, MilvusClient -from everos.component.utils.datetime import ( - ensure_utc, - from_iso_format, - from_timestamp, - to_timestamp_ms, -) +from everos.component.utils.datetime import ensure_utc, from_timestamp, to_timestamp_ms from everos.config import load_settings from everos.core.observability.logging import get_logger -from everos.core.persistence import BaseLanceTable - -from .milvus_manager import ( - MilvusSchemaMismatchError, - collection_name, - get_client, +from everos.infra.persistence.index.predicate import ( + Predicate, + all_of, + eq, + one_of, ) +from everos.infra.persistence.index.schema import ( + IndexField, + IndexFieldKind, + IndexSchema, + schema_for, +) + +from .milvus_manager import MilvusSchemaMismatchError, collection_name, get_client +from .predicate import render_predicate logger = get_logger(__name__) _DUMMY_VECTOR_FIELD = "_everos_dummy_vector" +_DUMMY_VECTOR_DIMENSION = 2 _SPARSE_SUFFIX = "__sparse" -_MAX_VARCHAR_LENGTH = 65_535 -_ID_MAX_LENGTH = 512 -_ARRAY_MAX_CAPACITY = 256 +_PRESENT_SUFFIX = "__present" +_UPDATE_FETCH_LIMIT = 10_000 -def _q(value: str) -> str: - """Escape a string for a Milvus single-quoted expression literal.""" - return value.replace("\\", "\\\\").replace("'", "\\'") +class MilvusValueLimitError(ValueError): + """A row exceeds a documented Milvus VARCHAR, array, or vector limit.""" -class MilvusRepoBase[T: BaseLanceTable]: - """Generic Milvus repository for one EverOS derived index collection.""" +class MilvusRepoBase[T: BaseModel]: + """Generic Milvus repository backed by one neutral index schema.""" schema: type[T] _write_locks: ClassVar[dict[str, asyncio.Lock]] = {} + _collection_locks: ClassVar[dict[str, asyncio.Lock]] = {} + _ready_collections: ClassVar[set[str]] = set() + + @property + def index_schema(self) -> IndexSchema: + return schema_for(self.schema) @property def table_name(self) -> str: - return self.schema.TABLE_NAME + return self.index_schema.table_name @property def collection_name(self) -> str: @@ -59,37 +66,58 @@ def collection_name(self) -> str: def _write_lock(cls, name: str) -> asyncio.Lock: return cls._write_locks.setdefault(name, asyncio.Lock()) + @classmethod + def _collection_lock(cls, name: str) -> asyncio.Lock: + return cls._collection_locks.setdefault(name, asyncio.Lock()) + + @classmethod + def _reset_collection_cache(cls) -> None: + cls._ready_collections.clear() + cls._collection_locks.clear() + @classmethod def _reset_locks_for_tests(cls) -> None: cls._write_locks.clear() + cls._reset_collection_cache() async def ensure_collection(self) -> None: - client = await get_client() - if await _run(client.has_collection, self.collection_name): - await self.verify_collection() + """Create or verify the collection once per process.""" + name = self.collection_name + if name in self._ready_collections: return + async with self._collection_lock(name): + if name in self._ready_collections: + return + client = await get_client() + if await _run(client.has_collection, name): + await self.verify_collection() + else: + await self._create_collection(client) + self._ready_collections.add(name) + async def _create_collection(self, client: MilvusClient) -> None: schema = self._build_collection_schema() index_params = client.prepare_index_params() - if "vector" in self.schema.model_fields: - index_params.add_index( - field_name="vector", - index_type="AUTOINDEX", - metric_type="COSINE", - ) + vector_fields = self.index_schema.vector_fields + if vector_fields: + for field in vector_fields: + index_params.add_index( + field_name=field.name, + index_type="AUTOINDEX", + metric_type="COSINE", + ) else: index_params.add_index( field_name=_DUMMY_VECTOR_FIELD, index_type="AUTOINDEX", metric_type="COSINE", ) - for field in self.schema.BM25_FIELDS: + for field in self.index_schema.bm25_fields: index_params.add_index( field_name=_sparse_field(field), index_type="AUTOINDEX", metric_type="BM25", ) - settings = load_settings().milvus await _run( client.create_collection, @@ -110,21 +138,20 @@ async def verify_collection(self) -> None: actual = {field["name"] for field in description.get("fields", [])} expected = set(self._stored_field_names()) missing = expected - actual - if missing: + stale = actual - expected + if missing or stale: raise MilvusSchemaMismatchError( f"Milvus collection {self.collection_name!r} schema drift: " - f"missing={sorted(missing)}. The index is rebuildable from md; " - "drop the collection and restart to rebuild it." + f"missing={sorted(missing)}, stale={sorted(stale)}. The index is " + "rebuildable from markdown; run `everos cascade rebuild`." ) - # ── Create / update ───────────────────────────────────────────── - async def add(self, records: Sequence[T]) -> None: if not records: return await self.ensure_collection() - client = await get_client() payload = [self._to_milvus_record(record) for record in records] + client = await get_client() async with self._write_lock(self.collection_name): await _run(client.insert, self.collection_name, payload) @@ -134,52 +161,57 @@ async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: if not records: return await self.ensure_collection() - client = await get_client() payload = [self._to_milvus_record(record) for record in records] + client = await get_client() async with self._write_lock(self.collection_name): await _run(client.upsert, self.collection_name, payload) - async def update(self, updates: dict[str, Any], *, where: Any) -> None: - await self.ensure_collection() - rows = await self._query_raw(where, limit=10_000, include_vectors=True) + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: + rows = await self._query_raw( + where, limit=_UPDATE_FETCH_LIMIT, include_vectors=True + ) if not rows: return - patched = [] + if len(rows) == _UPDATE_FETCH_LIMIT: + logger.warning( + "milvus_update_truncated", + table=self.table_name, + limit=_UPDATE_FETCH_LIMIT, + ) + patched: list[dict[str, Any]] = [] for row in rows: merged = dict(row) for key, value in updates.items(): self._write_field_value(merged, key, value) + self._validate_raw_record(merged) patched.append(merged) client = await get_client() async with self._write_lock(self.collection_name): await _run(client.upsert, self.collection_name, patched) - # ── Maintenance ──────────────────────────────────────────────── - async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None: - """Milvus indexes are managed by the service; no per-write compaction.""" + """Milvus indexes and compaction are service-managed.""" async def rebuild_indexes(self) -> None: """Milvus AUTOINDEX maintenance is service-managed.""" - # ── Reads ────────────────────────────────────────────────────── - async def count(self) -> int: + return await self._count_where(None) + + async def _count_where(self, where: Predicate | None) -> int: await self.ensure_collection() client = await get_client() rows = await _run( client.query, self.collection_name, - filter="", + filter=self._expr(where), output_fields=["count(*)"], ) - if not rows: - return 0 - return int(rows[0].get("count(*)", 0)) + return int(rows[0].get("count(*)", 0)) if rows else 0 async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: if id_field != "id": - rows = await self.find_where(f"{id_field} = '{_q(id_value)}'", limit=1) + rows = await self.find_where(eq(id_field, id_value), limit=1) return rows[0] if rows else None await self.ensure_collection() client = await get_client() @@ -189,21 +221,19 @@ async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: ids=[id_value], output_fields=self._output_fields(include_vectors=True), ) - if not rows: - return None - return self._model_from_milvus(rows[0]) + return self._model_from_milvus(rows[0]) if rows else None - async def find_where(self, where: Any, *, limit: int = 100) -> list[T]: + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: rows = await self._query_raw(where, limit=limit, include_vectors=True) return [self._model_from_milvus(row) for row in rows] - async def find_one_where(self, where: Any) -> T | None: + async def find_one_where(self, where: Predicate) -> T | None: rows = await self.find_where(where, limit=1) return rows[0] if rows else None async def find_where_paginated( self, - where: Any, + where: Predicate, *, sort_by: str, descending: bool = True, @@ -211,9 +241,9 @@ async def find_where_paginated( page_size: int = 20, max_fetch: int = 20_000, ) -> tuple[list[T], int]: + total = await self._count_where(where) raw = await self._query_raw(where, limit=max_fetch, include_vectors=True) - total = len(raw) - if total >= max_fetch: + if total > len(raw): logger.warning( "milvus_find_where_paginated_truncated", table=self.table_name, @@ -229,16 +259,16 @@ async def find_where_paginated( return rows[offset : offset + page_size], total async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: - return await self.find_where(f"owner_id = '{_q(owner_id)}'", limit=limit) + return await self.find_where(eq("owner_id", owner_id), limit=limit) async def find_by_md_path(self, md_path: str) -> T | None: - return await self.find_one_where(f"md_path = '{_q(md_path)}'") + return await self.find_one_where(eq("md_path", md_path)) async def search( self, *, vector: Sequence[float] | None = None, - where: Any = None, + where: Predicate | None = None, limit: int = 10, ) -> list[dict[str, Any]]: if vector is None: @@ -248,7 +278,7 @@ async def search( async def sparse_search( self, query_terms: Sequence[str], - where: Any, + where: Predicate | None, *, columns: Sequence[str] | None = None, limit: int, @@ -256,11 +286,13 @@ async def sparse_search( if not query_terms: return [] await self.ensure_collection() - fields = list(columns or self.schema.BM25_FIELDS) + fields = list(columns or self.index_schema.bm25_fields) if not fields: return [] + unknown = set(fields) - set(self.index_schema.bm25_fields) + if unknown: + raise ValueError(f"unknown BM25 fields: {sorted(unknown)}") client = await get_client() - expr = self._expr(where) query = " ".join(term for term in query_terms if term) best: dict[str, dict[str, Any]] = {} for field in fields: @@ -269,7 +301,7 @@ async def sparse_search( self.collection_name, data=[query], anns_field=_sparse_field(field), - filter=expr, + filter=self._expr(where), limit=limit, output_fields=self._output_fields(include_vectors=False), search_params={"metric_type": "BM25"}, @@ -291,20 +323,26 @@ async def sparse_search( async def dense_search( self, vector: Sequence[float], - where: Any, + where: Predicate | None, *, limit: int, + vector_field: str = "vector", ) -> list[dict[str, Any]]: - if not vector or "vector" not in self.schema.model_fields: + if not vector: return [] + field = self.index_schema.field(vector_field) + if field.kind is not IndexFieldKind.DENSE_VECTOR: + raise ValueError(f"{vector_field!r} is not a dense-vector field") + self._validate_vector(field, vector) await self.ensure_collection() client = await get_client() + present = eq(_present_field(vector_field), True) results = await _run( client.search, self.collection_name, data=[list(vector)], - anns_field="vector", - filter=self._expr(where), + anns_field=vector_field, + filter=self._expr(all_of(where, present)), limit=limit, output_fields=self._output_fields(include_vectors=False), search_params={"metric_type": "COSINE"}, @@ -314,9 +352,7 @@ async def dense_search( for row in _first_result_set(results) ] - # ── Deletes ──────────────────────────────────────────────────── - - async def delete(self, predicate: Any) -> None: + async def delete(self, predicate: Predicate) -> None: await self.ensure_collection() client = await get_client() async with self._write_lock(self.collection_name): @@ -327,14 +363,11 @@ async def delete(self, predicate: Any) -> None: ) async def delete_by_md_path(self, md_path: str) -> int: - await self.ensure_collection() - before = await self.find_where(f"md_path = '{_q(md_path)}'", limit=10_000) - if not before: - return 0 - await self.delete(f"md_path = '{_q(md_path)}'") - return len(before) - - # ── Daily-log helpers ────────────────────────────────────────── + predicate = eq("md_path", md_path) + count = await self._count_where(predicate) + if count: + await self.delete(predicate) + return count async def find_by_owner_entry( self, @@ -345,8 +378,12 @@ async def find_by_owner_entry( project_id: str = "default", ) -> T | None: return await self.find_one_where( - f"owner_id = '{_q(owner_id)}' AND entry_id = '{_q(entry_id)}' " - f"AND app_id = '{_q(app_id)}' AND project_id = '{_q(project_id)}'" + all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), + ) ) async def find_by_owner_entries( @@ -359,10 +396,13 @@ async def find_by_owner_entries( ) -> list[T]: if not entry_ids: return [] - quoted = ", ".join(f"'{_q(entry_id)}'" for entry_id in entry_ids) return await self.find_where( - f"owner_id = '{_q(owner_id)}' AND entry_id IN ({quoted}) " - f"AND app_id = '{_q(app_id)}' AND project_id = '{_q(project_id)}'", + all_of( + eq("owner_id", owner_id), + one_of("entry_id", list(entry_ids)), + eq("app_id", app_id), + eq("project_id", project_id), + ), limit=len(entry_ids), ) @@ -370,7 +410,7 @@ async def find_by_session( self, owner_id: str, session_id: str, *, limit: int = 100 ) -> list[T]: return await self.find_where( - f"owner_id = '{_q(owner_id)}' AND session_id = '{_q(session_id)}'", + all_of(eq("owner_id", owner_id), eq("session_id", session_id)), limit=limit, ) @@ -378,82 +418,24 @@ async def find_by_parent( self, parent_type: str, parent_id: str, *, limit: int = 100 ) -> list[T]: return await self.find_where( - f"parent_type = '{_q(parent_type)}' AND parent_id = '{_q(parent_id)}'", + all_of(eq("parent_type", parent_type), eq("parent_id", parent_id)), limit=limit, ) - # ── Field conversion ─────────────────────────────────────────── - def _build_collection_schema(self): # type: ignore[no-untyped-def] - settings = load_settings().milvus schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False) - has_vector = False - for name, field in self.schema.model_fields.items(): - if name == "subject_vector": - continue - annotation = field.annotation - if name == "id": - schema.add_field( - field_name=name, - datatype=DataType.VARCHAR, - is_primary=True, - max_length=_ID_MAX_LENGTH, - ) - elif name == "vector": - has_vector = True - schema.add_field( - field_name=name, - datatype=DataType.FLOAT_VECTOR, - dim=settings.dimension, - ) - elif _is_datetime(annotation): - schema.add_field( - field_name=_datetime_storage_field(name), - datatype=DataType.INT64, - nullable=_is_optional(annotation), - ) - elif _is_list(annotation): - schema.add_field( - field_name=name, - datatype=DataType.ARRAY, - element_type=DataType.VARCHAR, - max_capacity=_ARRAY_MAX_CAPACITY, - max_length=_ID_MAX_LENGTH, - nullable=_is_optional(annotation), - ) - elif _is_float(annotation): - schema.add_field( - field_name=name, - datatype=DataType.DOUBLE, - nullable=_is_optional(annotation), - ) - elif _is_int(annotation): - schema.add_field( - field_name=name, - datatype=DataType.INT64, - nullable=_is_optional(annotation), - ) - else: - kwargs: dict[str, Any] = { - "field_name": name, - "datatype": DataType.VARCHAR, - "max_length": _MAX_VARCHAR_LENGTH, - } - if name in self.schema.BM25_FIELDS: - kwargs["enable_analyzer"] = True - elif _is_optional(annotation): - kwargs["nullable"] = True - schema.add_field(**kwargs) - - if not has_vector: + for field in self.index_schema.fields: + self._add_schema_field(schema, field) + if not self.index_schema.vector_fields: schema.add_field( field_name=_DUMMY_VECTOR_FIELD, datatype=DataType.FLOAT_VECTOR, - dim=1, + dim=_DUMMY_VECTOR_DIMENSION, ) - for field in self.schema.BM25_FIELDS: + for field in self.index_schema.bm25_fields: + sparse_name = _sparse_field(field) schema.add_field( - field_name=_sparse_field(field), + field_name=sparse_name, datatype=DataType.SPARSE_FLOAT_VECTOR, ) schema.add_function( @@ -461,35 +443,91 @@ def _build_collection_schema(self): # type: ignore[no-untyped-def] name=f"{field}_bm25", function_type=FunctionType.BM25, input_field_names=[field], - output_field_names=[_sparse_field(field)], + output_field_names=[sparse_name], ) ) return schema + def _add_schema_field(self, schema: Any, field: IndexField) -> None: + if field.kind is IndexFieldKind.STRING: + kwargs: dict[str, Any] = { + "field_name": field.name, + "datatype": DataType.VARCHAR, + "max_length": field.max_length, + } + if field.primary: + kwargs["is_primary"] = True + elif field.nullable and field.name not in self.index_schema.bm25_fields: + kwargs["nullable"] = True + if field.name in self.index_schema.bm25_fields: + kwargs["enable_analyzer"] = True + schema.add_field(**kwargs) + elif field.kind is IndexFieldKind.STRING_ARRAY: + schema.add_field( + field_name=field.name, + datatype=DataType.ARRAY, + element_type=DataType.VARCHAR, + max_capacity=field.max_capacity, + max_length=field.max_length, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.FLOAT: + schema.add_field( + field_name=field.name, + datatype=DataType.DOUBLE, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.INTEGER: + schema.add_field( + field_name=field.name, + datatype=DataType.INT64, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.DATETIME: + schema.add_field( + field_name=_datetime_storage_field(field.name), + datatype=DataType.INT64, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + schema.add_field( + field_name=field.name, + datatype=DataType.FLOAT_VECTOR, + dim=field.dimension, + ) + schema.add_field( + field_name=_present_field(field.name), + datatype=DataType.BOOL, + ) + else: # pragma: no cover - enum exhaustiveness guard + raise TypeError(f"unsupported index field kind: {field.kind}") + def _stored_field_names(self) -> list[str]: names: list[str] = [] - has_vector = False - for name, field in self.schema.model_fields.items(): - if name == "subject_vector": - continue - if name == "vector": - has_vector = True - names.append(name) - elif _is_datetime(field.annotation): - names.append(_datetime_storage_field(name)) + for field in self.index_schema.fields: + if field.kind is IndexFieldKind.DATETIME: + names.append(_datetime_storage_field(field.name)) else: - names.append(name) - if not has_vector: + names.append(field.name) + if field.kind is IndexFieldKind.DENSE_VECTOR: + names.append(_present_field(field.name)) + if not self.index_schema.vector_fields: names.append(_DUMMY_VECTOR_FIELD) - names.extend(_sparse_field(field) for field in self.schema.BM25_FIELDS) + names.extend(_sparse_field(field) for field in self.index_schema.bm25_fields) return names def _output_fields(self, *, include_vectors: bool) -> list[str]: fields: list[str] = [] + vector_names = {field.name for field in self.index_schema.vector_fields} + present_names = {_present_field(name) for name in vector_names} for name in self._stored_field_names(): - if name.endswith(_SPARSE_SUFFIX) or name == _DUMMY_VECTOR_FIELD: + if name.endswith(_SPARSE_SUFFIX): + continue + if name == _DUMMY_VECTOR_FIELD: + if include_vectors: + fields.append(name) continue - if not include_vectors and name == "vector": + if not include_vectors and (name in vector_names or name in present_names): continue fields.append(name) return fields @@ -497,35 +535,80 @@ def _output_fields(self, *, include_vectors: bool) -> list[str]: def _to_milvus_record(self, record: T) -> dict[str, Any]: raw = record.model_dump(mode="python") out: dict[str, Any] = {} - for name, field in self.schema.model_fields.items(): - if name == "subject_vector": - continue - value = raw.get(name) - if name in self.schema.BM25_FIELDS and value is None: + for field in self.index_schema.fields: + value = raw.get(field.name) + if field.name in self.index_schema.bm25_fields and value is None: value = "" - if _is_datetime(field.annotation): - storage_name = _datetime_storage_field(name) - out[storage_name] = ( + if field.kind is IndexFieldKind.DATETIME: + out[_datetime_storage_field(field.name)] = ( _datetime_to_ms(value) if value is not None else None ) - elif name == "vector": - out[name] = list(value or []) - elif _is_list(field.annotation): - out[name] = [str(item) for item in (value or [])] + elif field.kind is IndexFieldKind.DENSE_VECTOR: + present = value is not None + out[field.name] = ( + list(value) if present else [0.0] * int(field.dimension or 0) + ) + out[_present_field(field.name)] = present + elif field.kind is IndexFieldKind.STRING_ARRAY: + out[field.name] = [str(item) for item in (value or [])] else: - out[name] = value - if "vector" not in self.schema.model_fields: - out[_DUMMY_VECTOR_FIELD] = [0.0] + out[field.name] = value + if not self.index_schema.vector_fields: + out[_DUMMY_VECTOR_FIELD] = [0.0] * _DUMMY_VECTOR_DIMENSION + self._validate_raw_record(out) return out + def _validate_raw_record(self, row: dict[str, Any]) -> None: + for field in self.index_schema.fields: + storage_name = ( + _datetime_storage_field(field.name) + if field.kind is IndexFieldKind.DATETIME + else field.name + ) + value = row.get(storage_name) + if value is None: + if not field.nullable: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} cannot be null" + ) + continue + if field.kind is IndexFieldKind.STRING: + size = len(str(value).encode("utf-8")) + if field.max_length is not None and size > field.max_length: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} is {size} UTF-8 bytes; " + f"Milvus limit is {field.max_length}" + ) + elif field.kind is IndexFieldKind.STRING_ARRAY: + if field.max_capacity is not None and len(value) > field.max_capacity: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} has {len(value)} items; " + f"Milvus limit is {field.max_capacity}" + ) + for position, item in enumerate(value): + size = len(str(item).encode("utf-8")) + if field.max_length is not None and size > field.max_length: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name}[{position}] is {size} " + f"UTF-8 bytes; Milvus limit is {field.max_length}" + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + self._validate_vector(field, value) + + def _validate_vector(self, field: IndexField, value: Sequence[float]) -> None: + if len(value) != field.dimension: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} has dimension {len(value)}; " + f"expected {field.dimension}" + ) + def _model_from_milvus(self, row: dict[str, Any]) -> T: - shaped = self._restore_row(row, include_distance=False) - return self.schema.model_validate(shaped) + return self.schema.model_validate(self._restore_row(row)) def _candidate_row_from_search( self, row: dict[str, Any], *, normalize_cosine: bool = False ) -> dict[str, Any]: - shaped = self._restore_row(row.get("entity", {}), include_distance=False) + shaped = self._restore_row(row.get("entity", {})) raw_distance = row.get("distance") shaped["_distance"] = ( _cosine_distance_from_milvus(raw_distance) @@ -534,49 +617,53 @@ def _candidate_row_from_search( ) return shaped - def _restore_row( - self, row: dict[str, Any], *, include_distance: bool - ) -> dict[str, Any]: + def _restore_row(self, row: dict[str, Any]) -> dict[str, Any]: out: dict[str, Any] = {} - for name, field in self.schema.model_fields.items(): - if name == "subject_vector": - continue - if _is_datetime(field.annotation): - storage_name = _datetime_storage_field(name) - value = row.get(storage_name) - if value is None: - out[name] = None - else: - out[name] = from_timestamp(int(value)) - elif name in row: - value = row[name] + for field in self.index_schema.fields: + if field.kind is IndexFieldKind.DATETIME: + value = row.get(_datetime_storage_field(field.name)) + out[field.name] = None if value is None else from_timestamp(int(value)) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + if field.name in row: + out[field.name] = ( + row[field.name] + if row.get(_present_field(field.name), True) + else None + ) + elif field.name in row: + value = row[field.name] if ( - name in self.schema.BM25_FIELDS + field.name in self.index_schema.bm25_fields and value == "" - and _is_optional(field.annotation) + and field.nullable ): - out[name] = None - else: - out[name] = value - if include_distance and "distance" in row: - out["_distance"] = row["distance"] + value = None + out[field.name] = value return out def _write_field_value( self, row: dict[str, Any], field_name: str, value: Any ) -> None: - field = self.schema.model_fields.get(field_name) - if field is None: - row[field_name] = value - elif _is_datetime(field.annotation): + field = self.index_schema.field(field_name) + if field.kind is IndexFieldKind.DATETIME: row[_datetime_storage_field(field_name)] = ( _datetime_to_ms(value) if value is not None else None ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + present = value is not None + row[field_name] = ( + list(value) if present else [0.0] * int(field.dimension or 0) + ) + row[_present_field(field_name)] = present else: row[field_name] = value async def _query_raw( - self, where: Any, *, limit: int, include_vectors: bool + self, + where: Predicate | None, + *, + limit: int, + include_vectors: bool, ) -> list[dict[str, Any]]: await self.ensure_collection() client = await get_client() @@ -589,49 +676,31 @@ async def _query_raw( ) async def _query_candidate_rows( - self, where: Any, *, limit: int + self, where: Predicate | None, *, limit: int ) -> list[dict[str, Any]]: rows = await self._query_raw(where, limit=limit, include_vectors=False) - return [self._restore_row(row, include_distance=False) for row in rows] - - def _expr(self, where: Any) -> str: - if where is None: - return "" - milvus_expr = getattr(where, "milvus", None) - if isinstance(milvus_expr, str): - return milvus_expr - if not isinstance(where, str): - return str(where) - return _lancedb_expr_to_milvus(where) - - -def _lancedb_expr_to_milvus(expr: str) -> str: - """Best-effort conversion for internal EverOS LanceDB predicates.""" - converted = expr - converted = re.sub(r"\barray_has\s*\(", "array_contains(", converted) - converted = re.sub(r"\bIS\s+NULL\b", "is null", converted, flags=re.IGNORECASE) - converted = re.sub( - r"\b([A-Za-z_][A-Za-z0-9_]*)\s+IN\s+\(([^)]*)\)", - lambda m: f"{m.group(1)} in [{m.group(2)}]", - converted, - flags=re.IGNORECASE, - ) - converted = re.sub(r"(?=])=(?!=)", "==", converted) - converted = re.sub(r"\btimestamp\b", "timestamp_ms", converted) - converted = re.sub(r"\bcreated_at\b", "created_at_ms", converted) - converted = re.sub(r"\bupdated_at\b", "updated_at_ms", converted) - converted = re.sub( - r"TIMESTAMP\s+'([^']*)'", - lambda m: str(_iso_to_ms(m.group(1))), - converted, - ) - return converted + return [self._restore_row(row) for row in rows] + + def _expr(self, where: Predicate | None) -> str: + if where is not None and not isinstance(where, Predicate): + raise TypeError( + "Milvus repository predicates must use the neutral Predicate AST, " + f"got {type(where).__name__}" + ) + return render_predicate( + where, + datetime_fields=self.index_schema.datetime_fields, + ) def _datetime_storage_field(name: str) -> str: return f"{name}_ms" +def _present_field(name: str) -> str: + return f"{name}{_PRESENT_SUFFIX}" + + def _sparse_field(name: str) -> str: return f"{name}{_SPARSE_SUFFIX}" @@ -646,46 +715,6 @@ def _datetime_to_ms(value: Any) -> int: raise TypeError(f"expected datetime or epoch ms, got {type(value).__name__}") -def _iso_to_ms(value: str) -> int: - aware = ensure_utc(from_iso_format(value)) - assert aware is not None - return to_timestamp_ms(aware) - - -def _is_optional(annotation: Any) -> bool: - args = get_args(annotation) - return type(None) in args - - -def _non_none_args(annotation: Any) -> tuple[Any, ...]: - return tuple(arg for arg in get_args(annotation) if arg is not type(None)) - - -def _is_datetime(annotation: Any) -> bool: - if annotation is dt.datetime: - return True - return any(arg is dt.datetime for arg in _non_none_args(annotation)) - - -def _is_list(annotation: Any) -> bool: - origin = get_origin(annotation) - if origin is list: - return True - return any(get_origin(arg) is list for arg in _non_none_args(annotation)) - - -def _is_float(annotation: Any) -> bool: - if annotation is float: - return True - return any(arg is float for arg in _non_none_args(annotation)) - - -def _is_int(annotation: Any) -> bool: - if annotation is int: - return True - return any(arg is int for arg in _non_none_args(annotation)) - - def _sort_value(value: Any) -> Any: if value is None: fallback = ensure_utc(dt.datetime.min) @@ -695,64 +724,25 @@ def _sort_value(value: Any) -> Any: def _bm25_score_from_distance(distance: Any) -> float: - """Convert the Milvus BM25 search distance into a higher-is-better score.""" - if distance is None: - return 0.0 - return abs(float(distance)) + """Milvus BM25 is higher-is-better; expose a non-negative score.""" + return 0.0 if distance is None else max(0.0, float(distance)) def _cosine_distance_from_milvus(distance: Any) -> float | None: - """Normalize Milvus COSINE results to LanceDB-style distance. - - Milvus server / Zilliz Cloud return cosine similarity through the - ``distance`` field. Milvus Lite 3.0 reports cosine distance instead; keep - this version-gated workaround narrow because the upstream issue is expected - to be fixed after 3.0: - https://github.com/milvus-io/milvus-lite/issues/343 - - Recaller code expects LanceDB's ``1 - similarity`` distance contract. - """ + """Convert Milvus Server / Zilliz similarity to Lance-style distance.""" if distance is None: return None - value = float(distance) - if not _uses_milvus_lite_3_0_cosine_distance_bug(): - value = 1.0 - value - if value < 0.0: - return 0.0 - if value > 1.0: - return 1.0 - return value - - -def _uses_milvus_lite() -> bool: - uri = load_settings().milvus.uri - if not uri: - return True - return "://" not in uri - - -def _uses_milvus_lite_3_0_cosine_distance_bug() -> bool: - if not _uses_milvus_lite(): - return False - return _milvus_lite_version() in {"3.0", "3.0.0"} - - -def _milvus_lite_version() -> str | None: - try: - return version("milvus-lite") - except PackageNotFoundError: - return None + return min(1.0, max(0.0, 1.0 - float(distance))) def _first_result_set(results: Any) -> list[dict[str, Any]]: if not results: return [] - first = results[0] - return list(first or []) + return list(results[0] or []) async def _run(func: Any, /, *args: Any, **kwargs: Any) -> Any: return await asyncio.to_thread(func, *args, **kwargs) -__all__ = ["MilvusRepoBase"] +__all__ = ["MilvusRepoBase", "MilvusValueLimitError"] diff --git a/src/everos/infra/persistence/predicate.py b/src/everos/infra/persistence/predicate.py new file mode 100644 index 000000000..def7ca69f --- /dev/null +++ b/src/everos/infra/persistence/predicate.py @@ -0,0 +1,138 @@ +"""Backend-neutral predicate tree shared by derived-index adapters.""" + +from __future__ import annotations + +import datetime as dt +from dataclasses import dataclass +from typing import Literal + +type Scalar = str | int | float | bool | dt.datetime +type ComparisonOperator = Literal["eq", "ne", "gt", "gte", "lt", "lte"] + + +class Predicate: + """Marker base class for derived-index predicates.""" + + +@dataclass(frozen=True) +class Comparison(Predicate): + field: str + operator: ComparisonOperator + value: Scalar + + +@dataclass(frozen=True) +class In(Predicate): + field: str + values: tuple[Scalar, ...] + + +@dataclass(frozen=True) +class Contains(Predicate): + field: str + value: str + + +@dataclass(frozen=True) +class IsNull(Predicate): + field: str + + +@dataclass(frozen=True) +class All(Predicate): + children: tuple[Predicate, ...] + + +@dataclass(frozen=True) +class AnyOf(Predicate): + children: tuple[Predicate, ...] + + +def compare(field: str, operator: ComparisonOperator, value: Scalar) -> Predicate: + return Comparison(field, operator, value) + + +def eq(field: str, value: Scalar) -> Predicate: + return compare(field, "eq", value) + + +def ne(field: str, value: Scalar) -> Predicate: + return compare(field, "ne", value) + + +def gt(field: str, value: Scalar) -> Predicate: + return compare(field, "gt", value) + + +def gte(field: str, value: Scalar) -> Predicate: + return compare(field, "gte", value) + + +def lt(field: str, value: Scalar) -> Predicate: + return compare(field, "lt", value) + + +def lte(field: str, value: Scalar) -> Predicate: + return compare(field, "lte", value) + + +def one_of(field: str, values: list[Scalar] | tuple[Scalar, ...]) -> Predicate: + if not values: + raise ValueError("one_of requires at least one value") + return In(field, tuple(values)) + + +def contains(field: str, value: str) -> Predicate: + return Contains(field, value) + + +def is_null(field: str) -> Predicate: + return IsNull(field) + + +def all_of(*predicates: Predicate | None) -> Predicate: + children: list[Predicate] = [] + for predicate in predicates: + if predicate is None: + continue + if isinstance(predicate, All): + children.extend(predicate.children) + else: + children.append(predicate) + return All(tuple(children)) + + +def any_of(*predicates: Predicate | None) -> Predicate: + children: list[Predicate] = [] + for predicate in predicates: + if predicate is None: + continue + if isinstance(predicate, AnyOf): + children.extend(predicate.children) + else: + children.append(predicate) + return AnyOf(tuple(children)) + + +__all__ = [ + "All", + "AnyOf", + "Comparison", + "Contains", + "In", + "IsNull", + "Predicate", + "Scalar", + "all_of", + "any_of", + "compare", + "contains", + "eq", + "gt", + "gte", + "is_null", + "lt", + "lte", + "ne", + "one_of", +] diff --git a/src/everos/memory/cascade/_backfill.py b/src/everos/memory/cascade/_backfill.py index 767ee6d24..d78bd27ba 100644 --- a/src/everos/memory/cascade/_backfill.py +++ b/src/everos/memory/cascade/_backfill.py @@ -46,8 +46,7 @@ from everos.infra.ome.config import OMEConfig from everos.infra.ome.engine import OfflineEngine from everos.infra.ome.exceptions import EngineLockHeldError -from everos.infra.persistence.lancedb import ( - BUSINESS_SCHEMAS_WITH_VECTOR, +from everos.infra.persistence.index import ( AgentCase, AgentSkill, AtomicFact, @@ -58,10 +57,11 @@ agent_skill_repo, atomic_fact_repo, episode_repo, + eq, foresight_repo, - get_table, knowledge_topic_repo, ) +from everos.infra.persistence.lancedb import BUSINESS_SCHEMAS_WITH_VECTOR, get_table from everos.infra.persistence.markdown import AgentSkillFrontmatter from everos.infra.persistence.sqlite import cluster_repo, get_engine from everos.memory.cascade.worker import ( @@ -697,7 +697,7 @@ async def _backfill_table( and row.id not in subject_vectors ) try: - await backlog.spec.repo.update(updates, where=f"id = '{_q(row.id)}'") + await backlog.spec.repo.update(updates, where=eq("id", row.id)) except Exception: result.rows_failed += 1 logger.warning( diff --git a/src/everos/memory/cascade/handlers/_daily_log_base.py b/src/everos/memory/cascade/handlers/_daily_log_base.py index 8791a418f..52f67b781 100644 --- a/src/everos/memory/cascade/handlers/_daily_log_base.py +++ b/src/everos/memory/cascade/handlers/_daily_log_base.py @@ -30,6 +30,7 @@ from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, StructuredEntry +from everos.infra.persistence.index import all_of, eq, one_of from ..types import HandlerOutcome from ._common import content_sha256 as compute_content_sha256 @@ -108,7 +109,7 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: ] existing = await self.lance_repo.find_where( - f"md_path = '{_q(md_path)}'", + eq("md_path", md_path), limit=10_000, ) owner_id, owner_type = resolve_owner(parsed.frontmatter, md_path) @@ -198,9 +199,11 @@ async def _apply_lance_changes( if to_upsert: await self.lance_repo.upsert(to_upsert) if to_delete_ids: - in_list = ", ".join(f"'{eid}'" for eid in to_delete_ids) await self.lance_repo.delete( - f"md_path = '{_q(md_path)}' AND entry_id IN ({in_list})" + all_of( + eq("md_path", md_path), + one_of("entry_id", to_delete_ids), + ) ) async def handle_deleted(self, md_path: str) -> HandlerOutcome: @@ -252,11 +255,11 @@ async def _mark_deprecated( entry may have been deleted or not yet indexed. """ app_id, project_id = scope - predicate = ( - f"owner_id = '{_q(owner_id)}' " - f"AND entry_id = '{_q(entry_id)}' " - f"AND app_id = '{_q(app_id)}' " - f"AND project_id = '{_q(project_id)}'" + predicate = all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), ) try: await self.lance_repo.update( @@ -290,8 +293,3 @@ async def _build_row( ``"default"`` so white-box callers exercising only the field mapping can omit them. """ - - -def _q(text: str) -> str: - """Defensive SQL-quote escape (mirrors lancedb chassis convention).""" - return text.replace("'", "''") diff --git a/src/everos/memory/cascade/handlers/agent_skill.py b/src/everos/memory/cascade/handlers/agent_skill.py index 25315d35b..1e492eec1 100644 --- a/src/everos/memory/cascade/handlers/agent_skill.py +++ b/src/everos/memory/cascade/handlers/agent_skill.py @@ -42,7 +42,13 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader -from everos.infra.persistence.index import AgentSkill, agent_skill_repo +from everos.infra.persistence.index import ( + AgentSkill, + agent_skill_repo, + all_of, + eq, + ne, +) from everos.infra.persistence.markdown import AgentSkillFrontmatter from ..types import HandlerOutcome @@ -136,7 +142,7 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: # cascade's contract — skip the lookup. deleted = 0 if prior is None: - orphan_clause = f"md_path = '{_q(md_path)}' AND id != '{_q(skill_id)}'" + orphan_clause = all_of(eq("md_path", md_path), ne("id", skill_id)) orphans = await agent_skill_repo.find_where(orphan_clause, limit=1000) deleted = len(orphans) if deleted: @@ -224,8 +230,3 @@ def _join_body_and_references(body: str, references: str) -> str: if not body: return references return f"{body}\n\n{references}" - - -def _q(value: str) -> str: - """Defensive SQL-quote escape (mirrors lancedb chassis convention).""" - return value.replace("'", "''") diff --git a/src/everos/memory/get/filters_adapter.py b/src/everos/memory/get/filters_adapter.py index 46b1e6305..cd23dcc67 100644 --- a/src/everos/memory/get/filters_adapter.py +++ b/src/everos/memory/get/filters_adapter.py @@ -15,8 +15,9 @@ from __future__ import annotations +from everos.infra.persistence.index.predicate import Predicate from everos.memory.search import FilterNode, compile_filters -from everos.memory.search.filters import BackendFilters, compile_filters_for_backends +from everos.memory.search.filters import compile_filters_for_backends def compile_filters_for_get( @@ -26,7 +27,7 @@ def compile_filters_for_get( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> str: +) -> Predicate: """Compile ``/get`` filters via the shared ``compile_filters`` path. Kept as a named wrapper so ``memory.get`` consumers depend on a @@ -48,7 +49,7 @@ def compile_filters_for_get_backends( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> BackendFilters: +) -> Predicate: """Compile ``/get`` filters for every derived index backend.""" return compile_filters_for_backends( filters, diff --git a/src/everos/memory/reflection/orchestrator.py b/src/everos/memory/reflection/orchestrator.py index 547adc643..9c9e8c54d 100644 --- a/src/everos/memory/reflection/orchestrator.py +++ b/src/everos/memory/reflection/orchestrator.py @@ -32,6 +32,7 @@ from everos.core.observability.tracing import memory_span from everos.core.persistence import MemoryRoot from everos.infra.ome.context import StrategyContext +from everos.infra.persistence.index import all_of, eq, is_null from everos.memory._partition_locks import get_partition_lock from everos.memory.events import EpisodeExtracted @@ -41,21 +42,6 @@ _WAIT_TIMEOUT_SECONDS = 120.0 -def _escape_sql(value: str) -> str: - """Escape single quotes for LanceDB SQL-like ``where`` predicates. - - LanceDB has no parameterised query API; doubling the quote - (``'`` -> ``''``) is the SQL-standard escape. - - Args: - value: Raw string to escape. - - Returns: - Escaped string safe for interpolation into a WHERE clause. - """ - return value.replace("'", "''") - - class ReflectionOrchestrator: """Run one Reflection cycle for a single owner scope. @@ -531,12 +517,13 @@ async def _detect_orphans( app_id: Application scope. project_id: Project scope. """ - where = ( - f"parent_type = 'cluster' AND parent_id = '{_escape_sql(cluster_id)}' " - f"AND deprecated_by IS NULL " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND app_id = '{_escape_sql(app_id)}' " - f"AND project_id = '{_escape_sql(project_id)}'" + where = all_of( + eq("parent_type", "cluster"), + eq("parent_id", cluster_id), + is_null("deprecated_by"), + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), ) orphans = await self._episode_store.find_where(where, limit=10) if orphans: @@ -814,14 +801,14 @@ async def _apply_deprecation_writes( to_deprecate=to_deprecate, merged_entry_id=merged_entry_id, ) - deprecated_ep_count = await self._deprecate_lance_episodes( + deprecated_ep_count = await self._deprecate_index_episodes( entry_ids=to_deprecate, owner_id=owner_id, app_id=app_id, project_id=project_id, merged_entry_id=merged_entry_id, ) - deprecated_fact_count = await self._deprecate_lance_facts( + deprecated_fact_count = await self._deprecate_index_facts( parent_ids=to_deprecate, owner_id=owner_id, merged_entry_id=merged_entry_id, @@ -848,7 +835,7 @@ async def _resolve_deprecation_targets( original_ids = {mid for mid, _ in original_members} return original_ids & current_ids - async def _deprecate_lance_episodes( + async def _deprecate_index_episodes( self, *, entry_ids: set[str], @@ -857,19 +844,19 @@ async def _deprecate_lance_episodes( project_id: str, merged_entry_id: str, ) -> int: - """Mark deprecated episodes in LanceDB by entry_id. + """Mark deprecated episodes in the active derived index by entry_id. Returns: - Number of LanceDB update calls issued. + Number of derived-index update calls issued. """ coros: list[Any] = [ self._episode_store.update( {"deprecated_by": merged_entry_id}, - where=( - f"entry_id = '{_escape_sql(eid)}' " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND app_id = '{_escape_sql(app_id)}' " - f"AND project_id = '{_escape_sql(project_id)}'" + where=all_of( + eq("entry_id", eid), + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), ), ) for eid in entry_ids @@ -878,14 +865,14 @@ async def _deprecate_lance_episodes( await asyncio.gather(*coros) return len(coros) - async def _deprecate_lance_facts( + async def _deprecate_index_facts( self, *, parent_ids: set[str], owner_id: str, merged_entry_id: str, ) -> int: - """Mark deprecated atomic facts in LanceDB. + """Mark deprecated atomic facts in the active derived index. Args: parent_ids: Parent IDs (memcell or episode) whose facts to deprecate. @@ -893,7 +880,7 @@ async def _deprecate_lance_facts( merged_entry_id: Entry ID of the replacement merged episode. Returns: - Total number of LanceDB update calls issued. + Total number of derived-index update calls issued. """ if not parent_ids: return 0 @@ -901,10 +888,10 @@ async def _deprecate_lance_facts( coros = [ self._atomic_fact_store.update( {"deprecated_by": merged_entry_id}, - where=( - f"parent_id = '{_escape_sql(pid)}' " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND deprecated_by IS NULL" + where=all_of( + eq("parent_id", pid), + eq("owner_id", owner_id), + is_null("deprecated_by"), ), ) for pid in parent_ids diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 0f417afc0..74ceeb2d5 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -1,86 +1,37 @@ -"""Filters DSL → derived-index filter compiler. - -The Filters DSL is intentionally permissive at the JSON layer (so callers -can pass whatever they like and get a clean 400 if it is not supported) -and rigid at compile time. Field names are validated against a small -allow-list; operators against a closed enum; string literals are -single-quote-escaped. Timestamps are accepted as epoch milliseconds and -rendered as DataFusion ``TIMESTAMP ''`` literals. - -``owner_id`` and ``owner_type`` are the hard partition keys; they are -not part of the DSL at all. :func:`compile_filters` injects them at the -top of the compiled string from :class:`SearchRequest` and rejects any -attempt to override them inside ``filters``. - -Public surface --------------- - -The compiler exposes three primitives so adjacent subpackages -(notably ``memory.get``) can build narrower DSLs without forking the -field allow-list: - -* :data:`ALLOWED_FIELDS` — mapping ``field_name → _FieldSpec`` (column + - kind). Iterate / membership-test only; do not mutate. -* :data:`RESERVED_FIELDS` — names rejected inside any ``filters`` block. -* :func:`compile_predicate` — render one ``{field: value}`` clause to - a backend-specific predicate. Operator-map and equality-shorthand are - both handled. - -The high-level :func:`compile_filters` remains the entry point for -``/search`` (combinator-aware). +"""Validate the public filter DSL and build a backend-neutral predicate. + +Storage-specific syntax is intentionally absent from this module. LanceDB and +Milvus render the resulting predicate in their own persistence packages. """ from __future__ import annotations -import dataclasses -import datetime as _dt -import json -from typing import Any, Final - -from everos.component.utils.datetime import ( - ensure_utc, - from_iso_format, - from_timestamp, - to_iso_format, - to_timestamp_ms, -) +import datetime as dt +from dataclasses import dataclass +from typing import Any, Final, Literal + +from everos.component.utils.datetime import ensure_utc, from_iso_format, from_timestamp from everos.core.errors import FilterError as FilterError +from everos.infra.persistence.index.predicate import ( + Predicate, + all_of, + any_of, + compare, + contains, + eq, + is_null, + one_of, +) from .dto import FilterNode -# ── Allow-lists ────────────────────────────────────────────────────────── - -_OP_MAP: Final[dict[str, str]] = { - "eq": "=", - "ne": "!=", - "gt": ">", - "gte": ">=", - "lt": "<", - "lte": "<=", - "in": "IN", -} -_MILVUS_OP_MAP: Final[dict[str, str]] = { - "eq": "==", - "ne": "!=", - "gt": ">", - "gte": ">=", - "lt": "<", - "lte": "<=", - "in": "in", -} - -# Field kinds: ``str`` rendered as ``''``; ``ts`` rendered as -# ``TIMESTAMP ''`` (DataFusion timestamp literal); ``array_str`` -# uses DataFusion's ``array_has`` on a list column. -_FieldKind = str # one of: "str" | "ts" | "array_str" +_FieldKind = Literal["str", "ts", "array_str"] +@dataclass(frozen=True) class _FieldSpec: - __slots__ = ("column", "kind") - - def __init__(self, column: str, kind: _FieldKind) -> None: - self.column = column - self.kind = kind + column: str + kind: _FieldKind ALLOWED_FIELDS: Final[dict[str, _FieldSpec]] = { @@ -91,29 +42,11 @@ def __init__(self, column: str, kind: _FieldKind) -> None: "sender_id": _FieldSpec("sender_ids", "array_str"), } -# Fields the caller is explicitly **not** allowed to place inside -# ``filters``; they live at the top of :class:`SearchRequest` and are -# injected by :func:`compile_filters`. Rejecting them here turns a -# silent override into a 400. RESERVED_FIELDS: Final[frozenset[str]] = frozenset( {"owner_id", "owner_type", "app_id", "project_id"} ) -@dataclasses.dataclass(frozen=True) -class BackendFilters: - """Compiled filters for every supported derived index backend.""" - - lancedb: str - milvus: str - - def __str__(self) -> str: - return self.lancedb - - -# ── Public API ─────────────────────────────────────────────────────────── - - def compile_filters( node: FilterNode | None, *, @@ -121,24 +54,21 @@ def compile_filters( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> str: - """Compile a request's filters into a single LanceDB ``where`` string. - - The base clause always pins the hard partition keys (``owner_id`` / - ``owner_type`` and the ``app_id`` / ``project_id`` scope segments) to - the request's top-level values; anything in ``node`` is appended with - an ``AND``. Pinning app/project here is what isolates one space's rows - from another — omitting it would let a query bleed across spaces. Both - ``/search`` and ``/get`` share this compile path. - """ - return _compile_filters_backend( - node, - owner_id=owner_id, - owner_type=owner_type, - app_id=app_id, - project_id=project_id, - backend="lancedb", - ) +) -> Predicate: + """Validate request filters and return one normalized predicate tree.""" + base: list[Predicate] = [ + eq("owner_id", owner_id), + eq("owner_type", owner_type), + eq("app_id", app_id), + eq("project_id", project_id), + ] + if owner_type == "user": + base.append(is_null("deprecated_by")) + if node is not None: + compiled = _compile_node(node.model_dump(exclude_none=True)) + if compiled is not None: + base.append(compiled) + return all_of(*base) def compile_filters_for_backends( @@ -148,74 +78,29 @@ def compile_filters_for_backends( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> BackendFilters: - """Compile request filters for every supported derived index backend.""" - return BackendFilters( - lancedb=_compile_filters_backend( - node, - owner_id=owner_id, - owner_type=owner_type, - app_id=app_id, - project_id=project_id, - backend="lancedb", - ), - milvus=_compile_filters_backend( - node, - owner_id=owner_id, - owner_type=owner_type, - app_id=app_id, - project_id=project_id, - backend="milvus", - ), - ) - - -def _compile_filters_backend( - node: FilterNode | None, - *, - owner_id: str, - owner_type: str, - app_id: str, - project_id: str, - backend: str, -) -> str: - base = _base_clauses( +) -> Predicate: + """Compatibility name; one predicate is rendered by the active backend.""" + return compile_filters( + node, owner_id=owner_id, owner_type=owner_type, app_id=app_id, project_id=project_id, - backend=backend, ) - # Only episode / atomic_fact tables carry the ``deprecated_by`` column - # (Reflection V1 marks superseded entries). Agent tables don't have it. - if owner_type == "user": - base.append( - "deprecated_by IS NULL" if backend == "lancedb" else "deprecated_by is null" - ) - if node is None: - return " AND ".join(base) - compiled = _compile_node(node.model_dump(exclude_none=True), backend=backend) - if not compiled: - return " AND ".join(base) - return " AND ".join([*base, compiled]) - -# ── Internals ──────────────────────────────────────────────────────────── - -def _compile_node(raw: dict[str, Any], *, backend: str = "lancedb") -> str: - """Walk one DSL node; return the matching SQL fragment (no leading parens). - - Empty nodes yield ``""`` so :func:`compile_filters` can skip the - trailing ``AND``. - """ - raw = dict(raw) # never mutate the caller's dict - parts: list[str] = [] +def _compile_node(raw: dict[str, Any]) -> Predicate | None: + raw = dict(raw) + parts: list[Predicate] = [] if (and_list := raw.pop("AND", None)) is not None: - parts.append(_compile_combinator(and_list, "AND", backend=backend)) + combinator = _compile_combinator(and_list, "AND") + if combinator is not None: + parts.append(combinator) if (or_list := raw.pop("OR", None)) is not None: - parts.append(_compile_combinator(or_list, "OR", backend=backend)) + combinator = _compile_combinator(or_list, "OR") + if combinator is not None: + parts.append(combinator) for field, value in raw.items(): if field in RESERVED_FIELDS: @@ -224,195 +109,101 @@ def _compile_node(raw: dict[str, Any], *, backend: str = "lancedb") -> str: ) if field not in ALLOWED_FIELDS: raise FilterError(f"unsupported filter field: {field!r}") - parts.append(compile_predicate(field, value, backend=backend)) + parts.append(compile_predicate(field, value)) - # Drop empty fragments coming from empty AND/OR arrays. - parts = [p for p in parts if p] - if not parts: - return "" - if len(parts) == 1: - return parts[0] - return " AND ".join(parts) + return all_of(*parts) if parts else None def _compile_combinator( - children: list[dict[str, Any]], op: str, *, backend: str = "lancedb" -) -> str: - """Render an ``AND`` / ``OR`` array of child nodes.""" + children: list[dict[str, Any]], op: Literal["AND", "OR"] +) -> Predicate | None: if not isinstance(children, list): raise FilterError(f"{op} expects an array of nodes") - fragments: list[str] = [] + fragments: list[Predicate] = [] for child in children: if not isinstance(child, dict): raise FilterError(f"{op} children must be objects") - compiled = _compile_node(child, backend=backend) - if compiled: - fragments.append(f"({compiled})") + compiled = _compile_node(child) + if compiled is not None: + fragments.append(compiled) if not fragments: - return "" - if len(fragments) == 1: - # No need for the surrounding combinator when only one effective child. - return fragments[0] - glue = f" {op} " - return "(" + glue.join(fragments) + ")" - - -def compile_predicate(field: str, value: Any, *, backend: str = "lancedb") -> str: - """Render one ``"": `` clause to SQL. - - Public primitive — :mod:`memory.get` builds a flat (no AND/OR) - DSL on top of it. Callers must pre-validate ``field`` against - :data:`ALLOWED_FIELDS` and :data:`RESERVED_FIELDS`; this function - will ``KeyError`` on unknown fields. + return None + return all_of(*fragments) if op == "AND" else any_of(*fragments) - ``value`` is either a scalar (equality shorthand) or an - ``{"": }`` map. Mixing multiple operators in one - dict is allowed and folds with ``AND``:: - "timestamp": {"gte": 1, "lt": 2} - → (timestamp >= TIMESTAMP '...' AND timestamp < TIMESTAMP '...') - """ +def compile_predicate(field: str, value: Any) -> Predicate: + """Validate and normalize one field clause into the neutral AST.""" spec = ALLOWED_FIELDS[field] if isinstance(value, dict): if not value: raise FilterError(f"empty operator map for field {field!r}") - clauses = [ - _compile_op_clause(spec, field, op, op_val, backend=backend) - for op, op_val in value.items() - ] - if len(clauses) == 1: - return clauses[0] - return "(" + " AND ".join(clauses) + ")" - # Equality shorthand. - return _compile_op_clause(spec, field, "eq", value, backend=backend) - - -def _compile_op_clause( - spec: _FieldSpec, field: str, op: str, value: Any, *, backend: str = "lancedb" -) -> str: - """Render a single `` `` clause.""" - if op not in _OP_MAP: + return all_of( + *( + _compile_op_clause(spec, field, op, op_value) + for op, op_value in value.items() + ) + ) + return _compile_op_clause(spec, field, "eq", value) + + +def _compile_op_clause(spec: _FieldSpec, field: str, op: str, value: Any) -> Predicate: + if op not in {"eq", "ne", "gt", "gte", "lt", "lte", "in"}: raise FilterError(f"unsupported operator {op!r} on field {field!r}") - op_map = _MILVUS_OP_MAP if backend == "milvus" else _OP_MAP - sql_op = op_map[op] - column = _column_for_backend(spec, backend) if spec.kind == "array_str": - # Only equality / membership make sense on a list column. - fn = "array_contains" if backend == "milvus" else "array_has" if op == "eq": - literal = _render_str_literal(_require_str(value, field), backend) - return f"{fn}({column}, {literal})" + return contains(spec.column, _require_str(value, field)) if op == "in": - items = _require_list(value, field) - literals = [ - _render_str_literal(_require_str(v, field), backend) for v in items - ] - inner = " OR ".join(f"{fn}({column}, {lit})" for lit in literals) - return f"({inner})" + values = _require_list(value, field) + return any_of( + *(contains(spec.column, _require_str(item, field)) for item in values) + ) raise FilterError(f"operator {op!r} is not supported on array field {field!r}") if op == "in": - items = _require_list(value, field) - literals = [ - _render_literal(v, spec.kind, field, backend=backend) for v in items - ] - if backend == "milvus": - return f"{column} in [{', '.join(literals)}]" - return f"{column} IN ({', '.join(literals)})" - - literal = _render_literal(value, spec.kind, field, backend=backend) - return f"{column} {sql_op} {literal}" - - -# ── Literal rendering ──────────────────────────────────────────────────── + values = _require_list(value, field) + return one_of( + spec.column, + [_normalize_literal(item, spec.kind, field) for item in values], + ) + return compare( + spec.column, + op, # type: ignore[arg-type] + _normalize_literal(value, spec.kind, field), + ) -def _render_literal( - value: Any, kind: _FieldKind, field: str, *, backend: str = "lancedb" -) -> str: +def _normalize_literal(value: Any, kind: _FieldKind, field: str): # type: ignore[no-untyped-def] if kind == "str": - return _render_str_literal(_require_str(value, field), backend) + return _require_str(value, field) if kind == "ts": - if backend == "milvus": - return str(_render_ts_ms(value, field)) - return f"TIMESTAMP '{_render_ts(value, field)}'" + return _normalize_timestamp(value, field) raise FilterError(f"unsupported field kind {kind!r} for field {field!r}") -def _render_ts(value: Any, field: str) -> str: - """Accept epoch ms (int / float) or an ISO 8601 string; emit ISO.""" - if isinstance(value, bool): # bools subclass int — reject early - raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") - if isinstance(value, (int, float)): - return to_iso_format(from_timestamp(int(value))) - if isinstance(value, str): - # Trust the caller-supplied ISO string but escape quotes defensively. - if "'" in value: - raise FilterError(f"timestamp string for {field!r} contains a quote") - return value - if isinstance(value, _dt.datetime): - return to_iso_format(value) - raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") - - -def _render_ts_ms(value: Any, field: str) -> int: +def _normalize_timestamp(value: Any, field: str) -> dt.datetime: if isinstance(value, bool): raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") - if isinstance(value, (int, float)): - return int(value) - if isinstance(value, str): - if "'" in value: - raise FilterError(f"timestamp string for {field!r} contains a quote") - try: - aware = ensure_utc(from_iso_format(value)) - except (TypeError, ValueError) as exc: - raise FilterError( - f"timestamp value for {field!r} must be ms or ISO string" - ) from exc - assert aware is not None - return to_timestamp_ms(aware) - if isinstance(value, _dt.datetime): - aware = ensure_utc(value) - assert aware is not None - return to_timestamp_ms(aware) + try: + if isinstance(value, (int, float)): + return from_timestamp(int(value)) + if isinstance(value, str): + if "'" in value: + raise FilterError(f"timestamp string for {field!r} contains a quote") + parsed = ensure_utc(from_iso_format(value)) + assert parsed is not None + return parsed + if isinstance(value, dt.datetime): + parsed = ensure_utc(value) + assert parsed is not None + return parsed + except (TypeError, ValueError) as exc: + raise FilterError( + f"timestamp value for {field!r} must be ms or ISO string" + ) from exc raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") -def _render_str_literal(value: str, backend: str) -> str: - if backend == "milvus": - return json.dumps(value, ensure_ascii=False) - return f"'{_escape_str(value)}'" - - -def _base_clauses( - *, - owner_id: str, - owner_type: str, - app_id: str, - project_id: str, - backend: str, -) -> list[str]: - eq = "==" if backend == "milvus" else "=" - return [ - f"owner_id {eq} {_render_str_literal(owner_id, backend)}", - f"owner_type {eq} {_render_str_literal(owner_type, backend)}", - f"app_id {eq} {_render_str_literal(app_id, backend)}", - f"project_id {eq} {_render_str_literal(project_id, backend)}", - ] - - -def _column_for_backend(spec: _FieldSpec, backend: str) -> str: - if backend == "milvus" and spec.kind == "ts": - return f"{spec.column}_ms" - return spec.column - - -def _escape_str(value: str) -> str: - """Double single quotes — SQL-standard escape for a single-quoted literal.""" - return value.replace("'", "''") - - def _require_str(value: Any, field: str) -> str: if not isinstance(value, str): raise FilterError(f"value for {field!r} must be a string") @@ -423,3 +214,12 @@ def _require_list(value: Any, field: str) -> list[Any]: if not isinstance(value, list) or not value: raise FilterError(f"value for {field!r} with 'in' must be a non-empty list") return value + + +__all__ = [ + "ALLOWED_FIELDS", + "RESERVED_FIELDS", + "compile_filters", + "compile_filters_for_backends", + "compile_predicate", +] diff --git a/src/everos/memory/search/recall/agent_skill.py b/src/everos/memory/search/recall/agent_skill.py index 0c1831c79..324084829 100644 --- a/src/everos/memory/search/recall/agent_skill.py +++ b/src/everos/memory/search/recall/agent_skill.py @@ -14,7 +14,14 @@ from everalgo.types import Candidate -from everos.infra.persistence.index import AgentSkill, agent_skill_repo +from everos.infra.persistence.index import ( + AgentSkill, + Predicate, + agent_skill_repo, + all_of, + any_of, + contains, +) from .base import ( RecallerDeps, @@ -23,10 +30,6 @@ ) -def _q(value: str) -> str: - return value.replace("'", "''") - - class AgentSkillRecaller: """BM25 + vector recall over the LanceDB ``agent_skill`` table.""" @@ -38,7 +41,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Dual-column BM25 recall via OR-mode BooleanQuery per column. @@ -61,7 +64,7 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: if not vector: return [] @@ -76,7 +79,7 @@ async def dense_recall( ] async def fetch_by_case_ids( - self, case_ids: Sequence[str], where: str, *, limit: int + self, case_ids: Sequence[str], where: Predicate, *, limit: int ) -> list[Candidate]: """Skills whose ``source_case_ids`` intersect ``case_ids``. Filter is ``array_has`` OR-ed per id (same as @@ -88,7 +91,9 @@ async def fetch_by_case_ids( """ if not case_ids: return [] - clause = " OR ".join(f"array_has(source_case_ids, '{_q(c)}')" for c in case_ids) - full_where = f"({where}) AND ({clause})" + full_where = all_of( + where, + any_of(*(contains("source_case_ids", case_id) for case_id in case_ids)), + ) rows = await agent_skill_repo.search(where=full_where, limit=limit) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index f78b3f50a..885457e45 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -21,7 +21,13 @@ from everalgo.types import Candidate, FactCandidate -from everos.infra.persistence.index import AtomicFact, atomic_fact_repo +from everos.infra.persistence.index import ( + AtomicFact, + Predicate, + all_of, + atomic_fact_repo, + one_of, +) from .base import ( RecallerDeps, @@ -46,7 +52,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery (see EpisodeRecaller docstring).""" terms = [term for term in self._deps.tokenizer.tokenize(query) if term] @@ -64,7 +70,7 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Cosine ANN recall over the atomic_fact table. @@ -91,7 +97,7 @@ async def dense_recall( async def facts_for_episodes( self, ep_to_parents: Mapping[str, Sequence[str]], - where: str, + where: Predicate, *, per_episode: int, query_vector: Sequence[float] | None = None, @@ -152,15 +158,13 @@ async def facts_for_episodes( async def _query_facts_for_parents( self, parent_to_eps: dict[str, list[str]], - where: str, + where: Predicate, *, per_episode: int, query_vector: Sequence[float] | None, ) -> list[dict[str, Any]]: """Construct and execute the LanceDB query for parent_id IN (...).""" - quoted = ", ".join(f"'{_q(pid)}'" for pid in parent_to_eps) - clause = f"parent_id IN ({quoted})" - full_where = f"({where}) AND ({clause})" + full_where = all_of(where, one_of("parent_id", list(parent_to_eps))) # Milvus server / Zilliz Cloud reject search topK values above 1024. # The fact expansion only needs a bounded candidate pool for top-N # competition, so keep the same cap for every derived-index backend. @@ -182,7 +186,3 @@ def _build_parent_to_episode_map( if pid: parent_to_eps[pid].append(ep_id) return parent_to_eps - - -def _q(value: str) -> str: - return value.replace("'", "''") diff --git a/src/everos/memory/search/recall/episode.py b/src/everos/memory/search/recall/episode.py index 1f2c13821..f24e8ef8a 100644 --- a/src/everos/memory/search/recall/episode.py +++ b/src/everos/memory/search/recall/episode.py @@ -7,7 +7,14 @@ from everalgo.types import Candidate -from everos.infra.persistence.index import Episode, active_backend, episode_repo +from everos.core.observability.logging import get_logger +from everos.infra.persistence.index import ( + Episode, + Predicate, + all_of, + episode_repo, + one_of, +) from .base import ( RecallerDeps, @@ -29,8 +36,8 @@ def _inject_parent_id(candidates: list[Candidate]) -> list[Candidate]: ] -def _q(value: str) -> str: - return value.replace("'", "''") +logger = get_logger(__name__) +_FETCH_ALL_OWNER_LIMIT = 10_000 class EpisodeRecaller: @@ -44,7 +51,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery. @@ -69,7 +76,7 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: if not vector: return [] @@ -84,19 +91,19 @@ async def dense_recall( ] async def sparse_recall_as_child( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Sparse recall returning episodes as MaxSim child candidates.""" return _inject_parent_id(await self.sparse_recall(query, where, limit=limit)) async def dense_recall_as_child( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Dense recall (body vector ANN) returning as MaxSim children.""" return _inject_parent_id(await self.dense_recall(vector, where, limit=limit)) async def dense_recall_subject( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """ANN over the ``subject_vector`` column. @@ -105,19 +112,11 @@ async def dense_recall_subject( """ if not vector: return [] - if active_backend() == "milvus": - return await self.dense_recall(vector, where, limit=limit) - from everos.infra.persistence.lancedb import get_table - - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query() - .nearest_to(list(vector)) - .column("subject_vector") - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() + rows = await episode_repo.dense_search( + vector, + where, + limit=limit, + vector_field="subject_vector", ) return [ row_to_candidate( @@ -129,23 +128,28 @@ async def dense_recall_subject( ] async def dense_recall_subject_as_child( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Subject-vector ANN returning as MaxSim children.""" candidates = await self.dense_recall_subject(vector, where, limit=limit) return _inject_parent_id(candidates) - async def fetch_all_for_owner(self, where: str) -> list[Candidate]: + async def fetch_all_for_owner(self, where: Predicate) -> list[Candidate]: """Flat scan — all episodes for this owner, keyed by entry_id. Cluster membership matching in ``acluster_retrieve`` compares ``Candidate.id`` against ``Cluster.members``. Both are now episode entry_ids regardless of parent_type. - No ``limit`` — the full owner partition is required for cluster - membership matching. + The scan is capped at :data:`_FETCH_ALL_OWNER_LIMIT` to bound memory. + Hitting the cap is logged because cluster matching may be incomplete. """ - rows = await episode_repo.search(where=where, limit=10_000) + rows = await episode_repo.search(where=where, limit=_FETCH_ALL_OWNER_LIMIT) + if len(rows) == _FETCH_ALL_OWNER_LIMIT: + logger.warning( + "episode_owner_scan_truncated", + limit=_FETCH_ALL_OWNER_LIMIT, + ) result: list[Candidate] = [] for r in rows: entry_id = r.get("entry_id") @@ -163,12 +167,11 @@ async def fetch_all_for_owner(self, where: str) -> list[Candidate]: return result async def fetch_by_entry_ids( - self, entry_ids: list[str], where: str + self, entry_ids: list[str], where: Predicate ) -> list[Candidate]: """Fetch episodes by entry_id (for facts whose parent_id is an entry_id).""" if not entry_ids: return [] - quoted = ", ".join(f"'{_q(eid)}'" for eid in entry_ids) - full_where = f"({where}) AND (entry_id IN ({quoted}))" + full_where = all_of(where, one_of("entry_id", entry_ids)) rows = await episode_repo.search(where=full_where, limit=len(entry_ids)) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/service/knowledge.py b/src/everos/service/knowledge.py index 2040ace17..6ee1a4d23 100644 --- a/src/everos/service/knowledge.py +++ b/src/everos/service/knowledge.py @@ -1148,15 +1148,15 @@ def _validate_scope_id(value: str, name: str) -> None: raise ValueError(f"{name} contains invalid characters: {value!r}") -def compile_knowledge_where(app_id: str, project_id: str) -> str: - """Build a LanceDB ``where`` clause scoped to the given tenant. +def compile_knowledge_where(app_id: str, project_id: str): # type: ignore[no-untyped-def] + """Build a backend-neutral predicate scoped to the given tenant. Args: app_id: Tenant application identifier. project_id: Tenant project identifier. Returns: - SQL-style predicate string safe for use in LanceDB ``where`` parameter. + Predicate safe for either derived-index backend. Raises: ValueError: If either id contains invalid characters. @@ -1164,10 +1164,9 @@ def compile_knowledge_where(app_id: str, project_id: str) -> str: _validate_scope_id(app_id, "app_id") _validate_scope_id(project_id, "project_id") - def _esc(v: str) -> str: - return v.replace("'", "''") + from everos.infra.persistence.index import all_of, eq - return f"app_id = '{_esc(app_id)}' AND project_id = '{_esc(project_id)}'" + return all_of(eq("app_id", app_id), eq("project_id", project_id)) # ── Recall helpers ─────────────────────────────────────────────────────────── diff --git a/tests/integration/test_cascade_cli_integration.py b/tests/integration/test_cascade_cli_integration.py index 22511ddf1..c89ad7f84 100644 --- a/tests/integration/test_cascade_cli_integration.py +++ b/tests/integration/test_cascade_cli_integration.py @@ -124,7 +124,7 @@ def test_sync_with_path_outside_root_errors( # between the two tokens. output = result.stdout + (result.stderr or "") plain_output = _strip_ansi(output) - assert re.search(r"not under[^\w]+memory root", plain_output), output + assert re.search(r"not[^\w]+under[^\w]+memory[^\w]+root", plain_output), output def test_sync_with_unmatched_path( diff --git a/tests/integration/test_milvus_remote.py b/tests/integration/test_milvus_remote.py new file mode 100644 index 000000000..99734acd8 --- /dev/null +++ b/tests/integration/test_milvus_remote.py @@ -0,0 +1,217 @@ +"""Cross-backend behavior against Milvus Server or Zilliz Cloud. + +Set ``EVEROS_TEST_MILVUS_URI`` and, when required, +``EVEROS_TEST_MILVUS_TOKEN``. The same test is used for self-hosted and cloud +endpoints and creates uniquely prefixed, disposable collections. +""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import os +import uuid + +import pytest +import pytest_asyncio + +from everos.config import load_settings + +_URI = os.environ.get("EVEROS_TEST_MILVUS_URI", "") + +pytestmark = pytest.mark.skipif( + not _URI, + reason="EVEROS_TEST_MILVUS_URI is not configured", +) + + +@pytest_asyncio.fixture(autouse=True) +async def _remote_milvus(monkeypatch: pytest.MonkeyPatch): + prefix = f"everos_e2e_{uuid.uuid4().hex}" + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", _URI) + monkeypatch.setenv( + "EVEROS_MILVUS__TOKEN", + os.environ.get("EVEROS_TEST_MILVUS_TOKEN", ""), + ) + monkeypatch.setenv( + "EVEROS_MILVUS__DB_NAME", + os.environ.get("EVEROS_TEST_MILVUS_DB_NAME", ""), + ) + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", prefix) + load_settings.cache_clear() + + from everos.infra.persistence.index import episode_repo, startup + + try: + if os.environ.get("EVEROS_TEST_MILVUS_FULL_STARTUP") == "1": + await startup() + else: + await episode_repo._repo().ensure_collection() # type: ignore[attr-defined] + yield + finally: + from everos.infra.persistence.index import drop_business_tables, shutdown + + await drop_business_tables() + await shutdown() + load_settings.cache_clear() + + +def _episode( + *, + row_id: str, + entry_id: str, + session_id: str, + text: str, + vector_axis: int, + subject_axis: int, + timestamp: dt.datetime, +): # type: ignore[no-untyped-def] + from everos.infra.persistence.index import Episode + + vector = [0.0] * 1024 + vector[vector_axis] = 1.0 + subject_vector = [0.0] * 1024 + subject_vector[subject_axis] = 1.0 + return Episode( + id=row_id, + entry_id=entry_id, + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + session_id=session_id, + timestamp=timestamp, + parent_id=f"mc_{entry_id}", + sender_ids=["user"], + subject=f"subject {text}", + episode=text, + episode_tokens=text, + md_path="test_app/test_project/users/u1/episodes/day.md", + content_sha256=entry_id, + vector=vector, + subject_vector=subject_vector, + ) + + +async def test_remote_milvus_matches_derived_index_contract() -> None: + from everos.infra.persistence.index import ( + Episode, + UserProfile, + episode_repo, + eq, + user_profile_repo, + ) + from everos.memory.search import FilterNode + from everos.memory.search.filters import compile_filters + + first_vector = [1.0] + [0.0] * 1023 + first_subject = [0.0, 1.0] + [0.0] * 1022 + now = dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + await episode_repo.upsert( + [ + _episode( + row_id="u1_ep1", + entry_id="ep1", + session_id="abc=", + text="red apple memory", + vector_axis=0, + subject_axis=1, + timestamp=now, + ), + _episode( + row_id="u1_ep2", + entry_id="ep2", + session_id="other", + text="blue banana memory", + vector_axis=1, + subject_axis=0, + timestamp=now + dt.timedelta(seconds=1), + ), + ] + ) + + where = compile_filters( + None, + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + ) + rows = await episode_repo.find_where(where, limit=10) + assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} + assert await episode_repo.count() == 2 + + equals_filter = compile_filters( + FilterNode.model_validate({"session_id": "abc="}), + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + ) + equals_rows = await episode_repo.find_where(equals_filter, limit=10) + assert [row.id for row in equals_rows] == ["u1_ep1"] + + sparse = await episode_repo.sparse_search( + ["apple"], where, columns=Episode.BM25_FIELDS, limit=5 + ) + assert sparse[0]["id"] == "u1_ep1" + assert sparse[0]["_score"] > 0 + + dense = await episode_repo.dense_search(first_vector, where, limit=5) + assert dense[0]["id"] == "u1_ep1" + assert dense[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + + by_subject = await episode_repo.dense_search( + first_subject, + where, + limit=5, + vector_field="subject_vector", + ) + assert by_subject[0]["id"] == "u1_ep1" + assert by_subject[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + + page, total = await episode_repo.find_where_paginated( + where, + sort_by="timestamp", + page=1, + page_size=1, + max_fetch=10, + ) + assert total == 2 + assert len(page) == 1 + + concurrent = await asyncio.gather( + *(episode_repo.find_where(where, limit=10) for _ in range(4)) + ) + assert all(len(result) == 2 for result in concurrent) + + if os.environ.get("EVEROS_TEST_MILVUS_FULL_STARTUP") == "1": + profile = UserProfile( + id="u1", + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + summary="initial profile", + explicit_info_json="[]", + implicit_traits_json="[]", + profile_timestamp_ms=1, + md_path="test_app/test_project/users/u1/user.md", + content_sha256="profile-v1", + ) + await user_profile_repo.upsert([profile]) + await user_profile_repo.update( + {"summary": "updated profile"}, where=eq("id", "u1") + ) + updated_profile = await user_profile_repo.get_by_id("u1") + assert updated_profile is not None + assert updated_profile.summary == "updated profile" + + assert ( + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) + == 2 + ) + assert await episode_repo.count() == 0 diff --git a/tests/integration/test_reflection_integration.py b/tests/integration/test_reflection_integration.py index 63b91c2fd..7c7fa5362 100644 --- a/tests/integration/test_reflection_integration.py +++ b/tests/integration/test_reflection_integration.py @@ -19,6 +19,7 @@ import hashlib import json from pathlib import Path +from typing import Any import numpy as np import pytest @@ -34,9 +35,11 @@ ) from everos.core.persistence.lancedb import LanceDailyLogRepoBase, LanceRepoBase from everos.infra.ome.testing import FakeStrategyContext +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.lancedb.tables.atomic_fact import AtomicFact from everos.infra.persistence.lancedb.tables.episode import Episode as LanceEpisode from everos.infra.persistence.markdown.writers.episode_writer import EpisodeWriter +from everos.infra.persistence.predicate import Predicate from everos.infra.persistence.sqlite import cluster_repo, reflection_report_repo from everos.memory._partition_locks import _reset_for_tests from everos.memory.reflection.orchestrator import ReflectionOrchestrator @@ -69,10 +72,24 @@ async def embed(self, text: str) -> list[float]: class _EpisodeRepo(LanceDailyLogRepoBase[LanceEpisode]): schema = LanceEpisode + async def find_where( + self, where: str | Predicate, *, limit: int = 100 + ) -> list[LanceEpisode]: + rendered = render_predicate(where) if isinstance(where, Predicate) else where + return await super().find_where(rendered, limit=limit) + + async def update(self, updates: dict[str, Any], *, where: str | Predicate) -> None: + rendered = render_predicate(where) if isinstance(where, Predicate) else where + await super().update(updates, where=rendered) + class _AtomicFactRepo(LanceDailyLogRepoBase[AtomicFact]): schema = AtomicFact + async def update(self, updates: dict[str, Any], *, where: str | Predicate) -> None: + rendered = render_predicate(where) if isinstance(where, Predicate) else where + await super().update(updates, where=rendered) + # --------------------------------------------------------------------------- # Fixtures diff --git a/tests/integration/test_tiers/test_upgrade_path.py b/tests/integration/test_tiers/test_upgrade_path.py index 7d8c2425f..72a35bb5a 100644 --- a/tests/integration/test_tiers/test_upgrade_path.py +++ b/tests/integration/test_tiers/test_upgrade_path.py @@ -16,6 +16,7 @@ from __future__ import annotations +import asyncio import hashlib from pathlib import Path @@ -41,6 +42,18 @@ async def _episode_rows(owner_id: str) -> list[dict]: return await table.query().where(f"owner_id = '{owner_id}'").to_list() +async def _wait_for_episode_rows( + owner_id: str, expected: int, *, deadline_seconds: float +) -> list[dict]: + """Wait for every logical entry in a debounced daily-log update.""" + async with asyncio.timeout(deadline_seconds): + while True: + rows = await _episode_rows(owner_id) + if len(rows) >= expected: + return rows + await asyncio.sleep(0.1) + + async def _atomic_fact_rows(owner_id: str) -> list[dict]: table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) return await table.query().where(f"owner_id = '{owner_id}'").to_list() @@ -118,7 +131,10 @@ async def test_tier1_to_tier2_upgrade_via_backfill( # below directly against the LanceDB table. await wait_drained(deadline_seconds=40.0) - rows = await _episode_rows("u_alice") + # The queue drains file-level work, but another append to the same + # daily log can arrive just after that drain. Wait for the logical + # entry count as the end-to-end completion condition. + rows = await _wait_for_episode_rows("u_alice", _N_ITEMS, deadline_seconds=40.0) assert len(rows) == _N_ITEMS assert all(r["vector"] is None for r in rows), ( "Tier 1 (no embed) must write every episode with vector=NULL" diff --git a/tests/unit/test_config/test_settings.py b/tests/unit/test_config/test_settings.py index b02d2e78b..6d3f2c59d 100644 --- a/tests/unit/test_config/test_settings.py +++ b/tests/unit/test_config/test_settings.py @@ -127,7 +127,6 @@ def test_index_milvus_defaults() -> None: assert s.milvus.token.get_secret_value() == "" assert s.milvus.db_name == "" assert s.milvus.consistency_level == "Session" - assert s.milvus.dimension == 1024 assert s.milvus.collection_prefix == "everos" @@ -137,14 +136,12 @@ def test_index_milvus_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("EVEROS_MILVUS__TOKEN", "secret") monkeypatch.setenv("EVEROS_MILVUS__DB_NAME", "tenant_a") monkeypatch.setenv("EVEROS_MILVUS__CONSISTENCY_LEVEL", "Strong") - monkeypatch.setenv("EVEROS_MILVUS__DIMENSION", "1536") s = Settings() assert s.index.backend == "milvus" assert s.milvus.uri == "http://localhost:19530" assert s.milvus.token.get_secret_value() == "secret" assert s.milvus.db_name == "tenant_a" assert s.milvus.consistency_level == "Strong" - assert s.milvus.dimension == 1536 def test_resolve_root_default(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/unit/test_core/test_persistence/test_memory_root.py b/tests/unit/test_core/test_persistence/test_memory_root.py index efca98227..6da6ccd9a 100644 --- a/tests/unit/test_core/test_persistence/test_memory_root.py +++ b/tests/unit/test_core/test_persistence/test_memory_root.py @@ -91,8 +91,6 @@ def test_dotfile_paths(tmp_path: Path) -> None: mr = MemoryRoot(tmp_path) assert mr.index_dir == tmp_path / ".index" assert mr.lancedb_dir == tmp_path / ".index" / "lancedb" - assert mr.milvus_dir == tmp_path / ".index" / "milvus" - assert mr.milvus_db == tmp_path / ".index" / "milvus" / "milvus.db" assert mr.sqlite_dir == tmp_path / ".index" / "sqlite" assert mr.system_db == tmp_path / ".index" / "sqlite" / "system.db" assert mr.lock_file == tmp_path / ".lock" @@ -106,7 +104,6 @@ def test_ensure_creates_required_dirs(tmp_path: Path) -> None: assert mr.index_dir.is_dir() assert mr.sqlite_dir.is_dir() assert mr.lancedb_dir.is_dir() - assert mr.milvus_dir.is_dir() assert mr.tmp_dir.is_dir() # User-visible dirs are NOT pre-created. assert not mr.agents_dir().exists() diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index 8d265d81f..d6fdbeb25 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -1,128 +1,148 @@ -"""Real Milvus Lite coverage for the optional derived index backend.""" +"""Unit coverage for the remote Milvus derived-index adapter.""" from __future__ import annotations import datetime as dt -import importlib.util -import uuid -from pathlib import Path import pytest -import pytest_asyncio from everos.config import load_settings - -pytestmark = pytest.mark.skipif( - importlib.util.find_spec("pymilvus") is None, - reason="pymilvus optional extra is not installed", +from everos.infra.persistence.index import Episode, episode_repo, user_profile_repo +from everos.infra.persistence.index.schema import schema_for +from everos.infra.persistence.milvus import repository +from everos.infra.persistence.milvus.milvus_manager import ( + MilvusConfigurationError, + _resolve_uri, +) +from everos.infra.persistence.milvus.repository import ( + MilvusRepoBase, + MilvusValueLimitError, ) -def test_milvus_score_normalization_handles_lite_and_cloud( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from everos.infra.persistence.milvus import repository - - monkeypatch.delenv("EVEROS_MILVUS__URI", raising=False) - monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0") - load_settings.cache_clear() - assert repository._cosine_distance_from_milvus(0.25) == pytest.approx(0.25) - assert repository._bm25_score_from_distance(-1.5) == pytest.approx(1.5) - - monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0.0") - assert repository._cosine_distance_from_milvus(0.25) == pytest.approx(0.25) - - monkeypatch.setenv("EVEROS_MILVUS__URI", "https://example.zillizcloud.com") - load_settings.cache_clear() - assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) - assert repository._bm25_score_from_distance(1.5) == pytest.approx(1.5) - - monkeypatch.delenv("EVEROS_MILVUS__URI", raising=False) - monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0.1") - load_settings.cache_clear() - assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) - - -@pytest_asyncio.fixture(autouse=True) -async def _milvus_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): - monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) +@pytest.fixture(autouse=True) +def _reset_state(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") - monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", f"test_{uuid.uuid4().hex}") + monkeypatch.setenv("EVEROS_MILVUS__URI", "http://milvus.example:19530") + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", "unit_test") load_settings.cache_clear() + MilvusRepoBase._reset_locks_for_tests() yield - from everos.infra.persistence.index import shutdown - - await shutdown() + MilvusRepoBase._reset_locks_for_tests() load_settings.cache_clear() -async def test_milvus_episode_repo_upsert_search_filter_and_delete() -> None: - from everos.infra.persistence.index import Episode, episode_repo - from everos.memory.search.filters import compile_filters_for_backends - - v_apple = [0.0] * 1024 - v_apple[0] = 1.0 - v_banana = [0.0] * 1024 - v_banana[1] = 1.0 - now = dt.datetime(2026, 1, 1, tzinfo=dt.UTC) - - await episode_repo.upsert( - [ - Episode( - id="u1_ep1", - entry_id="ep1", - owner_id="u1", - owner_type="user", - app_id="default", - project_id="default", - session_id="s1", - timestamp=now, - parent_id="mc1", - sender_ids=["user"], - episode="red apple memory", - episode_tokens="red apple memory", - md_path="default_app/default_project/users/u1/episodes/day.md", - content_sha256="a", - vector=v_apple, - ), - Episode( - id="u1_ep2", - entry_id="ep2", - owner_id="u1", - owner_type="user", - app_id="default", - project_id="default", - session_id="s1", - timestamp=now + dt.timedelta(seconds=1), - parent_id="mc2", - sender_ids=["assistant"], - episode="blue banana memory", - episode_tokens="blue banana memory", - md_path="default_app/default_project/users/u1/episodes/day.md", - content_sha256="b", - vector=v_banana, - ), - ] +def _episode(**overrides): # type: ignore[no-untyped-def] + values = { + "id": "u1_ep1", + "entry_id": "ep1", + "owner_id": "u1", + "owner_type": "user", + "session_id": "abc=", + "timestamp": dt.datetime(2026, 1, 1, tzinfo=dt.UTC), + "parent_id": "mc1", + "sender_ids": ["user"], + "episode": "red apple memory", + "episode_tokens": "red apple memory", + "md_path": "default_app/default_project/users/u1/episodes/day.md", + "content_sha256": "a", + "vector": [1.0] + [0.0] * 1023, + "subject_vector": [0.0, 1.0] + [0.0] * 1022, + } + values.update(overrides) + return Episode(**values) + + +def test_remote_uri_is_required_and_local_paths_are_rejected() -> None: + settings = load_settings().milvus.model_copy(update={"uri": ""}) + with pytest.raises(MilvusConfigurationError, match="requires"): + _resolve_uri(settings) + + settings = settings.model_copy(update={"uri": "/tmp/milvus.db"}) + with pytest.raises(MilvusConfigurationError, match="remote http"): + _resolve_uri(settings) + + settings = settings.model_copy(update={"uri": "file:///tmp/milvus.db"}) + with pytest.raises(MilvusConfigurationError, match=r"http\(s\)"): + _resolve_uri(settings) + + +def test_neutral_schema_tracks_every_model_field_and_dense_vector() -> None: + schema = schema_for(Episode) + assert {field.name for field in schema.fields} == set(Episode.model_fields) + assert [field.name for field in schema.vector_fields] == [ + "vector", + "subject_vector", + ] + + +def test_record_conversion_stores_every_dense_vector_with_presence() -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + record = milvus_repo._to_milvus_record(_episode()) + assert len(record["vector"]) == 1024 + assert len(record["subject_vector"]) == 1024 + assert record["vector__present"] is True + assert record["subject_vector__present"] is True + + missing = milvus_repo._to_milvus_record(_episode(subject_vector=None)) + assert missing["subject_vector__present"] is False + assert missing["subject_vector"] == [0.0] * 1024 + + +def test_update_fetch_preserves_dummy_vector_for_scalar_only_tables() -> None: + milvus_repo = user_profile_repo._repo() # type: ignore[attr-defined] + assert "_everos_dummy_vector" in milvus_repo._output_fields(include_vectors=True) + assert "_everos_dummy_vector" not in milvus_repo._output_fields( + include_vectors=False ) - filters = compile_filters_for_backends(None, owner_id="u1", owner_type="user") - rows = await episode_repo.find_where(filters, limit=10) - assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} - assert rows[0].timestamp.utcoffset() == dt.timedelta(0) - assert await episode_repo.count() == 2 - sparse = await episode_repo.sparse_search( - ["apple"], filters, columns=Episode.BM25_FIELDS, limit=5 - ) - assert [row["id"] for row in sparse] == ["u1_ep1"] - assert sparse[0]["_score"] > 0 +def test_record_conversion_reports_varchar_array_and_vector_limits() -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + with pytest.raises(MilvusValueLimitError, match=r"episode is .* UTF-8 bytes"): + milvus_repo._to_milvus_record(_episode(episode="x" * 65_536)) + with pytest.raises(MilvusValueLimitError, match="sender_ids has 257 items"): + milvus_repo._to_milvus_record(_episode(sender_ids=["u"] * 257)) + with pytest.raises(MilvusValueLimitError, match="dimension 2"): + milvus_repo._validate_vector( + milvus_repo.index_schema.field("vector"), + [1.0, 0.0], + ) - dense = await episode_repo.dense_search(v_apple, filters, limit=5) - assert dense[0]["id"] == "u1_ep1" - assert dense[0]["_distance"] == pytest.approx(0.0) - deleted = await episode_repo.delete_by_md_path( - "default_app/default_project/users/u1/episodes/day.md" - ) - assert deleted == 2 - assert await episode_repo.count() == 0 +def test_server_score_normalization() -> None: + assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) + assert repository._bm25_score_from_distance(1.5) == pytest.approx(1.5) + + +async def test_collection_metadata_is_cached_after_startup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + + class _FakeClient: + has_calls = 0 + describe_calls = 0 + + def has_collection(self, name: str) -> bool: + self.has_calls += 1 + return True + + def describe_collection(self, name: str): # type: ignore[no-untyped-def] + self.describe_calls += 1 + return { + "fields": [ + {"name": field} for field in milvus_repo._stored_field_names() + ] + } + + client = _FakeClient() + + async def _fake_get_client(): # type: ignore[no-untyped-def] + return client + + monkeypatch.setattr(repository, "get_client", _fake_get_client) + await milvus_repo.ensure_collection() + await milvus_repo.ensure_collection() + assert client.has_calls == 1 + assert client.describe_calls == 1 diff --git a/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py b/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py index 76990db7e..8450e3979 100644 --- a/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py +++ b/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py @@ -29,6 +29,7 @@ from typing import Any +from everos.infra.persistence.predicate import Comparison, Predicate from everos.memory.cascade._backfill import ( NullBackfillPresenter, _backfill_table, @@ -59,9 +60,9 @@ class _RecordingRepo: assert on the exact ``{col: value}`` shape written back.""" def __init__(self) -> None: - self.updates: list[tuple[dict[str, Any], str]] = [] + self.updates: list[tuple[dict[str, Any], Predicate]] = [] - async def update(self, values: dict[str, Any], *, where: str) -> None: + async def update(self, values: dict[str, Any], *, where: Predicate) -> None: self.updates.append((values, where)) @@ -200,11 +201,12 @@ async def test_orthogonal_partial_states_all_recover() -> None: assert result.rows_processed == 3 assert result.rows_failed == 0 - # Recover per-id updates by parsing the where clause - # (``id = 'xxx'``) — the repo double preserves call order and shape. + # Recover per-id updates from the neutral predicate tree. updates_by_id: dict[str, dict[str, Any]] = {} for values, where in repo.updates: - row_id = where.split("'")[1] + assert isinstance(where, Comparison) + assert where.field == "id" + row_id = str(where.value) updates_by_id[row_id] = values assert set(updates_by_id) == {"both", "subj_only", "prim_only"} diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py index 9f79cc562..21722c0d9 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py @@ -24,7 +24,9 @@ from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot from everos.infra.persistence.lancedb import AgentCase +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.markdown import AgentCaseWriter +from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import HandlerDeps from everos.memory.cascade.handlers.agent_case import AgentCaseHandler @@ -61,10 +63,13 @@ def __init__(self) -> None: self.deletes: list[str] = [] self.rows: list[AgentCase] = [] - async def find_where(self, where: str, *, limit: int = 100) -> list[AgentCase]: + async def find_where( + self, where: Predicate, *, limit: int = 100 + ) -> list[AgentCase]: + rendered = render_predicate(where) prefix = "md_path = '" - if where.startswith(prefix): - md_path = where[len(prefix) :].rstrip("'") + if rendered.startswith(prefix): + md_path = rendered[len(prefix) :].rstrip("'") return [r for r in self.rows if r.md_path == md_path] return [] @@ -75,8 +80,8 @@ async def upsert(self, rows: list[AgentCase]) -> None: by_id[r.id] = r self.rows = list(by_id.values()) - async def delete(self, predicate: str) -> None: - self.deletes.append(predicate) + async def delete(self, predicate: Predicate) -> None: + self.deletes.append(render_predicate(predicate)) async def delete_by_md_path(self, md_path: str) -> int: before = len(self.rows) diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py index 8ed467d45..b5caa773a 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py @@ -19,7 +19,9 @@ from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot from everos.infra.persistence.lancedb import AgentSkill +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.markdown import AgentSkillWriter +from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import AgentSkillHandler, HandlerDeps @@ -72,29 +74,31 @@ async def delete_by_md_path(self, md_path: str) -> int: self.deletes.append(md_path) return 1 - async def find_where(self, predicate: str, *, limit: int) -> list[AgentSkill]: + async def find_where(self, predicate: Predicate, *, limit: int) -> list[AgentSkill]: """In-memory equivalent — handles only the ``md_path = '...' AND id != '...'`` shape the handler emits.""" - if "md_path = " in predicate and "id != " in predicate: - md_lit = predicate.split("md_path = '")[1].split("'", 1)[0] - id_lit = predicate.split("id != '")[1].split("'", 1)[0] + rendered = render_predicate(predicate) + if "md_path = " in rendered and "id != " in rendered: + md_lit = rendered.split("md_path = '")[1].split("'", 1)[0] + id_lit = rendered.split("id != '")[1].split("'", 1)[0] return [ r for r in self.rows.values() if r.md_path == md_lit and r.id != id_lit ][:limit] - raise NotImplementedError(f"fake repo doesn't handle {predicate!r}") - - async def delete(self, predicate: str) -> None: - self.predicate_deletes.append(predicate) - if "md_path = " in predicate and "id != " in predicate: - md_lit = predicate.split("md_path = '")[1].split("'", 1)[0] - id_lit = predicate.split("id != '")[1].split("'", 1)[0] + raise NotImplementedError(f"fake repo doesn't handle {rendered!r}") + + async def delete(self, predicate: Predicate) -> None: + rendered = render_predicate(predicate) + self.predicate_deletes.append(rendered) + if "md_path = " in rendered and "id != " in rendered: + md_lit = rendered.split("md_path = '")[1].split("'", 1)[0] + id_lit = rendered.split("id != '")[1].split("'", 1)[0] self.rows = { rid: row for rid, row in self.rows.items() if not (row.md_path == md_lit and row.id != id_lit) } return - raise NotImplementedError(f"fake repo doesn't handle {predicate!r}") + raise NotImplementedError(f"fake repo doesn't handle {rendered!r}") @pytest.fixture @@ -233,7 +237,7 @@ async def test_renaming_skill_via_frontmatter_clears_old_row( assert list(fake_repo.rows.keys()) == ["a1_new_name"] # The sweep predicate references the *new* id with the same md_path. assert fake_repo.predicate_deletes == [ - f"md_path = '{md_path}' AND id != 'a1_new_name'" + f"((md_path = '{md_path}') AND (id != 'a1_new_name'))" ] diff --git a/tests/unit/test_memory/test_cascade/test_handler_episode.py b/tests/unit/test_memory/test_cascade/test_handler_episode.py index a8af7fe1a..e23855c5d 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_episode.py +++ b/tests/unit/test_memory/test_cascade/test_handler_episode.py @@ -19,7 +19,9 @@ from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot from everos.infra.persistence.lancedb import Episode +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.markdown import EpisodeWriter +from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import HandlerDeps from everos.memory.cascade.handlers.episode import EpisodeHandler @@ -58,11 +60,12 @@ def __init__(self) -> None: self.deletes: list[str] = [] self.rows: list[Episode] = [] - async def find_where(self, where: str, *, limit: int = 100) -> list[Episode]: + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[Episode]: # Honour only the md_path = '...' filter the handler emits. + rendered = render_predicate(where) prefix = "md_path = '" - if where.startswith(prefix): - md_path = where[len(prefix) :].rstrip("'") + if rendered.startswith(prefix): + md_path = rendered[len(prefix) :].rstrip("'") return [r for r in self.rows if r.md_path == md_path] return [] @@ -74,8 +77,8 @@ async def upsert(self, rows: list[Episode]) -> None: by_id[r.id] = r self.rows = list(by_id.values()) - async def delete(self, predicate: str) -> None: - self.deletes.append(predicate) + async def delete(self, predicate: Predicate) -> None: + self.deletes.append(render_predicate(predicate)) async def delete_by_md_path(self, md_path: str) -> int: before = len(self.rows) diff --git a/tests/unit/test_memory/test_get/test_filters_adapter.py b/tests/unit/test_memory/test_get/test_filters_adapter.py index 4d6c48f4d..7fe6c76b3 100644 --- a/tests/unit/test_memory/test_get/test_filters_adapter.py +++ b/tests/unit/test_memory/test_get/test_filters_adapter.py @@ -19,17 +19,25 @@ import pytest -from everos.memory.get.filters_adapter import compile_filters_for_get +from everos.infra.persistence.lancedb.predicate import render_predicate +from everos.memory.get.filters_adapter import ( + compile_filters_for_get as compile_filter_predicate_for_get, +) from everos.memory.search import FilterError, FilterNode +def compile_filters_for_get(*args: object, **kwargs: object) -> str: + """Render the neutral predicate with the legacy LanceDB test oracle.""" + return render_predicate(compile_filter_predicate_for_get(*args, **kwargs)) # type: ignore[arg-type] + + def test_no_filters_emits_base_clause() -> None: """``filters=None`` → owner + app/project scope clauses AND-joined.""" where = compile_filters_for_get(None, owner_id="u1", owner_type="user") assert where == ( - "owner_id = 'u1' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" + "((owner_id = 'u1') AND (owner_type = 'user') " + "AND (app_id = 'default') AND (project_id = 'default') " + "AND (deprecated_by IS NULL))" ) @@ -43,9 +51,9 @@ def test_owner_id_quote_is_escaped() -> None: """SQL-standard double-quote escape on ``owner_id``.""" where = compile_filters_for_get(None, owner_id="o'reilly", owner_type="user") assert where == ( - "owner_id = 'o''reilly' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" + "((owner_id = 'o''reilly') AND (owner_type = 'user') " + "AND (app_id = 'default') AND (project_id = 'default') " + "AND (deprecated_by IS NULL))" ) @@ -93,7 +101,7 @@ def test_timestamp_range_renders_and_folded() -> None: assert "timestamp < TIMESTAMP '" in where # The two clauses are AND-joined inside one parenthesised group. assert "(timestamp >= TIMESTAMP" in where - assert " AND timestamp < TIMESTAMP" in where + assert ") AND (timestamp < TIMESTAMP" in where def test_sender_id_in_list_renders_array_has() -> None: @@ -125,7 +133,7 @@ def test_top_level_and_renders_grouped_clause() -> None: ) where = compile_filters_for_get(node, owner_id="u1", owner_type="user") # Base clause is always first; combinator output appended. - assert where.startswith("owner_id = 'u1' AND owner_type = 'user' AND ") + assert where.startswith("((owner_id = 'u1') AND (owner_type = 'user') AND ") assert "session_id = 'sess_a'" in where assert "parent_id = 'mc_x'" in where diff --git a/tests/unit/test_memory/test_get/test_manager.py b/tests/unit/test_memory/test_get/test_manager.py index 4cc5e9bfb..ea040f63c 100644 --- a/tests/unit/test_memory/test_get/test_manager.py +++ b/tests/unit/test_memory/test_get/test_manager.py @@ -27,6 +27,7 @@ Episode, UserProfile, ) +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.memory.get import ( GetManager, GetMemoryType, @@ -65,7 +66,7 @@ async def find_where_paginated( max_fetch: int = 20000, ) -> tuple[list[Any], int]: self.last = _CallRecord( - where=str(where), + where=render_predicate(where), sort_by=sort_by, descending=descending, page=page, diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index 59e85dd2e..f2c7041de 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -4,32 +4,40 @@ import pytest +from everos.infra.persistence.lancedb.predicate import render_predicate as render_lance +from everos.infra.persistence.milvus.predicate import render_predicate as render_milvus from everos.memory.search import ( FilterError, FilterNode, - compile_filters, +) +from everos.memory.search import ( + compile_filters as compile_filter_ast, ) from everos.memory.search.filters import compile_filters_for_backends + +def compile_filters(*args, **kwargs): # type: ignore[no-untyped-def] + """Render the neutral result through LanceDB for legacy syntax assertions.""" + return render_lance(compile_filter_ast(*args, **kwargs)) + + # ── Base injection ─────────────────────────────────────────────────────── def test_no_filters_emits_base_clause() -> None: where = compile_filters(None, owner_id="alice", owner_type="user") - assert where == ( - "owner_id = 'alice' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" - ) + assert "owner_id = 'alice'" in where + assert "owner_type = 'user'" in where + assert "app_id = 'default'" in where + assert "project_id = 'default'" in where + assert "deprecated_by IS NULL" in where def test_no_filters_agent_omits_deprecated_by() -> None: where = compile_filters(None, owner_id="bot_42", owner_type="agent") assert "deprecated_by" not in where - assert where == ( - "owner_id = 'bot_42' AND owner_type = 'agent' " - "AND app_id = 'default' AND project_id = 'default'" - ) + assert "owner_id = 'bot_42'" in where + assert "owner_type = 'agent'" in where def test_owner_type_agent_pinned() -> None: @@ -249,10 +257,10 @@ def test_empty_and_array_skips_combinator() -> None: """Empty AND/OR arrays compile to no clauses — only the base remains.""" node = FilterNode.model_validate({"AND": []}) where = compile_filters(node, owner_id="alice", owner_type="user") - assert where == ( - "owner_id = 'alice' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" + assert where == compile_filters( + None, + owner_id="alice", + owner_type="user", ) @@ -274,11 +282,12 @@ def test_compile_filters_omits_deprecated_by_for_agent() -> None: def test_compile_filters_for_backends_preserves_lancedb_default() -> None: filters = compile_filters_for_backends(None, owner_id="u_a", owner_type="user") - assert str(filters) == filters.lancedb - assert "owner_id = 'u_a'" in filters.lancedb - assert 'owner_id == "u_a"' in filters.milvus - assert "deprecated_by IS NULL" in filters.lancedb - assert "deprecated_by is null" in filters.milvus + lancedb = render_lance(filters) + milvus = render_milvus(filters) + assert "owner_id = 'u_a'" in lancedb + assert 'owner_id == "u_a"' in milvus + assert "deprecated_by IS NULL" in lancedb + assert "deprecated_by is null" in milvus def test_compile_filters_for_milvus_timestamp_and_array() -> None: @@ -286,16 +295,20 @@ def test_compile_filters_for_milvus_timestamp_and_array() -> None: {"timestamp": {"gte": 1704067200000}, "sender_id": "u_jason"} ) filters = compile_filters_for_backends(node, owner_id="u_a", owner_type="user") - assert "timestamp_ms >= 1704067200000" in filters.milvus - assert 'array_contains(sender_ids, "u_jason")' in filters.milvus - assert "TIMESTAMP '" in filters.lancedb - assert "array_has(sender_ids, 'u_jason')" in filters.lancedb + milvus = render_milvus(filters, datetime_fields={"timestamp"}) + lancedb = render_lance(filters) + assert "timestamp_ms >= 1704067200000" in milvus + assert 'array_contains(sender_ids, "u_jason")' in milvus + assert "TIMESTAMP '" in lancedb + assert "array_has(sender_ids, 'u_jason')" in lancedb def test_compile_filters_for_milvus_escapes_string_literals() -> None: node = FilterNode.model_validate({"session_id": "ses's"}) filters = compile_filters_for_backends(node, owner_id="al'ice", owner_type="user") - assert 'owner_id == "al\'ice"' in filters.milvus - assert 'session_id == "ses\'s"' in filters.milvus - assert "owner_id = 'al''ice'" in filters.lancedb - assert "session_id = 'ses''s'" in filters.lancedb + milvus = render_milvus(filters) + lancedb = render_lance(filters) + assert 'owner_id == "al\'ice"' in milvus + assert 'session_id == "ses\'s"' in milvus + assert "owner_id = 'al''ice'" in lancedb + assert "session_id = 'ses''s'" in lancedb diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index 9ee62af13..a84980d51 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -28,6 +28,7 @@ from everos.component.embedding import EmbeddingCapability from everos.component.rerank import RerankCapability from everos.core.errors import ProviderNotConfiguredError +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.memory.search.dto import SearchMethod, SearchRequest from everos.memory.search.manager import SearchManager @@ -108,13 +109,13 @@ def __init__(self, sparse: list[Candidate], dense: list[Candidate]) -> None: async def sparse_recall( self, query: str, where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = str(where) + self.last_where = render_predicate(where) return list(self._sparse[:limit]) async def dense_recall( self, vector: Sequence[float], where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = str(where) + self.last_where = render_predicate(where) return list(self._dense[:limit]) async def fetch_by_parent_ids( diff --git a/tests/unit/test_memory/test_search/test_recall_agent_skill.py b/tests/unit/test_memory/test_search/test_recall_agent_skill.py index 9239fa34a..661e58a0b 100644 --- a/tests/unit/test_memory/test_search/test_recall_agent_skill.py +++ b/tests/unit/test_memory/test_search/test_recall_agent_skill.py @@ -26,6 +26,7 @@ agent_skill_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq from everos.memory.search.recall.agent_skill import AgentSkillRecaller from everos.memory.search.recall.base import RecallerDeps @@ -76,7 +77,7 @@ def _recaller() -> AgentSkillRecaller: return AgentSkillRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) -_OWNER_WHERE = "owner_id = 'agt' AND owner_type = 'agent'" +_OWNER_WHERE = all_of(eq("owner_id", "agt"), eq("owner_type", "agent")) async def test_fetch_by_case_ids_matches_any_lineage_case() -> None: diff --git a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py index c3c71d433..6e81f755f 100644 --- a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py +++ b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py @@ -29,6 +29,7 @@ atomic_fact_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq from everos.memory.search.recall import atomic_fact as atomic_fact_module from everos.memory.search.recall.atomic_fact import AtomicFactRecaller from everos.memory.search.recall.base import RecallerDeps @@ -84,6 +85,9 @@ def _recaller() -> AtomicFactRecaller: return AtomicFactRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) +_ALICE_WHERE = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) + + async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: """Two episodes sharing one memcell both see the same fact pool. @@ -105,8 +109,9 @@ async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: "alice_ep_b": ["mc_shared"], "alice_ep_c": ["mc_other"], } - where = "owner_id = 'alice' AND owner_type = 'user'" - out = await _recaller().facts_for_episodes(ep_to_parents, where, per_episode=10) + out = await _recaller().facts_for_episodes( + ep_to_parents, _ALICE_WHERE, per_episode=10 + ) assert sorted(out.keys()) == ["alice_ep_a", "alice_ep_b", "alice_ep_c"] assert sorted(f.id for f in out["alice_ep_a"]) == ["alice_af_1", "alice_af_2"] @@ -121,9 +126,7 @@ async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: async def test_facts_for_episodes_returns_empty_for_no_episodes() -> None: - out: dict = await _recaller().facts_for_episodes( - {}, "owner_id = 'alice'", per_episode=10 - ) + out: dict = await _recaller().facts_for_episodes({}, _ALICE_WHERE, per_episode=10) assert out == {} @@ -135,7 +138,7 @@ async def test_facts_for_episodes_skips_unknown_memcells() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"], "alice_ep_b": ["mc_missing"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert "alice_ep_a" in out @@ -164,7 +167,7 @@ async def test_facts_for_episodes_filters_by_where_clause() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert [f.id for f in out["alice_ep_a"]] == ["alice_af_1"] @@ -184,7 +187,7 @@ async def test_facts_for_episodes_drops_empty_parent_ids() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": [""]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert out == {} @@ -232,7 +235,7 @@ async def test_facts_for_episodes_assigns_real_cosine_score_with_query_vector() out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_shared"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, query_vector=_unit_vector(0), ) @@ -260,7 +263,7 @@ async def test_facts_for_episodes_score_zero_without_query_vector() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, # no query_vector ) @@ -287,7 +290,7 @@ async def fake_dense_search(vector, where, *, limit): # type: ignore[no-untyped out = await _recaller().facts_for_episodes( ep_to_parents, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=20, query_vector=_unit_vector(0), ) @@ -316,7 +319,7 @@ async def test_facts_for_episodes_dual_parent_id_finds_both_eras() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["ep_entry_1", "mc_1"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) @@ -344,7 +347,7 @@ async def test_facts_for_episodes_multiple_parent_ids_dedup_across_episodes() -> "alice_ep_a": ["ep_entry_1", "mc_shared"], "alice_ep_b": ["ep_entry_2", "mc_shared"], }, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) diff --git a/tests/unit/test_memory/test_search/test_recall_episode.py b/tests/unit/test_memory/test_search/test_recall_episode.py index 2cde9f368..0ae51992c 100644 --- a/tests/unit/test_memory/test_search/test_recall_episode.py +++ b/tests/unit/test_memory/test_search/test_recall_episode.py @@ -8,6 +8,7 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.predicate import eq from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.episode import EpisodeRecaller @@ -42,6 +43,9 @@ def _mock_table(rows: list[dict[str, Any]]) -> MagicMock: return tbl +_ALICE_WHERE = eq("owner_id", "alice") + + @pytest.fixture() def recaller() -> EpisodeRecaller: tok = MagicMock(spec=Tokenizer) @@ -62,7 +66,7 @@ async def test_fetch_all_for_owner_returns_entry_id_keyed_candidates( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 2 ids = {c.id for c in result} @@ -79,7 +83,7 @@ async def test_fetch_all_for_owner_stores_episode_id_in_metadata( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert result[0].metadata["episode_id"] == "ep_abc" assert result[0].metadata["parent_id"] == "mc_xyz" @@ -108,7 +112,7 @@ async def test_fetch_all_for_owner_skips_rows_without_entry_id( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert result == [] @@ -135,7 +139,7 @@ async def test_fetch_all_for_owner_merged_episode_uses_entry_id( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 1 assert result[0].id == "entry_xyz", "merged episode id must be entry_id" @@ -160,7 +164,7 @@ async def test_fetch_all_for_owner_mixed_regular_and_merged( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 2 ids = {c.id for c in result} @@ -184,7 +188,7 @@ async def test_fetch_by_entry_ids_returns_candidates( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_by_entry_ids(["entry_xyz"], "owner_id = 'alice'") + result = await recaller.fetch_by_entry_ids(["entry_xyz"], _ALICE_WHERE) assert len(result) == 1 assert result[0].id == "ep_merged" @@ -194,7 +198,7 @@ async def test_fetch_by_entry_ids_empty_input_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty entry_ids list short-circuits without querying.""" - result = await recaller.fetch_by_entry_ids([], "owner_id = 'alice'") + result = await recaller.fetch_by_entry_ids([], _ALICE_WHERE) assert result == [] @@ -226,9 +230,7 @@ async def test_sparse_recall_as_child_injects_parent_id( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.sparse_recall_as_child( - "hello", "owner_id = 'alice'", limit=10 - ) + result = await recaller.sparse_recall_as_child("hello", _ALICE_WHERE, limit=10) assert len(result) == 1 assert result[0].metadata["parent_id"] == "entry_1" @@ -256,9 +258,7 @@ async def test_sparse_recall_as_child_falls_back_to_id_when_no_entry_id( new_callable=AsyncMock, return_value=[row], ): - result = await recaller.sparse_recall_as_child( - "hello", "owner_id = 'alice'", limit=10 - ) + result = await recaller.sparse_recall_as_child("hello", _ALICE_WHERE, limit=10) assert len(result) == 1 cand = result[0] @@ -270,7 +270,7 @@ async def test_sparse_recall_as_child_empty_query_returns_empty( ) -> None: """Empty query token list short-circuits; no table call needed.""" recaller._deps.tokenizer.tokenize.return_value = [] - result = await recaller.sparse_recall_as_child("", "owner_id = 'alice'", limit=10) + result = await recaller.sparse_recall_as_child("", _ALICE_WHERE, limit=10) assert result == [] @@ -287,7 +287,7 @@ async def test_dense_recall_as_child_injects_parent_id( return_value=rows, ): result = await recaller.dense_recall_as_child( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 @@ -298,7 +298,7 @@ async def test_dense_recall_as_child_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_as_child([], "owner_id = 'alice'", limit=10) + result = await recaller.dense_recall_as_child([], _ALICE_WHERE, limit=10) assert result == [] @@ -321,31 +321,31 @@ async def test_dense_recall_subject_returns_subject_vector_source( rows = [ {**_make_row("ep_s1", "mc_s1", entry_id="entry_s1"), "_distance": 0.2}, ] - with ( - patch( - "everos.memory.search.recall.episode.active_backend", - return_value="lancedb", - ), - patch( - "everos.infra.persistence.lancedb.get_table", - new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), - ), - ): + with patch( + "everos.memory.search.recall.episode.episode_repo.dense_search", + new_callable=AsyncMock, + return_value=rows, + ) as dense_search: result = await recaller.dense_recall_subject( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 assert result[0].source == "vector" assert result[0].score == pytest.approx(0.8) + dense_search.assert_awaited_once_with( + [0.1] * 1024, + _ALICE_WHERE, + limit=10, + vector_field="subject_vector", + ) async def test_dense_recall_subject_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_subject([], "owner_id = 'alice'", limit=10) + result = await recaller.dense_recall_subject([], _ALICE_WHERE, limit=10) assert result == [] @@ -356,19 +356,13 @@ async def test_dense_recall_subject_as_child_injects_parent_id( rows = [ {**_make_row("ep_s2", "mc_s2", entry_id="entry_s2"), "_distance": 0.15}, ] - with ( - patch( - "everos.memory.search.recall.episode.active_backend", - return_value="lancedb", - ), - patch( - "everos.infra.persistence.lancedb.get_table", - new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), - ), + with patch( + "everos.memory.search.recall.episode.episode_repo.dense_search", + new_callable=AsyncMock, + return_value=rows, ): result = await recaller.dense_recall_subject_as_child( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 @@ -380,7 +374,5 @@ async def test_dense_recall_subject_as_child_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_subject_as_child( - [], "owner_id = 'alice'", limit=10 - ) + result = await recaller.dense_recall_subject_as_child([], _ALICE_WHERE, limit=10) assert result == [] diff --git a/tests/unit/test_memory/test_search/test_recall_or_semantics.py b/tests/unit/test_memory/test_search/test_recall_or_semantics.py index 3e35cde83..0c2fe4a7f 100644 --- a/tests/unit/test_memory/test_search/test_recall_or_semantics.py +++ b/tests/unit/test_memory/test_search/test_recall_or_semantics.py @@ -30,6 +30,7 @@ episode_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq from everos.memory.search.recall.base import RecallerDeps, build_or_query from everos.memory.search.recall.episode import EpisodeRecaller @@ -148,7 +149,7 @@ async def test_or_semantics_poison_token_does_not_kill_query() -> None: tbl = await get_table(Episode.TABLE_NAME, Episode) await tbl.optimize() - where = "owner_id = 'alice' AND owner_type = 'user'" + where = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) cands = await _recaller().sparse_recall("alice support group", where, limit=10) assert cands, "alice + support + group should recall ep_1 via SHOULD" # ep_1 is the support-group episode; should rank above ep_2 (no support). @@ -177,7 +178,7 @@ async def test_or_semantics_single_informative_token() -> None: tbl = await get_table(Episode.TABLE_NAME, Episode) await tbl.optimize() - where = "owner_id = 'alice' AND owner_type = 'user'" + where = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) cands = await _recaller().sparse_recall("painting", where, limit=10) assert cands, "single informative token must recall the matching episode" assert cands[0].id == "alice_ep_2" @@ -185,5 +186,5 @@ async def test_or_semantics_single_informative_token() -> None: async def test_or_semantics_empty_query_returns_empty() -> None: """Tokenisation yields nothing → recall returns ``[]`` without hitting LanceDB.""" - cands = await _recaller().sparse_recall(" ", "owner_id = 'alice'", limit=10) + cands = await _recaller().sparse_recall(" ", eq("owner_id", "alice"), limit=10) assert cands == [] diff --git a/tests/unit/test_service/test_knowledge_search.py b/tests/unit/test_service/test_knowledge_search.py index 73500924f..037b39bd6 100644 --- a/tests/unit/test_service/test_knowledge_search.py +++ b/tests/unit/test_service/test_knowledge_search.py @@ -17,6 +17,7 @@ from everos.component.utils.datetime import get_utc_now from everos.core.errors import ProviderNotConfiguredError +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.sqlite.tables.knowledge import ( KnowledgeDocumentRow, ) @@ -134,11 +135,11 @@ def _patch_stack( class TestCompileKnowledgeWhere: def test_basic_clause(self) -> None: - result = compile_knowledge_where("myapp", "myproj") - assert result == "app_id = 'myapp' AND project_id = 'myproj'" + result = render_predicate(compile_knowledge_where("myapp", "myproj")) + assert result == "((app_id = 'myapp') AND (project_id = 'myproj'))" def test_defaults(self) -> None: - result = compile_knowledge_where("default", "default") + result = render_predicate(compile_knowledge_where("default", "default")) assert "app_id = 'default'" in result assert "project_id = 'default'" in result @@ -159,12 +160,12 @@ def test_rejects_empty_project_id(self) -> None: compile_knowledge_where("app", "") def test_accepts_valid_ids_with_special_chars(self) -> None: - result = compile_knowledge_where("my_app.v2", "project-1") + result = render_predicate(compile_knowledge_where("my_app.v2", "project-1")) assert "my_app.v2" in result assert "project-1" in result def test_accepts_valid_ids_with_at_plus(self) -> None: - result = compile_knowledge_where("app@org+v1", "proj_1") + result = render_predicate(compile_knowledge_where("app@org+v1", "proj_1")) assert "app@org+v1" in result assert "proj_1" in result diff --git a/uv.lock b/uv.lock index c9c12429c..f0757aa71 100644 --- a/uv.lock +++ b/uv.lock @@ -109,11 +109,11 @@ wheels = [ [[package]] name = "cachetools" -version = "7.1.6" +version = "7.1.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/55/af/861ebc2e318a5c3300e3eb63bc4d30f3d70a46d13b360093728ac0705eed/cachetools-7.1.6.tar.gz", hash = "sha256:c7a79e7f30ba9943c1cefd08cc36f006aaae086e017af9166f1d59d6170c47e1", size = 40572, upload-time = "2026-07-23T22:47:53.737Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/d2/47e8bc06fe2a06d3f5bdf20f1126ab66c4e99dc48d940e7ba873f7ac7131/cachetools-7.1.7.tar.gz", hash = "sha256:a3e2a00b14d8f8a6b70c1dae7b4685e7ad3bc965c5b42124a2d6ce895da6cf50", size = 40680, upload-time = "2026-08-01T21:20:40.434Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/f2/2086ba18a925a73586c4d4e61d25f4a6058e56fd00d77ce8f1d361ab4c9b/cachetools-7.1.6-py3-none-any.whl", hash = "sha256:2c12e255780330af28b91bb7fb96cce4c766f04e38396b9a24510190a5827096", size = 16954, upload-time = "2026-07-23T22:47:52.397Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d8/767faeda872075724b95dd675466a645f1b92aadcdcf2d1429dcfd76c176/cachetools-7.1.7-py3-none-any.whl", hash = "sha256:ef98ef375ad188819ef2f9b3645e3987f4b8c5b7550e436ad998c2de78296df0", size = 16830, upload-time = "2026-08-01T21:20:38.977Z" }, ] [[package]] @@ -613,7 +613,7 @@ dependencies = [ [package.optional-dependencies] milvus = [ - { name = "pymilvus", extra = ["milvus-lite"] }, + { name = "pymilvus" }, ] multimodal = [ { name = "everalgo-parser", extra = ["svg"] }, @@ -631,6 +631,7 @@ dev = [ { name = "opentelemetry-sdk" }, { name = "pre-commit" }, { name = "pyinstrument" }, + { name = "pymilvus" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -661,7 +662,7 @@ requires-dist = [ { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "pydantic", specifier = ">=2.7.1" }, { name = "pydantic-settings", specifier = ">=2.0.0" }, - { name = "pymilvus", extras = ["milvus-lite"], marker = "extra == 'milvus'", specifier = ">=3.0.0" }, + { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=3.0.0" }, { name = "python-multipart", specifier = ">=0.0.7" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "sqlmodel", specifier = ">=0.0.22" }, @@ -682,6 +683,7 @@ dev = [ { name = "opentelemetry-sdk", specifier = ">=1.27.0" }, { name = "pre-commit", specifier = ">=4.0.0" }, { name = "pyinstrument", specifier = ">=5.0.0" }, + { name = "pymilvus", specifier = ">=3.0.0" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-asyncio", specifier = ">=1.1.0" }, { name = "pytest-cov", specifier = ">=6.0.0" }, @@ -698,23 +700,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] -[[package]] -name = "faiss-cpu" -version = "1.14.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "packaging" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/b0/48c083d01b7b68c463c1d56507147a9d733f791e1c469a77215a872a9fb5/faiss_cpu-1.14.3-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9369863290a3f0e033757e4c10577b6ef7431f1cede394dabd0a137e4e2ed45", size = 4768290, upload-time = "2026-06-13T02:19:03.427Z" }, - { url = "https://files.pythonhosted.org/packages/ab/34/6b04ef5bae3eada6b5a9457d7875cce041c040d53c890815cbd1e9821c65/faiss_cpu-1.14.3-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:f9d0e84d909194f63f027bbd3c1e35e905e48c9345c2db6e6f24da09a6bc5906", size = 6925734, upload-time = "2026-06-13T02:19:05.232Z" }, - { url = "https://files.pythonhosted.org/packages/7c/8a/b451af4b3c6dd18749ecfb58ccb68503b77e49c9aa4a89d950e9d521e058/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d734cfa9ac90b6a5dfed3a27cb706d05f22824703dafc3969b4e2071877a31c", size = 9661210, upload-time = "2026-06-13T02:19:06.99Z" }, - { url = "https://files.pythonhosted.org/packages/a0/ed/57335bc18c9e18677587bec9bf070c675b29c8e683e13f4def0440731ca0/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8780b526c06e57aad90a8c4655dfba2ff1b3195bd600ff4499752fc45159c9fc", size = 18506292, upload-time = "2026-06-13T02:19:09.621Z" }, - { url = "https://files.pythonhosted.org/packages/e7/d3/c6ca8c44a63b909e78aa8a69e14501c79c89613e62261d58455ea603d710/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b28ba083e8c02f2c9be03783402537fd3f00d27e68799c44bf88931500ee12ec", size = 11238800, upload-time = "2026-06-13T02:19:12.821Z" }, - { url = "https://files.pythonhosted.org/packages/93/5f/b405692913a301251749cb175cb3f564ed257fdaa80a22c9a36444d0095d/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cdcb90850cb4b7c27d270839b37bcc0dc8d1fb6a62d1e13053e51ac95061ca25", size = 19237637, upload-time = "2026-06-13T02:19:15.531Z" }, -] - [[package]] name = "fastapi" version = "0.139.0" @@ -1365,21 +1350,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] -[[package]] -name = "milvus-lite" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "faiss-cpu" }, - { name = "grpcio" }, - { name = "numpy" }, - { name = "pyarrow" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7f/58/54c72981958073d365febeae8fed3115c87761f45905f2d4725f506b94d1/milvus_lite-3.1.0.tar.gz", hash = "sha256:8d467ae81173f1ede93723a80f08d85b91988b451cd2b7bbf9b5a7cfb875e04b", size = 636180, upload-time = "2026-07-15T06:40:58.177Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/c2/fb5c2b73618973f943c8064e2d1f076f81731e2fafd5575a6ddaf3deda68/milvus_lite-3.1.0-py3-none-any.whl", hash = "sha256:0d20fffbee3c683eca69c4732a091273123368452159581ed050c933b41a59be", size = 252267, upload-time = "2026-07-15T06:40:52.538Z" }, -] - [[package]] name = "nodeenv" version = "1.10.0" @@ -1542,55 +1512,54 @@ wheels = [ [[package]] name = "orjson" -version = "3.11.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515, upload-time = "2026-05-06T15:09:57.265Z" }, - { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409, upload-time = "2026-05-06T15:09:59.063Z" }, - { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106, upload-time = "2026-05-06T15:10:00.798Z" }, - { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864, upload-time = "2026-05-06T15:10:02.15Z" }, - { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213, upload-time = "2026-05-06T15:10:03.515Z" }, - { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994, upload-time = "2026-05-06T15:10:05.083Z" }, - { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744, upload-time = "2026-05-06T15:10:06.853Z" }, - { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014, upload-time = "2026-05-06T15:10:08.213Z" }, - { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509, upload-time = "2026-05-06T15:10:09.595Z" }, - { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127, upload-time = "2026-05-06T15:10:11.049Z" }, - { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025, upload-time = "2026-05-06T15:10:12.842Z" }, - { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943, upload-time = "2026-05-06T15:10:14.405Z" }, - { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606, upload-time = "2026-05-06T15:10:15.791Z" }, - { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101, upload-time = "2026-05-06T15:10:17.129Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, - { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, - { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, - { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, - { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, - { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, - { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, - { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, - { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, - { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, - { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, - { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, - { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, - { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, - { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, - { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, - { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, - { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, - { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, - { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, - { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, - { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, - { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, - { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, - { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, - { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, - { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, - { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, - { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +version = "3.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/f3/742fb1f62b825f2c010697eaf4e828004bc2a81e7e806666989c132c7c42/orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5", size = 4142915, upload-time = "2026-08-14T16:13:30.607Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/4a/295da39c651c2faac8bd351a2a346f0fdedd9d50b847ee9dfc27d2207ef6/orjson-3.12.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:aa3e43a6846e91d7bde3d5a9c66090fcd8744f569a9b6cffc5e1ca38f6a461c0", size = 223427, upload-time = "2026-08-14T16:12:28.525Z" }, + { url = "https://files.pythonhosted.org/packages/29/98/758cf90fbeaaafb7f8141bfac75a432099959f3a2f5db93a412e876415d8/orjson-3.12.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:11edb4660a6680abee9788a3a9072208a2c96538cc1322bd79542065229d8e54", size = 123725, upload-time = "2026-08-14T16:12:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/32/b5/5b934d251f8651f7e41df180ad0c57a6e1cabe15c7bd331638413a50ebc9/orjson-3.12.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d3a9da945a4d96ae758fdaaca56742e6b73b6fd554c5d8876f252a6dad70b83", size = 113375, upload-time = "2026-08-14T16:12:31.209Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d2/37efb5b12a176ce3ced29f4144f20da57d02757f78ce549637dc1b4e1fc8/orjson-3.12.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:92ffc09e07233a6ab6d4e067f7841edcbcc134cb4812155cf171ea5255a421d7", size = 129983, upload-time = "2026-08-14T16:12:32.721Z" }, + { url = "https://files.pythonhosted.org/packages/50/22/0644b87c73f13e0092df8f35a1fe280d991e5e90072087411e0dd7e44e0c/orjson-3.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf44e374aadde77b1f6109f1030be51433eb61984379852766b6f4e187db7b1e", size = 130629, upload-time = "2026-08-14T16:12:34.084Z" }, + { url = "https://files.pythonhosted.org/packages/8c/57/80b986ebfecd9c6a177ddf1c2319717f0cd8feffb2b78946595a18a2fc88/orjson-3.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1192a7021b6d071aaf909864f6e924d6a2675ca360485b972b8401749311750b", size = 131245, upload-time = "2026-08-14T16:12:35.713Z" }, + { url = "https://files.pythonhosted.org/packages/80/3d/75c5ac5a69161f44492a68fbdde66f4cc4ce48cd5e1fb05918e46f0c8848/orjson-3.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:53c0c474a9d9aff9aebfc0c88de1f28f843d940e6e3a80729abdf6a20274356f", size = 135397, upload-time = "2026-08-14T16:12:37.128Z" }, + { url = "https://files.pythonhosted.org/packages/71/93/4d71f2df314a97ff0d27a4559bf5888fc8406e3c6dec90e92291e3511215/orjson-3.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:532ff8cd4bd59a327a953a7dcde922c7fc25b85e29721bb8633265430d3a3873", size = 127693, upload-time = "2026-08-14T16:12:38.627Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1d/0dbc6be5adfd1730491072fb60beb6bcdf5d7b2596ee41b7fc2e298bfc09/orjson-3.12.0-cp312-cp312-win32.whl", hash = "sha256:a6cf4b18e7de173f209f2084ffbd736dd72389a396326ee80a7022168be232e5", size = 128000, upload-time = "2026-08-14T16:12:39.954Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c9/97b1ce0112ebf5e949c775ed5b1755e562233179f3584579673cc24d6378/orjson-3.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:010811c1b69773450a01cef97727a67b223242f350b77d4ca000e59a9ef2155a", size = 122106, upload-time = "2026-08-14T16:12:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/a8/6a/facd8b312e4a0d3a7fa978c7e15821f74a336adf1d65529faec33b48e18b/orjson-3.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:ad29eece0c601737f2a60edc2752a84e7a0785df3efb62e3012834700a5afe0d", size = 126869, upload-time = "2026-08-14T16:12:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/54/cb/d7b78218a987eb8a8ce4eeae0286b1bb679333eb631ea0eeaf6371680bfc/orjson-3.12.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9a36ec60f1796f9a3f13e3b98390295e17a1c7c10155b448d264098bf9ee5900", size = 223397, upload-time = "2026-08-14T16:12:44.003Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4a/bc87c45e7ec639d35ebefd62618e01939531ac8e171426606a01bda05914/orjson-3.12.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ad0422b92d5195443a39f80c3bcf731cc2e00f153bd32063a47b73b057bd0f03", size = 123662, upload-time = "2026-08-14T16:12:45.433Z" }, + { url = "https://files.pythonhosted.org/packages/94/ee/c9a4ff3f2dbedbbe9e635d0fa72c8866adede09b6335ef9644f53752f0d8/orjson-3.12.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5a0fdbc216388f653d3752ff310e710f59253bd4ed6a2bfb3f4f06b84714bbd8", size = 113374, upload-time = "2026-08-14T16:12:46.755Z" }, + { url = "https://files.pythonhosted.org/packages/75/09/3f330a026a796c8b4c97a6f429652a5e912e7065039bf96ed25e42aa7b25/orjson-3.12.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2eb5c56e534127b2b8fa38d2363c8b1b8190367ee0d1d16c041517d880843b94", size = 130029, upload-time = "2026-08-14T16:12:48.06Z" }, + { url = "https://files.pythonhosted.org/packages/7d/40/094cc53126a3d22f76cdf83b6ea67338bed01d774037621a785aa8e6e5ea/orjson-3.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784106539f4b9d4b930e0b4eb8d45168507dae001945e71b4675a367f1e5e806", size = 130528, upload-time = "2026-08-14T16:12:49.362Z" }, + { url = "https://files.pythonhosted.org/packages/bc/74/89bb236deb9565f99434b13052bb40ddfcce4adf3afbfa3132ee7e421468/orjson-3.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c680706fc8396d95e7c4c1f9482563f552137aef91b57237a3ad5aaf64629df", size = 131075, upload-time = "2026-08-14T16:12:50.692Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ac/1176360d762c01b5bd34acd56fc098e936c491363d8b6b397ad4aa475547/orjson-3.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83445adc40cba26d6d621185a45128ce455b766af368cad2ab64b970603a7978", size = 135321, upload-time = "2026-08-14T16:12:52.114Z" }, + { url = "https://files.pythonhosted.org/packages/7a/02/bbd881c8b9276d50b998de38b4e97de8ace1aac940b0ee545aedbf65ed00/orjson-3.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:644d005bc82f917337a95ce270c9f6f92f9834c2bed7b1477572f8db00784222", size = 127472, upload-time = "2026-08-14T16:12:53.517Z" }, + { url = "https://files.pythonhosted.org/packages/8e/02/a0934d7503e6dcbedd6afac3e7f3f8597fd09389949ad94d0f7540e9dbca/orjson-3.12.0-cp313-cp313-win32.whl", hash = "sha256:d8e78d3d93705e3d27cc17cdb209e44d7a8ea203010cac6ce9c7ffc1ae1996f1", size = 128000, upload-time = "2026-08-14T16:12:55.14Z" }, + { url = "https://files.pythonhosted.org/packages/52/87/69f98f8d40faff103a965a5fbb83f08241b01beaf92badb5413fbc9358cc/orjson-3.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b85931be5b6763c31283805c9bdaae1ca03ad9f6f12a15f1cbf6745b907932c2", size = 121841, upload-time = "2026-08-14T16:12:56.507Z" }, + { url = "https://files.pythonhosted.org/packages/e6/07/b83046a4e3cadcc0987d0f160696107c4af706a619b56e4ad01940cadadf/orjson-3.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:6a31348d7dfa64cd9c78bd1f510ff44c48fe64d71094e6b90e364dba3b55949e", size = 126765, upload-time = "2026-08-14T16:12:57.806Z" }, + { url = "https://files.pythonhosted.org/packages/12/9d/3931253e6f3148abf2cbe14830367042a4806b362ea520df2303db188fb9/orjson-3.12.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9e6fee342a48760e854d743e7a81534d8e2925a6f46e09f750cf56b50fd1de5d", size = 223391, upload-time = "2026-08-14T16:12:59.184Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0e/b4a4f1e305367245877b967a0bad70fcf001d77c54ac4339a120b66fdae4/orjson-3.12.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:8c3bb86dd10f39b3fbf434b7d5dc7cac77d6fc8ac572ae30a10731ede2c4b647", size = 123659, upload-time = "2026-08-14T16:13:00.548Z" }, + { url = "https://files.pythonhosted.org/packages/96/f3/6782c6fa85e2702bc66be183c3b421486167dcf266ee4dc1403fe3824870/orjson-3.12.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2bb3ce43203936072dd8b4917b01d3aecfc02329bfb42510cb7cfb24708adc9c", size = 113337, upload-time = "2026-08-14T16:13:02.009Z" }, + { url = "https://files.pythonhosted.org/packages/bf/79/b32ab64bacda9d0fa4942ef483bd03cabf0eaf2be819ca9fb7ff610c559d/orjson-3.12.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:6a2a79c89984dc719817d388c8709e0efc2a2795a934eaa746b4882eb6045adc", size = 130112, upload-time = "2026-08-14T16:13:03.404Z" }, + { url = "https://files.pythonhosted.org/packages/ee/49/6e6142999ca01509219be5e5a9c338a3e5ea011f63e91ff473fbbf3734ed/orjson-3.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f06dd838d1e07d9b1de0932ec0485ec92c4d5f5d1ad4817a656268c3e88be1e1", size = 130520, upload-time = "2026-08-14T16:13:04.798Z" }, + { url = "https://files.pythonhosted.org/packages/49/d0/3745af0a4cc9867784f29722929cec4d10bd1c877cd754b01ba6d96eb21a/orjson-3.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6b11be792c3d2c6a4be2af4ebf97a68d0bf5f580aca6e86a418a354f6cc846a", size = 131053, upload-time = "2026-08-14T16:13:06.14Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f4/6fe5a22fa478fffb190e65c338c84df5c311ef597b363150a17cc57063c0/orjson-3.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:477ecaf6b9f88f873341b91fcc736119ca81b5e002a9f7f308ff5b4f2ce2a70e", size = 135321, upload-time = "2026-08-14T16:13:07.544Z" }, + { url = "https://files.pythonhosted.org/packages/ff/41/b1b0ec30289646a81a76e2dbaae2686b96fcccb7cb0323dc1dd78cbc7875/orjson-3.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3c0683136acdc29afdf88a5bc2f7d3d0e34087788d1d63c0144b805a87a196f", size = 127485, upload-time = "2026-08-14T16:13:08.88Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2b/277404bdcc21c93b112b963655b76443ebfe828f8a3ff1de7d90f8850eb3/orjson-3.12.0-cp314-cp314-win32.whl", hash = "sha256:d39f3f5c3927e2dc0913fe5bbc1a2f6b1b9d1bba1de6358340d0ad0d0c00ca92", size = 128048, upload-time = "2026-08-14T16:13:10.305Z" }, + { url = "https://files.pythonhosted.org/packages/41/2b/395b36fa2b4ce7af70b651d715e88f80d884b2c2b14a6b53e84d554fb5f0/orjson-3.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:0b1ac5bf6609b2716c7954011c5fef6254922df029f45d032ee4ebf5d363cbed", size = 121858, upload-time = "2026-08-14T16:13:11.634Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a3/833e895ff452859eebe75093d26691fe9108f1a7a6a08435d7a5780ea652/orjson-3.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:50fae885cb073eac7556353ff3df93312b0d5137b0a5056b2bb63f97ed9a93c7", size = 126749, upload-time = "2026-08-14T16:13:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/58/64/99c8947ece10c17176af9aae85c4948f1d109da77440ec14d87239efaf73/orjson-3.12.0-cp315-cp315-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:01efac2074fffb4cb1ea3fab7861e9d0f2a26913854a972f5ac760525dbdaf6e", size = 223398, upload-time = "2026-08-14T16:13:14.694Z" }, + { url = "https://files.pythonhosted.org/packages/3e/30/cf983fe09f2731420fda097a9f7ef4343f47fa216c228961ad8f6da44f3d/orjson-3.12.0-cp315-cp315-macosx_15_0_arm64.whl", hash = "sha256:ed4ca42bd55955aa34deedcfdfd0e0c31abf51143aae158ae2bc3520b626e517", size = 123655, upload-time = "2026-08-14T16:13:16.221Z" }, + { url = "https://files.pythonhosted.org/packages/11/50/9cb8ae73fa4749dbbc20f617004213b5ff01c20aaeec34c3f31124f2c1d8/orjson-3.12.0-cp315-cp315-manylinux_2_39_aarch64.whl", hash = "sha256:40f92192227505acca4e2533ce565f8e6b9535f7d0d09b0968452f18b7376b38", size = 130515, upload-time = "2026-08-14T16:13:17.601Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0a/adb6ce1a5b5fbf9cb1790f9961bb668a0dd5429aadaf6cee044724681795/orjson-3.12.0-cp315-cp315-manylinux_2_39_armv7l.whl", hash = "sha256:33efefcf5d88eaf400b47e2eba02f91f319bb9951be61ca500b7d536d3f2079d", size = 113327, upload-time = "2026-08-14T16:13:18.927Z" }, + { url = "https://files.pythonhosted.org/packages/51/5c/d17f61581d8dbdde7048f87a330fa24915edec38db4d72b381fec14fbb56/orjson-3.12.0-cp315-cp315-manylinux_2_39_i686.whl", hash = "sha256:8e386b0bc0ddd7cd2056f884b5a0af33592bd01ac66a7ca4b42a65a7e7774a13", size = 130105, upload-time = "2026-08-14T16:13:20.317Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b7/938befcf33bee4704a92ecec6a2731224c539d939bf9429fd39396d28931/orjson-3.12.0-cp315-cp315-manylinux_2_39_x86_64.whl", hash = "sha256:58c58e1de0006ffb580368d6793c36c7b0b021db066479cf281bf5061e732328", size = 131049, upload-time = "2026-08-14T16:13:21.719Z" }, + { url = "https://files.pythonhosted.org/packages/b0/15/cfa2021d64d5aa8bb5c9f604ef375e00ec8b657651b5dd650b1b7ad13df1/orjson-3.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:08231552159be266a7269555bd9f7c016aee7d9ad6dab06eb58796c5ccb7101c", size = 135320, upload-time = "2026-08-14T16:13:23.415Z" }, + { url = "https://files.pythonhosted.org/packages/1a/50/3e75dfe357c1e8f9e287c7a5740260ef15bd23a5299eae8d0835dcad5375/orjson-3.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a15f9a891bce5f5cc5d210e3ad8614d4d1b489a56448c099d6d2a7168b2d954a", size = 127488, upload-time = "2026-08-14T16:13:24.791Z" }, + { url = "https://files.pythonhosted.org/packages/11/a6/79aed402eb3ab284dc5b4791a7ad62c5875127de01b8e3f04bd92d551298/orjson-3.12.0-cp315-cp315-win32.whl", hash = "sha256:03091c8a64db4be38746597ceea68f33c238e27acd9bfe99fb59420224ae7a55", size = 128048, upload-time = "2026-08-14T16:13:26.217Z" }, + { url = "https://files.pythonhosted.org/packages/64/f7/2723e264aab7248c1ed6ecaad8e5d0cb866c0cffde75442102ffa7491aba/orjson-3.12.0-cp315-cp315-win_amd64.whl", hash = "sha256:2b7bcefb9f40fa242fa6b06377232c048e655747790829609168c01162f60578", size = 121860, upload-time = "2026-08-14T16:13:27.577Z" }, + { url = "https://files.pythonhosted.org/packages/82/56/630c9113ec8996778f1f0304b364b091b9a9db5fef5fdc17cca622f5ea24/orjson-3.12.0-cp315-cp315-win_arm64.whl", hash = "sha256:859fc4196855890150bb08e649b30d2c93b249b3e3edd0d3bb2231abf8aa8adc", size = 126754, upload-time = "2026-08-14T16:13:28.962Z" }, ] [[package]] @@ -2073,7 +2042,7 @@ wheels = [ [[package]] name = "pymilvus" -version = "3.0.0" +version = "3.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cachetools" }, @@ -2083,15 +2052,10 @@ dependencies = [ { name = "protobuf" }, { name = "python-dotenv" }, { name = "requests" }, - { name = "setuptools" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/c0/78/6bd0dba340706bc63346af96f0ebe36ff17f75404f5de935fda94d476c98/pymilvus-3.0.1.tar.gz", hash = "sha256:c02389059088b18d6e598cd175541e445c772fab4926c5e527c4913be34887f1", size = 347593, upload-time = "2026-07-29T14:55:43.061Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/c1/01647e61f3a82fd881382746b6dde3401d65b88cd4f75bd059901fb2392b/pymilvus-3.0.0-1-py3-none-any.whl", hash = "sha256:57c8e7c87fbbf579f122b4df893949bc78e50bca2988527864891bd544817b05", size = 344817, upload-time = "2026-05-07T14:57:45.235Z" }, -] - -[package.optional-dependencies] -milvus-lite = [ - { name = "milvus-lite", marker = "sys_platform != 'win32'" }, + { url = "https://files.pythonhosted.org/packages/60/9d/7011887b29f452905745e8bd321f404068d5bfe78fe84e42c0b7cd81a065/pymilvus-3.0.1-py3-none-any.whl", hash = "sha256:c5a8d5c1fa1de7b416e3529d383d8cc2e7da2170433ffa4a2d9087e14f70171a", size = 386820, upload-time = "2026-07-29T14:55:44.279Z" }, ] [[package]] @@ -2399,15 +2363,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, ] -[[package]] -name = "setuptools" -version = "83.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, -] - [[package]] name = "shellingham" version = "1.5.4" From 79b629fd99e940418517ae7f33f9e5fe5d7ecc19 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Wed, 19 Aug 2026 04:02:51 +0000 Subject: [PATCH 05/27] docs(milvus): add remote connection examples Signed-off-by: Cheney Zhang --- docs/configuration.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 0608170e9..f30af50ca 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -120,6 +120,32 @@ paths are not supported. | `consistency_level` | `"Strong"` \| `"Session"` \| `"Bounded"` \| `"Eventually"` | `"Session"` | Milvus consistency level for created collections. | | `collection_prefix` | string | `"everos"` | Prefix for EverOS collections. Must start with a letter or underscore and contain only letters, digits, and underscores. Make it unique when deployments share a database. | +For a self-hosted Milvus Server, a minimal configuration is: + +```toml +[index] +backend = "milvus" + +[milvus] +uri = "http://localhost:19530" +collection_prefix = "everos" +``` + +Zilliz Cloud uses the same backend. Credentials can be supplied through +environment variables: + +```bash +export EVEROS_INDEX__BACKEND=milvus +export EVEROS_MILVUS__URI="https://your-cluster-endpoint" +export EVEROS_MILVUS__TOKEN="" +export EVEROS_MILVUS__DB_NAME="default" +export EVEROS_MILVUS__COLLECTION_PREFIX="everos_prod" +``` + +Use a deployment secret rather than committing the token to a configuration +file. If the memory root already contains Markdown records, stop the EverOS +server and run `everos cascade rebuild --yes` to populate the derived index. + EverOS creates seven collections and supports every dense field in the shared index schema, including `episode.subject_vector`. The vector dimension is owned by that schema (currently 1024), rather than duplicated in Milvus config. @@ -261,3 +287,6 @@ Examples: | `[memory] timezone = "Asia/Tokyo"` | `EVEROS_MEMORY__TIMEZONE=Asia/Tokyo` | | `[index] backend = "milvus"` | `EVEROS_INDEX__BACKEND=milvus` | | `[milvus] uri = "http://localhost:19530"` | `EVEROS_MILVUS__URI=http://localhost:19530` | +| `[milvus] token = "..."` | `EVEROS_MILVUS__TOKEN=...` | +| `[milvus] db_name = "default"` | `EVEROS_MILVUS__DB_NAME=default` | +| `[milvus] collection_prefix = "everos_prod"` | `EVEROS_MILVUS__COLLECTION_PREFIX=everos_prod` | From 7849535144ab7f7e677232a2287b932455f56faa Mon Sep 17 00:00:00 2001 From: zhanghui Date: Sat, 29 Aug 2026 20:25:12 +0800 Subject: [PATCH 06/27] refactor(index): separate derived-index contracts, routing, and adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds on the Milvus derived-index work in #332. That branch introduced the Predicate AST, the portable schema, and the Milvus repository, and migrated the filter DSL and the recallers off rendered LanceDB SQL. This commit restructures the layer those pieces sit in. Three gaps motivated it. The repository surface did not include the maintenance operations, so the cascade worker could not schedule optimize / prune / rebuild without knowing which backend was active. Backfill still reached LanceDB directly for its null-vector scan and its cluster-phase full-table read, so it could not run against any other backend. And there was no typed port a second adapter could be checked against. What changes: * `index/protocols.py` — IndexRepository and IndexBackend, the typed ports both adapters implement. Maintenance is deliberately part of the repository contract: an embedded engine does physical work, a service-managed engine implements a successful no-op, and the worker stays free of backend branches. * `index/router.py` — stable repository identities whose backend is resolved per call, so importers hold one object for the process lifetime while configuration still decides where reads and writes land. * `index/lancedb.py` — the LanceDB adapter behind the same port, including the predicate renderer that used to live in `lancedb/predicate.py`. * `index/schema.py` and `index/__init__.py` — reduced to the portable schema derivation and the facade the outer layers import, now that contracts and routing have their own modules. * Backfill reads through `scan()` and `repo_for_schema()`; the cascade worker looks for `index_repo`; the BM25 helpers move from `recall/base.py` into the Lance adapter that owns that syntax. Markdown remains the source of truth and SQLite remains the system-state store. This boundary owns only the rebuildable business indexes. Co-Authored-By: Cheney Zhang --- .github/workflows/ci.yml | 22 +- config.example.toml | 18 +- docs/configuration.md | 63 +-- pyproject.toml | 3 - src/everos/config/__init__.py | 3 +- src/everos/config/default.toml | 10 +- src/everos/config/settings.py | 9 +- .../entrypoints/api/lifespans/lancedb.py | 26 +- .../entrypoints/cli/commands/cascade.py | 41 +- .../infra/persistence/index/__init__.py | 409 ++++----------- src/everos/infra/persistence/index/lancedb.py | 464 ++++++++++++++++++ src/everos/infra/persistence/index/milvus.py | 51 ++ .../infra/persistence/index/predicate.py | 2 +- .../infra/persistence/index/protocols.py | 148 ++++++ src/everos/infra/persistence/index/router.py | 269 ++++++++++ src/everos/infra/persistence/index/schema.py | 328 +++---------- .../infra/persistence/lancedb/predicate.py | 83 ---- .../infra/persistence/milvus/predicate.py | 15 +- .../infra/persistence/milvus/repository.py | 53 +- src/everos/infra/persistence/predicate.py | 7 +- src/everos/memory/cascade/_backfill.py | 42 +- .../cascade/handlers/_daily_log_base.py | 21 +- .../memory/cascade/handlers/agent_case.py | 2 +- .../memory/cascade/handlers/agent_skill.py | 2 +- .../memory/cascade/handlers/atomic_fact.py | 2 +- src/everos/memory/cascade/handlers/episode.py | 2 +- .../memory/cascade/handlers/foresight.py | 2 +- .../cascade/handlers/knowledge_topic.py | 2 +- .../memory/cascade/handlers/user_profile.py | 2 +- src/everos/memory/cascade/registry.py | 38 +- src/everos/memory/cascade/worker.py | 8 +- src/everos/memory/get/filters_adapter.py | 1 - src/everos/memory/get/manager.py | 16 +- src/everos/memory/reflection/orchestrator.py | 16 +- src/everos/memory/search/filters.py | 6 +- src/everos/memory/search/manager.py | 3 +- src/everos/memory/search/recall/agent_case.py | 11 +- .../memory/search/recall/agent_skill.py | 5 +- .../memory/search/recall/atomic_fact.py | 15 +- src/everos/memory/search/recall/base.py | 98 +--- src/everos/memory/search/recall/episode.py | 29 +- .../memory/search/recall/knowledge_topic.py | 10 +- src/everos/service/get.py | 4 +- src/everos/service/knowledge.py | 2 +- tests/_consistency_assertions.py | 6 +- .../test_cascade_all_kinds_consistency.py | 2 +- tests/integration/test_milvus_remote.py | 25 +- .../test_reflection_integration.py | 34 +- .../test_lancedb_unbackfilled_hint.py | 61 ++- tests/unit/test_infra/test_index_contract.py | 107 ++++ .../unit/test_infra/test_milvus/test_repo.py | 18 +- .../test_backfill_subject_null_recovery.py | 8 +- .../test_cascade/test_backfill_table_specs.py | 27 +- .../test_cascade/test_handler_agent_case.py | 14 +- .../test_cascade/test_handler_agent_skill.py | 47 +- .../test_cascade/test_handler_episode.py | 17 +- .../test_cascade/test_registry_knowledge.py | 12 +- .../test_memory/test_cascade/test_worker.py | 10 +- .../test_get/test_filters_adapter.py | 33 +- .../unit/test_memory/test_get/test_manager.py | 14 +- .../test_memory/test_search/test_filters.py | 45 +- .../test_memory/test_search/test_manager.py | 24 +- .../test_search/test_recall_episode.py | 29 +- .../test_recall_knowledge_topic.py | 4 +- .../test_search/test_recall_or_semantics.py | 21 +- .../test_service/test_knowledge_search.py | 34 +- uv.lock | 2 +- 67 files changed, 1707 insertions(+), 1250 deletions(-) create mode 100644 src/everos/infra/persistence/index/lancedb.py create mode 100644 src/everos/infra/persistence/index/milvus.py create mode 100644 src/everos/infra/persistence/index/protocols.py create mode 100644 src/everos/infra/persistence/index/router.py delete mode 100644 src/everos/infra/persistence/lancedb/predicate.py create mode 100644 tests/unit/test_infra/test_index_contract.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d1fdd3b3..476c84cb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,8 +126,8 @@ jobs: - name: Integration tests run: make integration - milvus-integration: - name: Milvus Server integration + integration-milvus: + name: integration tests (Milvus 2.6) runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -144,14 +144,14 @@ jobs: - name: Install dependencies (frozen) run: make install-deps - - name: Start Milvus Server + - name: Start Milvus 2.6.22 run: | - curl -fsSL \ - https://github.com/milvus-io/milvus/releases/download/v2.6.22/milvus-standalone-docker-compose.yml \ - -o /tmp/milvus-compose.yml + curl --fail --location --retry 3 \ + --output /tmp/milvus-compose.yml \ + https://github.com/milvus-io/milvus/releases/download/v2.6.22/milvus-standalone-docker-compose.yml docker compose -f /tmp/milvus-compose.yml up -d - for attempt in $(seq 1 90); do - if curl -fsS http://127.0.0.1:9091/healthz; then + for attempt in {1..60}; do + if curl --fail --silent http://127.0.0.1:9091/healthz >/dev/null; then exit 0 fi sleep 2 @@ -159,13 +159,13 @@ jobs: docker compose -f /tmp/milvus-compose.yml logs exit 1 - - name: Milvus derived-index contract + - name: Milvus repository contract env: EVEROS_TEST_MILVUS_URI: http://127.0.0.1:19530 EVEROS_TEST_MILVUS_FULL_STARTUP: "1" - run: uv run pytest tests/integration/test_milvus_remote.py -v + run: uv run --frozen pytest tests/integration/test_milvus_remote.py -v - - name: Stop Milvus Server + - name: Stop Milvus if: always() run: docker compose -f /tmp/milvus-compose.yml down -v diff --git a/config.example.toml b/config.example.toml index 00045b2a9..48cff25fd 100644 --- a/config.example.toml +++ b/config.example.toml @@ -56,19 +56,15 @@ max_concurrent = 5 # # [lancedb] # read_consistency_seconds = 5.0 - -# ── Optional Milvus derived index ───────────────────── -# Markdown remains the source of truth. Set the index backend to Milvus -# when you want the rebuildable vector/BM25 index to live in an external -# Milvus Server or Zilliz Cloud deployment. Embedded Milvus Lite and local -# database paths are not supported. # # [index] -# backend = "milvus" +# backend = "lancedb" # or "milvus" # +# # Required only when index.backend = "milvus". Install with: +# # pip install "everos[milvus]" # [milvus] -# uri = "http://localhost:19530" # required remote endpoint -# token = "" # required for Zilliz Cloud/auth-enabled Milvus -# db_name = "" # use a dedicated database when available +# uri = "http://127.0.0.1:19530" # Milvus Server or Zilliz Cloud endpoint +# token = "" # e.g. "root:Milvus" or cloud token +# db_name = "" # consistency_level = "Session" -# collection_prefix = "everos" # make unique on a shared database +# collection_prefix = "everos" diff --git a/docs/configuration.md b/docs/configuration.md index f30af50ca..e47fbc80e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -101,60 +101,26 @@ everos init --root /data/everos ### `[index]` +The derived vector/BM25 index is rebuildable; Markdown remains the source of +truth regardless of backend. + | Field | Type | Default | Description | |---|---|---|---| -| `backend` | `"lancedb"` \| `"milvus"` | `"lancedb"` | Rebuildable vector/BM25 index backend used by cascade, search, and get. Markdown remains the source of truth; SQLite remains the system state store. | +| `backend` | string | `"lancedb"` | Index implementation: `lancedb` or `milvus`. | ### `[milvus]` -Milvus is optional. Install with `everos[milvus]`, then set -`[index].backend = "milvus"`. The backend connects only to an external -Milvus Server or Zilliz Cloud endpoint; embedded Milvus Lite and local `.db` -paths are not supported. +Used only when `index.backend = "milvus"`. Install the optional client with +`pip install "everos[milvus]"`. EverOS supports a remote Milvus Server or +Zilliz Cloud endpoint; a Milvus Lite filesystem path is rejected. | Field | Type | Default | Description | |---|---|---|---| -| `uri` | string | `""` | Required when the Milvus backend is selected. Must be a remote Milvus Server or Zilliz Cloud endpoint. | -| `token` | string | `""` | Token for Zilliz Cloud or auth-enabled Milvus Server. | -| `db_name` | string | `""` | Optional Milvus database name. Prefer a dedicated database on shared clusters. | -| `consistency_level` | `"Strong"` \| `"Session"` \| `"Bounded"` \| `"Eventually"` | `"Session"` | Milvus consistency level for created collections. | -| `collection_prefix` | string | `"everos"` | Prefix for EverOS collections. Must start with a letter or underscore and contain only letters, digits, and underscores. Make it unique when deployments share a database. | - -For a self-hosted Milvus Server, a minimal configuration is: - -```toml -[index] -backend = "milvus" - -[milvus] -uri = "http://localhost:19530" -collection_prefix = "everos" -``` - -Zilliz Cloud uses the same backend. Credentials can be supplied through -environment variables: - -```bash -export EVEROS_INDEX__BACKEND=milvus -export EVEROS_MILVUS__URI="https://your-cluster-endpoint" -export EVEROS_MILVUS__TOKEN="" -export EVEROS_MILVUS__DB_NAME="default" -export EVEROS_MILVUS__COLLECTION_PREFIX="everos_prod" -``` - -Use a deployment secret rather than committing the token to a configuration -file. If the memory root already contains Markdown records, stop the EverOS -server and run `everos cascade rebuild --yes` to populate the derived index. - -EverOS creates seven collections and supports every dense field in the shared -index schema, including `episode.subject_vector`. The vector dimension is owned -by that schema (currently 1024), rather than duplicated in Milvus config. - -Milvus imposes limits that LanceDB does not: strings are capped at 65,535 -UTF-8 bytes, string arrays at 256 items, and each array item at 512 UTF-8 -bytes. EverOS validates these before a write and reports the exact table and -field instead of relying on a server-side insert error. Ensure the selected -Zilliz Cloud plan permits at least seven collections. +| `uri` | string | `""` | Milvus Server or Zilliz Cloud URI. | +| `token` | string | `""` | Authentication token, when required. | +| `db_name` | string | `""` | Optional Milvus database name. | +| `consistency_level` | string | `"Session"` | Milvus consistency level used by collections. | +| `collection_prefix` | string | `"everos"` | Prefix for the seven derived-index collections. | ### `[llm]` @@ -285,8 +251,3 @@ Examples: | `[llm] api_key = "sk-..."` | `EVEROS_LLM__API_KEY=sk-...` | | `[sqlite] busy_timeout_ms = 10000` | `EVEROS_SQLITE__BUSY_TIMEOUT_MS=10000` | | `[memory] timezone = "Asia/Tokyo"` | `EVEROS_MEMORY__TIMEZONE=Asia/Tokyo` | -| `[index] backend = "milvus"` | `EVEROS_INDEX__BACKEND=milvus` | -| `[milvus] uri = "http://localhost:19530"` | `EVEROS_MILVUS__URI=http://localhost:19530` | -| `[milvus] token = "..."` | `EVEROS_MILVUS__TOKEN=...` | -| `[milvus] db_name = "default"` | `EVEROS_MILVUS__DB_NAME=default` | -| `[milvus] collection_prefix = "everos_prod"` | `EVEROS_MILVUS__COLLECTION_PREFIX=everos_prod` | diff --git a/pyproject.toml b/pyproject.toml index 7006dcd7d..dc800b660 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -250,7 +250,6 @@ ignore_imports = [ "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.tables", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.repos", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.lancedb_manager", - "everos.infra.persistence.index -> everos.infra.persistence.lancedb.predicate", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.mds", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.writers", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.readers", @@ -282,7 +281,5 @@ dev = [ # [otel] stack is always present in the dev / CI environment. "opentelemetry-sdk>=1.27.0", "opentelemetry-exporter-otlp-proto-http>=1.27.0", - # Milvus tests must run in every dev/CI environment. The client package - # intentionally excludes the embedded milvus-lite extra. "pymilvus>=3.0.0", ] diff --git a/src/everos/config/__init__.py b/src/everos/config/__init__.py index 6ce3d9686..fb3108132 100644 --- a/src/everos/config/__init__.py +++ b/src/everos/config/__init__.py @@ -2,8 +2,7 @@ Public API: from everos.config import ( - Settings, MemorySettings, SqliteSettings, IndexSettings, LanceDBSettings, - MilvusSettings, + Settings, MemorySettings, SqliteSettings, LanceDBSettings, LLMSettings, EmbeddingSettings, RerankSettings, BoundaryDetectionSettings, CascadeSettings, load_settings, resolve_root, diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index 38d226823..38bf40030 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -51,17 +51,11 @@ cache_size_kb = 2048 # read_consistency_seconds = 5.0 [index] -# Rebuildable vector/BM25 index backend. Markdown remains the source of truth -# and SQLite remains the system state store. -# Override via EVEROS_INDEX__BACKEND. +# Rebuildable vector/BM25 index. Markdown remains the source of truth. backend = "lancedb" [milvus] -# Optional remote derived index backend for Milvus Server or Zilliz Cloud. -# `uri` is required when [index] backend = "milvus". Embedded Milvus Lite and -# local database paths are not supported. Use a unique db_name or -# collection_prefix when multiple EverOS deployments share a cluster. -# Override via EVEROS_MILVUS__URI, EVEROS_MILVUS__TOKEN, EVEROS_MILVUS__DB_NAME. +# Remote Milvus Server or Zilliz Cloud only; Milvus Lite paths are unsupported. uri = "" token = "" db_name = "" diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 1e910ac8e..5ad9a105e 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -377,18 +377,13 @@ class CascadeSettings(BaseModel): class IndexSettings(BaseModel): - """Derived-index backend selection.""" + """Rebuildable derived-index backend selection.""" backend: Literal["lancedb", "milvus"] = "lancedb" class MilvusSettings(BaseModel): - """Remote Milvus Server or Zilliz Cloud connection settings. - - Embedded Milvus Lite is intentionally unsupported. Deployments that select - the Milvus backend must configure an external endpoint and should use a - unique database or collection prefix when sharing a cluster. - """ + """Remote Milvus Server or Zilliz Cloud connection settings.""" uri: str = "" token: SecretStr = SecretStr("") diff --git a/src/everos/entrypoints/api/lifespans/lancedb.py b/src/everos/entrypoints/api/lifespans/lancedb.py index 56994f565..a2059cd7e 100644 --- a/src/everos/entrypoints/api/lifespans/lancedb.py +++ b/src/everos/entrypoints/api/lifespans/lancedb.py @@ -1,4 +1,4 @@ -"""Derived-index lifespan provider (HTTP API entrypoint). +"""LanceDB lifespan provider (HTTP API entrypoint). Startup: Open the connection via ``get_connection`` (lazy, idempotent). @@ -32,7 +32,9 @@ from everos.core.lifespan import LifespanProvider from everos.core.observability.logging import get_logger from everos.infra.persistence.index import ( + ALL_REPOS, active_backend, + is_null, ) from everos.infra.persistence.index import ( shutdown as shutdown_index, @@ -40,10 +42,7 @@ from everos.infra.persistence.index import ( startup as startup_index, ) -from everos.infra.persistence.lancedb import ( - BUSINESS_SCHEMAS_WITH_VECTOR, - get_table, -) +from everos.infra.persistence.index.schema import schema_for logger = get_logger(__name__) @@ -63,14 +62,16 @@ async def _log_unbackfilled_hint() -> None: startup. """ total_null = 0 - for schema in BUSINESS_SCHEMAS_WITH_VECTOR: + for repo in ALL_REPOS: + logical_schema = schema_for(repo.schema) + if not any(field.name == "vector" for field in logical_schema.vector_fields): + continue try: - table = await get_table(schema.TABLE_NAME, schema) - count = await table.count_rows(filter="vector IS NULL") + count = await repo.count_where(is_null("vector")) except Exception as exc: logger.warning( "unbackfilled_check_failed", - schema=schema.__name__, + schema=repo.schema.__name__, error=repr(exc), ) continue @@ -88,7 +89,7 @@ async def _log_unbackfilled_hint() -> None: class LanceDBLifespanProvider(LifespanProvider): - """Manage the configured derived index for the app lifecycle. + """Manage the LanceDB connection + table cache for the app lifecycle. Startup runs four steps: @@ -105,11 +106,10 @@ def __init__(self, order: int = 11) -> None: super().__init__(name="lancedb", order=order) async def startup(self, app: FastAPI) -> Any: - backend = active_backend() conn = await startup_index() - if backend == "lancedb": + if active_backend() == "lancedb": await _log_unbackfilled_hint() - logger.info("derived_index_ready", backend=backend) + logger.info("derived_index_ready", backend=active_backend()) return conn async def shutdown(self, app: FastAPI) -> None: diff --git a/src/everos/entrypoints/cli/commands/cascade.py b/src/everos/entrypoints/cli/commands/cascade.py index d39f46944..7452c5aef 100644 --- a/src/everos/entrypoints/cli/commands/cascade.py +++ b/src/everos/entrypoints/cli/commands/cascade.py @@ -15,7 +15,7 @@ vectors, build clusters, extract skills. See :func:`everos.entrypoints.cli.commands._backfill_cmd.run_backfill` for the phase orchestration. -- ``cascade rebuild`` — drop every business derived-index table and re-index +- ``cascade rebuild`` — drop every business LanceDB table and re-index all md from scratch. Recovery for a drifted / corrupt index; safe because md is the source of truth and un-extracted buffered messages are preserved. Skips the schema-verify guard (which the drift would @@ -47,15 +47,12 @@ from everos.entrypoints.cli._log_setup import configure_cli_logging from everos.entrypoints.cli.commands._backfill_cmd import run_backfill from everos.infra.persistence.index import ( - active_backend, connect, drop_business_tables, ensure_business_indexes, + shutdown, verify_business_schemas, ) -from everos.infra.persistence.index import ( - shutdown as shutdown_index, -) from everos.infra.persistence.sqlite import ( dispose_engine, get_engine, @@ -71,7 +68,7 @@ app = typer.Typer( name="cascade", - help="Inspect and operate the markdown → derived-index sync queue", + help="Inspect and operate the md → LanceDB sync queue", no_args_is_help=True, ) @@ -141,7 +138,7 @@ def _apply_verbose_logging(verbose: bool | None) -> None: async def _runtime( # type: ignore[no-untyped-def] *, verify: bool = True, ensure: bool = True ): - """Stand up SQLite + the configured index the way the API lifespan would. + """Stand up sqlite + lancedb the same way the API lifespan would. The CLI uses the same lazy, process-wide singletons the API lifespan does. They are **per-process**: a running daemon has its own @@ -167,20 +164,14 @@ async def _runtime( # type: ignore[no-untyped-def] async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) await connect() - if active_backend() == "milvus": - if ensure: - await ensure_business_indexes() - if verify: - await verify_business_schemas() - else: - if verify: - await verify_business_schemas() - if ensure: - await ensure_business_indexes() + if verify: + await verify_business_schemas() + if ensure: + await ensure_business_indexes() try: yield finally: - await shutdown_index() + await shutdown() await dispose_engine() @@ -460,7 +451,7 @@ def rebuild( typer.Option("--yes", "-y", help="Skip the confirmation prompt."), ] = False, ) -> None: - """Rebuild the configured derived index from markdown. + """Rebuild the LanceDB index from markdown (recover from schema drift). **Stop the ``everos server`` first** — this is the one cascade command that is not safe alongside a live daemon. It drops and recreates the @@ -468,13 +459,13 @@ def rebuild( the dropped dataset; the command refuses to start while a server holds the OME lock. - Drops every business table or collection and re-indexes all markdown from + Drops every business LanceDB table and re-indexes all md from scratch. Markdown is the source of truth, so no memory content is lost, and this is the safe recovery from a drifted / corrupt index (e.g. the ``verify_business_schemas`` startup failure): - - unlike deleting ``~/.everos/.index/lancedb`` on the default backend, it - re-populates already-indexed entries (that command leaves the cascade queue + - unlike ``rm -rf ~/.everos/.index/lancedb``, it re-populates + already-indexed entries (that command leaves the cascade queue marked ``done``, so nothing re-indexes and the index comes back empty); - unlike ``rm -rf ~/.everos/.index``, it preserves SQLite state that @@ -485,7 +476,7 @@ def rebuild( typer.echo( "error: a server (or another exclusive CLI phase) is running on " "this memory root.\n" - " cascade rebuild drops and recreates the derived index; a live " + " cascade rebuild drops and recreates the LanceDB tables; a live " "daemon holds cached\n" " table handles and would keep writing to the dropped dataset. " "Stop `everos server`\n" @@ -495,7 +486,7 @@ def rebuild( raise typer.Exit(code=3) if not yes: typer.confirm( - "Drop all derived-index business data and re-index from markdown? " + "Drop all LanceDB business tables and re-index from markdown? " "(requires the server to be stopped)", abort=True, ) @@ -516,7 +507,7 @@ async def _run() -> None: typer.echo(f"reset {cleared} cascade queue row(s)") dropped = await drop_business_tables() typer.echo( - f"dropped {len(dropped)} derived-index table(s): " + f"dropped {len(dropped)} LanceDB table(s): " f"{', '.join(dropped) or '(none)'}" ) # Recreate the tables (current schema) + FTS indexes. diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py index dfe2bcab3..c3f614315 100644 --- a/src/everos/infra/persistence/index/__init__.py +++ b/src/everos/infra/persistence/index/__init__.py @@ -1,38 +1,50 @@ -"""Backend-neutral derived index persistence facade. +"""Backend-neutral facade for the rebuildable BM25/vector index. -Markdown is the source of truth and SQLite stores system state. This package -selects the rebuildable vector/BM25 index backend used by cascade, search, and -get. The default backend remains LanceDB; Milvus is opt-in through -``[index] backend = "milvus"``. +Markdown remains the source of truth and SQLite remains the system-state +store. This boundary owns only the derived business indexes used by cascade, +search, and get. LanceDB and Milvus implement the same typed ports, so callers +do not branch on physical storage. """ from __future__ import annotations -from collections.abc import Sequence from typing import Any -from lancedb.query import BooleanQuery, FullTextQuery, MatchQuery - -try: - from lancedb.query import Occur -except ImportError: # pragma: no cover - from lancedb._lancedb import Occur # type: ignore[attr-defined,no-redef] - from everos.config import load_settings from everos.infra.persistence import lancedb as _lancedb -from everos.infra.persistence.lancedb import ( - AgentCase, - AgentSkill, - AtomicFact, - Episode, - Foresight, - KnowledgeTopic, - ParentType, - UserProfile, -) -from everos.infra.persistence.lancedb.predicate import render_predicate as _render_lance +from .lancedb import ( + agent_case_repo as _lance_agent_case_repo, +) +from .lancedb import ( + agent_skill_repo as _lance_agent_skill_repo, +) +from .lancedb import ( + atomic_fact_repo as _lance_atomic_fact_repo, +) +from .lancedb import ( + episode_repo as _lance_episode_repo, +) +from .lancedb import ( + foresight_repo as _lance_foresight_repo, +) +from .lancedb import ( + knowledge_topic_repo as _lance_knowledge_topic_repo, +) +from .lancedb import ( + lance_index_backend, +) +from .lancedb import ( + user_profile_repo as _lance_user_profile_repo, +) +from .milvus import milvus_index_backend from .predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, Predicate, all_of, any_of, @@ -46,323 +58,103 @@ ne, one_of, ) -from .schema import schema_for +from .protocols import IndexBackend, IndexRepository +from .router import ( + RoutedAgentSkillRepository, + RoutedEpisodeRepository, + RoutedIndexRepository, +) + +AgentCase = _lancedb.AgentCase +AgentSkill = _lancedb.AgentSkill +AtomicFact = _lancedb.AtomicFact +Episode = _lancedb.Episode +Foresight = _lancedb.Foresight +KnowledgeTopic = _lancedb.KnowledgeTopic +ParentType = _lancedb.ParentType +UserProfile = _lancedb.UserProfile + +episode_repo = RoutedEpisodeRepository(_lance_episode_repo, "episode_repo") +atomic_fact_repo = RoutedIndexRepository(_lance_atomic_fact_repo, "atomic_fact_repo") +foresight_repo = RoutedIndexRepository(_lance_foresight_repo, "foresight_repo") +agent_case_repo = RoutedIndexRepository(_lance_agent_case_repo, "agent_case_repo") +agent_skill_repo = RoutedAgentSkillRepository( + _lance_agent_skill_repo, "agent_skill_repo" +) +user_profile_repo = RoutedIndexRepository(_lance_user_profile_repo, "user_profile_repo") +knowledge_topic_repo = RoutedIndexRepository( + _lance_knowledge_topic_repo, "knowledge_topic_repo" +) + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) def active_backend() -> str: - """Return the configured derived index backend name.""" + """Name of the configured derived-index backend.""" return load_settings().index.backend -async def startup() -> Any: - """Initialise the configured derived index backend.""" +def _backend() -> IndexBackend: if active_backend() == "milvus": - milvus = _milvus() - await milvus.get_client() - await milvus.ensure_business_indexes() - return "milvus" - conn = await _lancedb.get_connection() - await _lancedb.verify_business_schemas() - await _lancedb.ensure_business_indexes() - return conn + return milvus_index_backend + return lance_index_backend async def connect() -> Any: - """Open the active backend without creating or verifying indexes.""" - if active_backend() == "milvus": - return await _milvus().get_client() - return await _lancedb.get_connection() + return await _backend().connect() + + +async def startup() -> Any: + return await _backend().startup() async def shutdown() -> None: - """Dispose the configured derived index backend.""" - if active_backend() == "milvus": - await _milvus().dispose_connection() - else: - await _lancedb.dispose_connection() + await _backend().shutdown() async def ensure_business_indexes() -> None: - if active_backend() == "milvus": - await _milvus().ensure_business_indexes() - else: - await _lancedb.ensure_business_indexes() + await _backend().ensure_business_indexes() async def verify_business_schemas() -> None: - if active_backend() == "milvus": - await _milvus().verify_business_schemas() - else: - await _lancedb.verify_business_schemas() + await _backend().verify_business_schemas() async def drop_business_tables() -> list[str]: - """Drop all business indexes for the active backend.""" - if active_backend() == "milvus": - return await _milvus().drop_business_tables() - return await _lancedb.drop_business_tables() - - -class _LanceIndexRepoAdapter: - """Add backend-neutral recall helpers to an existing LanceDB repo.""" - - def __init__(self, repo: Any, schema: type[Any]) -> None: - self._repo = repo - self.schema = schema - self.index_schema = schema_for(schema) - - def __getattr__(self, name: str) -> Any: - return getattr(self._repo, name) - - async def sparse_search( - self, - query_terms: Sequence[str], - where: Any, - *, - columns: Sequence[str] | None = None, - limit: int, - ) -> list[dict[str, Any]]: - fields = list(columns or self.index_schema.bm25_fields) - if not query_terms or not fields: - return [] - table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) - expr = _lance_expr(where) - best: dict[str, dict[str, Any]] = {} - for field in fields: - query = _build_or_query(query_terms, field) - rows = ( - await table.query() - .nearest_to_text(query) - .where(expr) - .limit(limit) - .to_list() - ) - for row in rows: - rid = row.get("id") - if not isinstance(rid, str): - continue - score = float(row.get("_score", 0.0)) - prior = best.get(rid) - if prior is None or score > float(prior.get("_score", 0.0)): - shaped = dict(row) - shaped["_score"] = score - best[rid] = shaped - return sorted( - best.values(), key=lambda row: float(row.get("_score", 0.0)), reverse=True - )[:limit] - - async def dense_search( - self, - vector: Sequence[float], - where: Any, - *, - limit: int, - vector_field: str = "vector", - ) -> list[dict[str, Any]]: - if not vector: - return [] - table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) - return ( - await table.query() - .nearest_to(list(vector)) - .column(vector_field) - .distance_type("cosine") - .where(_lance_expr(where)) - .limit(limit) - .to_list() - ) - - -class _IndexRepoRouter: - """Route repo calls to the configured derived index backend.""" - - def __init__( - self, lance_repo: Any, milvus_repo_name: str, schema: type[Any] - ) -> None: - self._lance = _LanceIndexRepoAdapter(lance_repo, schema) - self._milvus_repo_name = milvus_repo_name - self.schema = schema - self.index_schema = schema_for(schema) + return await _backend().drop_business_tables() - @property - def table_name(self) -> str: - return self.index_schema.table_name - def _repo(self) -> Any: - if active_backend() == "milvus": - return getattr(_milvus(), self._milvus_repo_name) - return self._lance +def repo_for_schema(schema: type[Any]) -> IndexRepository[Any]: + """Resolve a registered repository by its logical record model.""" + for repo in ALL_REPOS: + if repo.schema is schema: + return repo + raise KeyError(f"no derived-index repository for {schema!r}") - def __getattr__(self, name: str) -> Any: - return getattr(self._repo(), name) - - async def add(self, records: Sequence[Any]) -> None: - await self._repo().add(records) - - async def upsert(self, records: Sequence[Any], *, by: str = "id") -> None: - await self._repo().upsert(records, by=by) - - async def count(self) -> int: - return await self._repo().count() - - async def get_by_id(self, id_value: str, *, id_field: str = "id") -> Any: - return await self._repo().get_by_id(id_value, id_field=id_field) - - async def find_where(self, where: Any, *, limit: int = 100) -> list[Any]: - return await self._repo().find_where(_where_for_backend(where), limit=limit) - - async def find_one_where(self, where: Any) -> Any: - return await self._repo().find_one_where(_where_for_backend(where)) - - async def find_where_paginated( - self, - where: Any, - *, - sort_by: str, - descending: bool = True, - page: int = 1, - page_size: int = 20, - max_fetch: int = 20_000, - ) -> tuple[list[Any], int]: - return await self._repo().find_where_paginated( - _where_for_backend(where), - sort_by=sort_by, - descending=descending, - page=page, - page_size=page_size, - max_fetch=max_fetch, - ) - - async def search( - self, - *, - vector: Sequence[float] | None = None, - where: Any = None, - limit: int = 10, - ) -> list[dict[str, Any]]: - return await self._repo().search( - vector=vector, where=_where_for_backend(where), limit=limit - ) - - async def update(self, updates: dict[str, Any], *, where: Any) -> None: - await self._repo().update(updates, where=_where_for_backend(where)) - - async def delete(self, predicate: Any) -> None: - await self._repo().delete(_where_for_backend(predicate)) - - async def delete_by_md_path(self, md_path: str) -> int: - return await self._repo().delete_by_md_path(md_path) - - async def optimize(self, **kwargs: Any) -> None: - await self._repo().optimize(**kwargs) - - async def rebuild_indexes(self) -> None: - await self._repo().rebuild_indexes() - - async def sparse_search( - self, - query_terms: Sequence[str], - where: Any, - *, - columns: Sequence[str] | None = None, - limit: int, - ) -> list[dict[str, Any]]: - return await self._repo().sparse_search( - query_terms, - _where_for_backend(where), - columns=columns, - limit=limit, - ) - - async def dense_search( - self, - vector: Sequence[float], - where: Any, - *, - limit: int, - vector_field: str = "vector", - ) -> list[dict[str, Any]]: - return await self._repo().dense_search( - vector, - _where_for_backend(where), - limit=limit, - vector_field=vector_field, - ) - - -def _where_for_backend(where: Any) -> Any: - if where is None: - return None if active_backend() == "milvus" else "" - if not isinstance(where, Predicate): - raise TypeError( - "derived-index repository predicates must use the neutral " - f"Predicate AST, got {type(where).__name__}" - ) - if active_backend() == "milvus": - return where - return _render_lance(where) - - -def _lance_expr(where: Any) -> str: - if isinstance(where, Predicate): - return _render_lance(where) - if isinstance(where, str): - return where - raise TypeError(f"unsupported LanceDB predicate: {type(where).__name__}") - - -def _build_or_query(tokens: Sequence[str], column: str) -> FullTextQuery: - clean = [token for token in tokens if token] - if not clean: - return MatchQuery("", column=column) - if len(clean) == 1: - return MatchQuery(clean[0], column=column) - return BooleanQuery( - [(Occur.SHOULD, MatchQuery(token, column=column)) for token in clean] - ) - - -def _milvus() -> Any: - from everos.infra.persistence import milvus - - return milvus - - -episode_repo = _IndexRepoRouter(_lancedb.episode_repo, "episode_repo", Episode) -atomic_fact_repo = _IndexRepoRouter( - _lancedb.atomic_fact_repo, "atomic_fact_repo", AtomicFact -) -foresight_repo = _IndexRepoRouter(_lancedb.foresight_repo, "foresight_repo", Foresight) -agent_case_repo = _IndexRepoRouter( - _lancedb.agent_case_repo, "agent_case_repo", AgentCase -) -agent_skill_repo = _IndexRepoRouter( - _lancedb.agent_skill_repo, "agent_skill_repo", AgentSkill -) -user_profile_repo = _IndexRepoRouter( - _lancedb.user_profile_repo, "user_profile_repo", UserProfile -) -knowledge_topic_repo = _IndexRepoRouter( - _lancedb.knowledge_topic_repo, "knowledge_topic_repo", KnowledgeTopic -) - -ALL_REPOS = ( - episode_repo, - atomic_fact_repo, - foresight_repo, - agent_case_repo, - agent_skill_repo, - user_profile_repo, - knowledge_topic_repo, -) __all__ = [ "ALL_REPOS", "AgentCase", "AgentSkill", + "All", + "AnyOf", "AtomicFact", + "Comparison", + "Contains", "Episode", "Foresight", + "In", + "IndexBackend", + "IndexRepository", + "IsNull", "KnowledgeTopic", "ParentType", "Predicate", @@ -388,6 +180,7 @@ def _milvus() -> Any: "lte", "ne", "one_of", + "repo_for_schema", "shutdown", "startup", "user_profile_repo", diff --git a/src/everos/infra/persistence/index/lancedb.py b/src/everos/infra/persistence/index/lancedb.py new file mode 100644 index 000000000..7e62e3f51 --- /dev/null +++ b/src/everos/infra/persistence/index/lancedb.py @@ -0,0 +1,464 @@ +"""LanceDB adapter for the backend-neutral derived-index ports.""" + +from __future__ import annotations + +import datetime as dt +import re +from collections.abc import Sequence +from typing import Any, ClassVar, Final + +from lancedb.query import BooleanQuery, FullTextQuery, MatchQuery +from pydantic import BaseModel + +try: + from lancedb.query import Occur +except ImportError: # pragma: no cover + from lancedb._lancedb import Occur # type: ignore[attr-defined,no-redef] + +from everos.component.utils.datetime import ensure_utc, to_iso_format +from everos.core.persistence import LanceRepoBase +from everos.infra.persistence import lancedb as _lancedb + +from .predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, + all_of, + eq, + one_of, +) + +_FIELD_NAME: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_OPERATORS: Final[dict[str, str]] = { + "eq": "=", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def render_predicate(predicate: Predicate | None) -> str: + """Render a neutral predicate as a LanceDB DataFusion expression.""" + if predicate is None: + return "" + if isinstance(predicate, Comparison): + return ( + f"{_field(predicate.field)} {_OPERATORS[predicate.operator]} " + f"{_literal(predicate.value)}" + ) + if isinstance(predicate, In): + values = ", ".join(_literal(value) for value in predicate.values) + return f"{_field(predicate.field)} IN ({values})" + if isinstance(predicate, Contains): + return f"array_has({_field(predicate.field)}, {_literal(predicate.value)})" + if isinstance(predicate, IsNull): + return f"{_field(predicate.field)} IS NULL" + if isinstance(predicate, All): + return _render_group(predicate.children, "AND") + if isinstance(predicate, AnyOf): + return _render_group(predicate.children, "OR") + raise TypeError(f"unsupported predicate: {type(predicate).__name__}") + + +def _render_group(children: tuple[Predicate, ...], operator: str) -> str: + rendered = [render_predicate(child) for child in children] + rendered = [item for item in rendered if item] + if not rendered: + return "" + if len(rendered) == 1: + return rendered[0] + return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" + + +def _field(value: str) -> str: + if not _FIELD_NAME.fullmatch(value): + raise ValueError(f"invalid predicate field: {value!r}") + return value + + +def _literal(value: Scalar) -> str: + if isinstance(value, str): + return f"'{value.replace(chr(39), chr(39) * 2)}'" + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return f"TIMESTAMP '{to_iso_format(aware)}'" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + return str(value) + + +class LanceIndexRepository[T: BaseModel]: + """Translate the neutral repository contract onto an existing Lance repo.""" + + def __init__(self, repo: LanceRepoBase[T], schema: type[T]) -> None: + self._repo = repo + self.schema = schema + + @property + def table_name(self) -> str: + return str(self.schema.TABLE_NAME) # type: ignore[attr-defined] + + async def add(self, records: Sequence[T]) -> None: + await self._repo.add(records) + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: + await self._repo.upsert(records, by=by) + + async def count(self) -> int: + return await self._repo.count() + + async def count_where(self, where: Predicate | None = None) -> int: + table = await _lancedb.get_table(self.table_name, self.schema) + return await table.count_rows(filter=_render_optional(where)) + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: + return await self._repo.get_by_id(id_value, id_field=id_field) + + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: + return await self._repo.find_where(render_predicate(where), limit=limit) + + async def find_one_where(self, where: Predicate) -> T | None: + return await self._repo.find_one_where(render_predicate(where)) + + async def find_where_paginated( + self, + where: Predicate, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: + return await self._repo.find_where_paginated( + render_predicate(where), + sort_by=sort_by, + descending=descending, + page=page, + page_size=page_size, + max_fetch=max_fetch, + ) + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Predicate | None = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + return await self._repo.search( + vector=vector, + where=_render_optional(where), + limit=limit, + ) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Predicate | None, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + fields = list(columns or getattr(self.schema, "BM25_FIELDS", ())) + clean_terms = [term for term in query_terms if term] + if not clean_terms or not fields: + return [] + table = await _lancedb.get_table(self.table_name, self.schema) + best: dict[str, dict[str, Any]] = {} + for field in fields: + query = build_or_query(clean_terms, field) + assert query is not None + builder = table.query().nearest_to_text(query) + expression = _render_optional(where) + if expression: + builder = builder.where(expression) + rows = await builder.limit(limit).to_list() + for row in rows: + rid = row.get("id") + if not isinstance(rid, str): + continue + score = float(row.get("_score", 0.0)) + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + shaped = dict(row) + shaped["_score"] = score + best[rid] = shaped + return sorted( + best.values(), + key=lambda row: float(row.get("_score", 0.0)), + reverse=True, + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Predicate | None, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: + if not vector: + return [] + table = await _lancedb.get_table(self.table_name, self.schema) + builder = ( + table.query() + .nearest_to(list(vector)) + .column(vector_field) + .distance_type("cosine") + ) + expression = _render_optional(where) + if expression: + builder = builder.where(expression) + return await builder.limit(limit).to_list() + + async def scan(self, where: Predicate | None = None) -> list[T]: + """Return all matching records without a hidden row cap.""" + table = await _lancedb.get_table(self.table_name, self.schema) + builder = table.query() + expression = _render_optional(where) + if expression: + builder = builder.where(expression) + rows = await builder.to_list() + return [self.schema.model_validate(row) for row in rows] + + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: + await self._repo.update(updates, where=render_predicate(where)) + + async def delete(self, predicate: Predicate) -> None: + await self._repo.delete(render_predicate(predicate)) + + async def delete_by_md_path(self, md_path: str) -> int: + return await self._repo.delete_by_md_path(md_path) + + async def optimize(self) -> None: + await self._repo.optimize() + + async def prune(self, older_than: dt.timedelta) -> None: + await self._repo.prune(older_than) + + async def rebuild_indexes(self) -> None: + await self._repo.rebuild_indexes() + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: + return await self.find_where(eq("owner_id", owner_id), limit=limit) + + async def find_by_md_path(self, md_path: str) -> T | None: + return await self.find_one_where(eq("md_path", md_path)) + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: + return await self.find_one_where( + all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), + ) + ) + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: + if not entry_ids: + return [] + return await self.find_where( + all_of( + eq("owner_id", owner_id), + one_of("entry_id", list(entry_ids)), + eq("app_id", app_id), + eq("project_id", project_id), + ), + limit=len(entry_ids), + ) + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + all_of(eq("owner_id", owner_id), eq("session_id", session_id)), + limit=limit, + ) + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + all_of(eq("parent_type", parent_type), eq("parent_id", parent_id)), + limit=limit, + ) + + +class LanceEpisodeRepository(LanceIndexRepository[_lancedb.Episode]): + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: + return await _lancedb.episode_repo.count_by_owner( + owner_id, + app_id=app_id, + project_id=project_id, + parent_type=parent_type, + ) + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[_lancedb.Episode] | list[dict[str, Any]]: + return await _lancedb.episode_repo.list_by_owner_after_ts( + owner_id=owner_id, + after_ts=after_ts, + parent_type=parent_type, + app_id=app_id, + project_id=project_id, + columns=columns, + limit=limit, + ) + + +class LanceAgentSkillRepository(LanceIndexRepository[_lancedb.AgentSkill]): + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: + return await _lancedb.agent_skill_repo.count_in_cluster( + owner_id=owner_id, cluster_id=cluster_id + ) + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[_lancedb.AgentSkill]: + return await _lancedb.agent_skill_repo.find_in_cluster( + owner_id=owner_id, cluster_id=cluster_id, limit=limit + ) + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[_lancedb.AgentSkill]: + return await _lancedb.agent_skill_repo.find_topk_relevant_in_cluster( + owner_id=owner_id, + cluster_id=cluster_id, + query_vector=query_vector, + top_k=top_k, + ) + + +episode_repo = LanceEpisodeRepository(_lancedb.episode_repo, _lancedb.Episode) +atomic_fact_repo = LanceIndexRepository(_lancedb.atomic_fact_repo, _lancedb.AtomicFact) +foresight_repo = LanceIndexRepository(_lancedb.foresight_repo, _lancedb.Foresight) +agent_case_repo = LanceIndexRepository(_lancedb.agent_case_repo, _lancedb.AgentCase) +agent_skill_repo = LanceAgentSkillRepository( + _lancedb.agent_skill_repo, _lancedb.AgentSkill +) +user_profile_repo = LanceIndexRepository( + _lancedb.user_profile_repo, _lancedb.UserProfile +) +knowledge_topic_repo = LanceIndexRepository( + _lancedb.knowledge_topic_repo, _lancedb.KnowledgeTopic +) + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + + +class LanceIndexBackend: + """Own LanceDB connection, schema, index, and repository lifecycle.""" + + name: ClassVar[str] = "lancedb" + repositories = ALL_REPOS + + async def connect(self) -> Any: + return await _lancedb.get_connection() + + async def startup(self) -> Any: + connection = await self.connect() + await self.verify_business_schemas() + await self.ensure_business_indexes() + return connection + + async def shutdown(self) -> None: + await _lancedb.dispose_connection() + + async def ensure_business_indexes(self) -> None: + await _lancedb.ensure_business_indexes() + + async def verify_business_schemas(self) -> None: + await _lancedb.verify_business_schemas() + + async def drop_business_tables(self) -> list[str]: + return await _lancedb.drop_business_tables() + + +def _render_optional(predicate: Predicate | None) -> str | None: + rendered = render_predicate(predicate) + return rendered or None + + +def build_or_query(tokens: Sequence[str], column: str) -> FullTextQuery | None: + """Build LanceDB's OR-mode BM25 query for already-tokenized terms.""" + clean = [token for token in tokens if token] + if not clean: + return None + if len(clean) == 1: + return MatchQuery(clean[0], column=column) + return BooleanQuery( + [(Occur.SHOULD, MatchQuery(token, column=column)) for token in clean] + ) + + +lance_index_backend = LanceIndexBackend() + +__all__ = [ + "ALL_REPOS", + "LanceIndexBackend", + "LanceIndexRepository", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "build_or_query", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "lance_index_backend", + "render_predicate", + "user_profile_repo", +] diff --git a/src/everos/infra/persistence/index/milvus.py b/src/everos/infra/persistence/index/milvus.py new file mode 100644 index 000000000..67a23594d --- /dev/null +++ b/src/everos/infra/persistence/index/milvus.py @@ -0,0 +1,51 @@ +"""Milvus lifecycle adapter for the derived-index backend port.""" + +from __future__ import annotations + +from typing import Any, ClassVar + + +class MilvusIndexBackend: + """Own remote Milvus connection, collection, and repository lifecycle.""" + + name: ClassVar[str] = "milvus" + + @property + def repositories(self): # type: ignore[no-untyped-def] + return _milvus().ALL_REPOS + + async def connect(self) -> Any: + return await _milvus().get_client() + + async def startup(self) -> Any: + client = await self.connect() + await self.verify_business_schemas() + await self.ensure_business_indexes() + return client + + async def shutdown(self) -> None: + await _milvus().dispose_connection() + + async def ensure_business_indexes(self) -> None: + await _milvus().ensure_business_indexes() + + async def verify_business_schemas(self) -> None: + # Verification tolerates absent collections; ensure_collection creates + # them immediately afterwards and verifies every existing collection. + for repo in self.repositories: + await repo.ensure_collection() + + async def drop_business_tables(self) -> list[str]: + return await _milvus().drop_business_tables() + + +def _milvus(): # type: ignore[no-untyped-def] + # Keep pymilvus genuinely optional for default LanceDB installations. + from everos.infra.persistence import milvus + + return milvus + + +milvus_index_backend = MilvusIndexBackend() + +__all__ = ["MilvusIndexBackend", "milvus_index_backend"] diff --git a/src/everos/infra/persistence/index/predicate.py b/src/everos/infra/persistence/index/predicate.py index 4892356eb..cf5f234fd 100644 --- a/src/everos/infra/persistence/index/predicate.py +++ b/src/everos/infra/persistence/index/predicate.py @@ -1,4 +1,4 @@ -"""Compatibility re-exports for the shared derived-index predicate AST.""" +"""Public predicate API for the derived-index boundary.""" from everos.infra.persistence.predicate import All as All from everos.infra.persistence.predicate import AnyOf as AnyOf diff --git a/src/everos/infra/persistence/index/protocols.py b/src/everos/infra/persistence/index/protocols.py new file mode 100644 index 000000000..d8e31f96f --- /dev/null +++ b/src/everos/infra/persistence/index/protocols.py @@ -0,0 +1,148 @@ +"""Typed ports for the rebuildable derived-index subsystem.""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Sequence +from typing import Any, Protocol, TypeVar, runtime_checkable + +from pydantic import BaseModel + +from .predicate import Predicate + +T_co = TypeVar("T_co", bound=BaseModel, covariant=True) +T = TypeVar("T", bound=BaseModel) + + +@runtime_checkable +class IndexRepository(Protocol[T]): + """Backend-neutral repository contract used by memory and cascade. + + Maintenance methods are deliberately part of the contract. Embedded + engines can perform physical work; service-managed engines implement + successful no-ops. This keeps scheduling and health semantics identical + without making the cascade worker know which backend is active. + """ + + schema: type[T] + + @property + def table_name(self) -> str: ... + + async def add(self, records: Sequence[T]) -> None: ... + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: ... + + async def count(self) -> int: ... + + async def count_where(self, where: Predicate | None = None) -> int: ... + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: ... + + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: ... + + async def find_one_where(self, where: Predicate) -> T | None: ... + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: ... + + async def find_by_md_path(self, md_path: str) -> T | None: ... + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: ... + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: ... + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: ... + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: ... + + async def find_where_paginated( + self, + where: Predicate, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: ... + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Predicate | None = None, + limit: int = 10, + ) -> list[dict[str, Any]]: ... + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Predicate | None, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: ... + + async def dense_search( + self, + vector: Sequence[float], + where: Predicate | None, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: ... + + async def scan(self, where: Predicate | None = None) -> list[T]: ... + + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: ... + + async def delete(self, predicate: Predicate) -> None: ... + + async def delete_by_md_path(self, md_path: str) -> int: ... + + async def optimize(self) -> None: ... + + async def prune(self, older_than: dt.timedelta) -> None: ... + + async def rebuild_indexes(self) -> None: ... + + +@runtime_checkable +class IndexBackend(Protocol): + """Lifecycle contract for one configured derived-index backend.""" + + name: str + repositories: tuple[IndexRepository[Any], ...] + + async def connect(self) -> object: ... + + async def startup(self) -> object: ... + + async def shutdown(self) -> None: ... + + async def ensure_business_indexes(self) -> None: ... + + async def verify_business_schemas(self) -> None: ... + + async def drop_business_tables(self) -> list[str]: ... + + +__all__ = ["IndexBackend", "IndexRepository"] diff --git a/src/everos/infra/persistence/index/router.py b/src/everos/infra/persistence/index/router.py new file mode 100644 index 000000000..0f772dcb7 --- /dev/null +++ b/src/everos/infra/persistence/index/router.py @@ -0,0 +1,269 @@ +"""Route stable repository objects to the configured derived-index backend.""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Sequence +from typing import Any + +from pydantic import BaseModel + +from everos.config import load_settings + +from .predicate import Predicate +from .protocols import IndexRepository + + +class RoutedIndexRepository[T: BaseModel]: + """Stable repository identity with a backend selected at call time.""" + + def __init__( + self, + lance_repo: IndexRepository[T], + milvus_repo_name: str, + ) -> None: + self._lance_repo = lance_repo + self._milvus_repo_name = milvus_repo_name + self.schema = lance_repo.schema + + @property + def table_name(self) -> str: + return self._lance_repo.table_name + + def _repo(self) -> IndexRepository[T]: + if load_settings().index.backend == "milvus": + from everos.infra.persistence import milvus + + return getattr(milvus, self._milvus_repo_name) + return self._lance_repo + + async def add(self, records: Sequence[T]) -> None: + await self._repo().add(records) + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: + await self._repo().upsert(records, by=by) + + async def count(self) -> int: + return await self._repo().count() + + async def count_where(self, where: Predicate | None = None) -> int: + return await self._repo().count_where(where) + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: + return await self._repo().get_by_id(id_value, id_field=id_field) + + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: + return await self._repo().find_where(where, limit=limit) + + async def find_one_where(self, where: Predicate) -> T | None: + return await self._repo().find_one_where(where) + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: + return await self._repo().find_by_owner(owner_id, limit=limit) + + async def find_by_md_path(self, md_path: str) -> T | None: + return await self._repo().find_by_md_path(md_path) + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: + return await self._repo().find_by_owner_entry( + owner_id, + entry_id, + app_id=app_id, + project_id=project_id, + ) + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: + return await self._repo().find_by_owner_entries( + owner_id, + entry_ids, + app_id=app_id, + project_id=project_id, + ) + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: + return await self._repo().find_by_session( + owner_id, + session_id, + limit=limit, + ) + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: + return await self._repo().find_by_parent( + parent_type, + parent_id, + limit=limit, + ) + + async def find_where_paginated( + self, + where: Predicate, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: + return await self._repo().find_where_paginated( + where, + sort_by=sort_by, + descending=descending, + page=page, + page_size=page_size, + max_fetch=max_fetch, + ) + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Predicate | None = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + return await self._repo().search(vector=vector, where=where, limit=limit) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Predicate | None, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + return await self._repo().sparse_search( + query_terms, + where, + columns=columns, + limit=limit, + ) + + async def dense_search( + self, + vector: Sequence[float], + where: Predicate | None, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: + return await self._repo().dense_search( + vector, + where, + limit=limit, + vector_field=vector_field, + ) + + async def scan(self, where: Predicate | None = None) -> list[T]: + return await self._repo().scan(where) + + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: + await self._repo().update(updates, where=where) + + async def delete(self, predicate: Predicate) -> None: + await self._repo().delete(predicate) + + async def delete_by_md_path(self, md_path: str) -> int: + return await self._repo().delete_by_md_path(md_path) + + async def optimize(self) -> None: + await self._repo().optimize() + + async def prune(self, older_than: dt.timedelta) -> None: + await self._repo().prune(older_than) + + async def rebuild_indexes(self) -> None: + await self._repo().rebuild_indexes() + + +class RoutedEpisodeRepository(RoutedIndexRepository[Any]): + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: + return await self._repo().count_by_owner( # type: ignore[attr-defined] + owner_id, + app_id=app_id, + project_id=project_id, + parent_type=parent_type, + ) + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[Any]: + return await self._repo().list_by_owner_after_ts( # type: ignore[attr-defined] + owner_id=owner_id, + after_ts=after_ts, + parent_type=parent_type, + app_id=app_id, + project_id=project_id, + columns=columns, + limit=limit, + ) + + +class RoutedAgentSkillRepository(RoutedIndexRepository[Any]): + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: + return await self._repo().count_in_cluster( # type: ignore[attr-defined] + owner_id=owner_id, + cluster_id=cluster_id, + ) + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[Any]: + return await self._repo().find_in_cluster( # type: ignore[attr-defined] + owner_id=owner_id, + cluster_id=cluster_id, + limit=limit, + ) + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[Any]: + return await self._repo().find_topk_relevant_in_cluster( # type: ignore[attr-defined] + owner_id=owner_id, + cluster_id=cluster_id, + query_vector=query_vector, + top_k=top_k, + ) + + +__all__ = [ + "RoutedAgentSkillRepository", + "RoutedEpisodeRepository", + "RoutedIndexRepository", +] diff --git a/src/everos/infra/persistence/index/schema.py b/src/everos/infra/persistence/index/schema.py index dd9294baa..0c6178b2b 100644 --- a/src/everos/infra/persistence/index/schema.py +++ b/src/everos/infra/persistence/index/schema.py @@ -1,9 +1,9 @@ -"""Backend-neutral schemas for rebuildable business indexes. +"""Logical schema descriptions shared by derived-index adapters. -Each table has one explicit field description consumed by every backend -adapter. The model-field parity check fails immediately when a domain field is -added without a storage decision, avoiding fallback coercions and silently -divergent schemas. +The record models remain the source of truth for logical fields. This module +normalizes their supported types into a small backend-neutral vocabulary and +fails loudly when a new field has no portable representation. Physical limits +and index options remain adapter responsibilities. """ from __future__ import annotations @@ -17,7 +17,6 @@ _DEFAULT_STRING_LENGTH = 65_535 _ID_LENGTH = 512 _ARRAY_CAPACITY = 256 -_VECTOR_DIMENSION = 1024 class IndexFieldKind(StrEnum): @@ -66,229 +65,16 @@ def datetime_fields(self) -> frozenset[str]: ) -def _s( - name: str, - *, - nullable: bool = False, - primary: bool = False, - max_length: int = _DEFAULT_STRING_LENGTH, -) -> IndexField: - return IndexField( - name, - IndexFieldKind.STRING, - nullable=nullable, - primary=primary, - max_length=max_length, - ) - - -def _id(name: str = "id") -> IndexField: - return _s(name, primary=True, max_length=_ID_LENGTH) - - -def _a(name: str, *, nullable: bool = False) -> IndexField: - return IndexField( - name, - IndexFieldKind.STRING_ARRAY, - nullable=nullable, - max_length=_ID_LENGTH, - max_capacity=_ARRAY_CAPACITY, - ) - - -def _f(name: str, *, nullable: bool = False) -> IndexField: - return IndexField(name, IndexFieldKind.FLOAT, nullable=nullable) - - -def _i(name: str, *, nullable: bool = False) -> IndexField: - return IndexField(name, IndexFieldKind.INTEGER, nullable=nullable) - - -def _d(name: str, *, nullable: bool = False) -> IndexField: - return IndexField(name, IndexFieldKind.DATETIME, nullable=nullable) - - -def _v(name: str, *, nullable: bool = True) -> IndexField: - return IndexField( - name, - IndexFieldKind.DENSE_VECTOR, - nullable=nullable, - dimension=_VECTOR_DIMENSION, - ) - - -_FIELDS: dict[str, tuple[IndexField, ...]] = { - "episode": ( - _id(), - _s("entry_id"), - _s("owner_id"), - _s("owner_type"), - _s("app_id"), - _s("project_id"), - _s("session_id", nullable=True), - _d("timestamp"), - _s("parent_type"), - _s("parent_id"), - _a("sender_ids"), - _s("subject", nullable=True), - _s("summary", nullable=True), - _s("episode"), - _s("episode_tokens"), - _s("md_path"), - _s("content_sha256"), - _s("deprecated_by", nullable=True), - _v("vector"), - _v("subject_vector"), - _d("created_at"), - _d("updated_at"), - ), - "atomic_fact": ( - _id(), - _s("entry_id"), - _s("owner_id"), - _s("owner_type"), - _s("app_id"), - _s("project_id"), - _s("session_id", nullable=True), - _d("timestamp"), - _s("parent_type"), - _s("parent_id"), - _a("sender_ids"), - _s("fact"), - _s("fact_tokens"), - _s("md_path"), - _s("content_sha256"), - _s("deprecated_by", nullable=True), - _v("vector"), - _d("created_at"), - _d("updated_at"), - ), - "foresight": ( - _id(), - _s("entry_id"), - _s("owner_id"), - _s("owner_type"), - _s("app_id"), - _s("project_id"), - _s("session_id", nullable=True), - _d("timestamp"), - _d("start_time", nullable=True), - _d("end_time", nullable=True), - _i("duration_days", nullable=True), - _s("parent_type"), - _s("parent_id"), - _a("sender_ids"), - _s("foresight"), - _s("foresight_tokens"), - _s("evidence", nullable=True), - _s("evidence_tokens", nullable=True), - _s("md_path"), - _s("content_sha256"), - _v("vector"), - _d("created_at"), - _d("updated_at"), - ), - "agent_case": ( - _id(), - _s("entry_id"), - _s("owner_id"), - _s("owner_type"), - _s("app_id"), - _s("project_id"), - _s("session_id"), - _d("timestamp"), - _s("parent_type"), - _s("parent_id"), - _f("quality_score"), - _s("task_intent"), - _s("task_intent_tokens"), - _s("approach"), - _s("approach_tokens"), - _s("key_insight", nullable=True), - _s("md_path"), - _s("content_sha256"), - _v("vector"), - _d("created_at"), - _d("updated_at"), - ), - "agent_skill": ( - _id(), - _s("owner_id"), - _s("owner_type"), - _s("app_id"), - _s("project_id"), - _s("name"), - _s("description"), - _s("description_tokens"), - _s("content"), - _s("content_tokens"), - _f("confidence"), - _f("maturity_score"), - _a("source_case_ids"), - _s("cluster_id", nullable=True), - _s("md_path"), - _s("content_sha256"), - _v("vector"), - _d("created_at"), - _d("updated_at"), - ), - "user_profile": ( - _id(), - _s("owner_id"), - _s("owner_type"), - _s("app_id"), - _s("project_id"), - _s("summary"), - _s("explicit_info_json"), - _s("implicit_traits_json"), - _i("profile_timestamp_ms"), - _s("md_path"), - _s("content_sha256"), - _d("created_at"), - _d("updated_at"), - ), - "knowledge_topic": ( - _id(), - _s("doc_id"), - _s("category_id"), - _s("app_id"), - _s("project_id"), - _s("topic_name"), - _s("topic_path"), - _i("depth"), - _s("parent_node_id"), - _s("summary"), - _s("summary_tokens"), - _s("content_tokens"), - _a("content_labels"), - _s("md_path"), - _s("content_sha256"), - _v("vector"), - _d("created_at"), - _d("updated_at"), - ), -} - - @cache def schema_for(model: type[Any]) -> IndexSchema: - """Return and validate the explicit neutral schema for a model.""" - table_name = model.TABLE_NAME - try: - fields = _FIELDS[table_name] - except KeyError as exc: - raise ValueError(f"no derived-index schema for {table_name!r}") from exc - + """Build and validate the portable logical schema for ``model``.""" + table_name = _class_var(model, "TABLE_NAME") + bm25_fields = tuple(_class_var(model, "BM25_FIELDS")) + fields = tuple( + _normalize_field(name, model_field.annotation) + for name, model_field in model.model_fields.items() + ) declared = {field.name for field in fields} - actual = set(model.model_fields) - if declared != actual: - raise ValueError( - f"derived-index schema drift for {table_name!r}: " - f"missing={sorted(actual - declared)}, stale={sorted(declared - actual)}" - ) - for field in fields: - _validate_model_field(table_name, field, model.model_fields[field.name]) - bm25_fields = tuple(model.BM25_FIELDS) unknown_bm25 = set(bm25_fields) - declared if unknown_bm25: raise ValueError( @@ -298,44 +84,66 @@ def schema_for(model: type[Any]) -> IndexSchema: return IndexSchema(table_name, model, fields, bm25_fields) -def _validate_model_field(table_name: str, field: IndexField, model_field: Any) -> None: - annotation = model_field.annotation +def _normalize_field(name: str, annotation: Any) -> IndexField: args = get_args(annotation) optional = type(None) in args - candidates = ( - tuple(arg for arg in args if arg is not type(None)) - if optional - else (annotation,) - ) - if optional != field.nullable: - raise ValueError( - f"derived-index schema {table_name}.{field.name} nullable drift: " - f"model={optional}, schema={field.nullable}" - ) - - valid = False - if field.kind is IndexFieldKind.STRING: - valid = candidates == (str,) - elif field.kind is IndexFieldKind.STRING_ARRAY: - valid = len(candidates) == 1 and ( - get_origin(candidates[0]) is list and get_args(candidates[0]) == (str,) - ) - elif field.kind is IndexFieldKind.FLOAT: - valid = candidates == (float,) - elif field.kind is IndexFieldKind.INTEGER: - valid = candidates == (int,) - elif field.kind is IndexFieldKind.DATETIME: - valid = candidates == (dt.datetime,) - elif field.kind is IndexFieldKind.DENSE_VECTOR: - dimension = getattr(candidates[0], "dim", None) if candidates else None + candidates = tuple(arg for arg in args if arg is not type(None)) if optional else () + value_type = candidates[0] if len(candidates) == 1 else annotation + + if value_type is str: + kind = IndexFieldKind.STRING + max_length = _ID_LENGTH if name == "id" else _DEFAULT_STRING_LENGTH + max_capacity = None + dimension = None + elif value_type is float: + kind = IndexFieldKind.FLOAT + max_length = None + max_capacity = None + dimension = None + elif value_type is int: + kind = IndexFieldKind.INTEGER + max_length = None + max_capacity = None + dimension = None + elif value_type is dt.datetime: + kind = IndexFieldKind.DATETIME + max_length = None + max_capacity = None + dimension = None + elif get_origin(value_type) is list and get_args(value_type) == (str,): + kind = IndexFieldKind.STRING_ARRAY + max_length = _ID_LENGTH + max_capacity = _ARRAY_CAPACITY + dimension = None + else: + dimension = getattr(value_type, "dim", None) if callable(dimension): dimension = dimension() - valid = len(candidates) == 1 and dimension == field.dimension - if not valid: - raise ValueError( - f"derived-index schema {table_name}.{field.name} type drift: " - f"model={annotation!r}, schema={field.kind.value}" - ) + if not isinstance(dimension, int) or dimension <= 0: + raise ValueError( + f"derived-index field {name!r} has no portable type mapping: " + f"{annotation!r}" + ) + kind = IndexFieldKind.DENSE_VECTOR + max_length = None + max_capacity = None + + return IndexField( + name=name, + kind=kind, + nullable=optional, + primary=name == "id", + max_length=max_length, + max_capacity=max_capacity, + dimension=dimension, + ) + + +def _class_var(model: type[Any], name: str) -> Any: + value = getattr(model, name, None) + if value is None: + raise ValueError(f"derived-index model {model.__name__} has no {name}") + return value __all__ = ["IndexField", "IndexFieldKind", "IndexSchema", "schema_for"] diff --git a/src/everos/infra/persistence/lancedb/predicate.py b/src/everos/infra/persistence/lancedb/predicate.py deleted file mode 100644 index 8a429d57e..000000000 --- a/src/everos/infra/persistence/lancedb/predicate.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Render backend-neutral predicates as LanceDB DataFusion expressions.""" - -from __future__ import annotations - -import datetime as dt -import re -from typing import Final - -from everos.component.utils.datetime import ensure_utc, to_iso_format -from everos.infra.persistence.predicate import ( - All, - AnyOf, - Comparison, - Contains, - In, - IsNull, - Predicate, - Scalar, -) - -_FIELD_NAME: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -_OPERATORS: Final[dict[str, str]] = { - "eq": "=", - "ne": "!=", - "gt": ">", - "gte": ">=", - "lt": "<", - "lte": "<=", -} - - -def render_predicate(predicate: Predicate | None) -> str: - """Render a predicate with LanceDB escaping and timestamp literals.""" - if predicate is None: - return "" - if isinstance(predicate, Comparison): - return ( - f"{_field(predicate.field)} {_OPERATORS[predicate.operator]} " - f"{_literal(predicate.value)}" - ) - if isinstance(predicate, In): - values = ", ".join(_literal(value) for value in predicate.values) - return f"{_field(predicate.field)} IN ({values})" - if isinstance(predicate, Contains): - return f"array_has({_field(predicate.field)}, {_literal(predicate.value)})" - if isinstance(predicate, IsNull): - return f"{_field(predicate.field)} IS NULL" - if isinstance(predicate, All): - return _render_group(predicate.children, "AND") - if isinstance(predicate, AnyOf): - return _render_group(predicate.children, "OR") - raise TypeError(f"unsupported predicate: {type(predicate).__name__}") - - -def _render_group(children: tuple[Predicate, ...], operator: str) -> str: - rendered = [render_predicate(child) for child in children] - rendered = [item for item in rendered if item] - if not rendered: - return "" - if len(rendered) == 1: - return rendered[0] - return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" - - -def _field(value: str) -> str: - if not _FIELD_NAME.fullmatch(value): - raise ValueError(f"invalid predicate field: {value!r}") - return value - - -def _literal(value: Scalar) -> str: - if isinstance(value, str): - return f"'{value.replace(chr(39), chr(39) * 2)}'" - if isinstance(value, dt.datetime): - aware = ensure_utc(value) - assert aware is not None - return f"TIMESTAMP '{to_iso_format(aware)}'" - if isinstance(value, bool): - return "TRUE" if value else "FALSE" - return str(value) - - -__all__ = ["render_predicate"] diff --git a/src/everos/infra/persistence/milvus/predicate.py b/src/everos/infra/persistence/milvus/predicate.py index 37f14b289..785e77605 100644 --- a/src/everos/infra/persistence/milvus/predicate.py +++ b/src/everos/infra/persistence/milvus/predicate.py @@ -35,6 +35,7 @@ def render_predicate( predicate: Predicate | None, *, datetime_fields: Collection[str] = (), + vector_fields: Collection[str] = (), ) -> str: """Render a predicate using Milvus operators and physical field names.""" if predicate is None: @@ -53,11 +54,13 @@ def render_predicate( f"{_literal(predicate.value)})" ) if isinstance(predicate, IsNull): + if predicate.field in vector_fields: + return f"{_field(predicate.field, ())}__present == false" return f"{_field(predicate.field, datetime_fields)} is null" if isinstance(predicate, All): - return _render_group(predicate.children, "and", datetime_fields) + return _render_group(predicate.children, "and", datetime_fields, vector_fields) if isinstance(predicate, AnyOf): - return _render_group(predicate.children, "or", datetime_fields) + return _render_group(predicate.children, "or", datetime_fields, vector_fields) raise TypeError(f"unsupported predicate: {type(predicate).__name__}") @@ -65,9 +68,15 @@ def _render_group( children: tuple[Predicate, ...], operator: str, datetime_fields: Collection[str], + vector_fields: Collection[str], ) -> str: rendered = [ - render_predicate(child, datetime_fields=datetime_fields) for child in children + render_predicate( + child, + datetime_fields=datetime_fields, + vector_fields=vector_fields, + ) + for child in children ] rendered = [item for item in rendered if item] if not rendered: diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index 44eaa11f5..cf8958665 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -192,12 +192,18 @@ async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None: """Milvus indexes and compaction are service-managed.""" + async def prune(self, older_than: dt.timedelta) -> None: + """Milvus compaction and retention are service-managed.""" + async def rebuild_indexes(self) -> None: """Milvus AUTOINDEX maintenance is service-managed.""" async def count(self) -> int: return await self._count_where(None) + async def count_where(self, where: Predicate | None = None) -> int: + return await self._count_where(where) + async def _count_where(self, where: Predicate | None) -> int: await self.ensure_collection() client = await get_client() @@ -231,6 +237,46 @@ async def find_one_where(self, where: Predicate) -> T | None: rows = await self.find_where(where, limit=1) return rows[0] if rows else None + async def scan(self, where: Predicate | None = None) -> list[T]: + """Stream every matching row without a fixed query-window cap.""" + rows = await self._scan_raw(where, include_vectors=True) + return [self._model_from_milvus(row) for row in rows] + + async def _scan_raw( + self, + where: Predicate | None, + *, + include_vectors: bool, + max_rows: int | None = None, + ) -> list[dict[str, Any]]: + """Read through Milvus' iterator, optionally stopping at a soft cap. + + A normal ``query(limit=...)`` cannot cross Milvus' 16,384-row result + window. The public pagination contract deliberately allows a 20,000 + candidate window, so both maintenance scans and paginated reads must + use the iterator API. + """ + await self.ensure_collection() + client = await get_client() + iterator = await _run( + client.query_iterator, + self.collection_name, + batch_size=1000, + limit=-1, + filter=self._expr(where), + output_fields=self._output_fields(include_vectors=include_vectors), + ) + rows: list[dict[str, Any]] = [] + try: + while batch := await _run(iterator.next): + rows.extend(batch) + if max_rows is not None and len(rows) >= max_rows: + del rows[max_rows:] + break + finally: + await _run(iterator.close) + return rows + async def find_where_paginated( self, where: Predicate, @@ -242,7 +288,11 @@ async def find_where_paginated( max_fetch: int = 20_000, ) -> tuple[list[T], int]: total = await self._count_where(where) - raw = await self._query_raw(where, limit=max_fetch, include_vectors=True) + raw = await self._scan_raw( + where, + include_vectors=True, + max_rows=max_fetch, + ) if total > len(raw): logger.warning( "milvus_find_where_paginated_truncated", @@ -690,6 +740,7 @@ def _expr(self, where: Predicate | None) -> str: return render_predicate( where, datetime_fields=self.index_schema.datetime_fields, + vector_fields={field.name for field in self.index_schema.vector_fields}, ) diff --git a/src/everos/infra/persistence/predicate.py b/src/everos/infra/persistence/predicate.py index def7ca69f..d030a10c6 100644 --- a/src/everos/infra/persistence/predicate.py +++ b/src/everos/infra/persistence/predicate.py @@ -1,4 +1,9 @@ -"""Backend-neutral predicate tree shared by derived-index adapters.""" +"""Backend-neutral predicate tree for rebuildable derived indexes. + +Application and memory code construct these nodes. A storage adapter owns +the rendering into its physical query language, so backend syntax never leaks +above :mod:`everos.infra.persistence`. +""" from __future__ import annotations diff --git a/src/everos/memory/cascade/_backfill.py b/src/everos/memory/cascade/_backfill.py index d78bd27ba..6f3df1002 100644 --- a/src/everos/memory/cascade/_backfill.py +++ b/src/everos/memory/cascade/_backfill.py @@ -42,26 +42,30 @@ from everos.core.errors import ProviderNotConfiguredError from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, MemoryRoot, SQLModel -from everos.core.persistence.lancedb import BaseLanceTable, LanceRepoBase from everos.infra.ome.config import OMEConfig from everos.infra.ome.engine import OfflineEngine from everos.infra.ome.exceptions import EngineLockHeldError from everos.infra.persistence.index import ( + ALL_REPOS, AgentCase, AgentSkill, AtomicFact, Episode, Foresight, + IndexRepository, KnowledgeTopic, agent_case_repo, agent_skill_repo, + any_of, atomic_fact_repo, episode_repo, eq, foresight_repo, + is_null, knowledge_topic_repo, + repo_for_schema, ) -from everos.infra.persistence.lancedb import BUSINESS_SCHEMAS_WITH_VECTOR, get_table +from everos.infra.persistence.index.schema import schema_for from everos.infra.persistence.markdown import AgentSkillFrontmatter from everos.infra.persistence.sqlite import cluster_repo, get_engine from everos.memory.cascade.worker import ( @@ -223,8 +227,8 @@ class _TableSpec: ``subject`` column — mirrors ``EpisodeHandler._build_row``. """ - schema: type[BaseLanceTable] - repo: LanceRepoBase[Any] + schema: type[Any] + repo: IndexRepository[Any] text_of: Callable[[dict[str, Any]], str] subject_of: Callable[[dict[str, Any]], str | None] | None = None @@ -257,7 +261,11 @@ def _agent_skill_embed_text(row: dict[str, Any]) -> str: # touches. Fail loud at import time so any drift shows up before a test # even starts. _TABLE_SPEC_NAMES = {spec.schema.TABLE_NAME for spec in _TABLE_SPECS} -_BUSINESS_SCHEMA_NAMES = {s.TABLE_NAME for s in BUSINESS_SCHEMAS_WITH_VECTOR} +_BUSINESS_SCHEMA_NAMES = { + repo.schema.TABLE_NAME + for repo in ALL_REPOS + if schema_for(repo.schema).vector_fields +} if _TABLE_SPEC_NAMES != _BUSINESS_SCHEMA_NAMES: raise RuntimeError( f"_TABLE_SPECS drift: BUSINESS_SCHEMAS_WITH_VECTOR has " @@ -340,12 +348,7 @@ class _PhaseResult: blocked_by_server: bool = False -def _q(value: str) -> str: - """Defensive SQL-quote escape (mirrors the lancedb repo convention).""" - return value.replace("'", "''") - - -def _null_filter(spec: _TableSpec) -> str: +def _null_filter(spec: _TableSpec): # type: ignore[no-untyped-def] """Where-clause the scan uses to spot rows that need (some) embedding. Episode carries a ``subject_vector`` alongside ``vector`` — the two @@ -362,8 +365,8 @@ def _null_filter(spec: _TableSpec) -> str: query, so they keep the round-1 filter. """ if spec.schema.TABLE_NAME == Episode.TABLE_NAME: - return "vector IS NULL OR subject_vector IS NULL" - return "vector IS NULL" + return any_of(is_null("vector"), is_null("subject_vector")) + return is_null("vector") class _RowSkipped: @@ -456,8 +459,8 @@ async def _scan_null_vector_backlog() -> tuple[list[_TableBacklog], int]: backlog: list[_TableBacklog] = [] scan_failed = 0 for spec in _TABLE_SPECS: - table = await get_table(spec.schema.TABLE_NAME, spec.schema) - raw_rows = await table.query().where(_null_filter(spec)).to_list() + records = await spec.repo.scan(_null_filter(spec)) + raw_rows = [record.model_dump(mode="python") for record in records] rows: list[_NullVectorRow] = [] for raw in raw_rows: row = _extract_row(raw, spec, tokenizer) @@ -917,7 +920,7 @@ class _ClusterPhaseResult: blocked_by_capability: str | None = None -async def _scan_all_rows(schema: type[BaseLanceTable]) -> list[dict[str, Any]]: +async def _scan_all_rows(schema: type[Any]) -> list[dict[str, Any]]: """Fetch every row of ``schema``'s table, no filter. Phase 2 doesn't care whether a row already carries a vector — the @@ -934,11 +937,8 @@ async def _scan_all_rows(schema: type[BaseLanceTable]) -> list[dict[str, Any]]: apply it themselves on the returned rows — this helper stays a plain unfiltered fetch shared by every business table. """ - table = await get_table(schema.TABLE_NAME, schema) - total = await table.count_rows() - if total == 0: - return [] - return await table.query().limit(total).to_list() + records = await repo_for_schema(schema).scan() + return [record.model_dump(mode="python") for record in records] def _episode_row_to_event(raw: dict[str, Any]) -> EpisodeExtracted: diff --git a/src/everos/memory/cascade/handlers/_daily_log_base.py b/src/everos/memory/cascade/handlers/_daily_log_base.py index 52f67b781..dc59287c5 100644 --- a/src/everos/memory/cascade/handlers/_daily_log_base.py +++ b/src/everos/memory/cascade/handlers/_daily_log_base.py @@ -15,7 +15,7 @@ — editing them does NOT propagate to LanceDB and does NOT waste an embed call. -Subclasses bind their ``kind`` / ``lance_repo`` / ``content_change_keys`` +Subclasses bind their ``kind`` / ``index_repo`` / ``content_change_keys`` as ClassVars and override :meth:`_build_row` to do the per-kind field mapping. Everything else — read, diff, embed call, upsert, delete — lives here. @@ -58,7 +58,7 @@ class BaseDailyLogHandler(Handler): Subclass requirements: - :attr:`kind` (ClassVar[str]) — registry name, surfaces in logs. - - :attr:`lance_repo` (ClassVar) — the LanceDB repo singleton for + - :attr:`index_repo` (ClassVar) — the LanceDB repo singleton for this kind (must expose ``find_where`` / ``upsert`` / ``delete`` / ``delete_by_md_path``). - :attr:`content_change_keys` (ClassVar[tuple[str, ...]]) — the @@ -71,7 +71,7 @@ class BaseDailyLogHandler(Handler): """ kind: ClassVar[str] = "" - lance_repo: ClassVar[Any] = None + index_repo: ClassVar[Any] = None content_change_keys: ClassVar[tuple[str, ...]] = () def _content_sha256(self, structured: StructuredEntry) -> str: @@ -108,7 +108,7 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: for entry in parsed.entries ] - existing = await self.lance_repo.find_where( + existing = await self.index_repo.find_where( eq("md_path", md_path), limit=10_000, ) @@ -197,17 +197,14 @@ async def _apply_lance_changes( ) -> None: """Flush upserts and deletes to LanceDB.""" if to_upsert: - await self.lance_repo.upsert(to_upsert) + await self.index_repo.upsert(to_upsert) if to_delete_ids: - await self.lance_repo.delete( - all_of( - eq("md_path", md_path), - one_of("entry_id", to_delete_ids), - ) + await self.index_repo.delete( + all_of(eq("md_path", md_path), one_of("entry_id", to_delete_ids)) ) async def handle_deleted(self, md_path: str) -> HandlerOutcome: - deleted = await self.lance_repo.delete_by_md_path(md_path) + deleted = await self.index_repo.delete_by_md_path(md_path) return HandlerOutcome( md_path=md_path, kind=self.kind, @@ -262,7 +259,7 @@ async def _mark_deprecated( eq("project_id", project_id), ) try: - await self.lance_repo.update( + await self.index_repo.update( {"deprecated_by": deprecated_by}, where=predicate, ) diff --git a/src/everos/memory/cascade/handlers/agent_case.py b/src/everos/memory/cascade/handlers/agent_case.py index 2f6bcca20..5ba0fae3b 100644 --- a/src/everos/memory/cascade/handlers/agent_case.py +++ b/src/everos/memory/cascade/handlers/agent_case.py @@ -42,7 +42,7 @@ class AgentCaseHandler(BaseDailyLogHandler): """Cascade handler for ``agents//.cases/agent_case-*.md``.""" kind = "agent_case" - lance_repo = agent_case_repo + index_repo = agent_case_repo content_change_keys = ( "section:TaskIntent", "section:Approach", diff --git a/src/everos/memory/cascade/handlers/agent_skill.py b/src/everos/memory/cascade/handlers/agent_skill.py index 1e492eec1..a72d1371b 100644 --- a/src/everos/memory/cascade/handlers/agent_skill.py +++ b/src/everos/memory/cascade/handlers/agent_skill.py @@ -64,7 +64,7 @@ class AgentSkillHandler(Handler): ``agents//skills/skill_/SKILL.md`` + ``references/*.md``.""" kind = "agent_skill" - lance_repo: ClassVar[Any] = agent_skill_repo + index_repo: ClassVar[Any] = agent_skill_repo """Exposed for ``CascadeWorker._optimize_touched_kinds`` — see the matching note on :class:`UserProfileHandler`.""" diff --git a/src/everos/memory/cascade/handlers/atomic_fact.py b/src/everos/memory/cascade/handlers/atomic_fact.py index 673423369..6dee3b94a 100644 --- a/src/everos/memory/cascade/handlers/atomic_fact.py +++ b/src/everos/memory/cascade/handlers/atomic_fact.py @@ -35,7 +35,7 @@ class AtomicFactHandler(BaseDailyLogHandler): """Cascade handler for ``users//.atomic_facts/atomic_fact-*.md``.""" kind = "atomic_fact" - lance_repo = atomic_fact_repo + index_repo = atomic_fact_repo content_change_keys = ("section:Fact",) """Only ``Fact`` matters — it's both the embedded text AND the BM25 source. Audit inline is excluded.""" diff --git a/src/everos/memory/cascade/handlers/episode.py b/src/everos/memory/cascade/handlers/episode.py index 57dd6e5ad..c8980650b 100644 --- a/src/everos/memory/cascade/handlers/episode.py +++ b/src/everos/memory/cascade/handlers/episode.py @@ -50,7 +50,7 @@ class EpisodeHandler(BaseDailyLogHandler): """Cascade handler for ``users//episodes/episode-*.md``.""" kind = "episode" - lance_repo = episode_repo + index_repo = episode_repo content_change_keys = ( "section:Subject", "section:Summary", diff --git a/src/everos/memory/cascade/handlers/foresight.py b/src/everos/memory/cascade/handlers/foresight.py index 02aeea926..2c618df18 100644 --- a/src/everos/memory/cascade/handlers/foresight.py +++ b/src/everos/memory/cascade/handlers/foresight.py @@ -47,7 +47,7 @@ class ForesightHandler(BaseDailyLogHandler): """Cascade handler for ``users//.foresights/foresight-*.md``.""" kind = "foresight" - lance_repo = foresight_repo + index_repo = foresight_repo content_change_keys = ( "section:Foresight", "section:Evidence", diff --git a/src/everos/memory/cascade/handlers/knowledge_topic.py b/src/everos/memory/cascade/handlers/knowledge_topic.py index 2535a0572..16fc443a2 100644 --- a/src/everos/memory/cascade/handlers/knowledge_topic.py +++ b/src/everos/memory/cascade/handlers/knowledge_topic.py @@ -57,7 +57,7 @@ class KnowledgeTopicHandler(Handler): ``knowledge/{category}/{doc_title}/_.md``.""" kind = "knowledge_topic" - lance_repo: ClassVar[Any] = knowledge_topic_repo + index_repo: ClassVar[Any] = knowledge_topic_repo content_change_keys: ClassVar[tuple[str, ...]] = ( "frontmatter:summary", diff --git a/src/everos/memory/cascade/handlers/user_profile.py b/src/everos/memory/cascade/handlers/user_profile.py index 8d7573521..460a017d6 100644 --- a/src/everos/memory/cascade/handlers/user_profile.py +++ b/src/everos/memory/cascade/handlers/user_profile.py @@ -39,7 +39,7 @@ class UserProfileHandler(Handler): """Cascade handler for ``users//user.md``.""" kind = "user_profile" - lance_repo: ClassVar[Any] = user_profile_repo + index_repo: ClassVar[Any] = user_profile_repo """Exposed for ``CascadeWorker._optimize_touched_kinds`` — the worker discovers the LanceDB repo to optimize via this attribute, mirroring the daily-log handlers that bind it through diff --git a/src/everos/memory/cascade/registry.py b/src/everos/memory/cascade/registry.py index 3054a17a3..feecebbe3 100644 --- a/src/everos/memory/cascade/registry.py +++ b/src/everos/memory/cascade/registry.py @@ -66,7 +66,7 @@ class KindSpec: ``frontmatter_schema`` carries the ``path_glob()`` classmethod the scanner uses to enumerate eligible files; the same schema is also the contract the reader / writer share at the markdown layer. - ``lance_schema`` + ``lance_repo`` describe the destination side. + ``index_schema`` + ``index_repo`` describe the destination side. ``handler_factory`` is a callable that receives the shared :class:`HandlerDeps` bundle and returns the kind's :class:`Handler`. """ @@ -74,8 +74,8 @@ class KindSpec: name: str frontmatter_schema: type[BaseFrontmatter] handler_factory: type[Handler] - lance_schema: type | None = None - lance_repo: object | None = None + index_schema: type | None = None + index_repo: object | None = None def path_glob(self) -> str: """Glob (relative to memory root) for every md this kind covers.""" @@ -96,58 +96,58 @@ def matches(self, rel_md_path: str) -> bool: KindSpec( name="episode", frontmatter_schema=EpisodeDailyFrontmatter, - lance_schema=Episode, - lance_repo=episode_repo, + index_schema=Episode, + index_repo=episode_repo, handler_factory=EpisodeHandler, ), KindSpec( name="atomic_fact", frontmatter_schema=AtomicFactDailyFrontmatter, - lance_schema=AtomicFact, - lance_repo=atomic_fact_repo, + index_schema=AtomicFact, + index_repo=atomic_fact_repo, handler_factory=AtomicFactHandler, ), KindSpec( name="foresight", frontmatter_schema=ForesightDailyFrontmatter, - lance_schema=Foresight, - lance_repo=foresight_repo, + index_schema=Foresight, + index_repo=foresight_repo, handler_factory=ForesightHandler, ), KindSpec( name="agent_case", frontmatter_schema=AgentCaseDailyFrontmatter, - lance_schema=AgentCase, - lance_repo=agent_case_repo, + index_schema=AgentCase, + index_repo=agent_case_repo, handler_factory=AgentCaseHandler, ), KindSpec( name="agent_skill", frontmatter_schema=AgentSkillFrontmatter, - lance_schema=AgentSkill, - lance_repo=agent_skill_repo, + index_schema=AgentSkill, + index_repo=agent_skill_repo, handler_factory=AgentSkillHandler, ), KindSpec( name="user_profile", frontmatter_schema=UserProfileFrontmatter, - lance_schema=UserProfile, - lance_repo=user_profile_repo, + index_schema=UserProfile, + index_repo=user_profile_repo, handler_factory=UserProfileHandler, ), KindSpec( name="knowledge_document", frontmatter_schema=KnowledgeDocumentFrontmatter, handler_factory=KnowledgeDocumentHandler, - lance_schema=None, - lance_repo=None, + index_schema=None, + index_repo=None, ), KindSpec( name="knowledge_topic", frontmatter_schema=KnowledgeTopicFrontmatter, handler_factory=KnowledgeTopicHandler, - lance_schema=KnowledgeTopic, - lance_repo=knowledge_topic_repo, + index_schema=KnowledgeTopic, + index_repo=knowledge_topic_repo, ), ) """Every cascade kind, evaluated in declaration order by :func:`match_kind`.""" diff --git a/src/everos/memory/cascade/worker.py b/src/everos/memory/cascade/worker.py index 42232469a..7ab050a05 100644 --- a/src/everos/memory/cascade/worker.py +++ b/src/everos/memory/cascade/worker.py @@ -879,7 +879,7 @@ def _schedule_optimize(self, kind: str) -> None: LanceDB's per-table manifest version expects. No-op when the handler for ``kind`` doesn't expose a - ``lance_repo`` (test stubs, handlers that intentionally skip + ``index_repo`` (test stubs, handlers that intentionally skip LanceDB). Idempotent and cheap (a single dict lookup + flag write in @@ -887,7 +887,7 @@ def _schedule_optimize(self, kind: str) -> None: heartbeat sweep. """ handler = self._handlers.get(kind) - repo = getattr(handler, "lance_repo", None) if handler else None + repo = getattr(handler, "index_repo", None) if handler else None if repo is None: return state = self._optimizer_states.setdefault(kind, _KindOptimizerState()) @@ -987,7 +987,7 @@ async def _run_optimize_once(self, kind: str) -> None: manifest, and prune is a write. """ handler = self._handlers.get(kind) - repo = getattr(handler, "lance_repo", None) if handler else None + repo = getattr(handler, "index_repo", None) if handler else None if repo is None: return state = self._optimizer_states.get(kind) @@ -1201,7 +1201,7 @@ async def _run_rebuild_once(self, kind: str) -> None: during the rebuild coalesce instead of racing on the manifest. """ handler = self._handlers.get(kind) - repo = getattr(handler, "lance_repo", None) if handler else None + repo = getattr(handler, "index_repo", None) if handler else None if repo is None: return state = self._optimizer_states.setdefault(kind, _KindOptimizerState()) diff --git a/src/everos/memory/get/filters_adapter.py b/src/everos/memory/get/filters_adapter.py index cd23dcc67..268834037 100644 --- a/src/everos/memory/get/filters_adapter.py +++ b/src/everos/memory/get/filters_adapter.py @@ -50,7 +50,6 @@ def compile_filters_for_get_backends( app_id: str = "default", project_id: str = "default", ) -> Predicate: - """Compile ``/get`` filters for every derived index backend.""" return compile_filters_for_backends( filters, owner_id=owner_id, diff --git a/src/everos/memory/get/manager.py b/src/everos/memory/get/manager.py index 7a9421797..10bb1837e 100644 --- a/src/everos/memory/get/manager.py +++ b/src/everos/memory/get/manager.py @@ -12,7 +12,7 @@ Reads only — never writes. Filters are compiled through :func:`compile_filters_for_get` so the column allow-list stays shared with :mod:`memory.search`. Pagination + in-memory sort -runs through the configured derived index repository. +runs through :meth:`LanceRepoBase.find_where_paginated`. """ from __future__ import annotations @@ -37,11 +37,11 @@ from .filters_adapter import compile_filters_for_get_backends if TYPE_CHECKING: - from everos.core.persistence.lancedb import LanceRepoBase from everos.infra.persistence.index import ( AgentCase, AgentSkill, Episode, + IndexRepository, UserProfile, ) @@ -49,16 +49,16 @@ class GetManager: - """Dispatch ``GetRequest`` to the matching derived index repo and + """Dispatch ``GetRequest`` to the matching LanceDB-backed repo and shape rows into the public DTO.""" def __init__( self, *, - episode_repo: LanceRepoBase[Episode], - agent_case_repo: LanceRepoBase[AgentCase], - agent_skill_repo: LanceRepoBase[AgentSkill], - user_profile_repo: LanceRepoBase[UserProfile], + episode_repo: IndexRepository[Episode], + agent_case_repo: IndexRepository[AgentCase], + agent_skill_repo: IndexRepository[AgentSkill], + user_profile_repo: IndexRepository[UserProfile], ) -> None: self._ep = episode_repo self._case = agent_case_repo @@ -187,7 +187,7 @@ def _shape_agent_skill(row: AgentSkill) -> GetAgentSkillItem: async def _fetch_profile(self, owner_id: str) -> list[GetProfileItem]: """Fetch the owner's single profile row from the ``user_profile`` - Derived index row (kept in sync with ``users//user.md`` by cascade). + LanceDB table (kept in sync with ``users//user.md`` by cascade). Profile is one-row-per-owner KV — there is no pagination / sort / filter surface, so at most one item is returned. Mirrors the diff --git a/src/everos/memory/reflection/orchestrator.py b/src/everos/memory/reflection/orchestrator.py index 9c9e8c54d..58b04d50c 100644 --- a/src/everos/memory/reflection/orchestrator.py +++ b/src/everos/memory/reflection/orchestrator.py @@ -801,14 +801,14 @@ async def _apply_deprecation_writes( to_deprecate=to_deprecate, merged_entry_id=merged_entry_id, ) - deprecated_ep_count = await self._deprecate_index_episodes( + deprecated_ep_count = await self._deprecate_lance_episodes( entry_ids=to_deprecate, owner_id=owner_id, app_id=app_id, project_id=project_id, merged_entry_id=merged_entry_id, ) - deprecated_fact_count = await self._deprecate_index_facts( + deprecated_fact_count = await self._deprecate_lance_facts( parent_ids=to_deprecate, owner_id=owner_id, merged_entry_id=merged_entry_id, @@ -835,7 +835,7 @@ async def _resolve_deprecation_targets( original_ids = {mid for mid, _ in original_members} return original_ids & current_ids - async def _deprecate_index_episodes( + async def _deprecate_lance_episodes( self, *, entry_ids: set[str], @@ -844,10 +844,10 @@ async def _deprecate_index_episodes( project_id: str, merged_entry_id: str, ) -> int: - """Mark deprecated episodes in the active derived index by entry_id. + """Mark deprecated episodes in LanceDB by entry_id. Returns: - Number of derived-index update calls issued. + Number of LanceDB update calls issued. """ coros: list[Any] = [ self._episode_store.update( @@ -865,14 +865,14 @@ async def _deprecate_index_episodes( await asyncio.gather(*coros) return len(coros) - async def _deprecate_index_facts( + async def _deprecate_lance_facts( self, *, parent_ids: set[str], owner_id: str, merged_entry_id: str, ) -> int: - """Mark deprecated atomic facts in the active derived index. + """Mark deprecated atomic facts in LanceDB. Args: parent_ids: Parent IDs (memcell or episode) whose facts to deprecate. @@ -880,7 +880,7 @@ async def _deprecate_index_facts( merged_entry_id: Entry ID of the replacement merged episode. Returns: - Total number of derived-index update calls issued. + Total number of LanceDB update calls issued. """ if not parent_ids: return 0 diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 74ceeb2d5..73cd7c2ae 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -1,7 +1,7 @@ """Validate the public filter DSL and build a backend-neutral predicate. -Storage-specific syntax is intentionally absent from this module. LanceDB and -Milvus render the resulting predicate in their own persistence packages. +Storage-specific syntax is intentionally absent from this module. Adapters +render the resulting predicate in their own persistence packages. """ from __future__ import annotations @@ -79,7 +79,7 @@ def compile_filters_for_backends( app_id: str = "default", project_id: str = "default", ) -> Predicate: - """Compatibility name; one predicate is rendered by the active backend.""" + """Compatibility name for callers migrating from rendered filters.""" return compile_filters( node, owner_id=owner_id, diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index 7bb15a6b4..d5f3f944f 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -22,8 +22,7 @@ no query-relevance score we can assign to a fact pulled by parent_id alone, and emitting ``score=0.0`` facts would muddy the contract. -The manager never writes to storage; it only reads the derived index + -markdown. +The manager never writes to storage; it only reads LanceDB + markdown. """ from __future__ import annotations diff --git a/src/everos/memory/search/recall/agent_case.py b/src/everos/memory/search/recall/agent_case.py index 9c17ba45e..bfa71f47f 100644 --- a/src/everos/memory/search/recall/agent_case.py +++ b/src/everos/memory/search/recall/agent_case.py @@ -18,7 +18,7 @@ from everalgo.types import Candidate -from everos.infra.persistence.index import AgentCase, agent_case_repo +from everos.infra.persistence.index import AgentCase, Predicate, agent_case_repo from .base import ( RecallerDeps, @@ -38,7 +38,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Dual-column BM25 recall via OR-mode BooleanQuery per column. @@ -52,10 +52,7 @@ async def sparse_recall( if not terms: return [] merged_rows = await agent_case_repo.sparse_search( - terms, - where, - columns=AgentCase.BM25_FIELDS, - limit=limit, + terms, where, columns=AgentCase.BM25_FIELDS, limit=limit ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -63,7 +60,7 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: if not vector: return [] diff --git a/src/everos/memory/search/recall/agent_skill.py b/src/everos/memory/search/recall/agent_skill.py index 324084829..c2ad6295d 100644 --- a/src/everos/memory/search/recall/agent_skill.py +++ b/src/everos/memory/search/recall/agent_skill.py @@ -53,10 +53,7 @@ async def sparse_recall( if not terms: return [] merged_rows = await agent_skill_repo.sparse_search( - terms, - where, - columns=AgentSkill.BM25_FIELDS, - limit=limit, + terms, where, columns=AgentSkill.BM25_FIELDS, limit=limit ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index 885457e45..c4fa38e84 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -59,10 +59,7 @@ async def sparse_recall( if not terms: return [] rows = await atomic_fact_repo.sparse_search( - terms, - where, - columns=AtomicFact.BM25_FIELDS, - limit=limit, + terms, where, columns=AtomicFact.BM25_FIELDS, limit=limit ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -165,10 +162,12 @@ async def _query_facts_for_parents( ) -> list[dict[str, Any]]: """Construct and execute the LanceDB query for parent_id IN (...).""" full_where = all_of(where, one_of("parent_id", list(parent_to_eps))) - # Milvus server / Zilliz Cloud reject search topK values above 1024. - # The fact expansion only needs a bounded candidate pool for top-N - # competition, so keep the same cap for every derived-index backend. - limit = min(per_episode * max(len(parent_to_eps), 1), _MAX_FACT_RECALL_LIMIT) + # Keep one portable ceiling: Milvus/Zilliz reject larger search topK + # values, while fact expansion only needs a bounded competition pool. + limit = min( + per_episode * max(len(parent_to_eps), 1), + _MAX_FACT_RECALL_LIMIT, + ) if query_vector: return await atomic_fact_repo.dense_search( query_vector, full_where, limit=limit diff --git a/src/everos/memory/search/recall/base.py b/src/everos/memory/search/recall/base.py index 9b62c10f9..e77fd2354 100644 --- a/src/everos/memory/search/recall/base.py +++ b/src/everos/memory/search/recall/base.py @@ -24,14 +24,9 @@ from typing import Any, ClassVar, Protocol, runtime_checkable from everalgo.types import Candidate -from lancedb.query import BooleanQuery, FullTextQuery, MatchQuery - -try: - from lancedb.query import Occur -except ImportError: # pragma: no cover — fallback for older LanceDB layouts - from lancedb._lancedb import Occur # type: ignore[attr-defined,no-redef] from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.index import Predicate # Columns that should never travel through the ranker / shaper. ``vector`` # is huge (1024 floats); ``_distance`` belongs to LanceDB's query engine @@ -66,11 +61,11 @@ class KindRecaller(Protocol): """Source column for cross-encoder rerank passages (display text).""" async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: ... async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: ... @@ -112,90 +107,3 @@ def cosine_score_from_distance(distance: float | None) -> float: if sim > 1.0: return 1.0 return sim - - -def tokenize_query(tokenizer: Tokenizer, query: str) -> str: - """Run the configured tokenizer over the query and join with spaces. - - Cascade joins tokens with a single space when writing the - ``*_tokens`` columns; LanceDB FTS expects a whitespace-tokenised - query string. Same function on both sides keeps BM25 scoring - symmetric. - - Note: prefer :func:`build_or_query` for new code — it sidesteps - the tantivy implicit-AND query-parser pitfall where a single - IDF≈0 token (e.g. an owner's own name on the owner's partition) - poisons the whole query into zero hits. - """ - tokens = tokenizer.tokenize(query) - return " ".join(t for t in tokens if t) - - -def build_or_query( - tokenizer: Tokenizer, - query: str, - column: str, -) -> FullTextQuery | None: - """Tokenise ``query`` and wrap in an OR-mode FTS query. - - Mirrors the enterprise ES design - (``bool.should + minimum_should_match=1``): - - - jieba tokenises the user query. - - Each token becomes its own :class:`MatchQuery` clause. - - Clauses combine under :class:`BooleanQuery` with - :attr:`Occur.SHOULD` so any single matching token surfaces - the document. - - LanceDB still computes a proper joint BM25 score from all - clauses; tokens with IDF ≈ 0 contribute ~0 but no longer - poison the rest of the query. - - Returns ``None`` when the query tokenises to nothing (the - caller must guard against this — there's no useful BM25 query - over an empty token set). - - Single-token queries collapse to a bare :class:`MatchQuery` - (skipping the boolean wrapper) so the FTS engine doesn't pay - for an unnecessary boolean layer. - """ - tokens = [t for t in tokenizer.tokenize(query) if t] - if not tokens: - return None - if len(tokens) == 1: - return MatchQuery(tokens[0], column=column) - clauses: list[tuple[Occur, FullTextQuery]] = [ - (Occur.SHOULD, MatchQuery(t, column=column)) for t in tokens - ] - return BooleanQuery(clauses) - - -def build_or_query_multi_column( - tokenizer: Tokenizer, - query: str, - columns: Sequence[str], -) -> dict[str, FullTextQuery] | None: - """Same as :func:`build_or_query` but emit one FTS query per column. - - ``MatchQuery`` is bound to a single column, and LanceDB FTS only - searches one column per ``nearest_to_text`` call. Dual-column - kinds (``agent_case`` over ``task_intent_tokens`` / - ``approach_tokens``, ``agent_skill`` over - ``description_tokens`` / ``content_tokens``, etc.) need one - OR-bundle per column and merge the results in the caller. - - Returns ``None`` on empty tokenisation; otherwise a dict - ``{column: FullTextQuery}`` ready to feed into separate - ``nearest_to_text`` calls. - """ - tokens = [t for t in tokenizer.tokenize(query) if t] - if not tokens: - return None - out: dict[str, FullTextQuery] = {} - for col in columns: - if len(tokens) == 1: - out[col] = MatchQuery(tokens[0], column=col) - else: - out[col] = BooleanQuery( - [(Occur.SHOULD, MatchQuery(t, column=col)) for t in tokens] - ) - return out diff --git a/src/everos/memory/search/recall/episode.py b/src/everos/memory/search/recall/episode.py index f24e8ef8a..47db9c1fb 100644 --- a/src/everos/memory/search/recall/episode.py +++ b/src/everos/memory/search/recall/episode.py @@ -7,7 +7,6 @@ from everalgo.types import Candidate -from everos.core.observability.logging import get_logger from everos.infra.persistence.index import ( Episode, Predicate, @@ -36,10 +35,6 @@ def _inject_parent_id(candidates: list[Candidate]) -> list[Candidate]: ] -logger = get_logger(__name__) -_FETCH_ALL_OWNER_LIMIT = 10_000 - - class EpisodeRecaller: """BM25 + vector recall over the LanceDB ``episode`` table.""" @@ -65,10 +60,7 @@ async def sparse_recall( if not terms: return [] rows = await episode_repo.sparse_search( - terms, - where, - columns=Episode.BM25_FIELDS, - limit=limit, + terms, where, columns=Episode.BM25_FIELDS, limit=limit ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -113,10 +105,7 @@ async def dense_recall_subject( if not vector: return [] rows = await episode_repo.dense_search( - vector, - where, - limit=limit, - vector_field="subject_vector", + vector, where, limit=limit, vector_field="subject_vector" ) return [ row_to_candidate( @@ -141,15 +130,13 @@ async def fetch_all_for_owner(self, where: Predicate) -> list[Candidate]: ``Candidate.id`` against ``Cluster.members``. Both are now episode entry_ids regardless of parent_type. - The scan is capped at :data:`_FETCH_ALL_OWNER_LIMIT` to bound memory. - Hitting the cap is logged because cluster matching may be incomplete. + No ``limit`` — the full owner partition is required for cluster + membership matching. """ - rows = await episode_repo.search(where=where, limit=_FETCH_ALL_OWNER_LIMIT) - if len(rows) == _FETCH_ALL_OWNER_LIMIT: - logger.warning( - "episode_owner_scan_truncated", - limit=_FETCH_ALL_OWNER_LIMIT, - ) + rows = [ + record.model_dump(mode="python") + for record in await episode_repo.scan(where) + ] result: list[Candidate] = [] for r in rows: entry_id = r.get("entry_id") diff --git a/src/everos/memory/search/recall/knowledge_topic.py b/src/everos/memory/search/recall/knowledge_topic.py index 9cc6a9bfb..3180ec81d 100644 --- a/src/everos/memory/search/recall/knowledge_topic.py +++ b/src/everos/memory/search/recall/knowledge_topic.py @@ -17,7 +17,11 @@ from everalgo.types import Candidate -from everos.infra.persistence.index import KnowledgeTopic, knowledge_topic_repo +from everos.infra.persistence.index import ( + KnowledgeTopic, + Predicate, + knowledge_topic_repo, +) from .base import ( RecallerDeps, @@ -66,7 +70,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Dual-column BM25 recall via OR-mode BooleanQuery per column. @@ -91,7 +95,7 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Cosine ANN over the ``summary`` vector (1024-d).""" if not vector: diff --git a/src/everos/service/get.py b/src/everos/service/get.py index 0a83ff746..4f246ddee 100644 --- a/src/everos/service/get.py +++ b/src/everos/service/get.py @@ -1,8 +1,8 @@ """Get use case — lazy singleton wiring for ``POST /api/v2/memory/get``. Mirrors :mod:`everos.service.search`: the :class:`GetManager` and its -derived index repo singletons are built on first call so the FastAPI module -import order stays decoupled from the lifespan that brings up the index backend. +LanceDB repo singletons are built on first call so the FastAPI module +import order stays decoupled from the lifespan that brings up LanceDB. ``/get`` is read-only and uses no embedding / LLM / rerank clients — it never blocks on optional components the way ``/search`` does. diff --git a/src/everos/service/knowledge.py b/src/everos/service/knowledge.py index 6ee1a4d23..80c1f8d8a 100644 --- a/src/everos/service/knowledge.py +++ b/src/everos/service/knowledge.py @@ -1156,7 +1156,7 @@ def compile_knowledge_where(app_id: str, project_id: str): # type: ignore[no-un project_id: Tenant project identifier. Returns: - Predicate safe for either derived-index backend. + Predicate safe for the active derived-index backend. Raises: ValueError: If either id contains invalid characters. diff --git a/tests/_consistency_assertions.py b/tests/_consistency_assertions.py index 5890cbaa1..41f0879a4 100644 --- a/tests/_consistency_assertions.py +++ b/tests/_consistency_assertions.py @@ -201,7 +201,7 @@ async def _check_daily_log( ) for entry in parsed.entries } - lance_rows = await spec.lance_repo.find_where( + lance_rows = await spec.index_repo.find_where( f"md_path = '{_q(md_path)}'", limit=10_000 ) lance_sha_by_id = {r.entry_id: r.content_sha256 for r in lance_rows} @@ -246,7 +246,7 @@ async def _check_user_profile( ), } ) - lance_row = await spec.lance_repo.get_by_id(owner_id) + lance_row = await spec.index_repo.get_by_id(owner_id) if lance_row is None: raise AssertionError( f"user_profile row missing for owner {owner_id!r} @ {md_path}" @@ -306,7 +306,7 @@ async def _check_agent_skill( "references_content": references_content, } ) - lance_row = await spec.lance_repo.get_by_id(skill_id) + lance_row = await spec.index_repo.get_by_id(skill_id) if lance_row is None: raise AssertionError( f"agent_skill row missing for skill {skill_id!r} @ {md_path}" diff --git a/tests/integration/test_cascade_all_kinds_consistency.py b/tests/integration/test_cascade_all_kinds_consistency.py index c2310ecf3..74ca489ef 100644 --- a/tests/integration/test_cascade_all_kinds_consistency.py +++ b/tests/integration/test_cascade_all_kinds_consistency.py @@ -6,7 +6,7 @@ * ``frontmatter.entry_count == N`` * number of ```` blocks == N -* ``lance_repo.count_rows(md_path=...) == N`` +* ``index_repo.count_rows(md_path=...) == N`` * lance ``entry_id`` set == md ``entry_id`` set This is the strict counterpart to the loose ``>=`` assertions in diff --git a/tests/integration/test_milvus_remote.py b/tests/integration/test_milvus_remote.py index 99734acd8..e47f3953c 100644 --- a/tests/integration/test_milvus_remote.py +++ b/tests/integration/test_milvus_remote.py @@ -100,6 +100,7 @@ async def test_remote_milvus_matches_derived_index_contract() -> None: UserProfile, episode_repo, eq, + is_null, user_profile_repo, ) from everos.memory.search import FilterNode @@ -176,7 +177,6 @@ async def test_remote_milvus_matches_derived_index_contract() -> None: sort_by="timestamp", page=1, page_size=1, - max_fetch=10, ) assert total == 2 assert len(page) == 1 @@ -208,10 +208,31 @@ async def test_remote_milvus_matches_derived_index_contract() -> None: assert updated_profile is not None assert updated_profile.summary == "updated profile" + # Exercise the logical-null mapping for physical Milvus vector fields and + # the iterator-backed scan path. This also guards against reintroducing the + # old implicit 100-row maintenance cap. + null_vector_rows = [] + for index in range(101): + row = _episode( + row_id=f"u1_null_{index}", + entry_id=f"null_{index}", + session_id="null-vectors", + text=f"unembedded memory {index}", + vector_axis=0, + subject_axis=0, + timestamp=now + dt.timedelta(minutes=index + 1), + ) + null_vector_rows.append( + row.model_copy(update={"vector": None, "subject_vector": None}) + ) + await episode_repo.upsert(null_vector_rows) + assert await episode_repo.count_where(is_null("vector")) == 101 + assert len(await episode_repo.scan()) == 103 + assert ( await episode_repo.delete_by_md_path( "test_app/test_project/users/u1/episodes/day.md" ) - == 2 + == 103 ) assert await episode_repo.count() == 0 diff --git a/tests/integration/test_reflection_integration.py b/tests/integration/test_reflection_integration.py index 7c7fa5362..401267ba7 100644 --- a/tests/integration/test_reflection_integration.py +++ b/tests/integration/test_reflection_integration.py @@ -19,7 +19,6 @@ import hashlib import json from pathlib import Path -from typing import Any import numpy as np import pytest @@ -35,11 +34,10 @@ ) from everos.core.persistence.lancedb import LanceDailyLogRepoBase, LanceRepoBase from everos.infra.ome.testing import FakeStrategyContext -from everos.infra.persistence.lancedb.predicate import render_predicate +from everos.infra.persistence.index.lancedb import LanceIndexRepository from everos.infra.persistence.lancedb.tables.atomic_fact import AtomicFact from everos.infra.persistence.lancedb.tables.episode import Episode as LanceEpisode from everos.infra.persistence.markdown.writers.episode_writer import EpisodeWriter -from everos.infra.persistence.predicate import Predicate from everos.infra.persistence.sqlite import cluster_repo, reflection_report_repo from everos.memory._partition_locks import _reset_for_tests from everos.memory.reflection.orchestrator import ReflectionOrchestrator @@ -72,24 +70,10 @@ async def embed(self, text: str) -> list[float]: class _EpisodeRepo(LanceDailyLogRepoBase[LanceEpisode]): schema = LanceEpisode - async def find_where( - self, where: str | Predicate, *, limit: int = 100 - ) -> list[LanceEpisode]: - rendered = render_predicate(where) if isinstance(where, Predicate) else where - return await super().find_where(rendered, limit=limit) - - async def update(self, updates: dict[str, Any], *, where: str | Predicate) -> None: - rendered = render_predicate(where) if isinstance(where, Predicate) else where - await super().update(updates, where=rendered) - class _AtomicFactRepo(LanceDailyLogRepoBase[AtomicFact]): schema = AtomicFact - async def update(self, updates: dict[str, Any], *, where: str | Predicate) -> None: - rendered = render_predicate(where) if isinstance(where, Predicate) else where - await super().update(updates, where=rendered) - # --------------------------------------------------------------------------- # Fixtures @@ -374,8 +358,8 @@ async def test_reflection_init_merges_cluster_episodes( # -- Build the orchestrator with real repos. orchestrator = ReflectionOrchestrator( cluster_repo=cluster_repo, - episode_store=ep_repo, - atomic_fact_store=af_repo, + episode_store=LanceIndexRepository(ep_repo, LanceEpisode), + atomic_fact_store=LanceIndexRepository(af_repo, AtomicFact), episode_writer=episode_writer, report_repo=reflection_report_repo, reflector=reflector, @@ -580,8 +564,8 @@ async def test_reflection_update_merges_new_episodes_with_existing_merged( orchestrator = ReflectionOrchestrator( cluster_repo=cluster_repo, - episode_store=ep_repo, - atomic_fact_store=af_repo, + episode_store=LanceIndexRepository(ep_repo, LanceEpisode), + atomic_fact_store=LanceIndexRepository(af_repo, AtomicFact), episode_writer=episode_writer, report_repo=reflection_report_repo, reflector=reflector, @@ -626,8 +610,8 @@ async def test_reflection_update_merges_new_episodes_with_existing_merged( # Fresh orchestrator, same FakeLLM (next pop = update_response). orchestrator2 = ReflectionOrchestrator( cluster_repo=cluster_repo, - episode_store=ep_repo, - atomic_fact_store=af_repo, + episode_store=LanceIndexRepository(ep_repo, LanceEpisode), + atomic_fact_store=LanceIndexRepository(af_repo, AtomicFact), episode_writer=episode_writer, report_repo=reflection_report_repo, reflector=reflector, @@ -815,8 +799,8 @@ async def test_reflected_episodes_visible_in_search_deprecated_excluded( orchestrator = ReflectionOrchestrator( cluster_repo=cluster_repo, - episode_store=ep_repo, - atomic_fact_store=af_repo, + episode_store=LanceIndexRepository(ep_repo, LanceEpisode), + atomic_fact_store=LanceIndexRepository(af_repo, AtomicFact), episode_writer=episode_writer, report_repo=reflection_report_repo, reflector=reflector, diff --git a/tests/unit/test_entrypoints/test_api/test_lifespans/test_lancedb_unbackfilled_hint.py b/tests/unit/test_entrypoints/test_api/test_lifespans/test_lancedb_unbackfilled_hint.py index f8a5cc84a..180b16cc4 100644 --- a/tests/unit/test_entrypoints/test_api/test_lifespans/test_lancedb_unbackfilled_hint.py +++ b/tests/unit/test_entrypoints/test_api/test_lifespans/test_lancedb_unbackfilled_hint.py @@ -29,18 +29,20 @@ import structlog.testing from everos.entrypoints.api.lifespans import lancedb as lancedb_lifespan -from everos.infra.persistence.lancedb import BUSINESS_SCHEMAS_WITH_VECTOR +from everos.infra.persistence.index import ALL_REPOS +from everos.infra.persistence.index.schema import schema_for -class _FakeTable: - """Minimal stand-in for :class:`BaseLanceTable` used by the hint.""" +class _FakeRepo: + """Minimal backend-neutral repository used by the hint.""" - def __init__(self, null_count: int) -> None: + def __init__(self, schema: Any, null_count: int) -> None: + self.schema = schema self._null_count = null_count - self.count_rows_calls = 0 + self.count_where_calls = 0 - async def count_rows(self, filter: str) -> int: - self.count_rows_calls += 1 + async def count_where(self, predicate: Any) -> int: + self.count_where_calls += 1 return self._null_count @@ -53,18 +55,15 @@ def _isolated_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: def _wire_tables( monkeypatch: pytest.MonkeyPatch, *, null_count: int -) -> dict[str, _FakeTable]: - """Replace ``get_table`` in the lifespan module with a per-table stub.""" - tables: dict[str, _FakeTable] = { - schema.TABLE_NAME: _FakeTable(null_count) - for schema in BUSINESS_SCHEMAS_WITH_VECTOR +) -> dict[str, _FakeRepo]: + """Replace the facade registry with per-schema repository stubs.""" + repos: dict[str, _FakeRepo] = { + repo.schema.TABLE_NAME: _FakeRepo(repo.schema, null_count) + for repo in ALL_REPOS + if schema_for(repo.schema).vector_fields } - - async def _fake_get_table(name: str, _schema: Any) -> _FakeTable: - return tables[name] - - monkeypatch.setattr(lancedb_lifespan, "get_table", _fake_get_table) - return tables + monkeypatch.setattr(lancedb_lifespan, "ALL_REPOS", tuple(repos.values())) + return repos async def test_hint_fires_when_null_vectors_exist( @@ -77,10 +76,10 @@ async def test_hint_fires_when_null_vectors_exist( emissions = [e for e in captured if e.get("event") == "unbackfilled_memory_rows"] assert len(emissions) == 1 - expected_total = 3 * len(BUSINESS_SCHEMAS_WITH_VECTOR) + expected_total = 3 * len(tables) assert emissions[0]["count"] == expected_total # Every business table contributes exactly one ``count_rows`` call. - assert all(t.count_rows_calls == 1 for t in tables.values()) + assert all(t.count_where_calls == 1 for t in tables.values()) async def test_hint_silent_when_no_null_vectors( @@ -95,24 +94,24 @@ async def test_hint_silent_when_no_null_vectors( assert emissions == [] # Every table was scanned (unconditional count) — none had rows to # report, so no banner. No marker involved. - assert all(t.count_rows_calls == 1 for t in tables.values()) + assert all(t.count_where_calls == 1 for t in tables.values()) async def test_per_table_failure_is_swallowed_and_logged( _isolated_root: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - tables: dict[str, _FakeTable] = { - schema.TABLE_NAME: _FakeTable(null_count=1) - for schema in BUSINESS_SCHEMAS_WITH_VECTOR + tables: dict[str, _FakeRepo] = { + repo.schema.TABLE_NAME: _FakeRepo(repo.schema, null_count=1) + for repo in ALL_REPOS + if schema_for(repo.schema).vector_fields } - poisoned = BUSINESS_SCHEMAS_WITH_VECTOR[0].TABLE_NAME + poisoned = next(iter(tables)) - async def _fake_get_table(name: str, _schema: Any) -> _FakeTable: - if name == poisoned: - raise RuntimeError("simulated LanceDB hiccup") - return tables[name] + async def _poisoned_count(_predicate: Any) -> int: + raise RuntimeError("simulated index hiccup") - monkeypatch.setattr(lancedb_lifespan, "get_table", _fake_get_table) + tables[poisoned].count_where = _poisoned_count # type: ignore[method-assign] + monkeypatch.setattr(lancedb_lifespan, "ALL_REPOS", tuple(tables.values())) with structlog.testing.capture_logs() as captured: await lancedb_lifespan._log_unbackfilled_hint() @@ -125,4 +124,4 @@ async def _fake_get_table(name: str, _schema: Any) -> _FakeTable: # out silently. emissions = [e for e in captured if e.get("event") == "unbackfilled_memory_rows"] assert len(emissions) == 1 - assert emissions[0]["count"] == len(BUSINESS_SCHEMAS_WITH_VECTOR) - 1 + assert emissions[0]["count"] == len(tables) - 1 diff --git a/tests/unit/test_infra/test_index_contract.py b/tests/unit/test_infra/test_index_contract.py new file mode 100644 index 000000000..10b3a274d --- /dev/null +++ b/tests/unit/test_infra/test_index_contract.py @@ -0,0 +1,107 @@ +"""Contract tests for the backend-neutral derived-index boundary.""" + +from __future__ import annotations + +import datetime as dt +from pathlib import Path + +import pytest + +from everos.infra.persistence.index import ( + ALL_REPOS, + Episode, + IndexBackend, + IndexRepository, + all_of, + episode_repo, + eq, +) +from everos.infra.persistence.index.lancedb import lance_index_backend, render_predicate +from everos.infra.persistence.index.schema import IndexFieldKind, schema_for +from everos.infra.persistence.lancedb import lancedb_manager + + +@pytest.fixture(autouse=True) +async def _isolated_index(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + lancedb_manager._conn = None + lancedb_manager._tables.clear() + yield + await lancedb_manager.dispose_connection() + + +def _episode(number: int, *, owner_id: str = "owner") -> Episode: + return Episode( + id=f"{owner_id}_ep_{number:04d}", + entry_id=f"ep_{number:04d}", + owner_id=owner_id, + owner_type="user", + session_id="session", + timestamp=dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + dt.timedelta(seconds=number), + parent_id="memcell", + sender_ids=[owner_id], + subject="subject", + summary="summary", + episode=f"portable record {number}", + episode_tokens=f"portable record {number}", + md_path=f"users/{owner_id}/episodes/episode.md", + content_sha256=f"{number:064x}", + ) + + +def test_all_registered_repositories_and_backend_satisfy_ports() -> None: + assert isinstance(lance_index_backend, IndexBackend) + assert {repo.table_name for repo in lance_index_backend.repositories} == { + repo.table_name for repo in ALL_REPOS + } + assert len({repo.table_name for repo in ALL_REPOS}) == len(ALL_REPOS) == 7 + assert all(isinstance(repo, IndexRepository) for repo in ALL_REPOS) + + +def test_all_logical_schemas_are_portable_and_validate_bm25_fields() -> None: + for repo in ALL_REPOS: + logical = schema_for(repo.schema) + assert logical.table_name == repo.table_name + declared = {field.name for field in logical.fields} + assert set(logical.bm25_fields) <= declared + for field in logical.vector_fields: + assert field.kind is IndexFieldKind.DENSE_VECTOR + assert field.dimension == 1024 + + +def test_lance_predicate_renderer_owns_escaping() -> None: + rendered = render_predicate( + all_of(eq("owner_id", "o'reilly"), eq("session_id", "session")) + ) + assert "owner_id = 'o''reilly'" in rendered + assert "session_id = 'session'" in rendered + + +async def test_repository_port_crud_count_and_uncapped_scan() -> None: + rows = [_episode(number) for number in range(101)] + await episode_repo.upsert(rows) + + owner = eq("owner_id", "owner") + assert await episode_repo.count() == 101 + assert await episode_repo.count_where(owner) == 101 + assert len(await episode_repo.scan(owner)) == 101 + + found = await episode_repo.find_one_where(eq("id", rows[0].id)) + assert found is not None and found.entry_id == rows[0].entry_id + + await episode_repo.update( + {"subject": "updated"}, + where=eq("id", rows[0].id), + ) + updated = await episode_repo.get_by_id(rows[0].id) + assert updated is not None and updated.subject == "updated" + + await episode_repo.delete(eq("id", rows[0].id)) + assert await episode_repo.count_where(owner) == 100 + + +async def test_repository_maintenance_contract_is_executable() -> None: + await episode_repo.upsert([_episode(1)]) + await episode_repo.optimize() + await episode_repo.prune(dt.timedelta(days=7)) + await episode_repo.rebuild_indexes() diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index d6fdbeb25..f4ef8fa9a 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -7,13 +7,21 @@ import pytest from everos.config import load_settings -from everos.infra.persistence.index import Episode, episode_repo, user_profile_repo +from everos.infra.persistence.index import ( + Episode, + IndexRepository, + episode_repo, + is_null, + user_profile_repo, +) from everos.infra.persistence.index.schema import schema_for from everos.infra.persistence.milvus import repository from everos.infra.persistence.milvus.milvus_manager import ( MilvusConfigurationError, _resolve_uri, ) +from everos.infra.persistence.milvus.predicate import render_predicate +from everos.infra.persistence.milvus.repos import ALL_REPOS from everos.infra.persistence.milvus.repository import ( MilvusRepoBase, MilvusValueLimitError, @@ -74,6 +82,14 @@ def test_neutral_schema_tracks_every_model_field_and_dense_vector() -> None: "vector", "subject_vector", ] + assert all(isinstance(repo, IndexRepository) for repo in ALL_REPOS) + + +def test_null_vector_predicate_targets_presence_marker() -> None: + assert ( + render_predicate(is_null("vector"), vector_fields={"vector"}) + == "vector__present == false" + ) def test_record_conversion_stores_every_dense_vector_with_presence() -> None: diff --git a/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py b/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py index 8450e3979..679925370 100644 --- a/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py +++ b/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py @@ -29,7 +29,7 @@ from typing import Any -from everos.infra.persistence.predicate import Comparison, Predicate +from everos.infra.persistence.predicate import AnyOf, Comparison, IsNull, Predicate from everos.memory.cascade._backfill import ( NullBackfillPresenter, _backfill_table, @@ -134,7 +134,7 @@ async def test_subject_only_failure_leaves_row_in_backlog() -> None: # The widened Episode scan filter would still match these rows # since their subject_vector remains NULL. - assert _null_filter(spec) == "vector IS NULL OR subject_vector IS NULL" + assert _null_filter(spec) == AnyOf((IsNull("vector"), IsNull("subject_vector"))) # Simulate a fresh scan encountering the residue: primary vector # is now populated (from the last write), subject_vector is None. @@ -224,14 +224,14 @@ def test_null_filter_widens_only_for_episode() -> None: ``subject_vector`` column, so referencing it would blow up).""" ep_repo = _RecordingRepo() ep_spec = _episode_spec(ep_repo) - assert _null_filter(ep_spec) == "vector IS NULL OR subject_vector IS NULL" + assert _null_filter(ep_spec) == AnyOf((IsNull("vector"), IsNull("subject_vector"))) fact_spec = _TableSpec( schema=_NonSubjectSchema, # type: ignore[arg-type] repo=_RecordingRepo(), # type: ignore[arg-type] text_of=lambda r: r["fact"], ) - assert _null_filter(fact_spec) == "vector IS NULL" + assert _null_filter(fact_spec) == IsNull("vector") def test_extract_row_skips_legitimate_null_subject() -> None: diff --git a/tests/unit/test_memory/test_cascade/test_backfill_table_specs.py b/tests/unit/test_memory/test_cascade/test_backfill_table_specs.py index d4ca1ea17..d76833bb0 100644 --- a/tests/unit/test_memory/test_cascade/test_backfill_table_specs.py +++ b/tests/unit/test_memory/test_cascade/test_backfill_table_specs.py @@ -18,7 +18,8 @@ import pytest -from everos.infra.persistence.lancedb import BUSINESS_SCHEMAS_WITH_VECTOR +from everos.infra.persistence.index import ALL_REPOS, Episode +from everos.infra.persistence.index.schema import schema_for from everos.memory.cascade import _backfill @@ -31,7 +32,11 @@ def test_table_specs_covers_business_schemas() -> None: tree: the two sets match at import time as well. """ spec_names = {spec.schema.TABLE_NAME for spec in _backfill._TABLE_SPECS} - schema_names = {schema.TABLE_NAME for schema in BUSINESS_SCHEMAS_WITH_VECTOR} + schema_names = { + repo.schema.TABLE_NAME + for repo in ALL_REPOS + if schema_for(repo.schema).vector_fields + } assert spec_names == schema_names @@ -50,19 +55,23 @@ def test_drift_scenario_actually_raises_at_import() -> None: """ import importlib from types import SimpleNamespace + from typing import ClassVar - import everos.infra.persistence.lancedb as lancedb_infra + import everos.infra.persistence.index as index_infra import everos.memory.cascade._backfill as backfill_mod - real_schemas = tuple(lancedb_infra.BUSINESS_SCHEMAS_WITH_VECTOR) - fake_schema = SimpleNamespace(TABLE_NAME="synthetic_drift_kind") - monkey_schemas = (*real_schemas, fake_schema) + class _SyntheticDriftSchema(Episode): + TABLE_NAME: ClassVar[str] = "synthetic_drift_kind" - original = lancedb_infra.BUSINESS_SCHEMAS_WITH_VECTOR - lancedb_infra.BUSINESS_SCHEMAS_WITH_VECTOR = monkey_schemas # type: ignore[misc] + fake_schema = _SyntheticDriftSchema + fake_repo = SimpleNamespace(schema=fake_schema) + monkey_repos = (*index_infra.ALL_REPOS, fake_repo) + + original = index_infra.ALL_REPOS + index_infra.ALL_REPOS = monkey_repos # type: ignore[misc] try: with pytest.raises(RuntimeError, match=r"synthetic_drift_kind|drift"): importlib.reload(backfill_mod) finally: - lancedb_infra.BUSINESS_SCHEMAS_WITH_VECTOR = original # type: ignore[misc] + index_infra.ALL_REPOS = original # type: ignore[misc] importlib.reload(backfill_mod) # restore module state diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py index 21722c0d9..c29782c60 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py @@ -23,10 +23,8 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot -from everos.infra.persistence.lancedb import AgentCase -from everos.infra.persistence.lancedb.predicate import render_predicate +from everos.infra.persistence.index import AgentCase, Comparison, Predicate from everos.infra.persistence.markdown import AgentCaseWriter -from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import HandlerDeps from everos.memory.cascade.handlers.agent_case import AgentCaseHandler @@ -66,10 +64,8 @@ def __init__(self) -> None: async def find_where( self, where: Predicate, *, limit: int = 100 ) -> list[AgentCase]: - rendered = render_predicate(where) - prefix = "md_path = '" - if rendered.startswith(prefix): - md_path = rendered[len(prefix) :].rstrip("'") + if isinstance(where, Comparison) and where.field == "md_path": + md_path = str(where.value) return [r for r in self.rows if r.md_path == md_path] return [] @@ -81,7 +77,7 @@ async def upsert(self, rows: list[AgentCase]) -> None: self.rows = list(by_id.values()) async def delete(self, predicate: Predicate) -> None: - self.deletes.append(render_predicate(predicate)) + self.deletes.append(predicate) async def delete_by_md_path(self, md_path: str) -> int: before = len(self.rows) @@ -116,7 +112,7 @@ def no_embedder(monkeypatch: pytest.MonkeyPatch) -> None: @pytest.fixture def fake_repo(monkeypatch: pytest.MonkeyPatch) -> _FakeAgentCaseRepo: repo = _FakeAgentCaseRepo() - monkeypatch.setattr(AgentCaseHandler, "lance_repo", repo) + monkeypatch.setattr(AgentCaseHandler, "index_repo", repo) return repo diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py index b5caa773a..290a7a673 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py @@ -18,10 +18,16 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot -from everos.infra.persistence.lancedb import AgentSkill -from everos.infra.persistence.lancedb.predicate import render_predicate +from everos.infra.persistence.index import ( + AgentSkill, + All, + Comparison, + Predicate, + all_of, + eq, + ne, +) from everos.infra.persistence.markdown import AgentSkillWriter -from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import AgentSkillHandler, HandlerDeps @@ -75,30 +81,37 @@ async def delete_by_md_path(self, md_path: str) -> int: return 1 async def find_where(self, predicate: Predicate, *, limit: int) -> list[AgentSkill]: - """In-memory equivalent — handles only the - ``md_path = '...' AND id != '...'`` shape the handler emits.""" - rendered = render_predicate(predicate) - if "md_path = " in rendered and "id != " in rendered: - md_lit = rendered.split("md_path = '")[1].split("'", 1)[0] - id_lit = rendered.split("id != '")[1].split("'", 1)[0] + """In-memory equivalent for the handler's neutral predicate.""" + if isinstance(predicate, All): + values = { + child.field: child.value + for child in predicate.children + if isinstance(child, Comparison) + } + md_lit = values.get("md_path") + id_lit = values.get("id") return [ r for r in self.rows.values() if r.md_path == md_lit and r.id != id_lit ][:limit] - raise NotImplementedError(f"fake repo doesn't handle {rendered!r}") + raise NotImplementedError(f"fake repo doesn't handle {predicate!r}") async def delete(self, predicate: Predicate) -> None: - rendered = render_predicate(predicate) - self.predicate_deletes.append(rendered) - if "md_path = " in rendered and "id != " in rendered: - md_lit = rendered.split("md_path = '")[1].split("'", 1)[0] - id_lit = rendered.split("id != '")[1].split("'", 1)[0] + self.predicate_deletes.append(predicate) + if isinstance(predicate, All): + values = { + child.field: child.value + for child in predicate.children + if isinstance(child, Comparison) + } + md_lit = values.get("md_path") + id_lit = values.get("id") self.rows = { rid: row for rid, row in self.rows.items() if not (row.md_path == md_lit and row.id != id_lit) } return - raise NotImplementedError(f"fake repo doesn't handle {rendered!r}") + raise NotImplementedError(f"fake repo doesn't handle {predicate!r}") @pytest.fixture @@ -237,7 +250,7 @@ async def test_renaming_skill_via_frontmatter_clears_old_row( assert list(fake_repo.rows.keys()) == ["a1_new_name"] # The sweep predicate references the *new* id with the same md_path. assert fake_repo.predicate_deletes == [ - f"((md_path = '{md_path}') AND (id != 'a1_new_name'))" + all_of(eq("md_path", md_path), ne("id", "a1_new_name")) ] diff --git a/tests/unit/test_memory/test_cascade/test_handler_episode.py b/tests/unit/test_memory/test_cascade/test_handler_episode.py index e23855c5d..d423aa64d 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_episode.py +++ b/tests/unit/test_memory/test_cascade/test_handler_episode.py @@ -18,10 +18,8 @@ from everos.component.embedding import EmbeddingCapability, EmbeddingProvider from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot -from everos.infra.persistence.lancedb import Episode -from everos.infra.persistence.lancedb.predicate import render_predicate +from everos.infra.persistence.index import Comparison, Episode, Predicate from everos.infra.persistence.markdown import EpisodeWriter -from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import HandlerDeps from everos.memory.cascade.handlers.episode import EpisodeHandler @@ -61,11 +59,8 @@ def __init__(self) -> None: self.rows: list[Episode] = [] async def find_where(self, where: Predicate, *, limit: int = 100) -> list[Episode]: - # Honour only the md_path = '...' filter the handler emits. - rendered = render_predicate(where) - prefix = "md_path = '" - if rendered.startswith(prefix): - md_path = rendered[len(prefix) :].rstrip("'") + if isinstance(where, Comparison) and where.field == "md_path": + md_path = str(where.value) return [r for r in self.rows if r.md_path == md_path] return [] @@ -78,7 +73,7 @@ async def upsert(self, rows: list[Episode]) -> None: self.rows = list(by_id.values()) async def delete(self, predicate: Predicate) -> None: - self.deletes.append(render_predicate(predicate)) + self.deletes.append(predicate) async def delete_by_md_path(self, md_path: str) -> int: before = len(self.rows) @@ -107,7 +102,7 @@ def stub_embedder(monkeypatch: pytest.MonkeyPatch) -> _StubEmbedder: @pytest.fixture def fake_repo(monkeypatch: pytest.MonkeyPatch) -> _FakeEpisodeRepo: - """Swap the class-level ``lance_repo`` on EpisodeHandler. + """Swap the class-level ``index_repo`` on EpisodeHandler. After the BaseDailyLogHandler refactor, the repo binding is a ClassVar resolved at class-definition time; patching the module @@ -116,7 +111,7 @@ def fake_repo(monkeypatch: pytest.MonkeyPatch) -> _FakeEpisodeRepo: from everos.memory.cascade.handlers.episode import EpisodeHandler repo = _FakeEpisodeRepo() - monkeypatch.setattr(EpisodeHandler, "lance_repo", repo) + monkeypatch.setattr(EpisodeHandler, "index_repo", repo) return repo diff --git a/tests/unit/test_memory/test_cascade/test_registry_knowledge.py b/tests/unit/test_memory/test_cascade/test_registry_knowledge.py index 3970250f7..f4dadd352 100644 --- a/tests/unit/test_memory/test_cascade/test_registry_knowledge.py +++ b/tests/unit/test_memory/test_cascade/test_registry_knowledge.py @@ -28,12 +28,12 @@ def test_match_topic_md(self) -> None: assert spec is not None assert spec.name == "knowledge_topic" - def test_knowledge_document_has_no_lance_schema(self) -> None: + def test_knowledge_document_has_no_index_schema(self) -> None: spec = next(k for k in KIND_REGISTRY if k.name == "knowledge_document") - assert spec.lance_schema is None - assert spec.lance_repo is None + assert spec.index_schema is None + assert spec.index_repo is None - def test_knowledge_topic_has_lance_schema(self) -> None: + def test_knowledge_topic_has_index_schema(self) -> None: spec = next(k for k in KIND_REGISTRY if k.name == "knowledge_topic") - assert spec.lance_schema is not None - assert spec.lance_repo is not None + assert spec.index_schema is not None + assert spec.index_repo is not None diff --git a/tests/unit/test_memory/test_cascade/test_worker.py b/tests/unit/test_memory/test_cascade/test_worker.py index f60fbbbe1..e7953c34a 100644 --- a/tests/unit/test_memory/test_cascade/test_worker.py +++ b/tests/unit/test_memory/test_cascade/test_worker.py @@ -335,17 +335,17 @@ async def rebuild_indexes(self) -> None: class _OkHandlerWithRepo(_OkHandler): - """OK handler exposing a fake ``lance_repo`` for scheduler tests.""" + """OK handler exposing a fake ``index_repo`` for scheduler tests.""" def __init__(self, repo: _FakeLanceRepo) -> None: super().__init__() - self.lance_repo = repo + self.index_repo = repo -async def test_schedule_optimize_noop_when_handler_has_no_lance_repo( +async def test_schedule_optimize_noop_when_handler_has_no_index_repo( patched_repo: _FakeRepo, ) -> None: - """Test stubs without ``lance_repo`` should not even register state.""" + """Test stubs without ``index_repo`` should not even register state.""" w = CascadeWorker( {"episode": _OkHandler()}, retry_backoff_seconds=0, @@ -515,7 +515,7 @@ async def prune(self, older_than: dt.timedelta) -> None: class _HandlerWithFailingRepo(_OkHandler): def __init__(self) -> None: super().__init__() - self.lance_repo = _FailingRepo() + self.index_repo = _FailingRepo() patched_repo.batch = [_Row(md_path="a.md")] w = CascadeWorker( diff --git a/tests/unit/test_memory/test_get/test_filters_adapter.py b/tests/unit/test_memory/test_get/test_filters_adapter.py index 7fe6c76b3..69833a29a 100644 --- a/tests/unit/test_memory/test_get/test_filters_adapter.py +++ b/tests/unit/test_memory/test_get/test_filters_adapter.py @@ -19,26 +19,26 @@ import pytest -from everos.infra.persistence.lancedb.predicate import render_predicate +from everos.infra.persistence.index.lancedb import render_predicate from everos.memory.get.filters_adapter import ( - compile_filters_for_get as compile_filter_predicate_for_get, + compile_filters_for_get as compile_filter_ast, ) from everos.memory.search import FilterError, FilterNode -def compile_filters_for_get(*args: object, **kwargs: object) -> str: - """Render the neutral predicate with the legacy LanceDB test oracle.""" - return render_predicate(compile_filter_predicate_for_get(*args, **kwargs)) # type: ignore[arg-type] +def compile_filters_for_get(*args, **kwargs): # type: ignore[no-untyped-def] + """Render the neutral result through LanceDB for syntax assertions.""" + return render_predicate(compile_filter_ast(*args, **kwargs)) def test_no_filters_emits_base_clause() -> None: """``filters=None`` → owner + app/project scope clauses AND-joined.""" where = compile_filters_for_get(None, owner_id="u1", owner_type="user") - assert where == ( - "((owner_id = 'u1') AND (owner_type = 'user') " - "AND (app_id = 'default') AND (project_id = 'default') " - "AND (deprecated_by IS NULL))" - ) + assert "owner_id = 'u1'" in where + assert "owner_type = 'user'" in where + assert "app_id = 'default'" in where + assert "project_id = 'default'" in where + assert "deprecated_by IS NULL" in where def test_no_filters_agent_omits_deprecated_by() -> None: @@ -50,11 +50,7 @@ def test_no_filters_agent_omits_deprecated_by() -> None: def test_owner_id_quote_is_escaped() -> None: """SQL-standard double-quote escape on ``owner_id``.""" where = compile_filters_for_get(None, owner_id="o'reilly", owner_type="user") - assert where == ( - "((owner_id = 'o''reilly') AND (owner_type = 'user') " - "AND (app_id = 'default') AND (project_id = 'default') " - "AND (deprecated_by IS NULL))" - ) + assert "owner_id = 'o''reilly'" in where def test_flat_multi_field_renders_implicit_and() -> None: @@ -99,9 +95,7 @@ def test_timestamp_range_renders_and_folded() -> None: where = compile_filters_for_get(node, owner_id="u1", owner_type="user") assert "timestamp >= TIMESTAMP '" in where assert "timestamp < TIMESTAMP '" in where - # The two clauses are AND-joined inside one parenthesised group. - assert "(timestamp >= TIMESTAMP" in where - assert ") AND (timestamp < TIMESTAMP" in where + assert " AND " in where def test_sender_id_in_list_renders_array_has() -> None: @@ -133,7 +127,8 @@ def test_top_level_and_renders_grouped_clause() -> None: ) where = compile_filters_for_get(node, owner_id="u1", owner_type="user") # Base clause is always first; combinator output appended. - assert where.startswith("((owner_id = 'u1') AND (owner_type = 'user') AND ") + assert "owner_id = 'u1'" in where + assert "owner_type = 'user'" in where assert "session_id = 'sess_a'" in where assert "parent_id = 'mc_x'" in where diff --git a/tests/unit/test_memory/test_get/test_manager.py b/tests/unit/test_memory/test_get/test_manager.py index ea040f63c..6efa3d55c 100644 --- a/tests/unit/test_memory/test_get/test_manager.py +++ b/tests/unit/test_memory/test_get/test_manager.py @@ -27,7 +27,6 @@ Episode, UserProfile, ) -from everos.infra.persistence.lancedb.predicate import render_predicate from everos.memory.get import ( GetManager, GetMemoryType, @@ -57,7 +56,7 @@ class _StubRepo: async def find_where_paginated( self, - where: Any, + where: str, *, sort_by: str, descending: bool = True, @@ -66,7 +65,7 @@ async def find_where_paginated( max_fetch: int = 20000, ) -> tuple[list[Any], int]: self.last = _CallRecord( - where=render_predicate(where), + where=where, sort_by=sort_by, descending=descending, page=page, @@ -255,9 +254,12 @@ async def test_episodic_memory_passes_where_and_sort_to_repo( filters=FilterNode.model_validate({"session_id": "sess_a"}), ) await mgr.get(req) - assert "owner_id = 'u1'" in ep.last.where - assert "owner_type = 'user'" in ep.last.where - assert "session_id = 'sess_a'" in ep.last.where + from everos.infra.persistence.index.lancedb import render_predicate + + rendered = render_predicate(ep.last.where) + assert "owner_id = 'u1'" in rendered + assert "owner_type = 'user'" in rendered + assert "session_id = 'sess_a'" in rendered assert ep.last.sort_by == "timestamp" assert ep.last.descending is False # asc assert ep.last.page == 2 diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index f2c7041de..4e879fd0b 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -4,8 +4,7 @@ import pytest -from everos.infra.persistence.lancedb.predicate import render_predicate as render_lance -from everos.infra.persistence.milvus.predicate import render_predicate as render_milvus +from everos.infra.persistence.index.lancedb import render_predicate from everos.memory.search import ( FilterError, FilterNode, @@ -13,12 +12,11 @@ from everos.memory.search import ( compile_filters as compile_filter_ast, ) -from everos.memory.search.filters import compile_filters_for_backends def compile_filters(*args, **kwargs): # type: ignore[no-untyped-def] - """Render the neutral result through LanceDB for legacy syntax assertions.""" - return render_lance(compile_filter_ast(*args, **kwargs)) + """Render the neutral AST through LanceDB for syntax assertions.""" + return render_predicate(compile_filter_ast(*args, **kwargs)) # ── Base injection ─────────────────────────────────────────────────────── @@ -275,40 +273,3 @@ def test_compile_filters_excludes_deprecated_by_for_user() -> None: def test_compile_filters_omits_deprecated_by_for_agent() -> None: result = compile_filters(None, owner_id="agent_1", owner_type="agent") assert "deprecated_by" not in result - - -# ── Backend-specific rendering ───────────────────────────────────────── - - -def test_compile_filters_for_backends_preserves_lancedb_default() -> None: - filters = compile_filters_for_backends(None, owner_id="u_a", owner_type="user") - lancedb = render_lance(filters) - milvus = render_milvus(filters) - assert "owner_id = 'u_a'" in lancedb - assert 'owner_id == "u_a"' in milvus - assert "deprecated_by IS NULL" in lancedb - assert "deprecated_by is null" in milvus - - -def test_compile_filters_for_milvus_timestamp_and_array() -> None: - node = FilterNode.model_validate( - {"timestamp": {"gte": 1704067200000}, "sender_id": "u_jason"} - ) - filters = compile_filters_for_backends(node, owner_id="u_a", owner_type="user") - milvus = render_milvus(filters, datetime_fields={"timestamp"}) - lancedb = render_lance(filters) - assert "timestamp_ms >= 1704067200000" in milvus - assert 'array_contains(sender_ids, "u_jason")' in milvus - assert "TIMESTAMP '" in lancedb - assert "array_has(sender_ids, 'u_jason')" in lancedb - - -def test_compile_filters_for_milvus_escapes_string_literals() -> None: - node = FilterNode.model_validate({"session_id": "ses's"}) - filters = compile_filters_for_backends(node, owner_id="al'ice", owner_type="user") - milvus = render_milvus(filters) - lancedb = render_lance(filters) - assert 'owner_id == "al\'ice"' in milvus - assert 'session_id == "ses\'s"' in milvus - assert "owner_id = 'al''ice'" in lancedb - assert "session_id = 'ses''s'" in lancedb diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index a84980d51..f2f16da2b 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -28,7 +28,6 @@ from everos.component.embedding import EmbeddingCapability from everos.component.rerank import RerankCapability from everos.core.errors import ProviderNotConfiguredError -from everos.infra.persistence.lancedb.predicate import render_predicate from everos.memory.search.dto import SearchMethod, SearchRequest from everos.memory.search.manager import SearchManager @@ -107,25 +106,25 @@ def __init__(self, sparse: list[Candidate], dense: list[Candidate]) -> None: self.last_where: str | None = None async def sparse_recall( - self, query: str, where: Any, *, limit: int + self, query: str, where: str, *, limit: int ) -> list[Candidate]: - self.last_where = render_predicate(where) + self.last_where = where return list(self._sparse[:limit]) async def dense_recall( - self, vector: Sequence[float], where: Any, *, limit: int + self, vector: Sequence[float], where: str, *, limit: int ) -> list[Candidate]: - self.last_where = render_predicate(where) + self.last_where = where return list(self._dense[:limit]) async def fetch_by_parent_ids( - self, parent_ids: Sequence[str], where: Any + self, parent_ids: Sequence[str], where: str ) -> list[Candidate]: by_parent = {str(c.metadata.get("parent_id", "")): c for c in self._dense} return [by_parent[p] for p in parent_ids if p in by_parent] async def fetch_by_entry_ids( - self, entry_ids: Sequence[str], where: Any + self, entry_ids: Sequence[str], where: str ) -> list[Candidate]: by_entry = {str(c.metadata.get("entry_id", "")): c for c in self._dense} return [by_entry[e] for e in entry_ids if e in by_entry] @@ -153,7 +152,7 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: async def facts_for_episodes( self, ep_to_parents: Mapping[str, Sequence[str]], - where: Any, + where: str, *, per_episode: int, query_vector: Any = None, @@ -204,7 +203,7 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: return list(self._dense) async def fetch_by_case_ids( - self, case_ids: Sequence[str], where: Any, *, limit: int + self, case_ids: Sequence[str], where: str, *, limit: int ) -> list[Candidate]: return list(self._by_case) @@ -394,8 +393,11 @@ async def test_user_keyword_filters_compile_pinned_owner() -> None: ) await mgr.search(_user_req()) assert recaller.last_where is not None - assert "owner_id = 'alice'" in recaller.last_where - assert "owner_type = 'user'" in recaller.last_where + from everos.infra.persistence.index.lancedb import render_predicate + + rendered = render_predicate(recaller.last_where) + assert "owner_id = 'alice'" in rendered + assert "owner_type = 'user'" in rendered def _atomic_fact_row(fid: str, *, parent_id: str, score: float) -> Candidate: diff --git a/tests/unit/test_memory/test_search/test_recall_episode.py b/tests/unit/test_memory/test_search/test_recall_episode.py index 0ae51992c..6f853a303 100644 --- a/tests/unit/test_memory/test_search/test_recall_episode.py +++ b/tests/unit/test_memory/test_search/test_recall_episode.py @@ -43,6 +43,15 @@ def _mock_table(rows: list[dict[str, Any]]) -> MagicMock: return tbl +def _mock_records(rows: list[dict[str, Any]]) -> list[MagicMock]: + records: list[MagicMock] = [] + for row in rows: + record = MagicMock() + record.model_dump.return_value = row + records.append(record) + return records + + _ALICE_WHERE = eq("owner_id", "alice") @@ -62,9 +71,9 @@ async def test_fetch_all_for_owner_returns_entry_id_keyed_candidates( _make_row("ep_2", "mc_2"), ] with patch( - "everos.memory.search.recall.episode.episode_repo.search", + "everos.memory.search.recall.episode.episode_repo.scan", new_callable=AsyncMock, - return_value=rows, + return_value=_mock_records(rows), ): result = await recaller.fetch_all_for_owner(_ALICE_WHERE) @@ -79,9 +88,9 @@ async def test_fetch_all_for_owner_stores_episode_id_in_metadata( """metadata['episode_id'] carries the real LanceDB episode id for final shaping.""" rows = [_make_row("ep_abc", "mc_xyz")] with patch( - "everos.memory.search.recall.episode.episode_repo.search", + "everos.memory.search.recall.episode.episode_repo.scan", new_callable=AsyncMock, - return_value=rows, + return_value=_mock_records(rows), ): result = await recaller.fetch_all_for_owner(_ALICE_WHERE) @@ -108,9 +117,9 @@ async def test_fetch_all_for_owner_skips_rows_without_entry_id( }, ] with patch( - "everos.memory.search.recall.episode.episode_repo.search", + "everos.memory.search.recall.episode.episode_repo.scan", new_callable=AsyncMock, - return_value=rows, + return_value=_mock_records(rows), ): result = await recaller.fetch_all_for_owner(_ALICE_WHERE) @@ -135,9 +144,9 @@ async def test_fetch_all_for_owner_merged_episode_uses_entry_id( ), ] with patch( - "everos.memory.search.recall.episode.episode_repo.search", + "everos.memory.search.recall.episode.episode_repo.scan", new_callable=AsyncMock, - return_value=rows, + return_value=_mock_records(rows), ): result = await recaller.fetch_all_for_owner(_ALICE_WHERE) @@ -160,9 +169,9 @@ async def test_fetch_all_for_owner_mixed_regular_and_merged( ), ] with patch( - "everos.memory.search.recall.episode.episode_repo.search", + "everos.memory.search.recall.episode.episode_repo.scan", new_callable=AsyncMock, - return_value=rows, + return_value=_mock_records(rows), ): result = await recaller.fetch_all_for_owner(_ALICE_WHERE) diff --git a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py index 3e98c58b4..ea855584d 100644 --- a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py +++ b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py @@ -17,7 +17,7 @@ import pytest from everos.component.tokenizer import Tokenizer -from everos.infra.persistence.index import KnowledgeTopic +from everos.infra.persistence.index import KnowledgeTopic, all_of, eq from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.knowledge_topic import KnowledgeTopicRecaller @@ -64,7 +64,7 @@ def recaller() -> KnowledgeTopicRecaller: return KnowledgeTopicRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) -_WHERE = "app_id = 'app' AND project_id = 'proj'" +_WHERE = all_of(eq("app_id", "app"), eq("project_id", "proj")) # --------------------------------------------------------------------------- diff --git a/tests/unit/test_memory/test_search/test_recall_or_semantics.py b/tests/unit/test_memory/test_search/test_recall_or_semantics.py index 0c2fe4a7f..3c55a3a4b 100644 --- a/tests/unit/test_memory/test_search/test_recall_or_semantics.py +++ b/tests/unit/test_memory/test_search/test_recall_or_semantics.py @@ -24,14 +24,14 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.index.lancedb import build_or_query from everos.infra.persistence.lancedb import ( Episode, ParentType, episode_repo, lancedb_manager, ) -from everos.infra.persistence.predicate import all_of, eq -from everos.memory.search.recall.base import RecallerDeps, build_or_query +from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.episode import EpisodeRecaller @@ -93,16 +93,15 @@ def _recaller() -> EpisodeRecaller: def test_build_or_query_empty_returns_none() -> None: """Empty / whitespace-only query → ``None`` (caller must short-circuit).""" - tk = _WhitespaceTokenizer() - assert build_or_query(tk, "", column="episode_tokens") is None - assert build_or_query(tk, " ", column="episode_tokens") is None + assert build_or_query([], column="episode_tokens") is None + assert build_or_query(["", ""], column="episode_tokens") is None def test_build_or_query_single_token_returns_match_query() -> None: """One token → bare MatchQuery (no boolean-wrapper overhead).""" from lancedb.query import MatchQuery - q = build_or_query(_WhitespaceTokenizer(), "hello", column="episode_tokens") + q = build_or_query(["hello"], column="episode_tokens") assert isinstance(q, MatchQuery) @@ -110,9 +109,7 @@ def test_build_or_query_multi_token_returns_boolean_query() -> None: """≥2 tokens → BooleanQuery with one SHOULD clause per token.""" from lancedb.query import BooleanQuery - q = build_or_query( - _WhitespaceTokenizer(), "alice support group", column="episode_tokens" - ) + q = build_or_query(["alice", "support", "group"], column="episode_tokens") assert isinstance(q, BooleanQuery) @@ -149,6 +146,8 @@ async def test_or_semantics_poison_token_does_not_kill_query() -> None: tbl = await get_table(Episode.TABLE_NAME, Episode) await tbl.optimize() + from everos.infra.persistence.index import all_of, eq + where = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) cands = await _recaller().sparse_recall("alice support group", where, limit=10) assert cands, "alice + support + group should recall ep_1 via SHOULD" @@ -178,6 +177,8 @@ async def test_or_semantics_single_informative_token() -> None: tbl = await get_table(Episode.TABLE_NAME, Episode) await tbl.optimize() + from everos.infra.persistence.index import all_of, eq + where = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) cands = await _recaller().sparse_recall("painting", where, limit=10) assert cands, "single informative token must recall the matching episode" @@ -186,5 +187,7 @@ async def test_or_semantics_single_informative_token() -> None: async def test_or_semantics_empty_query_returns_empty() -> None: """Tokenisation yields nothing → recall returns ``[]`` without hitting LanceDB.""" + from everos.infra.persistence.index import eq + cands = await _recaller().sparse_recall(" ", eq("owner_id", "alice"), limit=10) assert cands == [] diff --git a/tests/unit/test_service/test_knowledge_search.py b/tests/unit/test_service/test_knowledge_search.py index 037b39bd6..88bb7a0fb 100644 --- a/tests/unit/test_service/test_knowledge_search.py +++ b/tests/unit/test_service/test_knowledge_search.py @@ -17,7 +17,6 @@ from everos.component.utils.datetime import get_utc_now from everos.core.errors import ProviderNotConfiguredError -from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.sqlite.tables.knowledge import ( KnowledgeDocumentRow, ) @@ -135,13 +134,18 @@ def _patch_stack( class TestCompileKnowledgeWhere: def test_basic_clause(self) -> None: - result = render_predicate(compile_knowledge_where("myapp", "myproj")) - assert result == "((app_id = 'myapp') AND (project_id = 'myproj'))" + result = compile_knowledge_where("myapp", "myproj") + from everos.infra.persistence.index import all_of, eq + + assert result == all_of(eq("app_id", "myapp"), eq("project_id", "myproj")) def test_defaults(self) -> None: - result = render_predicate(compile_knowledge_where("default", "default")) - assert "app_id = 'default'" in result - assert "project_id = 'default'" in result + result = compile_knowledge_where("default", "default") + from everos.infra.persistence.index.lancedb import render_predicate + + rendered = render_predicate(result) + assert "app_id = 'default'" in rendered + assert "project_id = 'default'" in rendered def test_rejects_invalid_app_id_with_sql_injection(self) -> None: with pytest.raises(ValueError, match="app_id"): @@ -160,14 +164,20 @@ def test_rejects_empty_project_id(self) -> None: compile_knowledge_where("app", "") def test_accepts_valid_ids_with_special_chars(self) -> None: - result = render_predicate(compile_knowledge_where("my_app.v2", "project-1")) - assert "my_app.v2" in result - assert "project-1" in result + result = compile_knowledge_where("my_app.v2", "project-1") + from everos.infra.persistence.index.lancedb import render_predicate + + rendered = render_predicate(result) + assert "my_app.v2" in rendered + assert "project-1" in rendered def test_accepts_valid_ids_with_at_plus(self) -> None: - result = render_predicate(compile_knowledge_where("app@org+v1", "proj_1")) - assert "app@org+v1" in result - assert "proj_1" in result + result = compile_knowledge_where("app@org+v1", "proj_1") + from everos.infra.persistence.index.lancedb import render_predicate + + rendered = render_predicate(result) + assert "app@org+v1" in rendered + assert "proj_1" in rendered # ── search_knowledge ───────────────────────────────────────────────────────── diff --git a/uv.lock b/uv.lock index f0757aa71..5d1c15895 100644 --- a/uv.lock +++ b/uv.lock @@ -1631,7 +1631,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ From de563318c9da1298a163f30b6430d8ced8d27a52 Mon Sep 17 00:00:00 2001 From: zhanghui Date: Thu, 27 Aug 2026 15:14:48 +0800 Subject: [PATCH 07/27] refactor(index): stop the port leaking backend limits and syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the derived-index framework review. Behaviour-preserving except where the refactor had itself changed behaviour. Restore what the refactor changed by accident: * Drop _MAX_FACT_RECALL_LIMIT. A Milvus/Zilliz topK ceiling had been hard-coded into memory/search, which also shrank LanceDB's fact expansion pool from the caller's own limit down to 1024. The ceiling belongs to whichever adapter needs it. * Restore per-column BM25 concurrency. The move into the Lance adapter turned the recallers' asyncio.gather into a sequential loop, doubling the BM25 leg on the four dual-column kinds. Close gaps the port left open: * all_of() / any_of() reject an empty result instead of silently rendering to "" — an empty filter matches every row, and for OR it also inverts the meaning. Mirrors one_of()'s existing guard. * Give the five kind-specific repository methods real Protocols (EpisodeIndexRepository / AgentSkillIndexRepository) so the router narrows _repo() by cast instead of getattr + type: ignore. * Extend the import-privacy contract to index.** and milvus.**, the only persistence subpackages it did not cover, and route the four outer-layer modules that reached inside through the facade. * search(vector=...) delegates to dense_search so the metric is cosine on both backends rather than LanceDB's default L2. * Restore the return annotations that were replaced by # type: ignore[no-untyped-def], and fix two `where: str` parameters that now receive a Predicate. Assert the filter DSL against the predicate tree it actually produces. The tests had been kept green by re-rendering the AST through the LanceDB renderer, so the compiler's own output was never asserted and the domain suite imported a physical adapter. Escaping assertions move to the adapter contract test, where quoting belongs. Drop the dead _merge_bm25_results and the compile_filters_for_backends alias. Co-Authored-By: Claude Opus 5 (1M context) --- pyproject.toml | 8 + .../entrypoints/api/lifespans/lancedb.py | 2 +- .../entrypoints/cli/commands/cascade.py | 5 +- .../infra/persistence/index/__init__.py | 18 +- src/everos/infra/persistence/index/lancedb.py | 24 +- .../infra/persistence/index/protocols.py | 53 +++- src/everos/infra/persistence/index/router.py | 24 +- src/everos/infra/persistence/predicate.py | 16 ++ src/everos/memory/cascade/_backfill.py | 7 +- src/everos/memory/get/filters_adapter.py | 20 +- src/everos/memory/get/manager.py | 4 +- src/everos/memory/search/filters.py | 24 +- src/everos/memory/search/manager.py | 4 +- .../memory/search/recall/atomic_fact.py | 8 +- .../memory/search/recall/knowledge_topic.py | 25 -- src/everos/service/knowledge.py | 10 +- .../test_get/test_filters_adapter.py | 218 +++++++++-------- .../test_memory/test_search/test_filters.py | 228 ++++++++++-------- .../test_search/test_recall_atomic_fact.py | 29 --- 19 files changed, 399 insertions(+), 328 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dc800b660..26ecf6984 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -227,6 +227,8 @@ forbidden_modules = [ "everos.infra.persistence.markdown.**", "everos.infra.persistence.lancedb.**", "everos.infra.persistence.sqlite.**", + "everos.infra.persistence.index.**", + "everos.infra.persistence.milvus.**", ] # `forbidden` contracts run a *transitive closure* — any path from a # source module to a forbidden module fails the check, even when the @@ -253,6 +255,12 @@ ignore_imports = [ "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.mds", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.writers", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.readers", + "everos.infra.persistence.index -> everos.infra.persistence.index.lancedb", + "everos.infra.persistence.index -> everos.infra.persistence.index.milvus", + "everos.infra.persistence.index -> everos.infra.persistence.index.predicate", + "everos.infra.persistence.index -> everos.infra.persistence.index.protocols", + "everos.infra.persistence.index -> everos.infra.persistence.index.router", + "everos.infra.persistence.index -> everos.infra.persistence.index.schema", ] [[tool.importlinter.contracts]] diff --git a/src/everos/entrypoints/api/lifespans/lancedb.py b/src/everos/entrypoints/api/lifespans/lancedb.py index a2059cd7e..8f7a47df2 100644 --- a/src/everos/entrypoints/api/lifespans/lancedb.py +++ b/src/everos/entrypoints/api/lifespans/lancedb.py @@ -35,6 +35,7 @@ ALL_REPOS, active_backend, is_null, + schema_for, ) from everos.infra.persistence.index import ( shutdown as shutdown_index, @@ -42,7 +43,6 @@ from everos.infra.persistence.index import ( startup as startup_index, ) -from everos.infra.persistence.index.schema import schema_for logger = get_logger(__name__) diff --git a/src/everos/entrypoints/cli/commands/cascade.py b/src/everos/entrypoints/cli/commands/cascade.py index 7452c5aef..d8095f56e 100644 --- a/src/everos/entrypoints/cli/commands/cascade.py +++ b/src/everos/entrypoints/cli/commands/cascade.py @@ -32,6 +32,7 @@ import asyncio import enum import os +from collections.abc import AsyncIterator from contextlib import asynccontextmanager from pathlib import Path from typing import Annotated @@ -135,9 +136,7 @@ def _apply_verbose_logging(verbose: bool | None) -> None: @asynccontextmanager -async def _runtime( # type: ignore[no-untyped-def] - *, verify: bool = True, ensure: bool = True -): +async def _runtime(*, verify: bool = True, ensure: bool = True) -> AsyncIterator[None]: """Stand up sqlite + lancedb the same way the API lifespan would. The CLI uses the same lazy, process-wide singletons the API lifespan diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py index c3f614315..419df6b93 100644 --- a/src/everos/infra/persistence/index/__init__.py +++ b/src/everos/infra/persistence/index/__init__.py @@ -46,8 +46,10 @@ In, IsNull, Predicate, + Scalar, all_of, any_of, + compare, contains, eq, gt, @@ -58,12 +60,18 @@ ne, one_of, ) -from .protocols import IndexBackend, IndexRepository +from .protocols import ( + AgentSkillIndexRepository, + EpisodeIndexRepository, + IndexBackend, + IndexRepository, +) from .router import ( RoutedAgentSkillRepository, RoutedEpisodeRepository, RoutedIndexRepository, ) +from .schema import IndexField, IndexFieldKind, IndexSchema, schema_for AgentCase = _lancedb.AgentCase AgentSkill = _lancedb.AgentSkill @@ -144,20 +152,26 @@ def repo_for_schema(schema: type[Any]) -> IndexRepository[Any]: "ALL_REPOS", "AgentCase", "AgentSkill", + "AgentSkillIndexRepository", "All", "AnyOf", "AtomicFact", "Comparison", "Contains", "Episode", + "EpisodeIndexRepository", "Foresight", "In", "IndexBackend", + "IndexField", + "IndexFieldKind", "IndexRepository", + "IndexSchema", "IsNull", "KnowledgeTopic", "ParentType", "Predicate", + "Scalar", "UserProfile", "active_backend", "agent_case_repo", @@ -165,6 +179,7 @@ def repo_for_schema(schema: type[Any]) -> IndexRepository[Any]: "all_of", "any_of", "atomic_fact_repo", + "compare", "connect", "contains", "drop_business_tables", @@ -181,6 +196,7 @@ def repo_for_schema(schema: type[Any]) -> IndexRepository[Any]: "ne", "one_of", "repo_for_schema", + "schema_for", "shutdown", "startup", "user_profile_repo", diff --git a/src/everos/infra/persistence/index/lancedb.py b/src/everos/infra/persistence/index/lancedb.py index 7e62e3f51..2b284f571 100644 --- a/src/everos/infra/persistence/index/lancedb.py +++ b/src/everos/infra/persistence/index/lancedb.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import datetime as dt import re from collections.abc import Sequence @@ -154,11 +155,12 @@ async def search( where: Predicate | None = None, limit: int = 10, ) -> list[dict[str, Any]]: - return await self._repo.search( - vector=vector, - where=_render_optional(where), - limit=limit, - ) + if vector is not None: + # Route through dense_search so the metric is pinned to cosine. + # LanceRepoBase.search leaves distance_type at the table default + # (L2), which would not match the Milvus adapter's COSINE. + return await self.dense_search(vector, where, limit=limit) + return await self._repo.search(where=_render_optional(where), limit=limit) async def sparse_search( self, @@ -173,15 +175,19 @@ async def sparse_search( if not clean_terms or not fields: return [] table = await _lancedb.get_table(self.table_name, self.schema) - best: dict[str, dict[str, Any]] = {} - for field in fields: + expression = _render_optional(where) + + async def _query_one(field: str) -> list[dict[str, Any]]: query = build_or_query(clean_terms, field) assert query is not None builder = table.query().nearest_to_text(query) - expression = _render_optional(where) if expression: builder = builder.where(expression) - rows = await builder.limit(limit).to_list() + return await builder.limit(limit).to_list() + + per_column = await asyncio.gather(*(_query_one(field) for field in fields)) + best: dict[str, dict[str, Any]] = {} + for rows in per_column: for row in rows: rid = row.get("id") if not isinstance(rid, str): diff --git a/src/everos/infra/persistence/index/protocols.py b/src/everos/infra/persistence/index/protocols.py index d8e31f96f..8a9967bcf 100644 --- a/src/everos/infra/persistence/index/protocols.py +++ b/src/everos/infra/persistence/index/protocols.py @@ -125,6 +125,52 @@ async def prune(self, older_than: dt.timedelta) -> None: ... async def rebuild_indexes(self) -> None: ... +@runtime_checkable +class EpisodeIndexRepository(IndexRepository[T], Protocol[T]): + """Episode-only reads every backend must also provide.""" + + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: ... + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[Any]: ... + + +@runtime_checkable +class AgentSkillIndexRepository(IndexRepository[T], Protocol[T]): + """AgentSkill-only cluster reads every backend must also provide.""" + + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: ... + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[Any]: ... + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[Any]: ... + + @runtime_checkable class IndexBackend(Protocol): """Lifecycle contract for one configured derived-index backend.""" @@ -145,4 +191,9 @@ async def verify_business_schemas(self) -> None: ... async def drop_business_tables(self) -> list[str]: ... -__all__ = ["IndexBackend", "IndexRepository"] +__all__ = [ + "AgentSkillIndexRepository", + "EpisodeIndexRepository", + "IndexBackend", + "IndexRepository", +] diff --git a/src/everos/infra/persistence/index/router.py b/src/everos/infra/persistence/index/router.py index 0f772dcb7..83480c04d 100644 --- a/src/everos/infra/persistence/index/router.py +++ b/src/everos/infra/persistence/index/router.py @@ -4,14 +4,18 @@ import datetime as dt from collections.abc import Sequence -from typing import Any +from typing import Any, cast from pydantic import BaseModel from everos.config import load_settings from .predicate import Predicate -from .protocols import IndexRepository +from .protocols import ( + AgentSkillIndexRepository, + EpisodeIndexRepository, + IndexRepository, +) class RoutedIndexRepository[T: BaseModel]: @@ -193,6 +197,9 @@ async def rebuild_indexes(self) -> None: class RoutedEpisodeRepository(RoutedIndexRepository[Any]): + def _repo(self) -> EpisodeIndexRepository[Any]: + return cast(EpisodeIndexRepository[Any], super()._repo()) + async def count_by_owner( self, owner_id: str, @@ -201,7 +208,7 @@ async def count_by_owner( project_id: str = "default", parent_type: str | None = None, ) -> int: - return await self._repo().count_by_owner( # type: ignore[attr-defined] + return await self._repo().count_by_owner( owner_id, app_id=app_id, project_id=project_id, @@ -219,7 +226,7 @@ async def list_by_owner_after_ts( columns: Sequence[str] | None = None, limit: int | None = None, ) -> list[Any]: - return await self._repo().list_by_owner_after_ts( # type: ignore[attr-defined] + return await self._repo().list_by_owner_after_ts( owner_id=owner_id, after_ts=after_ts, parent_type=parent_type, @@ -231,8 +238,11 @@ async def list_by_owner_after_ts( class RoutedAgentSkillRepository(RoutedIndexRepository[Any]): + def _repo(self) -> AgentSkillIndexRepository[Any]: + return cast(AgentSkillIndexRepository[Any], super()._repo()) + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: - return await self._repo().count_in_cluster( # type: ignore[attr-defined] + return await self._repo().count_in_cluster( owner_id=owner_id, cluster_id=cluster_id, ) @@ -240,7 +250,7 @@ async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: async def find_in_cluster( self, *, owner_id: str, cluster_id: str, limit: int ) -> list[Any]: - return await self._repo().find_in_cluster( # type: ignore[attr-defined] + return await self._repo().find_in_cluster( owner_id=owner_id, cluster_id=cluster_id, limit=limit, @@ -254,7 +264,7 @@ async def find_topk_relevant_in_cluster( query_vector: Sequence[float], top_k: int, ) -> list[Any]: - return await self._repo().find_topk_relevant_in_cluster( # type: ignore[attr-defined] + return await self._repo().find_topk_relevant_in_cluster( owner_id=owner_id, cluster_id=cluster_id, query_vector=query_vector, diff --git a/src/everos/infra/persistence/predicate.py b/src/everos/infra/persistence/predicate.py index d030a10c6..0ca2da0a3 100644 --- a/src/everos/infra/persistence/predicate.py +++ b/src/everos/infra/persistence/predicate.py @@ -96,6 +96,12 @@ def is_null(field: str) -> Predicate: def all_of(*predicates: Predicate | None) -> Predicate: + """AND the given predicates, flattening nested ``All`` and dropping ``None``. + + An empty result is rejected rather than rendered: adapters emit ``""`` for + an empty group, and an empty filter means *match every row* — a silent + "delete everything" if it ever reached :meth:`IndexRepository.delete`. + """ children: list[Predicate] = [] for predicate in predicates: if predicate is None: @@ -104,10 +110,18 @@ def all_of(*predicates: Predicate | None) -> Predicate: children.extend(predicate.children) else: children.append(predicate) + if not children: + raise ValueError("all_of requires at least one non-None predicate") return All(tuple(children)) def any_of(*predicates: Predicate | None) -> Predicate: + """OR the given predicates, flattening nested ``AnyOf`` and dropping ``None``. + + Empty is rejected for the same reason as :func:`all_of`, and the inversion + is worse here: an empty OR means "match nothing" but would render as + "match everything". + """ children: list[Predicate] = [] for predicate in predicates: if predicate is None: @@ -116,6 +130,8 @@ def any_of(*predicates: Predicate | None) -> Predicate: children.extend(predicate.children) else: children.append(predicate) + if not children: + raise ValueError("any_of requires at least one non-None predicate") return AnyOf(tuple(children)) diff --git a/src/everos/memory/cascade/_backfill.py b/src/everos/memory/cascade/_backfill.py index 6f3df1002..11f2ac850 100644 --- a/src/everos/memory/cascade/_backfill.py +++ b/src/everos/memory/cascade/_backfill.py @@ -54,6 +54,7 @@ Foresight, IndexRepository, KnowledgeTopic, + Predicate, agent_case_repo, agent_skill_repo, any_of, @@ -64,8 +65,8 @@ is_null, knowledge_topic_repo, repo_for_schema, + schema_for, ) -from everos.infra.persistence.index.schema import schema_for from everos.infra.persistence.markdown import AgentSkillFrontmatter from everos.infra.persistence.sqlite import cluster_repo, get_engine from everos.memory.cascade.worker import ( @@ -268,7 +269,7 @@ def _agent_skill_embed_text(row: dict[str, Any]) -> str: } if _TABLE_SPEC_NAMES != _BUSINESS_SCHEMA_NAMES: raise RuntimeError( - f"_TABLE_SPECS drift: BUSINESS_SCHEMAS_WITH_VECTOR has " + f"_TABLE_SPECS drift: the vector-carrying index repos have " f"{_BUSINESS_SCHEMA_NAMES}, _TABLE_SPECS covers " f"{_TABLE_SPEC_NAMES}. Update _TABLE_SPECS to include every " f"business table with a nullable vector column." @@ -348,7 +349,7 @@ class _PhaseResult: blocked_by_server: bool = False -def _null_filter(spec: _TableSpec): # type: ignore[no-untyped-def] +def _null_filter(spec: _TableSpec) -> Predicate: """Where-clause the scan uses to spot rows that need (some) embedding. Episode carries a ``subject_vector`` alongside ``vector`` — the two diff --git a/src/everos/memory/get/filters_adapter.py b/src/everos/memory/get/filters_adapter.py index 268834037..63dd46846 100644 --- a/src/everos/memory/get/filters_adapter.py +++ b/src/everos/memory/get/filters_adapter.py @@ -15,9 +15,8 @@ from __future__ import annotations -from everos.infra.persistence.index.predicate import Predicate +from everos.infra.persistence.index import Predicate from everos.memory.search import FilterNode, compile_filters -from everos.memory.search.filters import compile_filters_for_backends def compile_filters_for_get( @@ -40,20 +39,3 @@ def compile_filters_for_get( app_id=app_id, project_id=project_id, ) - - -def compile_filters_for_get_backends( - filters: FilterNode | None, - *, - owner_id: str, - owner_type: str, - app_id: str = "default", - project_id: str = "default", -) -> Predicate: - return compile_filters_for_backends( - filters, - owner_id=owner_id, - owner_type=owner_type, - app_id=app_id, - project_id=project_id, - ) diff --git a/src/everos/memory/get/manager.py b/src/everos/memory/get/manager.py index 10bb1837e..8270323cc 100644 --- a/src/everos/memory/get/manager.py +++ b/src/everos/memory/get/manager.py @@ -34,7 +34,7 @@ GetRequest, GetResponse, ) -from .filters_adapter import compile_filters_for_get_backends +from .filters_adapter import compile_filters_for_get if TYPE_CHECKING: from everos.infra.persistence.index import ( @@ -70,7 +70,7 @@ def __init__( async def get(self, req: GetRequest) -> GetResponse: request_id = resolve_request_id() descending = req.sort_order == "desc" - where = compile_filters_for_get_backends( + where = compile_filters_for_get( req.filters, owner_id=req.owner_id, owner_type=req.owner_type, diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 73cd7c2ae..927a07052 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -12,8 +12,9 @@ from everos.component.utils.datetime import ensure_utc, from_iso_format, from_timestamp from everos.core.errors import FilterError as FilterError -from everos.infra.persistence.index.predicate import ( +from everos.infra.persistence.index import ( Predicate, + Scalar, all_of, any_of, compare, @@ -71,24 +72,6 @@ def compile_filters( return all_of(*base) -def compile_filters_for_backends( - node: FilterNode | None, - *, - owner_id: str, - owner_type: str, - app_id: str = "default", - project_id: str = "default", -) -> Predicate: - """Compatibility name for callers migrating from rendered filters.""" - return compile_filters( - node, - owner_id=owner_id, - owner_type=owner_type, - app_id=app_id, - project_id=project_id, - ) - - def _compile_node(raw: dict[str, Any]) -> Predicate | None: raw = dict(raw) parts: list[Predicate] = [] @@ -173,7 +156,7 @@ def _compile_op_clause(spec: _FieldSpec, field: str, op: str, value: Any) -> Pre ) -def _normalize_literal(value: Any, kind: _FieldKind, field: str): # type: ignore[no-untyped-def] +def _normalize_literal(value: Any, kind: _FieldKind, field: str) -> Scalar: if kind == "str": return _require_str(value, field) if kind == "ts": @@ -220,6 +203,5 @@ def _require_list(value: Any, field: str) -> list[Any]: "ALLOWED_FIELDS", "RESERVED_FIELDS", "compile_filters", - "compile_filters_for_backends", "compile_predicate", ] diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index d5f3f944f..1ae5bfd91 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -70,7 +70,7 @@ SearchResponse, UnprocessedMessageDTO, ) -from .filters import compile_filters_for_backends +from .filters import compile_filters from .hierarchy import build_ep_to_fact_parents, heap_expand from .shaper import ( reshape_hybrid_output, @@ -204,7 +204,7 @@ async def search(self, req: SearchRequest) -> SearchResponse: # Compile filters first: a malformed `filters` payload is a user # input error (422) and should surface before the server-side # component guard (500). The two steps are independent. - where = compile_filters_for_backends( + where = compile_filters( req.filters, owner_id=req.owner_id, owner_type=req.owner_type, diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index c4fa38e84..6b65f52cc 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -38,7 +38,6 @@ _NOISE_COLUMNS = frozenset( {"vector", "_distance", "_score", "created_at", "updated_at"} ) -_MAX_FACT_RECALL_LIMIT = 1024 class AtomicFactRecaller: @@ -162,12 +161,7 @@ async def _query_facts_for_parents( ) -> list[dict[str, Any]]: """Construct and execute the LanceDB query for parent_id IN (...).""" full_where = all_of(where, one_of("parent_id", list(parent_to_eps))) - # Keep one portable ceiling: Milvus/Zilliz reject larger search topK - # values, while fact expansion only needs a bounded competition pool. - limit = min( - per_episode * max(len(parent_to_eps), 1), - _MAX_FACT_RECALL_LIMIT, - ) + limit = per_episode * max(len(parent_to_eps), 1) if query_vector: return await atomic_fact_repo.dense_search( query_vector, full_where, limit=limit diff --git a/src/everos/memory/search/recall/knowledge_topic.py b/src/everos/memory/search/recall/knowledge_topic.py index 3180ec81d..f3ad1229a 100644 --- a/src/everos/memory/search/recall/knowledge_topic.py +++ b/src/everos/memory/search/recall/knowledge_topic.py @@ -30,31 +30,6 @@ ) -def _merge_bm25_results( - per_column: tuple[list[dict], ...], - *, - limit: int, -) -> list[dict]: - """Merge multi-column BM25 results by id, keeping max score.""" - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - return sorted( - best.values(), - key=lambda r: float(r.get("_score", 0.0)), - reverse=True, - )[:limit] - - class KnowledgeTopicRecaller: """BM25 (dual-column) + vector recall over the LanceDB ``knowledge_topic`` table. diff --git a/src/everos/service/knowledge.py b/src/everos/service/knowledge.py index 80c1f8d8a..e36941708 100644 --- a/src/everos/service/knowledge.py +++ b/src/everos/service/knowledge.py @@ -44,6 +44,7 @@ from everos.core.observability.logging import get_logger from everos.core.persistence import MemoryRoot from everos.core.persistence.markdown import dump_frontmatter, parse_frontmatter +from everos.infra.persistence.index import Predicate, all_of, eq from everos.infra.persistence.markdown import ( KnowledgeWriter, ensure_taxonomy, @@ -1148,7 +1149,7 @@ def _validate_scope_id(value: str, name: str) -> None: raise ValueError(f"{name} contains invalid characters: {value!r}") -def compile_knowledge_where(app_id: str, project_id: str): # type: ignore[no-untyped-def] +def compile_knowledge_where(app_id: str, project_id: str) -> Predicate: """Build a backend-neutral predicate scoped to the given tenant. Args: @@ -1163,9 +1164,6 @@ def compile_knowledge_where(app_id: str, project_id: str): # type: ignore[no-un """ _validate_scope_id(app_id, "app_id") _validate_scope_id(project_id, "project_id") - - from everos.infra.persistence.index import all_of, eq - return all_of(eq("app_id", app_id), eq("project_id", project_id)) @@ -1174,7 +1172,7 @@ def compile_knowledge_where(app_id: str, project_id: str): # type: ignore[no-un async def _base_retrieve( recaller: KnowledgeTopicRecaller, - where: str, + where: Predicate, *, method: str, query: str, @@ -1289,7 +1287,7 @@ def _require_search_providers() -> tuple[EmbeddingProvider, RerankProvider]: async def _run_category_pipeline( query: str, - where: str, + where: Predicate, *, method: str, vector: list[float], diff --git a/tests/unit/test_memory/test_get/test_filters_adapter.py b/tests/unit/test_memory/test_get/test_filters_adapter.py index 69833a29a..e5815dc73 100644 --- a/tests/unit/test_memory/test_get/test_filters_adapter.py +++ b/tests/unit/test_memory/test_get/test_filters_adapter.py @@ -4,66 +4,101 @@ :func:`everos.memory.search.compile_filters` — these tests pin the behaviour /get callers depend on: -* base clause shape (``owner_id = '...' AND owner_type = '...'``) -* flat multi-field → implicit ``AND`` +* base clause shape (owner / owner_type / app / project / deprecated_by) +* flat multi-field → implicit conjunction * reserved field (``owner_id`` / ``owner_type`` inside ``filters``) → :class:`FilterError` * unknown field → :class:`FilterError` * top-level ``AND`` / ``OR`` combinators are accepted (parity with ``/search`` — the wiki §附录 C restriction was dropped 2026-05-16) -* ``timestamp`` range (multi-op map) renders ``AND``-folded clauses -* ``sender_id`` is an array column → ``array_has(...)`` rendering +* ``timestamp`` range (multi-op map) folds into the conjunction +* ``sender_id`` is an array column → ``Contains`` nodes + +Assertions target the backend-neutral predicate tree; rendered syntax is an +adapter concern and is pinned in ``tests/unit/test_infra``. """ from __future__ import annotations import pytest -from everos.infra.persistence.index.lancedb import render_predicate -from everos.memory.get.filters_adapter import ( - compile_filters_for_get as compile_filter_ast, +from everos.component.utils.datetime import from_iso_format, from_timestamp +from everos.infra.persistence.index import ( + All, + AnyOf, + Comparison, + Contains, + Predicate, + contains, + eq, + is_null, ) +from everos.memory.get.filters_adapter import compile_filters_for_get from everos.memory.search import FilterError, FilterNode -def compile_filters_for_get(*args, **kwargs): # type: ignore[no-untyped-def] - """Render the neutral result through LanceDB for syntax assertions.""" - return render_predicate(compile_filter_ast(*args, **kwargs)) +def _clauses(predicate: Predicate) -> list[Predicate]: + """Flatten a compiled tree into its leaf clauses, descending into groups.""" + if isinstance(predicate, All | AnyOf): + return [leaf for child in predicate.children for leaf in _clauses(child)] + return [predicate] + + +def _groups(predicate: Predicate, kind: type[Predicate]) -> list[Predicate]: + """Every group node of ``kind`` in the tree, outermost first.""" + found: list[Predicate] = [predicate] if isinstance(predicate, kind) else [] + if isinstance(predicate, All | AnyOf): + for child in predicate.children: + found.extend(_groups(child, kind)) + return found + + +def _u1_filter(raw: dict[str, object]) -> Predicate: + node = FilterNode.model_validate(raw) + return compile_filters_for_get(node, owner_id="u1", owner_type="user") def test_no_filters_emits_base_clause() -> None: - """``filters=None`` → owner + app/project scope clauses AND-joined.""" + """``filters=None`` → owner + app/project scope clauses, AND-joined.""" where = compile_filters_for_get(None, owner_id="u1", owner_type="user") - assert "owner_id = 'u1'" in where - assert "owner_type = 'user'" in where - assert "app_id = 'default'" in where - assert "project_id = 'default'" in where - assert "deprecated_by IS NULL" in where + assert where == All( + ( + eq("owner_id", "u1"), + eq("owner_type", "user"), + eq("app_id", "default"), + eq("project_id", "default"), + is_null("deprecated_by"), + ) + ) def test_no_filters_agent_omits_deprecated_by() -> None: """Agent tables lack ``deprecated_by`` — clause must be absent.""" where = compile_filters_for_get(None, owner_id="bot", owner_type="agent") - assert "deprecated_by" not in where + assert is_null("deprecated_by") not in _clauses(where) -def test_owner_id_quote_is_escaped() -> None: - """SQL-standard double-quote escape on ``owner_id``.""" +def test_owner_id_is_carried_verbatim() -> None: + """The compiler does not pre-quote — escaping belongs to the adapter.""" where = compile_filters_for_get(None, owner_id="o'reilly", owner_type="user") - assert "owner_id = 'o''reilly'" in where + assert eq("owner_id", "o'reilly") in _clauses(where) def test_flat_multi_field_renders_implicit_and() -> None: - """Multiple top-level fields → implicit ``AND`` between predicates.""" - node = FilterNode.model_validate({"session_id": "sess_a", "parent_id": "mc_x"}) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - # Field iteration order follows insertion order, so both are present. - assert "owner_id = 'u1'" in where - assert "owner_type = 'user'" in where - assert "session_id = 'sess_a'" in where - assert "parent_id = 'mc_x'" in where - # 5 base scope clauses + 2 filter fields = 7 clauses → 6 ' AND ' joins. - assert where.count(" AND ") == 6 + """Multiple top-level fields → implicit conjunction with the base clauses.""" + where = _u1_filter({"session_id": "sess_a", "parent_id": "mc_x"}) + # 5 base scope clauses + 2 filter fields, flattened into one conjunction. + assert where == All( + ( + eq("owner_id", "u1"), + eq("owner_type", "user"), + eq("app_id", "default"), + eq("project_id", "default"), + is_null("deprecated_by"), + eq("session_id", "sess_a"), + eq("parent_id", "mc_x"), + ) + ) def test_reserved_owner_id_in_filters_raises() -> None: @@ -87,82 +122,74 @@ def test_unsupported_field_raises() -> None: compile_filters_for_get(node, owner_id="u1", owner_type="user") -def test_timestamp_range_renders_and_folded() -> None: - """Multi-op map on one field folds with ``AND`` (reused from /search).""" - node = FilterNode.model_validate( - {"timestamp": {"gte": 1704067200000, "lt": 1735689600000}} +def test_timestamp_range_folds_into_conjunction() -> None: + """Multi-op map on one field folds with AND (reused from /search).""" + where = _u1_filter({"timestamp": {"gte": 1704067200000, "lt": 1735689600000}}) + clauses = _clauses(where) + assert Comparison("timestamp", "gte", from_timestamp(1704067200000)) in clauses + assert Comparison("timestamp", "lt", from_timestamp(1735689600000)) in clauses + assert _groups(where, AnyOf) == [] + + +def test_sender_id_in_list_becomes_contains() -> None: + """``sender_id`` is an array column — ``in`` → OR of ``Contains``.""" + where = _u1_filter({"sender_id": {"in": ["alice", "bob"]}}) + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + assert disjunctions[0].children == ( # type: ignore[attr-defined] + Contains("sender_ids", "alice"), + Contains("sender_ids", "bob"), ) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "timestamp >= TIMESTAMP '" in where - assert "timestamp < TIMESTAMP '" in where - assert " AND " in where -def test_sender_id_in_list_renders_array_has() -> None: - """``sender_id`` is an array column — ``in`` → ``array_has(...) OR ...``.""" - node = FilterNode.model_validate({"sender_id": {"in": ["alice", "bob"]}}) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "array_has(sender_ids, 'alice')" in where - assert "array_has(sender_ids, 'bob')" in where +def test_sender_id_eq_shorthand_becomes_contains() -> None: + """Equality shorthand on an array column → a single ``Contains``.""" + where = _u1_filter({"sender_id": "alice"}) + assert contains("sender_ids", "alice") in _clauses(where) -def test_sender_id_eq_shorthand_renders_array_has() -> None: - """Equality shorthand on an array column → single ``array_has``.""" - node = FilterNode.model_validate({"sender_id": "alice"}) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "array_has(sender_ids, 'alice')" in where +def test_parent_id_eq_shorthand_stays_scalar_eq() -> None: + """``parent_id`` is a scalar string column → plain equality.""" + where = _u1_filter({"parent_id": "mc_42"}) + assert eq("parent_id", "mc_42") in _clauses(where) -def test_parent_id_eq_shorthand_renders_scalar_eq() -> None: - """``parent_id`` is a scalar string column → plain ``=``.""" - node = FilterNode.model_validate({"parent_id": "mc_42"}) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "parent_id = 'mc_42'" in where +def test_top_level_and_folds_into_conjunction() -> None: + """``AND`` combinator compiles like /search.""" + where = _u1_filter({"AND": [{"session_id": "sess_a"}, {"parent_id": "mc_x"}]}) + clauses = _clauses(where) + assert eq("owner_id", "u1") in clauses + assert eq("owner_type", "user") in clauses + assert eq("session_id", "sess_a") in clauses + assert eq("parent_id", "mc_x") in clauses + assert _groups(where, AnyOf) == [] -def test_top_level_and_renders_grouped_clause() -> None: - """``AND`` combinator compiles like /search — parens-grouped fragments.""" - node = FilterNode.model_validate( - {"AND": [{"session_id": "sess_a"}, {"parent_id": "mc_x"}]} - ) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - # Base clause is always first; combinator output appended. - assert "owner_id = 'u1'" in where - assert "owner_type = 'user'" in where - assert "session_id = 'sess_a'" in where - assert "parent_id = 'mc_x'" in where - - -def test_top_level_or_renders_grouped_clause() -> None: - """``OR`` combinator emits parens-grouped ``OR`` between sibling preds.""" - node = FilterNode.model_validate( - {"OR": [{"session_id": "sess_a"}, {"session_id": "sess_b"}]} - ) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "session_id = 'sess_a'" in where - assert "session_id = 'sess_b'" in where - assert " OR " in where +def test_top_level_or_emits_disjunction() -> None: + """``OR`` combinator emits a disjunction between sibling predicates.""" + where = _u1_filter({"OR": [{"session_id": "sess_a"}, {"session_id": "sess_b"}]}) + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + branches = [_clauses(child) for child in disjunctions[0].children] # type: ignore[attr-defined] + assert branches == [[eq("session_id", "sess_a")], [eq("session_id", "sess_b")]] -def test_ne_operator_renders_not_equal() -> None: - """``ne`` op compiles to ``!=`` on str fields.""" - node = FilterNode.model_validate({"session_id": {"ne": "sess_internal"}}) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "session_id != 'sess_internal'" in where +def test_ne_operator_kept_as_ne() -> None: + """``ne`` op compiles to a ``ne`` comparison on str fields.""" + where = _u1_filter({"session_id": {"ne": "sess_internal"}}) + assert Comparison("session_id", "ne", "sess_internal") in _clauses(where) -def test_timestamp_iso_string_renders_literal() -> None: - """ISO 8601 string is accepted as a timestamp literal (alongside epoch ms).""" - node = FilterNode.model_validate( - {"timestamp": {"gte": "2026-01-04T00:00:00+00:00"}} - ) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "timestamp >= TIMESTAMP '2026-01-04T00:00:00+00:00'" in where +def test_timestamp_iso_string_normalized_to_datetime() -> None: + """ISO 8601 strings are accepted alongside epoch ms and normalized.""" + where = _u1_filter({"timestamp": {"gte": "2026-01-04T00:00:00+00:00"}}) + expected = from_iso_format("2026-01-04T00:00:00+00:00") + assert Comparison("timestamp", "gte", expected) in _clauses(where) def test_nested_and_inside_or() -> None: """``AND`` nested inside ``OR`` — combinators compose recursively.""" - node = FilterNode.model_validate( + where = _u1_filter( { "OR": [ {"AND": [{"session_id": "sess_a"}, {"parent_id": "mc_x"}]}, @@ -170,12 +197,13 @@ def test_nested_and_inside_or() -> None: ] } ) - where = compile_filters_for_get(node, owner_id="u1", owner_type="user") - assert "session_id = 'sess_a'" in where - assert "parent_id = 'mc_x'" in where - assert "session_id = 'sess_b'" in where - assert " OR " in where - assert " AND " in where + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + branches = [_clauses(child) for child in disjunctions[0].children] # type: ignore[attr-defined] + assert branches == [ + [eq("session_id", "sess_a"), eq("parent_id", "mc_x")], + [eq("session_id", "sess_b")], + ] # ── Malformed value shapes ────────────────────────────────────────────── diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index 4e879fd0b..23e854601 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -1,22 +1,63 @@ -"""Unit tests for the Filters DSL compiler.""" +"""Unit tests for the Filters DSL compiler. + +The compiler's output is a backend-neutral :class:`Predicate` tree, so these +assertions are made against that tree rather than against any adapter's +rendered syntax. Quoting, literal formats and array-containment syntax belong +to the adapters and are pinned in ``tests/unit/test_infra`` instead. +""" from __future__ import annotations +import datetime as dt + import pytest -from everos.infra.persistence.index.lancedb import render_predicate +from everos.component.utils.datetime import from_timestamp +from everos.infra.persistence.index import ( + All, + AnyOf, + Comparison, + Contains, + Predicate, + contains, + eq, + is_null, + one_of, +) from everos.memory.search import ( FilterError, FilterNode, + compile_filters, ) -from everos.memory.search import ( - compile_filters as compile_filter_ast, + +_BASE_USER = ( + eq("owner_id", "alice"), + eq("owner_type", "user"), + eq("app_id", "default"), + eq("project_id", "default"), + is_null("deprecated_by"), ) -def compile_filters(*args, **kwargs): # type: ignore[no-untyped-def] - """Render the neutral AST through LanceDB for syntax assertions.""" - return render_predicate(compile_filter_ast(*args, **kwargs)) +def _clauses(predicate: Predicate) -> list[Predicate]: + """Flatten a compiled tree into its leaf clauses, descending into groups.""" + if isinstance(predicate, All | AnyOf): + return [leaf for child in predicate.children for leaf in _clauses(child)] + return [predicate] + + +def _groups(predicate: Predicate, kind: type[Predicate]) -> list[Predicate]: + """Every group node of ``kind`` in the tree, outermost first.""" + found: list[Predicate] = [predicate] if isinstance(predicate, kind) else [] + if isinstance(predicate, All | AnyOf): + for child in predicate.children: + found.extend(_groups(child, kind)) + return found + + +def _user_filter(raw: dict[str, object]) -> Predicate: + node = FilterNode.model_validate(raw) + return compile_filters(node, owner_id="alice", owner_type="user") # ── Base injection ─────────────────────────────────────────────────────── @@ -24,23 +65,25 @@ def compile_filters(*args, **kwargs): # type: ignore[no-untyped-def] def test_no_filters_emits_base_clause() -> None: where = compile_filters(None, owner_id="alice", owner_type="user") - assert "owner_id = 'alice'" in where - assert "owner_type = 'user'" in where - assert "app_id = 'default'" in where - assert "project_id = 'default'" in where - assert "deprecated_by IS NULL" in where + assert where == All(_BASE_USER) def test_no_filters_agent_omits_deprecated_by() -> None: where = compile_filters(None, owner_id="bot_42", owner_type="agent") - assert "deprecated_by" not in where - assert "owner_id = 'bot_42'" in where - assert "owner_type = 'agent'" in where + assert where == All( + ( + eq("owner_id", "bot_42"), + eq("owner_type", "agent"), + eq("app_id", "default"), + eq("project_id", "default"), + ) + ) + assert is_null("deprecated_by") not in _clauses(where) def test_owner_type_agent_pinned() -> None: where = compile_filters(None, owner_id="alice", owner_type="agent") - assert "owner_type = 'agent'" in where + assert eq("owner_type", "agent") in _clauses(where) def test_app_project_scope_pinned() -> None: @@ -51,55 +94,58 @@ def test_app_project_scope_pinned() -> None: app_id="claude_code", project_id="oss", ) - assert "app_id = 'claude_code'" in where - assert "project_id = 'oss'" in where + assert eq("app_id", "claude_code") in _clauses(where) + assert eq("project_id", "oss") in _clauses(where) -def test_owner_id_with_quote_is_escaped() -> None: +def test_owner_id_is_carried_verbatim() -> None: + """The compiler must not pre-escape — quoting is the adapter's job. + + Escaping itself is asserted against the LanceDB renderer in + ``tests/unit/test_infra/test_index_contract.py``. + """ where = compile_filters(None, owner_id="al'ice", owner_type="user") - assert "owner_id = 'al''ice'" in where + assert eq("owner_id", "al'ice") in _clauses(where) # ── Equality / shorthand ──────────────────────────────────────────────── def test_flat_equality_shorthand() -> None: - node = FilterNode(session_id="sess_a") # type: ignore[call-arg] - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "session_id = 'sess_a'" in where + where = _user_filter({"session_id": "sess_a"}) + assert eq("session_id", "sess_a") in _clauses(where) def test_multiple_flat_fields_join_with_and() -> None: - node = FilterNode.model_validate({"session_id": "sess_a", "parent_type": "memcell"}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "session_id = 'sess_a'" in where - assert "parent_type = 'memcell'" in where + where = _user_filter({"session_id": "sess_a", "parent_type": "memcell"}) + assert where == All( + (*_BASE_USER, eq("session_id", "sess_a"), eq("parent_type", "memcell")) + ) # ── Operators ─────────────────────────────────────────────────────────── -def test_timestamp_gte_renders_timestamp_literal() -> None: - node = FilterNode.model_validate({"timestamp": {"gte": 1704067200000}}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "timestamp >= TIMESTAMP '" in where +def test_timestamp_gte_normalized_to_datetime() -> None: + """Epoch ms become a real ``datetime`` in the AST, not a rendered literal.""" + where = _user_filter({"timestamp": {"gte": 1704067200000}}) + clause = Comparison("timestamp", "gte", from_timestamp(1704067200000)) + assert clause in _clauses(where) + assert isinstance(clause.value, dt.datetime) def test_timestamp_range_folds_with_and() -> None: - node = FilterNode.model_validate( - {"timestamp": {"gte": 1704067200000, "lt": 1740614399000}} - ) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "timestamp >= TIMESTAMP '" in where - assert "timestamp < TIMESTAMP '" in where - # Operators on the same field are wrapped in a single group. - assert " AND " in where + where = _user_filter({"timestamp": {"gte": 1704067200000, "lt": 1740614399000}}) + clauses = _clauses(where) + assert Comparison("timestamp", "gte", from_timestamp(1704067200000)) in clauses + assert Comparison("timestamp", "lt", from_timestamp(1740614399000)) in clauses + # Operators on the same field sit in the conjunction, not a disjunction. + assert _groups(where, AnyOf) == [] def test_in_operator_string_field() -> None: - node = FilterNode.model_validate({"parent_type": {"in": ["memcell", "episode"]}}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "parent_type IN ('memcell', 'episode')" in where + where = _user_filter({"parent_type": {"in": ["memcell", "episode"]}}) + assert one_of("parent_type", ["memcell", "episode"]) in _clauses(where) def test_in_operator_requires_non_empty_list() -> None: @@ -118,7 +164,7 @@ def test_invalid_operator_rejected() -> None: def test_and_combinator() -> None: - node = FilterNode.model_validate( + where = _user_filter( { "AND": [ {"timestamp": {"gte": 1704067200000}}, @@ -126,29 +172,24 @@ def test_and_combinator() -> None: ] } ) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "timestamp >= TIMESTAMP '" in where - assert "timestamp < TIMESTAMP '" in where - assert " AND " in where + clauses = _clauses(where) + assert Comparison("timestamp", "gte", from_timestamp(1704067200000)) in clauses + assert Comparison("timestamp", "lt", from_timestamp(1740614399000)) in clauses + assert _groups(where, AnyOf) == [] def test_or_combinator() -> None: - node = FilterNode.model_validate( - { - "OR": [ - {"parent_type": "memcell"}, - {"parent_type": "episode"}, - ] - } + where = _user_filter( + {"OR": [{"parent_type": "memcell"}, {"parent_type": "episode"}]} ) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert " OR " in where - assert "parent_type = 'memcell'" in where - assert "parent_type = 'episode'" in where + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + branches = [_clauses(child) for child in disjunctions[0].children] # type: ignore[attr-defined] + assert branches == [[eq("parent_type", "memcell")], [eq("parent_type", "episode")]] def test_nested_and_inside_or() -> None: - node = FilterNode.model_validate( + where = _user_filter( { "OR": [ {"AND": [{"parent_type": "memcell"}, {"session_id": "sa"}]}, @@ -156,41 +197,38 @@ def test_nested_and_inside_or() -> None: ] } ) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "parent_type = 'memcell'" in where - assert "session_id = 'sa'" in where - assert "parent_type = 'episode'" in where - assert " OR " in where - assert " AND " in where + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + branches = [_clauses(child) for child in disjunctions[0].children] # type: ignore[attr-defined] + assert branches == [ + [eq("parent_type", "memcell"), eq("session_id", "sa")], + [eq("parent_type", "episode")], + ] def test_flat_field_alongside_and_combinator() -> None: - node = FilterNode.model_validate( - { - "session_id": "sess_a", - "AND": [{"timestamp": {"gte": 1}}], - } - ) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "session_id = 'sess_a'" in where - assert "timestamp >= TIMESTAMP '" in where + where = _user_filter({"session_id": "sess_a", "AND": [{"timestamp": {"gte": 1}}]}) + clauses = _clauses(where) + assert eq("session_id", "sess_a") in clauses + assert Comparison("timestamp", "gte", from_timestamp(1)) in clauses # ── Array field (sender_id → sender_ids) ──────────────────────────────── -def test_sender_id_eq_uses_array_has() -> None: - node = FilterNode.model_validate({"sender_id": "u_jason"}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "array_has(sender_ids, 'u_jason')" in where +def test_sender_id_eq_becomes_contains() -> None: + where = _user_filter({"sender_id": "u_jason"}) + assert contains("sender_ids", "u_jason") in _clauses(where) -def test_sender_id_in_expands_to_or_array_has() -> None: - node = FilterNode.model_validate({"sender_id": {"in": ["u_a", "u_b"]}}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "array_has(sender_ids, 'u_a')" in where - assert "array_has(sender_ids, 'u_b')" in where - assert " OR " in where +def test_sender_id_in_expands_to_or_of_contains() -> None: + where = _user_filter({"sender_id": {"in": ["u_a", "u_b"]}}) + disjunctions = _groups(where, AnyOf) + assert len(disjunctions) == 1 + assert disjunctions[0].children == ( # type: ignore[attr-defined] + Contains("sender_ids", "u_a"), + Contains("sender_ids", "u_b"), + ) def test_sender_id_gt_rejected() -> None: @@ -220,10 +258,10 @@ def test_owner_type_in_filters_rejected() -> None: compile_filters(node, owner_id="alice", owner_type="user") -def test_string_with_single_quote_escaped() -> None: - node = FilterNode.model_validate({"session_id": "ses's"}) - where = compile_filters(node, owner_id="alice", owner_type="user") - assert "session_id = 'ses''s'" in where +def test_string_with_single_quote_is_carried_verbatim() -> None: + """Filter values reach the AST unmodified; the adapter does the quoting.""" + where = _user_filter({"session_id": "ses's"}) + assert eq("session_id", "ses's") in _clauses(where) def test_timestamp_string_with_quote_rejected() -> None: @@ -255,21 +293,17 @@ def test_empty_and_array_skips_combinator() -> None: """Empty AND/OR arrays compile to no clauses — only the base remains.""" node = FilterNode.model_validate({"AND": []}) where = compile_filters(node, owner_id="alice", owner_type="user") - assert where == compile_filters( - None, - owner_id="alice", - owner_type="user", - ) + assert where == compile_filters(None, owner_id="alice", owner_type="user") # ── Deprecated exclusion ────────────────────────────────────────────── def test_compile_filters_excludes_deprecated_by_for_user() -> None: - result = compile_filters(None, owner_id="u_a", owner_type="user") - assert "deprecated_by IS NULL" in result + where = compile_filters(None, owner_id="u_a", owner_type="user") + assert is_null("deprecated_by") in _clauses(where) def test_compile_filters_omits_deprecated_by_for_agent() -> None: - result = compile_filters(None, owner_id="agent_1", owner_type="agent") - assert "deprecated_by" not in result + where = compile_filters(None, owner_id="agent_1", owner_type="agent") + assert is_null("deprecated_by") not in _clauses(where) diff --git a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py index 6e81f755f..e3d5dfd9e 100644 --- a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py +++ b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py @@ -30,7 +30,6 @@ lancedb_manager, ) from everos.infra.persistence.predicate import all_of, eq -from everos.memory.search.recall import atomic_fact as atomic_fact_module from everos.memory.search.recall.atomic_fact import AtomicFactRecaller from everos.memory.search.recall.base import RecallerDeps @@ -271,34 +270,6 @@ async def test_facts_for_episodes_score_zero_without_query_vector() -> None: assert out["alice_ep_a"][0].score == 0.0 -async def test_facts_for_episodes_caps_dense_recall_limit( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Large parent fan-out must not exceed Milvus/Zilliz search topK limits.""" - captured: dict[str, int] = {} - - async def fake_dense_search(vector, where, *, limit): # type: ignore[no-untyped-def] - captured["limit"] = limit - return [] - - monkeypatch.setattr( - atomic_fact_module.atomic_fact_repo, - "dense_search", - fake_dense_search, - ) - ep_to_parents = {f"alice_ep_{i}": [f"parent_{i}"] for i in range(80)} - - out = await _recaller().facts_for_episodes( - ep_to_parents, - _ALICE_WHERE, - per_episode=20, - query_vector=_unit_vector(0), - ) - - assert out == {} - assert captured["limit"] == 1024 - - # ── Dual parent_id (post-1.5 migration) ──────────────────────────────── From 80294216dee4826c5fe1206b86b6793d8df8b71f Mon Sep 17 00:00:00 2001 From: zhanghui Date: Thu, 27 Aug 2026 15:14:48 +0800 Subject: [PATCH 08/27] fix(milvus): datetime round-trip, update race, and scan window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects in the Milvus derived-index adapter, found reviewing the framework change. datetime round-trip below the millisecond heuristic. _to_milvus_record always writes epoch milliseconds, but _restore_row read them back through from_timestamp, which treats anything under 1e12 as *seconds*. Every instant before 2001-09-09 came back in the year 30000, or raised "ValueError: year N is out of range" on the way out. Adds from_timestamp_ms as the exact inverse of to_timestamp_ms and reads the physical column through it. Lost update in update(). Milvus has no partial-column update, so the row is read back and re-upserted whole — but only the upsert held the write lock. Concurrent writers to one row (backfill setting `vector`, reflection setting `deprecated_by`) each overwrote the other's column with the value they had read before it landed. Both halves now hold the lock. This also removes the 10,000-row fetch ceiling that used to truncate silently and still report success; the read goes through the iterator instead. list_by_owner_after_ts overran the query window. It asked client.query for up to 20,000 rows, which cannot cross Milvus' 16,384-row result window — the same limit this package documents in _scan_raw and works around there. It now applies its ceiling through the iterator. Also stops _reset_collection_cache from clearing _collection_locks while other tasks may hold them, and drops the module-level verify_business_schemas, which had no caller. Tests: a unit round-trip over four instants spanning the 1e12 threshold (runs without a server), and two remote cases — update on a row that carries vectors, which only user_profile covered before, and a pre-2001 timestamp. Co-Authored-By: Claude Opus 5 (1M context) --- src/everos/component/utils/datetime.py | 11 +++ src/everos/infra/persistence/index/milvus.py | 21 ++++-- .../infra/persistence/milvus/__init__.py | 7 -- src/everos/infra/persistence/milvus/repos.py | 9 ++- .../infra/persistence/milvus/repository.py | 67 +++++++++-------- tests/integration/test_milvus_remote.py | 74 +++++++++++++++++++ .../unit/test_infra/test_milvus/test_repo.py | 21 ++++++ 7 files changed, 164 insertions(+), 46 deletions(-) diff --git a/src/everos/component/utils/datetime.py b/src/everos/component/utils/datetime.py index feccfecf1..063af50da 100644 --- a/src/everos/component/utils/datetime.py +++ b/src/everos/component/utils/datetime.py @@ -183,6 +183,17 @@ def from_timestamp(ts: int | float) -> _dt.datetime: return _dt.datetime.fromtimestamp(seconds, tz=_display_tz()) +def from_timestamp_ms(ts: int) -> _dt.datetime: + """Parse an epoch-milliseconds value into a timezone-aware datetime. + + Exact inverse of :func:`to_timestamp_ms`. Unlike :func:`from_timestamp` + this does **not** guess whether the value is seconds or milliseconds, so + pre-2001 instants (whose ms value falls under the 1e12 heuristic + threshold) round-trip instead of being read as seconds. + """ + return _dt.datetime.fromtimestamp(ts / 1000.0, tz=_display_tz()) + + def from_iso_format(value: _dt.datetime | int | float | str) -> _dt.datetime: """Parse a value into a timezone-aware datetime (strict). diff --git a/src/everos/infra/persistence/index/milvus.py b/src/everos/infra/persistence/index/milvus.py index 67a23594d..f3e1e81a4 100644 --- a/src/everos/infra/persistence/index/milvus.py +++ b/src/everos/infra/persistence/index/milvus.py @@ -2,8 +2,11 @@ from __future__ import annotations +from types import ModuleType from typing import Any, ClassVar +from .protocols import IndexRepository + class MilvusIndexBackend: """Own remote Milvus connection, collection, and repository lifecycle.""" @@ -11,15 +14,17 @@ class MilvusIndexBackend: name: ClassVar[str] = "milvus" @property - def repositories(self): # type: ignore[no-untyped-def] - return _milvus().ALL_REPOS + def repositories(self) -> tuple[IndexRepository[Any], ...]: + return tuple(_milvus().ALL_REPOS) async def connect(self) -> Any: return await _milvus().get_client() async def startup(self) -> Any: client = await self.connect() - await self.verify_business_schemas() + # ensure_collection creates a missing collection and drift-checks an + # existing one in a single round trip, so unlike the LanceDB backend + # there is no separate verify step to run first. await self.ensure_business_indexes() return client @@ -30,16 +35,16 @@ async def ensure_business_indexes(self) -> None: await _milvus().ensure_business_indexes() async def verify_business_schemas(self) -> None: - # Verification tolerates absent collections; ensure_collection creates - # them immediately afterwards and verifies every existing collection. - for repo in self.repositories: - await repo.ensure_collection() + # Same call as ensure_business_indexes: ensure_collection creates a + # missing collection and drift-checks an existing one. Kept distinct + # because the port and the CLI both address verification by name. + await self.ensure_business_indexes() async def drop_business_tables(self) -> list[str]: return await _milvus().drop_business_tables() -def _milvus(): # type: ignore[no-untyped-def] +def _milvus() -> ModuleType: # Keep pymilvus genuinely optional for default LanceDB installations. from everos.infra.persistence import milvus diff --git a/src/everos/infra/persistence/milvus/__init__.py b/src/everos/infra/persistence/milvus/__init__.py index 910f8e64b..d4f80dbdb 100644 --- a/src/everos/infra/persistence/milvus/__init__.py +++ b/src/everos/infra/persistence/milvus/__init__.py @@ -35,12 +35,6 @@ async def ensure_business_indexes() -> None: await repo.ensure_collection() -async def verify_business_schemas() -> None: - """Fail loud if existing Milvus collections drift from EverOS schemas.""" - for repo in ALL_REPOS: - await repo.verify_collection() - - async def drop_business_tables() -> list[str]: """Drop every configured Milvus collection and return their names.""" client = await get_client() @@ -81,5 +75,4 @@ async def drop_business_tables() -> list[str]: "get_client", "knowledge_topic_repo", "user_profile_repo", - "verify_business_schemas", ] diff --git a/src/everos/infra/persistence/milvus/repos.py b/src/everos/infra/persistence/milvus/repos.py index 9319a9d9e..a43095af6 100644 --- a/src/everos/infra/persistence/milvus/repos.py +++ b/src/everos/infra/persistence/milvus/repos.py @@ -6,7 +6,6 @@ from typing import Any from everos.component.utils.datetime import from_timestamp -from everos.infra.persistence.index.predicate import all_of, eq, gt, is_null from everos.infra.persistence.lancedb import ( AgentCase, AgentSkill, @@ -16,6 +15,7 @@ KnowledgeTopic, UserProfile, ) +from everos.infra.persistence.predicate import all_of, eq, gt, is_null from .repository import MilvusRepoBase @@ -60,7 +60,12 @@ async def list_by_owner_after_ts( eq("project_id", project_id), is_null("deprecated_by"), ) - rows = await self.find_where(predicate, limit=limit or 20_000) + # A plain query(limit=...) cannot cross Milvus' 16,384-row window, so + # the ceiling has to be applied through the iterator instead. + raw = await self._scan_raw( + predicate, include_vectors=True, max_rows=limit or 20_000 + ) + rows = [self._model_from_milvus(row) for row in raw] rows.sort(key=lambda row: row.timestamp) if columns is None: return rows diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index cf8958665..0fd2df3b9 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -10,21 +10,25 @@ from pydantic import BaseModel from pymilvus import DataType, Function, FunctionType, MilvusClient -from everos.component.utils.datetime import ensure_utc, from_timestamp, to_timestamp_ms +from everos.component.utils.datetime import ( + ensure_utc, + from_timestamp_ms, + to_timestamp_ms, +) from everos.config import load_settings from everos.core.observability.logging import get_logger -from everos.infra.persistence.index.predicate import ( - Predicate, - all_of, - eq, - one_of, -) from everos.infra.persistence.index.schema import ( IndexField, IndexFieldKind, IndexSchema, schema_for, ) +from everos.infra.persistence.predicate import ( + Predicate, + all_of, + eq, + one_of, +) from .milvus_manager import MilvusSchemaMismatchError, collection_name, get_client from .predicate import render_predicate @@ -35,7 +39,6 @@ _DUMMY_VECTOR_DIMENSION = 2 _SPARSE_SUFFIX = "__sparse" _PRESENT_SUFFIX = "__present" -_UPDATE_FETCH_LIMIT = 10_000 class MilvusValueLimitError(ValueError): @@ -72,12 +75,16 @@ def _collection_lock(cls, name: str) -> asyncio.Lock: @classmethod def _reset_collection_cache(cls) -> None: + # Only the readiness set is cleared. Dropping _collection_locks would + # hand a fresh Lock to the next caller while another task still holds + # the old one, so ensure_collection would stop being mutually + # exclusive exactly when a drop/rebuild is in flight. cls._ready_collections.clear() - cls._collection_locks.clear() @classmethod def _reset_locks_for_tests(cls) -> None: cls._write_locks.clear() + cls._collection_locks.clear() cls._reset_collection_cache() async def ensure_collection(self) -> None: @@ -167,26 +174,26 @@ async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: await _run(client.upsert, self.collection_name, payload) async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: - rows = await self._query_raw( - where, limit=_UPDATE_FETCH_LIMIT, include_vectors=True - ) - if not rows: - return - if len(rows) == _UPDATE_FETCH_LIMIT: - logger.warning( - "milvus_update_truncated", - table=self.table_name, - limit=_UPDATE_FETCH_LIMIT, - ) - patched: list[dict[str, Any]] = [] - for row in rows: - merged = dict(row) - for key, value in updates.items(): - self._write_field_value(merged, key, value) - self._validate_raw_record(merged) - patched.append(merged) + """Read-modify-write the matching rows. + + Milvus has no partial-column update, so the whole row is read back and + re-upserted. Both halves must hold the same lock: with the read + outside it, two concurrent updates to one row (backfill writing + ``vector`` while reflection writes ``deprecated_by``) each overwrite + the other's column with the value they read before it landed. + """ client = await get_client() async with self._write_lock(self.collection_name): + rows = await self._scan_raw(where, include_vectors=True) + if not rows: + return + patched: list[dict[str, Any]] = [] + for row in rows: + merged = dict(row) + for key, value in updates.items(): + self._write_field_value(merged, key, value) + self._validate_raw_record(merged) + patched.append(merged) await _run(client.upsert, self.collection_name, patched) async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None: @@ -472,7 +479,7 @@ async def find_by_parent( limit=limit, ) - def _build_collection_schema(self): # type: ignore[no-untyped-def] + def _build_collection_schema(self) -> Any: schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False) for field in self.index_schema.fields: self._add_schema_field(schema, field) @@ -672,7 +679,9 @@ def _restore_row(self, row: dict[str, Any]) -> dict[str, Any]: for field in self.index_schema.fields: if field.kind is IndexFieldKind.DATETIME: value = row.get(_datetime_storage_field(field.name)) - out[field.name] = None if value is None else from_timestamp(int(value)) + out[field.name] = ( + None if value is None else from_timestamp_ms(int(value)) + ) elif field.kind is IndexFieldKind.DENSE_VECTOR: if field.name in row: out[field.name] = ( diff --git a/tests/integration/test_milvus_remote.py b/tests/integration/test_milvus_remote.py index e47f3953c..a9b4695c5 100644 --- a/tests/integration/test_milvus_remote.py +++ b/tests/integration/test_milvus_remote.py @@ -236,3 +236,77 @@ async def test_remote_milvus_matches_derived_index_contract() -> None: == 103 ) assert await episode_repo.count() == 0 + + +async def test_update_preserves_vectors_on_a_row_that_has_them() -> None: + """Milvus has no partial-column update, so update() re-upserts whole rows. + + The only prior coverage ran against ``user_profile``, which carries no + vector column — so the read-back-and-re-upsert of a 1024-d vector (the + path backfill and reflection both take) was never exercised remotely. + """ + from everos.infra.persistence.index import episode_repo, eq + + now = dt.datetime(2026, 3, 1, 12, 0, tzinfo=dt.UTC) + row = _episode( + row_id="u1_vec_update", + entry_id="vec_update", + session_id="vector-update", + text="vector bearing episode", + vector_axis=7, + subject_axis=11, + timestamp=now, + ) + await episode_repo.upsert([row]) + + await episode_repo.update( + {"deprecated_by": "u1_replacement"}, where=eq("id", "u1_vec_update") + ) + + stored = await episode_repo.get_by_id("u1_vec_update") + assert stored is not None + assert stored.deprecated_by == "u1_replacement" + # The untouched columns must survive the round-trip intact. + assert stored.vector is not None + assert stored.subject_vector is not None + assert stored.vector[7] == pytest.approx(1.0) + assert stored.subject_vector[11] == pytest.approx(1.0) + assert stored.timestamp == now + assert stored.sender_ids == ["user"] + assert stored.episode == "vector bearing episode" + + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) + + +async def test_datetime_round_trip_survives_pre_2001_instants() -> None: + """Epoch-ms values below 1e12 must not be re-read as epoch seconds. + + ``to_timestamp_ms`` always writes milliseconds, so the read side has to + parse milliseconds unconditionally. A seconds-vs-ms heuristic sends any + pre-2001-09-09 instant into the year 30000 and raises on the way back. + """ + from everos.infra.persistence.index import episode_repo, eq + + old = dt.datetime(1999, 1, 1, tzinfo=dt.UTC) + row = _episode( + row_id="u1_pre2001", + entry_id="pre2001", + session_id="old-instants", + text="an episode from before the ms/seconds threshold", + vector_axis=3, + subject_axis=5, + timestamp=old, + ) + await episode_repo.upsert([row]) + + stored = await episode_repo.get_by_id("u1_pre2001") + assert stored is not None + assert stored.timestamp == old + + assert await episode_repo.count_where(eq("timestamp", old)) == 1 + + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index f4ef8fa9a..d8b908134 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -105,6 +105,27 @@ def test_record_conversion_stores_every_dense_vector_with_presence() -> None: assert missing["subject_vector"] == [0.0] * 1024 +def test_datetime_round_trip_is_exact_below_the_ms_heuristic() -> None: + """Epoch ms are written unconditionally, so they must be read the same way. + + ``from_timestamp`` treats anything under 1e12 as *seconds*, so a + pre-2001-09-09 instant stored as ms comes back in the year 30000 — or + raises ``ValueError: year ... is out of range`` on the way out. The + physical datetime column has to bypass that heuristic entirely. + """ + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + for moment in ( + dt.datetime(1970, 1, 2, tzinfo=dt.UTC), + dt.datetime(1999, 1, 1, tzinfo=dt.UTC), + dt.datetime(2001, 9, 8, tzinfo=dt.UTC), + dt.datetime(2026, 1, 1, tzinfo=dt.UTC), + ): + stored = milvus_repo._to_milvus_record(_episode(timestamp=moment)) + assert "timestamp_ms" in stored + restored = milvus_repo._restore_row(stored) + assert restored["timestamp"] == moment, f"round-trip broke at {moment}" + + def test_update_fetch_preserves_dummy_vector_for_scalar_only_tables() -> None: milvus_repo = user_profile_repo._repo() # type: ignore[attr-defined] assert "_everos_dummy_vector" in milvus_repo._output_fields(include_vectors=True) From f99f4cc537bf955cf533d23698c3403adbe25daa Mon Sep 17 00:00:00 2001 From: zhanghui Date: Fri, 28 Aug 2026 12:24:16 +0800 Subject: [PATCH 09/27] fix(milvus): verify the physical collection schema, not just field names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verify_collection() compared field *names* only, so a collection whose columns happened to share our names but carried the wrong datatype, primary key, nullability or vector dimension passed startup and then failed opaquely on the first write or search. That is the shape a stale model or a reused collection_prefix produces. Creation and verification now share one _PhysicalField descriptor, so a reported mismatch always means the server disagrees with us — never that the builder and the checker have drifted apart. The created schema is byte-identical to before for all seven tables. What is strict and what is not. describe_collection omits `nullable` and `is_primary` when false and omits `element_type` for non-array columns, so absence is unambiguous and those are compared exactly, along with datatype and vector dimension — the drift that fails silently and late. max_length and max_capacity are advisory: the server may normalize them, and a wrong length already surfaces as a loud write rejection, so blocking startup on them would trade a small gap for an outage risk. They are logged, never swallowed. Index and metric verification is deliberately not included. It needs a second descriptor shape (describe_index) that cannot be confirmed without a live server, and shipping an unverifiable check here would risk exactly the false positive this change is careful to avoid. Tests. The offline tests build their fake describe_collection reply from the same descriptor the verifier reads, so they can only prove creation and verification agree with each other; a hand-written snapshot of episode's physical layout is the independent statement of intent, and foresight.evidence_tokens — the only column that is both nullable and a BM25 input — pins the interaction neither episode nor user_profile can reach. Whether the descriptor matches what Milvus actually reports is asserted in the remote suite, which re-verifies every collection after clearing the process readiness cache. Co-Authored-By: Claude Opus 5 (1M context) --- .../infra/persistence/milvus/repository.py | 304 +++++++++++++----- tests/integration/test_milvus_remote.py | 47 +++ .../unit/test_infra/test_milvus/test_repo.py | 252 ++++++++++++++- 3 files changed, 516 insertions(+), 87 deletions(-) diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index 0fd2df3b9..fb9ce713f 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -5,6 +5,7 @@ import asyncio import datetime as dt from collections.abc import Sequence +from dataclasses import dataclass from typing import Any, ClassVar from pydantic import BaseModel @@ -45,6 +46,100 @@ class MilvusValueLimitError(ValueError): """A row exceeds a documented Milvus VARCHAR, array, or vector limit.""" +@dataclass(frozen=True) +class _PhysicalField: + """One Milvus column exactly as EverOS declares it. + + The same descriptor drives collection creation and startup verification, + so a reported mismatch always means the server disagrees with us — never + that the builder and the checker have drifted apart from each other. + """ + + name: str + datatype: DataType + is_primary: bool = False + nullable: bool = False + dim: int | None = None + element_type: DataType | None = None + max_length: int | None = None + max_capacity: int | None = None + enable_analyzer: bool = False + + def create_kwargs(self) -> dict[str, Any]: + kwargs: dict[str, Any] = {"field_name": self.name, "datatype": self.datatype} + if self.is_primary: + kwargs["is_primary"] = True + if self.element_type is not None: + kwargs["element_type"] = self.element_type + if self.max_length is not None: + kwargs["max_length"] = self.max_length + if self.max_capacity is not None: + kwargs["max_capacity"] = self.max_capacity + if self.dim is not None: + kwargs["dim"] = self.dim + if self.enable_analyzer: + kwargs["enable_analyzer"] = True + if self.nullable: + kwargs["nullable"] = True + return kwargs + + def mismatches(self, actual: dict[str, Any]) -> list[str]: + """Report how a server-reported field differs from this declaration. + + ``describe_collection`` omits ``nullable`` and ``is_primary`` when they + are false and omits ``element_type`` for non-array columns, so absence + is unambiguous and can be compared strictly. ``max_length`` and + ``max_capacity`` are reported only for the types that carry them and + the server may normalize them, so they are advisory: a wrong length + surfaces as a loud write rejection anyway, whereas a wrong datatype or + dimension is the kind that fails opaquely much later. + """ + params = actual.get("params") or {} + out: list[str] = [] + if actual.get("type") != self.datatype: + out.append( + f"{self.name}: datatype {_name_of(actual.get('type'))} " + f"!= expected {_name_of(self.datatype)}" + ) + if bool(actual.get("is_primary", False)) != self.is_primary: + out.append( + f"{self.name}: is_primary {actual.get('is_primary', False)} " + f"!= expected {self.is_primary}" + ) + if bool(actual.get("nullable", False)) != self.nullable: + out.append( + f"{self.name}: nullable {actual.get('nullable', False)} " + f"!= expected {self.nullable}" + ) + if self.dim is not None and params.get("dim") != self.dim: + out.append(f"{self.name}: dim {params.get('dim')} != expected {self.dim}") + if self.element_type is not None and actual.get("element_type") != ( + self.element_type + ): + out.append( + f"{self.name}: element_type {_name_of(actual.get('element_type'))} " + f"!= expected {_name_of(self.element_type)}" + ) + return out + + def soft_mismatches(self, actual: dict[str, Any]) -> list[str]: + """Advisory-only limit drift (see :meth:`mismatches`).""" + params = actual.get("params") or {} + out: list[str] = [] + for key, want in ( + ("max_length", self.max_length), + ("max_capacity", self.max_capacity), + ): + got = params.get(key) + if want is not None and got is not None and int(got) != want: + out.append(f"{self.name}: {key} {got} != declared {want}") + return out + + +def _name_of(datatype: Any) -> str: + return getattr(datatype, "name", repr(datatype)) + + class MilvusRepoBase[T: BaseModel]: """Generic Milvus repository backed by one neutral index schema.""" @@ -140,17 +235,48 @@ async def _create_collection(self, client: MilvusClient) -> None: ) async def verify_collection(self) -> None: + """Reject a collection whose physical schema disagrees with ours. + + A name-only check waves through a collection whose columns happen to + share our names but carry the wrong datatype, primary key, nullability + or vector dimension — a stale model, or a ``collection_prefix`` + collision with someone else's data. That collection starts fine and + then fails opaquely on the first write or search. Comparing the + physical shape turns it back into a startup error with a recovery + path. + """ client = await get_client() description = await _run(client.describe_collection, self.collection_name) - actual = {field["name"] for field in description.get("fields", [])} - expected = set(self._stored_field_names()) - missing = expected - actual - stale = actual - expected - if missing or stale: + reported = { + field["name"]: field + for field in description.get("fields", []) + if "name" in field + } + expected = {physical.name: physical for physical in self._physical_fields()} + + missing = sorted(set(expected) - set(reported)) + stale = sorted(set(reported) - set(expected)) + drift: list[str] = [] + advisory: list[str] = [] + for name, physical in expected.items(): + actual = reported.get(name) + if actual is None: + continue + drift.extend(physical.mismatches(actual)) + advisory.extend(physical.soft_mismatches(actual)) + + if advisory: + logger.warning( + "milvus_collection_limit_drift", + collection=self.collection_name, + details=advisory, + ) + if missing or stale or drift: raise MilvusSchemaMismatchError( f"Milvus collection {self.collection_name!r} schema drift: " - f"missing={sorted(missing)}, stale={sorted(stale)}. The index is " - "rebuildable from markdown; run `everos cascade rebuild`." + f"missing={missing}, stale={stale}, incompatible={drift}. " + "The index is rebuildable from markdown; run " + "`everos cascade rebuild`." ) async def add(self, records: Sequence[T]) -> None: @@ -481,20 +607,10 @@ async def find_by_parent( def _build_collection_schema(self) -> Any: schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False) - for field in self.index_schema.fields: - self._add_schema_field(schema, field) - if not self.index_schema.vector_fields: - schema.add_field( - field_name=_DUMMY_VECTOR_FIELD, - datatype=DataType.FLOAT_VECTOR, - dim=_DUMMY_VECTOR_DIMENSION, - ) + for physical in self._physical_fields(): + schema.add_field(**physical.create_kwargs()) for field in self.index_schema.bm25_fields: sparse_name = _sparse_field(field) - schema.add_field( - field_name=sparse_name, - datatype=DataType.SPARSE_FLOAT_VECTOR, - ) schema.add_function( Function( name=f"{field}_bm25", @@ -505,73 +621,97 @@ def _build_collection_schema(self) -> Any: ) return schema - def _add_schema_field(self, schema: Any, field: IndexField) -> None: - if field.kind is IndexFieldKind.STRING: - kwargs: dict[str, Any] = { - "field_name": field.name, - "datatype": DataType.VARCHAR, - "max_length": field.max_length, - } - if field.primary: - kwargs["is_primary"] = True - elif field.nullable and field.name not in self.index_schema.bm25_fields: - kwargs["nullable"] = True - if field.name in self.index_schema.bm25_fields: - kwargs["enable_analyzer"] = True - schema.add_field(**kwargs) - elif field.kind is IndexFieldKind.STRING_ARRAY: - schema.add_field( - field_name=field.name, - datatype=DataType.ARRAY, - element_type=DataType.VARCHAR, - max_capacity=field.max_capacity, - max_length=field.max_length, - nullable=field.nullable, - ) - elif field.kind is IndexFieldKind.FLOAT: - schema.add_field( - field_name=field.name, - datatype=DataType.DOUBLE, - nullable=field.nullable, - ) - elif field.kind is IndexFieldKind.INTEGER: - schema.add_field( - field_name=field.name, - datatype=DataType.INT64, - nullable=field.nullable, - ) - elif field.kind is IndexFieldKind.DATETIME: - schema.add_field( - field_name=_datetime_storage_field(field.name), - datatype=DataType.INT64, - nullable=field.nullable, - ) - elif field.kind is IndexFieldKind.DENSE_VECTOR: - schema.add_field( - field_name=field.name, - datatype=DataType.FLOAT_VECTOR, - dim=field.dimension, + def _physical_fields(self) -> tuple[_PhysicalField, ...]: + """Every Milvus column this table declares, in creation order.""" + out: list[_PhysicalField] = [] + for field in self.index_schema.fields: + out.extend(self._physical_for(field)) + if not self.index_schema.vector_fields: + out.append( + _PhysicalField( + name=_DUMMY_VECTOR_FIELD, + datatype=DataType.FLOAT_VECTOR, + dim=_DUMMY_VECTOR_DIMENSION, + ) ) - schema.add_field( - field_name=_present_field(field.name), - datatype=DataType.BOOL, + out.extend( + _PhysicalField( + name=_sparse_field(field), + datatype=DataType.SPARSE_FLOAT_VECTOR, ) - else: # pragma: no cover - enum exhaustiveness guard - raise TypeError(f"unsupported index field kind: {field.kind}") + for field in self.index_schema.bm25_fields + ) + return tuple(out) + + def _physical_for(self, field: IndexField) -> list[_PhysicalField]: + is_bm25 = field.name in self.index_schema.bm25_fields + if field.kind is IndexFieldKind.STRING: + return [ + _PhysicalField( + name=field.name, + datatype=DataType.VARCHAR, + is_primary=field.primary, + # A BM25 input is written as "" rather than null so the + # analyzer always has something to tokenize. + nullable=field.nullable and not field.primary and not is_bm25, + max_length=field.max_length, + enable_analyzer=is_bm25, + ) + ] + if field.kind is IndexFieldKind.STRING_ARRAY: + return [ + _PhysicalField( + name=field.name, + datatype=DataType.ARRAY, + nullable=field.nullable, + element_type=DataType.VARCHAR, + max_length=field.max_length, + max_capacity=field.max_capacity, + ) + ] + if field.kind is IndexFieldKind.FLOAT: + return [ + _PhysicalField( + name=field.name, + datatype=DataType.DOUBLE, + nullable=field.nullable, + ) + ] + if field.kind is IndexFieldKind.INTEGER: + return [ + _PhysicalField( + name=field.name, + datatype=DataType.INT64, + nullable=field.nullable, + ) + ] + if field.kind is IndexFieldKind.DATETIME: + return [ + _PhysicalField( + name=_datetime_storage_field(field.name), + datatype=DataType.INT64, + nullable=field.nullable, + ) + ] + if field.kind is IndexFieldKind.DENSE_VECTOR: + return [ + _PhysicalField( + name=field.name, + datatype=DataType.FLOAT_VECTOR, + dim=field.dimension, + ), + # Milvus cannot store a null dense vector, so a logical null is + # a zero vector plus this presence marker. + _PhysicalField( + name=_present_field(field.name), + datatype=DataType.BOOL, + ), + ] + # pragma: no cover - enum exhaustiveness guard + raise TypeError(f"unsupported index field kind: {field.kind}") def _stored_field_names(self) -> list[str]: - names: list[str] = [] - for field in self.index_schema.fields: - if field.kind is IndexFieldKind.DATETIME: - names.append(_datetime_storage_field(field.name)) - else: - names.append(field.name) - if field.kind is IndexFieldKind.DENSE_VECTOR: - names.append(_present_field(field.name)) - if not self.index_schema.vector_fields: - names.append(_DUMMY_VECTOR_FIELD) - names.extend(_sparse_field(field) for field in self.index_schema.bm25_fields) - return names + return [physical.name for physical in self._physical_fields()] def _output_fields(self, *, include_vectors: bool) -> list[str]: fields: list[str] = [] diff --git a/tests/integration/test_milvus_remote.py b/tests/integration/test_milvus_remote.py index a9b4695c5..f84763270 100644 --- a/tests/integration/test_milvus_remote.py +++ b/tests/integration/test_milvus_remote.py @@ -310,3 +310,50 @@ async def test_datetime_round_trip_survives_pre_2001_instants() -> None: await episode_repo.delete_by_md_path( "test_app/test_project/users/u1/episodes/day.md" ) + + +async def test_verify_accepts_a_collection_this_adapter_just_created() -> None: + """The decisive check for physical schema verification. + + Every offline test builds its fake ``describe_collection`` reply from the + same descriptor the verifier compares against, so they can only prove that + creation and verification agree with each other. Whether that descriptor + matches what Milvus actually stores and reports back — VARCHAR lengths it + may normalize, flags it may omit, the analyzer it attaches for BM25 — can + only be established against a real server. + + The fixture creates the collections with a fresh prefix, so verification + never runs during setup; dropping the process-local readiness cache forces + the existing-collection path. + """ + from everos.infra.persistence.index import ALL_REPOS, episode_repo + from everos.infra.persistence.milvus.repository import MilvusRepoBase + + await episode_repo.upsert( + [ + _episode( + row_id="u1_verify", + entry_id="verify", + session_id="verify", + text="a row so the collection is not empty", + vector_axis=1, + subject_axis=2, + timestamp=dt.datetime(2026, 5, 1, tzinfo=dt.UTC), + ) + ] + ) + + for repo in ALL_REPOS: + milvus_repo = repo._repo() # type: ignore[attr-defined] + await milvus_repo.ensure_collection() + + MilvusRepoBase._reset_collection_cache() + + for repo in ALL_REPOS: + # Raises MilvusSchemaMismatchError if our declaration and the server's + # description disagree on any field. + await repo._repo().verify_collection() # type: ignore[attr-defined] + + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index d8b908134..15c5a5e98 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -5,12 +5,14 @@ import datetime as dt import pytest +from pymilvus import DataType from everos.config import load_settings from everos.infra.persistence.index import ( Episode, IndexRepository, episode_repo, + foresight_repo, is_null, user_profile_repo, ) @@ -18,6 +20,7 @@ from everos.infra.persistence.milvus import repository from everos.infra.persistence.milvus.milvus_manager import ( MilvusConfigurationError, + MilvusSchemaMismatchError, _resolve_uri, ) from everos.infra.persistence.milvus.predicate import render_predicate @@ -40,6 +43,35 @@ def _reset_state(monkeypatch: pytest.MonkeyPatch): load_settings.cache_clear() +def _describe_response(milvus_repo, *, always_params: bool = False): # type: ignore[no-untyped-def] + """What Milvus reports for the collection this repo would create. + + Built from pymilvus' own ``CollectionSchema.to_dict()`` rather than a + hand-written literal, so the field shape comes from the library instead of + from our assumptions. ``describe_collection`` always emits ``params`` + while ``to_dict`` omits it when empty — ``always_params`` covers that one + known difference between the two serializations. + """ + fields = [ + dict(f) for f in milvus_repo._build_collection_schema().to_dict()["fields"] + ] + if always_params: + for field in fields: + field.setdefault("params", {}) + return {"fields": fields} + + +def _patch_describe(monkeypatch, response): # type: ignore[no-untyped-def] + class _FakeClient: + def describe_collection(self, name: str): # type: ignore[no-untyped-def] + return response + + async def _fake_get_client(): # type: ignore[no-untyped-def] + return _FakeClient() + + monkeypatch.setattr(repository, "get_client", _fake_get_client) + + def _episode(**overrides): # type: ignore[no-untyped-def] values = { "id": "u1_ep1", @@ -167,11 +199,7 @@ def has_collection(self, name: str) -> bool: def describe_collection(self, name: str): # type: ignore[no-untyped-def] self.describe_calls += 1 - return { - "fields": [ - {"name": field} for field in milvus_repo._stored_field_names() - ] - } + return _describe_response(milvus_repo) client = _FakeClient() @@ -183,3 +211,217 @@ async def _fake_get_client(): # type: ignore[no-untyped-def] await milvus_repo.ensure_collection() assert client.has_calls == 1 assert client.describe_calls == 1 + + +# ── Physical schema verification ──────────────────────────────────────── + + +@pytest.mark.parametrize("always_params", [False, True]) +async def test_verify_accepts_every_collection_it_would_create( + monkeypatch: pytest.MonkeyPatch, always_params: bool +) -> None: + """The false-positive guard: our own collections must always pass. + + A checker that rejects a healthy deployment is far worse than the + name-only check it replaces, so every table is verified against the exact + schema this adapter would have created for it. + """ + for repo in ALL_REPOS: + _patch_describe( + monkeypatch, _describe_response(repo, always_params=always_params) + ) + await repo.verify_collection() + + +async def test_verify_rejects_datatype_drift(monkeypatch: pytest.MonkeyPatch) -> None: + """Same field name, wrong physical type — the case a name check waves through.""" + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + for field in response["fields"]: + if field["name"] == "timestamp_ms": + field["type"] = DataType.VARCHAR + _patch_describe(monkeypatch, response) + + with pytest.raises(MilvusSchemaMismatchError, match="timestamp_ms: datatype"): + await milvus_repo.verify_collection() + + +async def test_verify_rejects_vector_dimension_drift( + monkeypatch: pytest.MonkeyPatch, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + for field in response["fields"]: + if field["name"] == "vector": + field["params"] = {"dim": 1} + _patch_describe(monkeypatch, response) + + with pytest.raises( + MilvusSchemaMismatchError, match=r"vector: dim 1 != expected 1024" + ): + await milvus_repo.verify_collection() + + +async def test_verify_rejects_primary_and_nullable_drift( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``describe_collection`` omits both flags when false, so absence is exact.""" + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + for field in response["fields"]: + if field["name"] == "id": + field.pop("is_primary", None) + if field["name"] == "deprecated_by": + field.pop("nullable", None) + _patch_describe(monkeypatch, response) + + with pytest.raises(MilvusSchemaMismatchError) as excinfo: + await milvus_repo.verify_collection() + assert "id: is_primary" in str(excinfo.value) + assert "deprecated_by: nullable" in str(excinfo.value) + + +async def test_verify_rejects_missing_and_stale_columns( + monkeypatch: pytest.MonkeyPatch, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + response["fields"] = [f for f in response["fields"] if f["name"] != "subject"] + response["fields"].append({"name": "left_over", "type": DataType.VARCHAR}) + _patch_describe(monkeypatch, response) + + with pytest.raises(MilvusSchemaMismatchError) as excinfo: + await milvus_repo.verify_collection() + assert "'subject'" in str(excinfo.value) + assert "'left_over'" in str(excinfo.value) + + +async def test_verify_treats_limit_drift_as_advisory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A wrong VARCHAR length fails loudly on write, so it must not block startup. + + Blocking here would turn a server-side normalization we cannot predict into + an outage; the datatype and dimension checks carry the load instead. + """ + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + for field in response["fields"]: + if field["name"] == "subject": + field["params"] = {"max_length": 4096} + _patch_describe(monkeypatch, response) + + warnings: list[tuple[str, dict]] = [] + monkeypatch.setattr( + repository.logger, + "warning", + lambda event, **kw: warnings.append((event, kw)), + ) + + await milvus_repo.verify_collection() + + # Tolerated, but never silently: the drift has to reach the operator. + assert [event for event, _ in warnings] == ["milvus_collection_limit_drift"] + assert warnings[0][1]["details"] == ["subject: max_length 4096 != declared 65535"] + + +# Written out by hand on purpose. _describe_response() builds its fake server +# reply from _build_collection_schema(), which shares _physical_fields() with +# the verifier — so those tests can only prove that creation and verification +# agree with *each other*, never that either matches what we intend. This +# literal is the independent statement of intent: a slip in _physical_for +# (a nullable flipped, a dim dropped, an analyzer lost) changes both sides at +# once and is invisible everywhere except here. +# +# Only a run against a real Milvus can prove the declaration matches what the +# server actually stores; see tests/integration/test_milvus_remote.py. +_EPISODE_PHYSICAL = ( + ("created_at_ms", DataType.INT64), + ("updated_at_ms", DataType.INT64), + ("id", DataType.VARCHAR, "is_primary", 512), + ("entry_id", DataType.VARCHAR, "", 65535), + ("owner_id", DataType.VARCHAR, "", 65535), + ("owner_type", DataType.VARCHAR, "", 65535), + ("app_id", DataType.VARCHAR, "", 65535), + ("project_id", DataType.VARCHAR, "", 65535), + ("session_id", DataType.VARCHAR, "nullable", 65535), + ("timestamp_ms", DataType.INT64), + ("parent_type", DataType.VARCHAR, "", 65535), + ("parent_id", DataType.VARCHAR, "", 65535), + ("sender_ids", DataType.ARRAY, "", 512), + ("subject", DataType.VARCHAR, "nullable", 65535), + ("summary", DataType.VARCHAR, "nullable", 65535), + ("episode", DataType.VARCHAR, "", 65535), + ("episode_tokens", DataType.VARCHAR, "enable_analyzer", 65535), + ("md_path", DataType.VARCHAR, "", 65535), + ("content_sha256", DataType.VARCHAR, "", 65535), + ("deprecated_by", DataType.VARCHAR, "nullable", 65535), + ("vector", DataType.FLOAT_VECTOR, "", None), + ("vector__present", DataType.BOOL), + ("subject_vector", DataType.FLOAT_VECTOR, "", None), + ("subject_vector__present", DataType.BOOL), + ("episode_tokens__sparse", DataType.SPARSE_FLOAT_VECTOR), +) + + +def _flag(field) -> str: # type: ignore[no-untyped-def] + for candidate in ("is_primary", "nullable", "enable_analyzer"): + if getattr(field, candidate): + return candidate + return "" + + +def test_episode_physical_layout_matches_the_declared_snapshot() -> None: + """Pin the physical column list against a hand-written expectation.""" + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + actual = milvus_repo._physical_fields() + + assert [f.name for f in actual] == [row[0] for row in _EPISODE_PHYSICAL] + for field, row in zip(actual, _EPISODE_PHYSICAL, strict=True): + assert field.datatype == row[1], field.name + if len(row) == 2: + assert not _flag(field), field.name + continue + assert _flag(field) == row[2], field.name + assert field.max_length == row[3], field.name + + sender_ids = next(f for f in actual if f.name == "sender_ids") + assert sender_ids.element_type is DataType.VARCHAR + assert sender_ids.max_capacity == 256 + for name in ("vector", "subject_vector"): + assert next(f for f in actual if f.name == name).dim == 1024 + + +def test_vectorless_table_declares_the_dummy_vector_column() -> None: + """Milvus needs at least one vector column, so scalar-only tables fake one.""" + milvus_repo = user_profile_repo._repo() # type: ignore[attr-defined] + actual = {f.name: f for f in milvus_repo._physical_fields()} + + assert "_everos_dummy_vector" in actual + assert actual["_everos_dummy_vector"].datatype is DataType.FLOAT_VECTOR + assert actual["_everos_dummy_vector"].dim == 2 + assert not [ + f for f in actual.values() if f.datatype is DataType.SPARSE_FLOAT_VECTOR + ] + assert actual["id"].is_primary is True + assert actual["summary"].nullable is False + + +def test_nullable_bm25_input_is_declared_not_null() -> None: + """``foresight.evidence_tokens`` is the only nullable BM25 column. + + A BM25 input feeds an analyzer, so it is stored as "" rather than null — + which means the physical column must be declared NOT NULL even though the + logical field is optional. Episode and user_profile cannot cover this + interaction, so it is asserted where it actually occurs. + """ + logical = schema_for(foresight_repo.schema) + assert logical.field("evidence_tokens").nullable is True + assert "evidence_tokens" in logical.bm25_fields + + physical = {f.name: f for f in foresight_repo._repo()._physical_fields()} # type: ignore[attr-defined] + assert physical["evidence_tokens"].nullable is False + assert physical["evidence_tokens"].enable_analyzer is True + # "evidence" is the same optional text without the analyzer attached, so + # it stays nullable — the NOT NULL above is caused by BM25, nothing else. + assert physical["evidence"].nullable is True From 6ab672249eb129c1483a469deac7ae55b8731111 Mon Sep 17 00:00:00 2001 From: zhanghui Date: Thu, 3 Sep 2026 16:13:50 +0800 Subject: [PATCH 10/27] test(integration): run the tiered API suites against both index backends The end-to-end suites drove LanceDB only, so nothing had ever exercised Milvus through the request path -- /add, /flush, cascade, the search methods, /get. The repository contract tests cover the port; they say nothing about whether the application works on top of it. Three of the tier modules carried the same helper reading episode rows straight off a Lance table. It moves into the conftest as episode_rows() and reads through the index facade instead, which is all the coupling that stood in the way -- the remaining direct LanceDB use in test_cascade_cli_integration inspects an Arrow column type and is genuinely engine-specific, so it stays. An index_backend fixture parametrizes the tier runtimes. LanceDB always runs; Milvus joins only when EVEROS_TEST_MILVUS_URI names a real server, because it is a remote service with no embedded fallback to degrade to -- a run without one proves nothing about Milvus and should not collect cases that suggest otherwise. Each Milvus case gets its own collection prefix and drops it afterwards, since collections outlive the process. Verified against Milvus 2.6.22: all 27 parametrized cases pass, though not in a single process on a 5.8 GiB Docker allocation -- 23 pass and the server is killed under the collection churn, and the remaining 4 pass on a restart. The CI job has more headroom; the new step there is what proves it holds in one run. Co-Authored-By: Cheney Zhang --- .github/workflows/ci.yml | 10 +++ tests/integration/test_tiers/conftest.py | 71 +++++++++++++++++-- .../test_tiers/test_tier1_keyword_only.py | 12 +--- .../test_tiers/test_tier2_no_rerank.py | 13 +--- .../integration/test_tiers/test_tier3_full.py | 18 +++-- 5 files changed, 90 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 476c84cb2..f84a3b139 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,6 +165,16 @@ jobs: EVEROS_TEST_MILVUS_FULL_STARTUP: "1" run: uv run --frozen pytest tests/integration/test_milvus_remote.py -v + # The contract tests above exercise the repository port directly. This + # drives the same request path the application does -- /add, /flush, + # cascade, every search method, /get -- against the remote backend, so a + # break anywhere between the API and Milvus surfaces here rather than in + # production. Deselecting the LanceDB half keeps the job to its subject. + - name: Milvus end-to-end (tiered API suites) + env: + EVEROS_TEST_MILVUS_URI: http://127.0.0.1:19530 + run: uv run --frozen pytest tests/integration/test_tiers -v -k milvus + - name: Stop Milvus if: always() run: docker compose -f /tmp/milvus-compose.yml down -v diff --git a/tests/integration/test_tiers/conftest.py b/tests/integration/test_tiers/conftest.py index b18e9d3f4..27197dfe8 100644 --- a/tests/integration/test_tiers/conftest.py +++ b/tests/integration/test_tiers/conftest.py @@ -40,6 +40,8 @@ import asyncio import importlib import json +import os +import uuid from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager from pathlib import Path @@ -175,6 +177,54 @@ def _reset_search_singletons(monkeypatch: pytest.MonkeyPatch) -> None: # --------------------------------------------------------------------------- +def _index_backends() -> list[str]: + """Backends the tier suites run against. + + LanceDB is always exercised. Milvus joins only when a real server is + configured — it is a remote service, so there is no embedded fallback to + silently degrade to. A run without ``EVEROS_TEST_MILVUS_URI`` therefore + proves nothing about Milvus, and says so by not collecting those cases. + """ + backends = ["lancedb"] + if os.environ.get("EVEROS_TEST_MILVUS_URI"): + backends.append("milvus") + return backends + + +@pytest_asyncio.fixture(params=_index_backends(), ids=lambda b: f"index={b}") +async def index_backend( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> AsyncIterator[str]: + """Point the derived index at one backend for the duration of a test.""" + backend = request.param + monkeypatch.setenv("EVEROS_INDEX__BACKEND", backend) + if backend == "milvus": + monkeypatch.setenv("EVEROS_MILVUS__URI", os.environ["EVEROS_TEST_MILVUS_URI"]) + monkeypatch.setenv( + "EVEROS_MILVUS__TOKEN", os.environ.get("EVEROS_TEST_MILVUS_TOKEN", "") + ) + monkeypatch.setenv( + "EVEROS_MILVUS__DB_NAME", os.environ.get("EVEROS_TEST_MILVUS_DB_NAME", "") + ) + # A fresh prefix per test: collections are remote and outlive the + # process, so two tests sharing one would leak state into each other. + monkeypatch.setenv( + "EVEROS_MILVUS__COLLECTION_PREFIX", f"everos_tier_{uuid.uuid4().hex}" + ) + + yield backend + + if backend == "milvus": + from everos.config import load_settings + from everos.infra.persistence.index import drop_business_tables, shutdown + + load_settings.cache_clear() + try: + await drop_business_tables() + finally: + await shutdown() + + @asynccontextmanager async def _tier_client( tmp_path: Path, @@ -234,7 +284,7 @@ async def _tier_client( @pytest_asyncio.fixture async def tier1_runtime( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, index_backend: str ) -> AsyncIterator[httpx.AsyncClient]: """Tier 1: LLM only. embed/rerank/multimodal all unavailable.""" async with _tier_client( @@ -245,7 +295,7 @@ async def tier1_runtime( @pytest_asyncio.fixture async def tier2_runtime( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, index_backend: str ) -> AsyncIterator[httpx.AsyncClient]: """Tier 2: LLM + embed. rerank unavailable.""" async with _tier_client( @@ -256,7 +306,7 @@ async def tier2_runtime( @pytest_asyncio.fixture async def tier3_runtime( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, index_backend: str ) -> AsyncIterator[httpx.AsyncClient]: """Tier 3: LLM + embed + rerank, all available.""" async with _tier_client( @@ -384,7 +434,7 @@ async def seed_atomic_fact_for_episode( import hashlib from everos.component.utils.datetime import get_utc_now - from everos.infra.persistence.lancedb import AtomicFact, atomic_fact_repo + from everos.infra.persistence.index import AtomicFact, atomic_fact_repo entry_id = f"af_seed_{episode_row['entry_id']}" owner_id = episode_row["owner_id"] @@ -434,3 +484,16 @@ def add_payload( } ], } + + +async def episode_rows(owner_id: str) -> list[dict[str, Any]]: + """Every episode row for one owner, as plain dicts. + + Backend-neutral on purpose: the tier suites assert on indexed content, not + on how a particular engine stores it, so they read through the same port + the application uses. + """ + from everos.infra.persistence.index import episode_repo, eq + + rows = await episode_repo.scan(eq("owner_id", owner_id)) + return [row.model_dump(mode="python") for row in rows] diff --git a/tests/integration/test_tiers/test_tier1_keyword_only.py b/tests/integration/test_tiers/test_tier1_keyword_only.py index 9dacbb506..2cdc0ce95 100644 --- a/tests/integration/test_tiers/test_tier1_keyword_only.py +++ b/tests/integration/test_tiers/test_tier1_keyword_only.py @@ -12,15 +12,7 @@ import pytest from httpx import AsyncClient -from everos.infra.persistence.lancedb import Episode, get_table - -from .conftest import add_and_flush - - -async def _episode_rows(owner_id: str) -> list[dict]: - table = await get_table(Episode.TABLE_NAME, Episode) - return await table.query().where(f"owner_id = '{owner_id}'").to_list() - +from .conftest import add_and_flush, episode_rows # --------------------------------------------------------------------------- # 1. POST /memory/add -> md write + LanceDB vector=NULL @@ -31,7 +23,7 @@ async def test_add_memory_writes_null_vector(tier1_runtime: AsyncClient) -> None body = await add_and_flush(tier1_runtime, session_id="s_tier1_add") assert body["data"]["status"] == "extracted" - rows = await _episode_rows("u_alice") + rows = await episode_rows("u_alice") assert rows, "expected cascade to index the new episode into LanceDB" assert rows[0]["vector"] is None, "Tier 1 (no embed) must write vector=NULL" diff --git a/tests/integration/test_tiers/test_tier2_no_rerank.py b/tests/integration/test_tiers/test_tier2_no_rerank.py index bb7574edd..ae20cc5ba 100644 --- a/tests/integration/test_tiers/test_tier2_no_rerank.py +++ b/tests/integration/test_tiers/test_tier2_no_rerank.py @@ -15,18 +15,11 @@ import pytest from httpx import AsyncClient -from everos.infra.persistence.lancedb import Episode, get_table - -from .conftest import add_and_flush, seed_atomic_fact_for_episode +from .conftest import add_and_flush, episode_rows, seed_atomic_fact_for_episode _STUB_VECTOR = [0.1] * 1024 -async def _episode_rows(owner_id: str) -> list[dict]: - table = await get_table(Episode.TABLE_NAME, Episode) - return await table.query().where(f"owner_id = '{owner_id}'").to_list() - - # --------------------------------------------------------------------------- # 1. Memory add + KEYWORD / VECTOR / HYBRID (user) all succeed # --------------------------------------------------------------------------- @@ -35,7 +28,7 @@ async def _episode_rows(owner_id: str) -> list[dict]: async def test_add_memory_writes_real_vector(tier2_runtime: AsyncClient) -> None: await add_and_flush(tier2_runtime, session_id="s_tier2_add") - rows = await _episode_rows("u_alice") + rows = await episode_rows("u_alice") assert rows, "expected cascade to index the new episode into LanceDB" assert rows[0]["vector"] is not None, "Tier 2 (embed available) must embed" assert len(rows[0]["vector"]) == 1024 @@ -48,7 +41,7 @@ async def test_user_search_methods_succeed( await add_and_flush(tier2_runtime, session_id=f"s_tier2_{method}") # VECTOR recalls via atomic_fact MaxSim, not a direct episode ANN scan # -- see seed_atomic_fact_for_episode's docstring. - episode_row = (await _episode_rows("u_alice"))[0] + episode_row = (await episode_rows("u_alice"))[0] await seed_atomic_fact_for_episode(episode_row, vector=_STUB_VECTOR) resp = await tier2_runtime.post( diff --git a/tests/integration/test_tiers/test_tier3_full.py b/tests/integration/test_tiers/test_tier3_full.py index 41dc0ecd8..01850e580 100644 --- a/tests/integration/test_tiers/test_tier3_full.py +++ b/tests/integration/test_tiers/test_tier3_full.py @@ -16,18 +16,16 @@ from everalgo.types import KnowledgeMemory from httpx import AsyncClient -from everos.infra.persistence.lancedb import Episode, get_table - -from .conftest import add_and_flush, cascade_progress, seed_atomic_fact_for_episode +from .conftest import ( + add_and_flush, + cascade_progress, + episode_rows, + seed_atomic_fact_for_episode, +) _STUB_VECTOR = [0.1] * 1024 -async def _episode_rows(owner_id: str) -> list[dict]: - table = await get_table(Episode.TABLE_NAME, Episode) - return await table.query().where(f"owner_id = '{owner_id}'").to_list() - - # --------------------------------------------------------------------------- # 1. Memory + every search method -> 200 # --------------------------------------------------------------------------- @@ -36,7 +34,7 @@ async def _episode_rows(owner_id: str) -> list[dict]: async def test_add_memory_writes_real_vector(tier3_runtime: AsyncClient) -> None: await add_and_flush(tier3_runtime, session_id="s_tier3_add") - rows = await _episode_rows("u_alice") + rows = await episode_rows("u_alice") assert rows and rows[0]["vector"] is not None @@ -45,7 +43,7 @@ async def test_user_search_methods_succeed( tier3_runtime: AsyncClient, method: str ) -> None: await add_and_flush(tier3_runtime, session_id=f"s_tier3_{method}") - episode_row = (await _episode_rows("u_alice"))[0] + episode_row = (await episode_rows("u_alice"))[0] await seed_atomic_fact_for_episode(episode_row, vector=_STUB_VECTOR) resp = await tier3_runtime.post( From b0a8ca3dbe68b2aad8873ea5fe34d17a5ceb55ac Mon Sep 17 00:00:00 2001 From: zhanghui Date: Thu, 3 Sep 2026 21:47:24 +0800 Subject: [PATCH 11/27] test(e2e): run the pipeline suites against both index backends The e2e suites named LanceDB directly, so they could only ever exercise one backend. Unlike the tier suites there was no raw table access to unpick -- only repository singletons, which the index facade already exposes -- so the switch is an import change plus the same `index_backend` fixture: LanceDB always, Milvus when EVEROS_TEST_MILVUS_URI names a real server. One LanceDB import stays. `test_get_endpoint_e2e` builds the app with no lifespan and swaps EVEROS_ROOT per test, so it has to drop Lance's process-local connection and table cache by hand or the next test reads the previous root. That cache is not part of the neutral port; the teardown that *is* now goes through the facade's shutdown(). Also corrects thirteen `where: str` annotations left over from the move to the predicate AST. They have been receiving a Predicate since the port landed, so the annotation said the opposite of what the code does. Nothing enforces them today -- there is no mypy in `make lint` -- which is exactly why they drifted. Co-Authored-By: Cheney Zhang --- src/everos/memory/search/agentic.py | 3 +- src/everos/memory/search/agentic_agent.py | 7 +-- src/everos/memory/search/callbacks.py | 3 +- src/everos/memory/search/manager.py | 17 ++++--- tests/e2e/conftest.py | 51 +++++++++++++++++++ .../e2e/test_add_flush_agent_pipeline_e2e.py | 2 +- tests/e2e/test_add_flush_user_pipeline_e2e.py | 6 +-- tests/e2e/test_get_endpoint_e2e.py | 12 +++-- 8 files changed, 81 insertions(+), 20 deletions(-) diff --git a/src/everos/memory/search/agentic.py b/src/everos/memory/search/agentic.py index 1b79e8a28..6ec2655f9 100644 --- a/src/everos/memory/search/agentic.py +++ b/src/everos/memory/search/agentic.py @@ -34,6 +34,7 @@ from everos.component.utils.datetime import from_timestamp, to_timestamp_ms from everos.core.observability.logging import get_logger from everos.core.observability.tracing import memory_span +from everos.infra.persistence.index import Predicate from everos.infra.persistence.sqlite import cluster_repo from everos.memory.search.callbacks import build_rerank_fn from everos.memory.search.shaper import shape_episode_from_candidate @@ -80,7 +81,7 @@ async def search_episodes_agentic( query: str, *, owner_id: str, - where: str, + where: Predicate, app_id: str = "default", project_id: str = "default", episode_recaller: EpisodeRecaller, diff --git a/src/everos/memory/search/agentic_agent.py b/src/everos/memory/search/agentic_agent.py index 584e1e9dd..140c8abe3 100644 --- a/src/everos/memory/search/agentic_agent.py +++ b/src/everos/memory/search/agentic_agent.py @@ -30,6 +30,7 @@ from everos.component.utils.datetime import from_timestamp, to_timestamp_ms from everos.core.observability.tracing import memory_span +from everos.infra.persistence.index import Predicate from everos.memory.search.callbacks import ( _format_case_passage_from_metadata, _format_skill_passage_from_metadata, @@ -127,7 +128,7 @@ def _restore_shaper_metadata(metadata: dict[str, Any]) -> dict[str, Any]: async def search_agent_cases_agentic( query: str, *, - where: str, + where: Predicate, case_recaller: AgentCaseRecaller, embed_query_fn: Callable[[str], Awaitable[list[float]]], reranker: RerankProvider, @@ -169,7 +170,7 @@ async def search_agent_cases_agentic( async def search_agent_skills_agentic( query: str, *, - where: str, + where: Predicate, skill_recaller: AgentSkillRecaller, embed_query_fn: Callable[[str], Awaitable[list[float]]], reranker: RerankProvider, @@ -211,7 +212,7 @@ async def search_agent_skills_agentic( async def _run_agentic_retrieve( query: str, *, - where: str, + where: Predicate, recaller: AgentCaseRecaller | AgentSkillRecaller, embed_query_fn: Callable[[str], Awaitable[list[float]]], reranker: RerankProvider, diff --git a/src/everos/memory/search/callbacks.py b/src/everos/memory/search/callbacks.py index a480a7804..7cf823c02 100644 --- a/src/everos/memory/search/callbacks.py +++ b/src/everos/memory/search/callbacks.py @@ -39,6 +39,7 @@ from everos.component.rerank import RerankProvider from everos.core.observability.tracing import memory_span +from everos.infra.persistence.index import Predicate if TYPE_CHECKING: from .recall import KindRecaller @@ -213,7 +214,7 @@ async def _rerank( def build_retrieve_fn( recaller: KindRecaller, *, - where: str, + where: Predicate, embed_query_fn: Callable[[str], Awaitable[list[float]]], rrf_k: int = 60, ) -> RetrieveFn: diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index 1ae5bfd91..27d77cd50 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -50,6 +50,7 @@ emit_recall_scores, memory_span, ) +from everos.infra.persistence.index import Predicate from everos.infra.persistence.sqlite import ( UnprocessedBuffer, unprocessed_buffer_repo, @@ -299,7 +300,7 @@ async def _load_unprocessed( # ── Agent partition ───────────────────────────────────────────── async def _search_cases_and_skills( - self, req: SearchRequest, where: str + self, req: SearchRequest, where: Predicate ) -> tuple[list[SearchAgentCaseItem], list[SearchAgentSkillItem]]: """Cases + skills, serial when bridging. @@ -328,7 +329,7 @@ async def _search_cases_and_skills( # ── Episodes ──────────────────────────────────────────────────── async def _search_episodes( - self, req: SearchRequest, where: str + self, req: SearchRequest, where: Predicate ) -> list[SearchEpisodeItem]: if req.method == SearchMethod.AGENTIC: return await search_episodes_agentic( @@ -436,7 +437,7 @@ async def _search_episodes( # ── Agent cases ───────────────────────────────────────────────── async def _search_agent_cases( - self, req: SearchRequest, where: str + self, req: SearchRequest, where: Predicate ) -> list[SearchAgentCaseItem]: if req.method == SearchMethod.AGENTIC: return await search_agent_cases_agentic( @@ -492,7 +493,7 @@ async def _search_agent_cases( async def _search_agent_skills( self, req: SearchRequest, - where: str, + where: Predicate, *, bridge_cases: list[Candidate] | None = None, ) -> list[SearchAgentSkillItem]: @@ -584,7 +585,7 @@ async def _single_route_recall( self, recaller: EpisodeRecaller | AgentCaseRecaller | AgentSkillRecaller, req: SearchRequest, - where: str, + where: Predicate, top_k: int, *, cap: int = _DEFAULT_TOP_K_CAP, @@ -608,7 +609,7 @@ async def _recall_sparse_dense( self, recaller: EpisodeRecaller | AgentCaseRecaller | AgentSkillRecaller, req: SearchRequest, - where: str, + where: Predicate, top_k: int, *, cap: int = _DEFAULT_TOP_K_CAP, @@ -638,7 +639,7 @@ async def _recall_sparse_dense( return sparse, dense, vector async def _maxsim_atomic_recall( - self, req: SearchRequest, where: str, top_k: int + self, req: SearchRequest, where: Predicate, top_k: int ) -> list[Candidate]: """MaxSim-style: ANN atomic_facts → max-pool by memcell → batch fetch episodes. @@ -680,7 +681,7 @@ async def _maxsim_atomic_recall( async def _case_bridged_skills( self, bridge_cases: list[Candidate] | None, - where: str, + where: Predicate, top_k: int, ) -> list[Candidate]: """Reverse-resolve lineage skills and max-pool their source-case diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index cb88e931a..bdd99af88 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -34,7 +34,9 @@ import asyncio import importlib import json +import os import shutil +import uuid from collections.abc import AsyncIterator, Awaitable, Callable from importlib.resources import files from pathlib import Path @@ -114,10 +116,59 @@ def search_seed() -> dict[str, list[dict]]: # --------------------------------------------------------------------------- +def _index_backends() -> list[str]: + """Backends the e2e suite runs against. + + LanceDB is always exercised. Milvus joins only when a real server is + configured — it is a remote service with no embedded fallback, so a run + without ``EVEROS_TEST_MILVUS_URI`` proves nothing about it and should not + collect cases that suggest otherwise. + """ + backends = ["lancedb"] + if os.environ.get("EVEROS_TEST_MILVUS_URI"): + backends.append("milvus") + return backends + + +@pytest_asyncio.fixture(params=_index_backends(), ids=lambda b: f"index={b}") +async def index_backend( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> AsyncIterator[str]: + """Point the derived index at one backend for the duration of a test.""" + backend = request.param + monkeypatch.setenv("EVEROS_INDEX__BACKEND", backend) + if backend == "milvus": + monkeypatch.setenv("EVEROS_MILVUS__URI", os.environ["EVEROS_TEST_MILVUS_URI"]) + monkeypatch.setenv( + "EVEROS_MILVUS__TOKEN", os.environ.get("EVEROS_TEST_MILVUS_TOKEN", "") + ) + monkeypatch.setenv( + "EVEROS_MILVUS__DB_NAME", os.environ.get("EVEROS_TEST_MILVUS_DB_NAME", "") + ) + # Collections are remote and outlive the process, so each test needs + # its own namespace or state leaks between them. + monkeypatch.setenv( + "EVEROS_MILVUS__COLLECTION_PREFIX", f"everos_e2e_{uuid.uuid4().hex}" + ) + + yield backend + + if backend == "milvus": + from everos.config import load_settings + from everos.infra.persistence.index import drop_business_tables, shutdown + + load_settings.cache_clear() + try: + await drop_business_tables() + finally: + await shutdown() + + @pytest_asyncio.fixture async def core_pipeline_runtime( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + index_backend: str, ) -> AsyncIterator[Path]: """Prepare clean memory root + reset memorize singletons. diff --git a/tests/e2e/test_add_flush_agent_pipeline_e2e.py b/tests/e2e/test_add_flush_agent_pipeline_e2e.py index 43d00212a..1491dc52f 100644 --- a/tests/e2e/test_add_flush_agent_pipeline_e2e.py +++ b/tests/e2e/test_add_flush_agent_pipeline_e2e.py @@ -47,7 +47,7 @@ import everos.component.embedding.accessor as _embedding_accessor from everos.infra.ome.records import RunStatus -from everos.infra.persistence.lancedb import agent_case_repo, agent_skill_repo +from everos.infra.persistence.index import agent_case_repo, agent_skill_repo from everos.infra.persistence.markdown import AgentCaseDailyFrontmatter from everos.service.memorize import _get_engine diff --git a/tests/e2e/test_add_flush_user_pipeline_e2e.py b/tests/e2e/test_add_flush_user_pipeline_e2e.py index 61e66e5b1..d147c01ba 100644 --- a/tests/e2e/test_add_flush_user_pipeline_e2e.py +++ b/tests/e2e/test_add_flush_user_pipeline_e2e.py @@ -242,7 +242,7 @@ async def test_long_conversation_produces_all_memory_types( ) # 5.2 episode → LanceDB - from everos.infra.persistence.lancedb import episode_repo + from everos.infra.persistence.index import episode_repo lance_episode_count = await episode_repo.count() assert lance_episode_count >= 15, ( @@ -253,7 +253,7 @@ async def test_long_conversation_produces_all_memory_types( af_files = _list_md_files(memory_root, _ATOMIC_FACT_DIR) assert af_files, "no atomic_fact md files — extract_atomic_facts did not emit" - from everos.infra.persistence.lancedb import atomic_fact_repo + from everos.infra.persistence.index import atomic_fact_repo lance_af_count = await atomic_fact_repo.count() assert lance_af_count >= 1, ( @@ -266,7 +266,7 @@ async def test_long_conversation_produces_all_memory_types( # yields explicit future-intent statements, so count is usually 0. # We assert the LanceDB table exists (count returns 0 cleanly) — not # that any row was emitted. - from everos.infra.persistence.lancedb import foresight_repo + from everos.infra.persistence.index import foresight_repo lance_fs_count = await foresight_repo.count() assert lance_fs_count >= 0, f"foresight table broken: count={lance_fs_count}" diff --git a/tests/e2e/test_get_endpoint_e2e.py b/tests/e2e/test_get_endpoint_e2e.py index 55af0989e..2c249877e 100644 --- a/tests/e2e/test_get_endpoint_e2e.py +++ b/tests/e2e/test_get_endpoint_e2e.py @@ -21,7 +21,7 @@ from everos.config import load_settings from everos.entrypoints.api.app import create_app -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase, AgentSkill, Episode, @@ -29,9 +29,14 @@ agent_case_repo, agent_skill_repo, episode_repo, - lancedb_manager, user_profile_repo, ) +from everos.infra.persistence.index import shutdown as shutdown_index + +# Lance keeps a process-local connection and table cache. It is not part of +# the neutral port, but this fixture swaps EVEROS_ROOT without a lifespan, so +# the cache has to be dropped by hand or the next test reads the old root. +from everos.infra.persistence.lancedb import lancedb_manager # ``everos.service.__init__`` re-exports the ``get`` function under the # same name as the submodule (``from .get import get as get``), which @@ -128,6 +133,7 @@ def _agent_skill( async def client( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + index_backend: str, ) -> AsyncIterator[AsyncClient]: """Build the FastAPI app against a tmp memory root with no lifespan.""" monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) @@ -143,7 +149,7 @@ async def client( async with AsyncClient(transport=transport, base_url="http://test") as c: yield c - await lancedb_manager.dispose_connection() + await shutdown_index() load_settings.cache_clear() From 91e35127b764a279bfd49169b5abaa9ceb73f615 Mon Sep 17 00:00:00 2001 From: zhanghui Date: Thu, 3 Sep 2026 22:05:03 +0800 Subject: [PATCH 12/27] test(e2e): assert the truncation warning by substance, not by its label Running the get suite against Milvus surfaced the one assertion that still named a backend. The adapters report a truncated sort window under different events -- "find_where_paginated truncated" on LanceDB, "milvus_find_where_paginated_truncated" on Milvus -- and the test matched the LanceDB spelling, so it failed on Milvus even though the behaviour was right (total=10 against max_fetch=5, duly warned). It now asserts that truncation is reported and that the warning carries the cap that caused it, which is the actual contract. The differing event names are worth noting separately: alerting built on the LanceDB label goes silent after a switch to Milvus. Renaming either one changes observable output, so it is left alone here. Co-Authored-By: Cheney Zhang --- tests/e2e/test_get_endpoint_e2e.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test_get_endpoint_e2e.py b/tests/e2e/test_get_endpoint_e2e.py index 2c249877e..87e6e27dc 100644 --- a/tests/e2e/test_get_endpoint_e2e.py +++ b/tests/e2e/test_get_endpoint_e2e.py @@ -725,7 +725,15 @@ async def low_cap(*args: object, **kwargs: object) -> object: # structlog now routes through stdlib's root logger (see # ``core/observability/logging/factory.py``); the warning surfaces via # the standard ``caplog`` fixture rather than direct stdout capture. - assert "find_where_paginated truncated" in caplog.text + # + # Assert the warning's substance, not its spelling: the two adapters name + # this event differently ("find_where_paginated truncated" on LanceDB, + # "milvus_find_where_paginated_truncated" on Milvus), and the contract + # here is that truncating the sort window is reported with the cap that + # caused it -- not that either adapter picked a particular label. + truncation = [r for r in caplog.records if "truncat" in r.getMessage()] + assert truncation, "truncating the sort window must be reported" + assert "'max_fetch': 5" in truncation[0].getMessage() # ── Concurrency ───────────────────────────────────────────────────────── From c4fd7206c9addabbad44ceb5f17029fd629c958c Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 3 Sep 2026 21:34:06 +0000 Subject: [PATCH 13/27] fix(milvus): verify derived metadata --- .github/workflows/ci.yml | 8 + .../infra/persistence/milvus/repository.py | 129 +++++++++++++- tests/e2e/test_get_endpoint_e2e.py | 13 +- tests/integration/test_milvus_remote.py | 8 +- .../unit/test_infra/test_milvus/test_repo.py | 165 +++++++++++++++++- 5 files changed, 301 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f84a3b139..6a39d42f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,6 +165,14 @@ jobs: EVEROS_TEST_MILVUS_FULL_STARTUP: "1" run: uv run --frozen pytest tests/integration/test_milvus_remote.py -v + - name: Milvus /get truncation telemetry + env: + EVEROS_TEST_MILVUS_URI: http://127.0.0.1:19530 + run: >- + uv run --frozen pytest + tests/e2e/test_get_endpoint_e2e.py::test_get_truncates_above_max_fetch + -v -k milvus + # The contract tests above exercise the repository port directly. This # drives the same request path the application does -- /add, /flush, # cascade, every search method, /get -- against the remote backend, so a diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index fb9ce713f..168943d19 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -120,6 +120,11 @@ def mismatches(self, actual: dict[str, Any]) -> list[str]: f"{self.name}: element_type {_name_of(actual.get('element_type'))} " f"!= expected {_name_of(self.element_type)}" ) + if self.enable_analyzer and not _is_true(params.get("enable_analyzer")): + out.append( + f"{self.name}: enable_analyzer " + f"{params.get('enable_analyzer', False)} != expected True" + ) return out def soft_mismatches(self, actual: dict[str, Any]) -> list[str]: @@ -140,6 +145,36 @@ def _name_of(datatype: Any) -> str: return getattr(datatype, "name", repr(datatype)) +def _is_true(value: Any) -> bool: + return value is True or (isinstance(value, str) and value.casefold() == "true") + + +def _function_key( + function: dict[str, Any], +) -> tuple[Any, tuple[str, ...], tuple[str, ...]]: + return ( + function.get("type"), + tuple(function.get("input_field_names") or ()), + tuple(function.get("output_field_names") or ()), + ) + + +def _format_function_key( + key: tuple[Any, tuple[str, ...], tuple[str, ...]], +) -> str: + function_type, inputs, outputs = key + return ( + f"type={_name_of(function_type)}, inputs={list(inputs)}, " + f"outputs={list(outputs)}" + ) + + +def _metric_name(metric: Any) -> str | None: + if metric is None: + return None + return str(getattr(metric, "name", metric)).upper() + + class MilvusRepoBase[T: BaseModel]: """Generic Milvus repository backed by one neutral index schema.""" @@ -235,15 +270,15 @@ async def _create_collection(self, client: MilvusClient) -> None: ) async def verify_collection(self) -> None: - """Reject a collection whose physical schema disagrees with ours. + """Reject a collection whose physical metadata disagrees with ours. A name-only check waves through a collection whose columns happen to share our names but carry the wrong datatype, primary key, nullability or vector dimension — a stale model, or a ``collection_prefix`` collision with someone else's data. That collection starts fine and then fails opaquely on the first write or search. Comparing the - physical shape turns it back into a startup error with a recovery - path. + physical shape, BM25 function mapping, and field-matched vector index + metrics turns it back into a startup error with a recovery path. """ client = await get_client() description = await _run(client.describe_collection, self.collection_name) @@ -265,16 +300,98 @@ async def verify_collection(self) -> None: drift.extend(physical.mismatches(actual)) advisory.extend(physical.soft_mismatches(actual)) + expected_functions = { + ( + FunctionType.BM25, + (field,), + (_sparse_field(field),), + ) + for field in self.index_schema.bm25_fields + } + reported_functions = { + _function_key(function) + for function in description.get("functions", []) + if isinstance(function, dict) + } + missing_functions = sorted( + _format_function_key(key) for key in expected_functions - reported_functions + ) + stale_functions = sorted( + _format_function_key(key) for key in reported_functions - expected_functions + ) + if advisory: logger.warning( "milvus_collection_limit_drift", collection=self.collection_name, details=advisory, ) - if missing or stale or drift: + if missing or stale or drift or missing_functions or stale_functions: raise MilvusSchemaMismatchError( f"Milvus collection {self.collection_name!r} schema drift: " - f"missing={missing}, stale={stale}, incompatible={drift}. " + f"missing={missing}, stale={stale}, incompatible={drift}, " + f"missing_functions={missing_functions}, " + f"stale_functions={stale_functions}. " + "The index is rebuildable from markdown; run " + "`everos cascade rebuild`." + ) + await self._verify_indexes(client) + + async def _verify_indexes(self, client: MilvusClient) -> None: + """Verify only the field and metric contracts of derived indexes.""" + expected = {field.name: "COSINE" for field in self.index_schema.vector_fields} + if not expected: + expected[_DUMMY_VECTOR_FIELD] = "COSINE" + expected.update( + {_sparse_field(field): "BM25" for field in self.index_schema.bm25_fields} + ) + + index_names = await _run(client.list_indexes, self.collection_name) + reported: dict[str, list[tuple[str, str | None]]] = {} + incompatible: list[str] = [] + physical = {field.name: field for field in self._physical_fields()} + vector_types = {DataType.FLOAT_VECTOR, DataType.SPARSE_FLOAT_VECTOR} + for index_name in index_names: + description = await _run( + client.describe_index, + self.collection_name, + index_name, + ) + if not isinstance(description, dict): + incompatible.append(f"index {index_name!r}: description unavailable") + continue + field_name = description.get("field_name") + if not isinstance(field_name, str): + incompatible.append(f"index {index_name!r}: field_name unavailable") + continue + declared = physical.get(field_name) + if field_name not in expected: + if declared is None or declared.datatype in vector_types: + incompatible.append( + f"index {index_name!r}: unexpected vector field {field_name!r}" + ) + continue + metric = description.get("metric_type") + if metric is None and isinstance(description.get("params"), dict): + metric = description["params"].get("metric_type") + reported.setdefault(field_name, []).append( + (str(index_name), _metric_name(metric)) + ) + + missing_indexes = sorted(set(expected) - set(reported)) + for field_name, indexes in reported.items(): + wanted = expected[field_name] + for index_name, metric in indexes: + if metric != wanted: + incompatible.append( + f"{field_name}: metric_type {metric!r} != expected {wanted!r} " + f"(index {index_name!r})" + ) + + if missing_indexes or incompatible: + raise MilvusSchemaMismatchError( + f"Milvus collection {self.collection_name!r} index drift: " + f"missing={missing_indexes}, incompatible={incompatible}. " "The index is rebuildable from markdown; run " "`everos cascade rebuild`." ) @@ -428,7 +545,7 @@ async def find_where_paginated( ) if total > len(raw): logger.warning( - "milvus_find_where_paginated_truncated", + "find_where_paginated truncated", table=self.table_name, total=total, max_fetch=max_fetch, diff --git a/tests/e2e/test_get_endpoint_e2e.py b/tests/e2e/test_get_endpoint_e2e.py index 87e6e27dc..950680b7c 100644 --- a/tests/e2e/test_get_endpoint_e2e.py +++ b/tests/e2e/test_get_endpoint_e2e.py @@ -726,12 +726,13 @@ async def low_cap(*args: object, **kwargs: object) -> object: # ``core/observability/logging/factory.py``); the warning surfaces via # the standard ``caplog`` fixture rather than direct stdout capture. # - # Assert the warning's substance, not its spelling: the two adapters name - # this event differently ("find_where_paginated truncated" on LanceDB, - # "milvus_find_where_paginated_truncated" on Milvus), and the contract - # here is that truncating the sort window is reported with the cap that - # caused it -- not that either adapter picked a particular label. - truncation = [r for r in caplog.records if "truncat" in r.getMessage()] + # Both adapters expose the same exact event so dashboards and alerts do + # not need backend-specific parsing. + truncation = [ + record + for record in caplog.records + if "find_where_paginated truncated" in record.getMessage() + ] assert truncation, "truncating the sort window must be reported" assert "'max_fetch': 5" in truncation[0].getMessage() diff --git a/tests/integration/test_milvus_remote.py b/tests/integration/test_milvus_remote.py index f84763270..d60a24432 100644 --- a/tests/integration/test_milvus_remote.py +++ b/tests/integration/test_milvus_remote.py @@ -312,8 +312,8 @@ async def test_datetime_round_trip_survives_pre_2001_instants() -> None: ) -async def test_verify_accepts_a_collection_this_adapter_just_created() -> None: - """The decisive check for physical schema verification. +async def test_verify_accepts_collection_schema_functions_and_indexes() -> None: + """The decisive check for physical metadata verification. Every offline test builds its fake ``describe_collection`` reply from the same descriptor the verifier compares against, so they can only prove that @@ -350,8 +350,8 @@ async def test_verify_accepts_a_collection_this_adapter_just_created() -> None: MilvusRepoBase._reset_collection_cache() for repo in ALL_REPOS: - # Raises MilvusSchemaMismatchError if our declaration and the server's - # description disagree on any field. + # Raises MilvusSchemaMismatchError if fields, BM25 functions, or + # vector index metrics disagree with our declaration. await repo._repo().verify_collection() # type: ignore[attr-defined] await episode_repo.delete_by_md_path( diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index 15c5a5e98..1177e4ddb 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -5,7 +5,7 @@ import datetime as dt import pytest -from pymilvus import DataType +from pymilvus import DataType, FunctionType from everos.config import load_settings from everos.infra.persistence.index import ( @@ -52,20 +52,53 @@ def _describe_response(milvus_repo, *, always_params: bool = False): # type: ig while ``to_dict`` omits it when empty — ``always_params`` covers that one known difference between the two serializations. """ - fields = [ - dict(f) for f in milvus_repo._build_collection_schema().to_dict()["fields"] - ] + schema = milvus_repo._build_collection_schema().to_dict() + fields = [dict(f) for f in schema["fields"]] + for field in fields: + params = field.get("params") or {} + if "enable_analyzer" in params: + field["params"] = {**params, "enable_analyzer": "true"} if always_params: for field in fields: field.setdefault("params", {}) - return {"fields": fields} + return { + "fields": fields, + "functions": [dict(function) for function in schema.get("functions", [])], + } -def _patch_describe(monkeypatch, response): # type: ignore[no-untyped-def] +def _index_descriptions(response): # type: ignore[no-untyped-def] + indexes = {} + for position, field in enumerate(response["fields"]): + if field["type"] not in {DataType.FLOAT_VECTOR, DataType.SPARSE_FLOAT_VECTOR}: + continue + name = f"opaque_index_{position}" + indexes[name] = { + "field_name": field["name"], + "index_name": name, + "index_type": "AUTOINDEX", + "metric_type": ( + "BM25" if field["type"] == DataType.SPARSE_FLOAT_VECTOR else "COSINE" + ), + "state": "Finished", + "params": {}, + } + return indexes + + +def _patch_describe(monkeypatch, response, *, indexes=None): # type: ignore[no-untyped-def] + index_descriptions = _index_descriptions(response) if indexes is None else indexes + class _FakeClient: def describe_collection(self, name: str): # type: ignore[no-untyped-def] return response + def list_indexes(self, name: str): # type: ignore[no-untyped-def] + return list(index_descriptions) + + def describe_index(self, name: str, index_name: str): # type: ignore[no-untyped-def] + return index_descriptions.get(index_name) + async def _fake_get_client(): # type: ignore[no-untyped-def] return _FakeClient() @@ -201,6 +234,12 @@ def describe_collection(self, name: str): # type: ignore[no-untyped-def] self.describe_calls += 1 return _describe_response(milvus_repo) + def list_indexes(self, name: str): # type: ignore[no-untyped-def] + return list(_index_descriptions(_describe_response(milvus_repo))) + + def describe_index(self, name: str, index_name: str): # type: ignore[no-untyped-def] + return _index_descriptions(_describe_response(milvus_repo))[index_name] + client = _FakeClient() async def _fake_get_client(): # type: ignore[no-untyped-def] @@ -281,6 +320,52 @@ async def test_verify_rejects_primary_and_nullable_drift( assert "deprecated_by: nullable" in str(excinfo.value) +@pytest.mark.parametrize("reported", [None, False, "false"]) +async def test_verify_rejects_missing_or_disabled_bm25_input_analyzer( + monkeypatch: pytest.MonkeyPatch, + reported: object, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + for field in response["fields"]: + if field["name"] == "episode_tokens": + if reported is None: + field["params"].pop("enable_analyzer") + else: + field["params"]["enable_analyzer"] = reported + _patch_describe(monkeypatch, response) + + with pytest.raises( + MilvusSchemaMismatchError, match="episode_tokens: enable_analyzer" + ): + await milvus_repo.verify_collection() + + +@pytest.mark.parametrize( + ("attribute", "replacement"), + [ + ("type", FunctionType.TEXTEMBEDDING), + ("input_field_names", ["episode"]), + ("output_field_names", ["vector"]), + ], +) +async def test_verify_rejects_bm25_function_mapping_drift( + monkeypatch: pytest.MonkeyPatch, + attribute: str, + replacement: object, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + response["functions"][0][attribute] = replacement + _patch_describe(monkeypatch, response) + + with pytest.raises(MilvusSchemaMismatchError) as excinfo: + await milvus_repo.verify_collection() + message = str(excinfo.value) + assert "missing_functions=" in message + assert "stale_functions=" in message + + async def test_verify_rejects_missing_and_stale_columns( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -325,6 +410,74 @@ async def test_verify_treats_limit_drift_as_advisory( assert warnings[0][1]["details"] == ["subject: max_length 4096 != declared 65535"] +@pytest.mark.parametrize( + ("repo_factory", "field_name", "metric"), + [ + (lambda: episode_repo._repo(), "vector", "L2"), # type: ignore[attr-defined] + ( + lambda: episode_repo._repo(), # type: ignore[attr-defined] + "episode_tokens__sparse", + "COSINE", + ), + ( + lambda: user_profile_repo._repo(), # type: ignore[attr-defined] + "_everos_dummy_vector", + None, + ), + ], +) +async def test_verify_rejects_missing_or_wrong_vector_index( + monkeypatch: pytest.MonkeyPatch, + repo_factory, # type: ignore[no-untyped-def] + field_name: str, + metric: str | None, +) -> None: + milvus_repo = repo_factory() + response = _describe_response(milvus_repo) + indexes = _index_descriptions(response) + index_name = next( + name + for name, description in indexes.items() + if description["field_name"] == field_name + ) + if metric is None: + indexes.pop(index_name) + else: + indexes[index_name]["metric_type"] = metric + _patch_describe(monkeypatch, response, indexes=indexes) + + with pytest.raises(MilvusSchemaMismatchError, match="index drift") as excinfo: + await milvus_repo.verify_collection() + assert field_name in str(excinfo.value) + + +async def test_verify_allows_scalar_indexes_and_ignores_index_implementation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + indexes = _index_descriptions(response) + for description in indexes.values(): + description.update( + { + "index_name": "server_selected_name", + "index_type": "server-selected implementation", + "state": "InProgress", + "params": {"nlist": 2048, "server_default": True}, + } + ) + indexes["extra_scalar"] = { + "field_name": "owner_id", + "index_name": "extra_scalar", + "index_type": "INVERTED", + "state": "Finished", + "params": {}, + } + _patch_describe(monkeypatch, response, indexes=indexes) + + await milvus_repo.verify_collection() + + # Written out by hand on purpose. _describe_response() builds its fake server # reply from _build_collection_schema(), which shares _physical_fields() with # the verifier — so those tests can only prove that creation and verification From 11e177f972b9506ee1808db05bd67f9adbeed4a8 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 3 Sep 2026 21:59:59 +0000 Subject: [PATCH 14/27] fix(milvus): tighten validation probes --- .../infra/persistence/milvus/repository.py | 4 + tests/e2e/test_get_endpoint_e2e.py | 22 ++- tests/integration/test_milvus_remote.py | 153 +++++++++++++++++- .../unit/test_infra/test_milvus/test_repo.py | 24 +++ 4 files changed, 198 insertions(+), 5 deletions(-) diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index 168943d19..0e3f7cdb9 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -381,6 +381,10 @@ async def _verify_indexes(self, client: MilvusClient) -> None: missing_indexes = sorted(set(expected) - set(reported)) for field_name, indexes in reported.items(): wanted = expected[field_name] + if len(indexes) != 1: + incompatible.append( + f"{field_name}: expected exactly one index, found {len(indexes)}" + ) for index_name, metric in indexes: if metric != wanted: incompatible.append( diff --git a/tests/e2e/test_get_endpoint_e2e.py b/tests/e2e/test_get_endpoint_e2e.py index 950680b7c..18f1c46ea 100644 --- a/tests/e2e/test_get_endpoint_e2e.py +++ b/tests/e2e/test_get_endpoint_e2e.py @@ -12,6 +12,8 @@ import asyncio import datetime as _dt +import os +import re from collections.abc import AsyncIterator from importlib import import_module from pathlib import Path @@ -677,6 +679,7 @@ async def test_get_episodes_top_level_and_filter(client: AsyncClient) -> None: async def test_get_truncates_above_max_fetch( client: AsyncClient, + index_backend: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: @@ -694,6 +697,16 @@ async def test_get_truncates_above_max_fetch( # wired up and ``caplog`` can observe the chassis warning. from everos.core.observability.logging import configure_logging + if index_backend == "milvus": + requested_prefix = os.environ.get("EVEROS_TEST_MILVUS_PREFIX") + if requested_prefix: + assert re.fullmatch(r"everos_e2e_[0-9a-f]{32}", requested_prefix) + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", requested_prefix) + load_settings.cache_clear() + prefix = load_settings().milvus.collection_prefix + assert re.fullmatch(r"everos_e2e_[0-9a-f]{32}", prefix) + print(f"EVEROS_E2E_PREFIX={prefix}") + configure_logging(level="WARNING") await episode_repo.add( @@ -729,12 +742,13 @@ async def low_cap(*args: object, **kwargs: object) -> object: # Both adapters expose the same exact event so dashboards and alerts do # not need backend-specific parsing. truncation = [ - record - for record in caplog.records - if "find_where_paginated truncated" in record.getMessage() + record for record in caplog.records if "truncated" in record.getMessage() ] assert truncation, "truncating the sort window must be reported" - assert "'max_fetch': 5" in truncation[0].getMessage() + message = truncation[0].getMessage() + assert "'event': 'find_where_paginated truncated'" in message + assert "'total': 10" in message + assert "'max_fetch': 5" in message # ── Concurrency ───────────────────────────────────────────────────────── diff --git a/tests/integration/test_milvus_remote.py b/tests/integration/test_milvus_remote.py index d60a24432..dd3f0da3d 100644 --- a/tests/integration/test_milvus_remote.py +++ b/tests/integration/test_milvus_remote.py @@ -9,7 +9,9 @@ import asyncio import datetime as dt +import json import os +import re import uuid import pytest @@ -27,7 +29,10 @@ @pytest_asyncio.fixture(autouse=True) async def _remote_milvus(monkeypatch: pytest.MonkeyPatch): - prefix = f"everos_e2e_{uuid.uuid4().hex}" + prefix = os.environ.get( + "EVEROS_TEST_MILVUS_PREFIX", f"everos_e2e_{uuid.uuid4().hex}" + ) + assert re.fullmatch(r"everos_e2e_[0-9a-f]{32}", prefix) monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") monkeypatch.setenv("EVEROS_MILVUS__URI", _URI) monkeypatch.setenv( @@ -40,9 +45,12 @@ async def _remote_milvus(monkeypatch: pytest.MonkeyPatch): ) monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", prefix) load_settings.cache_clear() + print(f"EVEROS_E2E_PREFIX={prefix}") + from everos.core.observability.logging import configure_logging from everos.infra.persistence.index import episode_repo, startup + configure_logging(level="WARNING") try: if os.environ.get("EVEROS_TEST_MILVUS_FULL_STARTUP") == "1": await startup() @@ -238,6 +246,149 @@ async def test_remote_milvus_matches_derived_index_contract() -> None: assert await episode_repo.count() == 0 +async def test_episode_only_probe_covers_metadata_search_and_http() -> None: + """One collection covers metadata, dense/BM25 search, and HTTP ``/get``.""" + from importlib import import_module + + from httpx import ASGITransport, AsyncClient + + from everos.entrypoints.api.app import create_app + from everos.infra.persistence.index import Episode, episode_repo + from everos.infra.persistence.milvus.milvus_manager import get_client + from everos.memory.search import FilterNode + from everos.memory.search.filters import compile_filters + + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + await milvus_repo.verify_collection() + + client = await get_client() + description = await asyncio.to_thread( + client.describe_collection, milvus_repo.collection_name + ) + analyzers = [ + { + "field_name": field["name"], + "enable_analyzer": str( + (field.get("params") or {}).get("enable_analyzer") + ).casefold() + == "true", + } + for field in description.get("fields", []) + if field.get("name") == "episode_tokens" + ] + functions = [ + { + "type": getattr(function.get("type"), "name", str(function.get("type"))), + "input_field_names": list(function.get("input_field_names") or []), + "output_field_names": list(function.get("output_field_names") or []), + } + for function in description.get("functions", []) + ] + indexes = [] + for index_name in await asyncio.to_thread( + client.list_indexes, milvus_repo.collection_name + ): + index = await asyncio.to_thread( + client.describe_index, milvus_repo.collection_name, index_name + ) + if not isinstance(index, dict): + continue + field_name = index.get("field_name") + if field_name not in {"vector", "subject_vector", "episode_tokens__sparse"}: + continue + metric = index.get("metric_type") + if metric is None and isinstance(index.get("params"), dict): + metric = index["params"].get("metric_type") + indexes.append({"field_name": field_name, "metric_type": str(metric).upper()}) + indexes.sort(key=lambda item: str(item["field_name"])) + + assert analyzers == [{"field_name": "episode_tokens", "enable_analyzer": True}] + assert functions == [ + { + "type": "BM25", + "input_field_names": ["episode_tokens"], + "output_field_names": ["episode_tokens__sparse"], + } + ] + assert indexes == [ + {"field_name": "episode_tokens__sparse", "metric_type": "BM25"}, + {"field_name": "subject_vector", "metric_type": "COSINE"}, + {"field_name": "vector", "metric_type": "COSINE"}, + ] + + now = dt.datetime(2026, 6, 1, tzinfo=dt.UTC) + await episode_repo.upsert( + [ + _episode( + row_id="probe_ep1", + entry_id="probe_ep1", + session_id="probe", + text="red apple probe", + vector_axis=0, + subject_axis=1, + timestamp=now, + ), + _episode( + row_id="probe_ep2", + entry_id="probe_ep2", + session_id="probe", + text="blue banana probe", + vector_axis=1, + subject_axis=0, + timestamp=now + dt.timedelta(seconds=1), + ), + ] + ) + where = compile_filters( + FilterNode.model_validate({"session_id": "probe"}), + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + ) + dense_query = [1.0] + [0.0] * 1023 + dense = await episode_repo.dense_search(dense_query, where, limit=2) + sparse = await episode_repo.sparse_search( + ["apple"], where, columns=Episode.BM25_FIELDS, limit=2 + ) + assert dense[0]["id"] == "probe_ep1" + assert dense[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + assert sparse[0]["id"] == "probe_ep1" + assert sparse[0]["_score"] > 0 + + get_service = import_module("everos.service.get") + get_service._manager = None + app = create_app(lifespan_providers=[]) + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as http: + response = await http.post( + "/api/v1/memory/get", + json={ + "user_id": "u1", + "memory_type": "episode", + "app_id": "test_app", + "project_id": "test_project", + "filters": {"session_id": "probe"}, + "page": 1, + "page_size": 2, + }, + ) + assert response.status_code == 200 + body = response.json() + assert body["data"]["count"] == 2 + assert body["data"]["total_count"] == 2 + + evidence = { + "analyzers": analyzers, + "functions": functions, + "indexes": indexes, + "dense_cosine": "passed", + "bm25": "passed", + "http_get": {"status": response.status_code, "count": 2, "total": 2}, + } + print(f"EVEROS_E2E_EVIDENCE={json.dumps(evidence, sort_keys=True)}") + + async def test_update_preserves_vectors_on_a_row_that_has_them() -> None: """Milvus has no partial-column update, so update() re-upserts whole rows. diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index 1177e4ddb..7a676c372 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -451,6 +451,30 @@ async def test_verify_rejects_missing_or_wrong_vector_index( assert field_name in str(excinfo.value) +async def test_verify_rejects_duplicate_same_metric_vector_indexes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + response = _describe_response(milvus_repo) + indexes = _index_descriptions(response) + original = next( + description + for description in indexes.values() + if description["field_name"] == "vector" + ) + indexes["duplicate_vector"] = { + **original, + "index_name": "duplicate_vector", + } + _patch_describe(monkeypatch, response, indexes=indexes) + + with pytest.raises( + MilvusSchemaMismatchError, + match=r"vector: expected exactly one index, found 2", + ): + await milvus_repo.verify_collection() + + async def test_verify_allows_scalar_indexes_and_ignores_index_implementation( monkeypatch: pytest.MonkeyPatch, ) -> None: From 05045aafe5555e7cbc2c756a8a443ada806fde6a Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 3 Sep 2026 22:19:51 +0000 Subject: [PATCH 15/27] test(milvus): guarantee fixture finalization --- tests/integration/test_milvus_remote.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_milvus_remote.py b/tests/integration/test_milvus_remote.py index dd3f0da3d..7c3f8d8a2 100644 --- a/tests/integration/test_milvus_remote.py +++ b/tests/integration/test_milvus_remote.py @@ -60,9 +60,13 @@ async def _remote_milvus(monkeypatch: pytest.MonkeyPatch): finally: from everos.infra.persistence.index import drop_business_tables, shutdown - await drop_business_tables() - await shutdown() - load_settings.cache_clear() + try: + await drop_business_tables() + finally: + try: + await shutdown() + finally: + load_settings.cache_clear() def _episode( From c96318854ff90394717d0ef049dc839e489f54aa Mon Sep 17 00:00:00 2001 From: zhanghui Date: Fri, 4 Sep 2026 10:27:32 +0800 Subject: [PATCH 16/27] test(index): check both adapters against the port contract The contract module asserted `isinstance(lance_index_backend, IndexBackend)` and stopped there, so the one place that says two implementations answer to the same contract was only ever looking at one of them. A second adapter could drop a port method or register a different table set and nothing would fail. The contract splits cleanly in two. Whether an adapter satisfies the ports and registers the same seven tables is structural -- no server involved -- so both backends are checked unconditionally, in plain unit tests. Driving records through one is not, so the CRUD, scan and maintenance cases take the same `index_backend` fixture as the other suites: LanceDB always, Milvus when EVEROS_TEST_MILVUS_URI names a server. Verified by breaking the Milvus adapter two ways -- dropping a repository from ALL_REPOS, and renaming `scan` off the repository -- each of which now fails the structural check without a Milvus running. Against a real server the behavioural cases pass on both backends. Co-Authored-By: Cheney Zhang --- tests/unit/test_infra/test_index_contract.py | 87 ++++++++++++++++++-- 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_infra/test_index_contract.py b/tests/unit/test_infra/test_index_contract.py index 10b3a274d..eefff803a 100644 --- a/tests/unit/test_infra/test_index_contract.py +++ b/tests/unit/test_infra/test_index_contract.py @@ -3,6 +3,8 @@ from __future__ import annotations import datetime as dt +import os +import uuid from pathlib import Path import pytest @@ -17,16 +19,62 @@ eq, ) from everos.infra.persistence.index.lancedb import lance_index_backend, render_predicate +from everos.infra.persistence.index.milvus import milvus_index_backend from everos.infra.persistence.index.schema import IndexFieldKind, schema_for from everos.infra.persistence.lancedb import lancedb_manager -@pytest.fixture(autouse=True) -async def _isolated_index(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): +def _behavioural_backends() -> list[str]: + """Backends whose behaviour this module can actually exercise. + + The structural half of the contract needs no server and covers both + adapters unconditionally. Driving records through one does: Milvus is + remote, so it joins only when EVEROS_TEST_MILVUS_URI points at a server. + """ + backends = ["lancedb"] + if os.environ.get("EVEROS_TEST_MILVUS_URI"): + backends.append("milvus") + return backends + + +@pytest.fixture(params=_behavioural_backends(), ids=lambda b: f"index={b}") +async def index_backend( + request: pytest.FixtureRequest, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """Route the shared repositories at one backend and clean up after.""" + backend = request.param monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_INDEX__BACKEND", backend) lancedb_manager._conn = None lancedb_manager._tables.clear() - yield + + if backend == "milvus": + monkeypatch.setenv("EVEROS_MILVUS__URI", os.environ["EVEROS_TEST_MILVUS_URI"]) + monkeypatch.setenv( + "EVEROS_MILVUS__TOKEN", os.environ.get("EVEROS_TEST_MILVUS_TOKEN", "") + ) + monkeypatch.setenv( + "EVEROS_MILVUS__DB_NAME", os.environ.get("EVEROS_TEST_MILVUS_DB_NAME", "") + ) + monkeypatch.setenv( + "EVEROS_MILVUS__COLLECTION_PREFIX", f"everos_ct_{uuid.uuid4().hex}" + ) + + from everos.config import load_settings + + load_settings.cache_clear() + yield backend + + if backend == "milvus": + from everos.infra.persistence.index import drop_business_tables + + load_settings.cache_clear() + try: + await drop_business_tables() + finally: + from everos.infra.persistence.index import shutdown + + await shutdown() await lancedb_manager.dispose_connection() @@ -49,11 +97,30 @@ def _episode(number: int, *, owner_id: str = "owner") -> Episode: ) -def test_all_registered_repositories_and_backend_satisfy_ports() -> None: - assert isinstance(lance_index_backend, IndexBackend) - assert {repo.table_name for repo in lance_index_backend.repositories} == { +@pytest.mark.parametrize( + "backend", + [ + pytest.param(lance_index_backend, id="lancedb"), + pytest.param(milvus_index_backend, id="milvus"), + ], +) +def test_every_backend_satisfies_the_ports(backend: IndexBackend) -> None: + """Both adapters, not just the default one. + + This is the only assertion that says the two implementations answer to the + same contract, so checking one of them was checking nothing -- a second + adapter could omit a method or register a different table set and no test + would notice. Needs no server: the ports and the registered repository set + are structural. + """ + assert isinstance(backend, IndexBackend) + assert {repo.table_name for repo in backend.repositories} == { repo.table_name for repo in ALL_REPOS } + assert all(isinstance(repo, IndexRepository) for repo in backend.repositories) + + +def test_the_registered_repository_set_is_the_seven_business_tables() -> None: assert len({repo.table_name for repo in ALL_REPOS}) == len(ALL_REPOS) == 7 assert all(isinstance(repo, IndexRepository) for repo in ALL_REPOS) @@ -77,7 +144,9 @@ def test_lance_predicate_renderer_owns_escaping() -> None: assert "session_id = 'session'" in rendered -async def test_repository_port_crud_count_and_uncapped_scan() -> None: +async def test_repository_port_crud_count_and_uncapped_scan( + index_backend: str, +) -> None: rows = [_episode(number) for number in range(101)] await episode_repo.upsert(rows) @@ -100,7 +169,9 @@ async def test_repository_port_crud_count_and_uncapped_scan() -> None: assert await episode_repo.count_where(owner) == 100 -async def test_repository_maintenance_contract_is_executable() -> None: +async def test_repository_maintenance_contract_is_executable( + index_backend: str, +) -> None: await episode_repo.upsert([_episode(1)]) await episode_repo.optimize() await episode_repo.prune(dt.timedelta(days=7)) From fd3dad1ec5ee9c69f501ab595e8d3d571607e35a Mon Sep 17 00:00:00 2001 From: zhanghui Date: Fri, 4 Sep 2026 10:27:44 +0800 Subject: [PATCH 17/27] ci: run the port contract against Milvus too The behavioural contract cases need a server to have a second backend to compare against, and the Milvus job is the only place one exists. The structural cases already cover both adapters in the ordinary unit job. Co-Authored-By: Cheney Zhang --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f84a3b139..7337ed1a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,6 +175,14 @@ jobs: EVEROS_TEST_MILVUS_URI: http://127.0.0.1:19530 run: uv run --frozen pytest tests/integration/test_tiers -v -k milvus + # The structural half of the port contract runs in the ordinary unit job + # for both adapters. The behavioural half needs a server, so it only has + # a backend to compare against here. + - name: Derived-index port contract (both backends) + env: + EVEROS_TEST_MILVUS_URI: http://127.0.0.1:19530 + run: uv run --frozen pytest tests/unit/test_infra/test_index_contract.py -v + - name: Stop Milvus if: always() run: docker compose -f /tmp/milvus-compose.yml down -v From 7f972911d514e2a0880d7b06277276599b474905 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Fri, 4 Sep 2026 07:33:18 +0000 Subject: [PATCH 18/27] test(index): tighten Milvus contract isolation --- tests/unit/test_infra/test_index_contract.py | 67 ++++++++++++++++---- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/tests/unit/test_infra/test_index_contract.py b/tests/unit/test_infra/test_index_contract.py index eefff803a..1e4cd5e79 100644 --- a/tests/unit/test_infra/test_index_contract.py +++ b/tests/unit/test_infra/test_index_contract.py @@ -4,6 +4,7 @@ import datetime as dt import os +import re import uuid from pathlib import Path @@ -15,14 +16,18 @@ IndexBackend, IndexRepository, all_of, + drop_business_tables, episode_repo, eq, + shutdown, ) from everos.infra.persistence.index.lancedb import lance_index_backend, render_predicate from everos.infra.persistence.index.milvus import milvus_index_backend from everos.infra.persistence.index.schema import IndexFieldKind, schema_for from everos.infra.persistence.lancedb import lancedb_manager +_EXTERNAL_PREFIX_PATTERN = re.compile(r"everos_e2e_[0-9a-f]{32}") + def _behavioural_backends() -> list[str]: """Backends whose behaviour this module can actually exercise. @@ -37,6 +42,17 @@ def _behavioural_backends() -> list[str]: return backends +def _milvus_collection_prefix() -> str: + external_prefix = os.environ.get("EVEROS_TEST_MILVUS_PREFIX") + if external_prefix is None: + return f"everos_ct_{uuid.uuid4().hex}" + if _EXTERNAL_PREFIX_PATTERN.fullmatch(external_prefix) is None: + raise ValueError( + "EVEROS_TEST_MILVUS_PREFIX must match everos_e2e_<32 lowercase hex>" + ) + return external_prefix + + @pytest.fixture(params=_behavioural_backends(), ids=lambda b: f"index={b}") async def index_backend( request: pytest.FixtureRequest, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -48,7 +64,9 @@ async def index_backend( lancedb_manager._conn = None lancedb_manager._tables.clear() + collection_prefix: str | None = None if backend == "milvus": + collection_prefix = _milvus_collection_prefix() monkeypatch.setenv("EVEROS_MILVUS__URI", os.environ["EVEROS_TEST_MILVUS_URI"]) monkeypatch.setenv( "EVEROS_MILVUS__TOKEN", os.environ.get("EVEROS_TEST_MILVUS_TOKEN", "") @@ -56,26 +74,53 @@ async def index_backend( monkeypatch.setenv( "EVEROS_MILVUS__DB_NAME", os.environ.get("EVEROS_TEST_MILVUS_DB_NAME", "") ) - monkeypatch.setenv( - "EVEROS_MILVUS__COLLECTION_PREFIX", f"everos_ct_{uuid.uuid4().hex}" - ) + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", collection_prefix) from everos.config import load_settings load_settings.cache_clear() - yield backend - if backend == "milvus": - from everos.infra.persistence.index import drop_business_tables + assert collection_prefix is not None + milvus_episode_repo = next( + repo + for repo in milvus_index_backend.repositories + if repo.table_name == "episode" + ) + assert milvus_episode_repo.collection_name == f"{collection_prefix}_episode" - load_settings.cache_clear() + try: + yield backend + finally: try: - await drop_business_tables() + if backend == "milvus": + try: + load_settings.cache_clear() + finally: + try: + await drop_business_tables() + finally: + await shutdown() finally: - from everos.infra.persistence.index import shutdown + try: + load_settings.cache_clear() + finally: + await lancedb_manager.dispose_connection() + + +def test_external_milvus_prefix_is_used_verbatim( + monkeypatch: pytest.MonkeyPatch, +) -> None: + prefix = "everos_e2e_0123456789abcdef0123456789abcdef" + monkeypatch.setenv("EVEROS_TEST_MILVUS_PREFIX", prefix) + assert _milvus_collection_prefix() == prefix + - await shutdown() - await lancedb_manager.dispose_connection() +def test_external_milvus_prefix_rejects_unbounded_names( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("EVEROS_TEST_MILVUS_PREFIX", "everos_ct_unreviewed") + with pytest.raises(ValueError, match="must match everos_e2e"): + _milvus_collection_prefix() def _episode(number: int, *, owner_id: str = "owner") -> Episode: From 41a26e4543244f97d49b234a52c7827b91aba2af Mon Sep 17 00:00:00 2001 From: zhanghui Date: Fri, 4 Sep 2026 11:20:51 +0800 Subject: [PATCH 19/27] refactor(persistence): split derived-index ports from their adapters `index/` is the package outer layers depend on, and it held both the abstraction and 515 lines implementing it -- the LanceDB adapter and the Milvus lifecycle shim sat next to the protocols they satisfy. Anyone opening it to add a third backend had to work out which files were the contract by reading them. The adapters move to `persistence/backends/`. `index/` keeps the ports, the portable schema, the router and the facade; nothing else changes, and no behaviour does -- the facade re-exports the same names, so every caller outside `persistence` is untouched. Two contracts make the split hold, because moving files does not stop anyone moving them back: * Outer layers may not import `backends.**`, matching the rule already covering the other storage subpackages. * `index.protocols`, `index.schema` and the shared `predicate` module may not import any concrete backend. The facade and the router resolve a configured backend and legitimately do; the port definitions must not, or "portable" stops being a checkable claim. Verified by breaking each rule in turn and confirming import-linter reports it. Six test modules referenced the adapters by their old path and move with them. Co-Authored-By: Cheney Zhang --- benchmarks/config_hybrid.toml | 33 +++++++++++++++++++ benchmarks/config_hybrid_judge4o.toml | 33 +++++++++++++++++++ benchmarks/config_keyword.toml | 33 +++++++++++++++++++ benchmarks/config_vector.toml | 33 +++++++++++++++++++ benchmarks/config_vector_judge4o.toml | 33 +++++++++++++++++++ pyproject.toml | 26 +++++++++++++-- .../infra/persistence/backends/__init__.py | 18 ++++++++++ .../{index => backends}/lancedb.py | 2 +- .../persistence/{index => backends}/milvus.py | 2 +- .../infra/persistence/index/__init__.py | 18 +++++----- .../test_reflection_integration.py | 2 +- tests/unit/test_infra/test_index_contract.py | 7 ++-- .../unit/test_memory/test_get/test_manager.py | 2 +- .../test_memory/test_search/test_manager.py | 2 +- .../test_search/test_recall_or_semantics.py | 2 +- .../test_service/test_knowledge_search.py | 6 ++-- 16 files changed, 230 insertions(+), 22 deletions(-) create mode 100644 benchmarks/config_hybrid.toml create mode 100644 benchmarks/config_hybrid_judge4o.toml create mode 100644 benchmarks/config_keyword.toml create mode 100644 benchmarks/config_vector.toml create mode 100644 benchmarks/config_vector_judge4o.toml create mode 100644 src/everos/infra/persistence/backends/__init__.py rename src/everos/infra/persistence/{index => backends}/lancedb.py (99%) rename src/everos/infra/persistence/{index => backends}/milvus.py (97%) diff --git a/benchmarks/config_hybrid.toml b/benchmarks/config_hybrid.toml new file mode 100644 index 000000000..a749f7d5f --- /dev/null +++ b/benchmarks/config_hybrid.toml @@ -0,0 +1,33 @@ +# Benchmark configuration — single source of truth. +# +# All parameters aligned with the upstream evaluation reference so that +# benchmark numbers are directly comparable to published LoCoMo results. +# BenchmarkConfig reads this file on every run. + +# --- EverOS server --- +cascade_timeout = 7200 # max seconds to wait for cascade + OME to complete +batch_size = 25 # messages per /add request (everos caps at 500) + +# --- Search --- +methods = "hybrid" # comma-separated: keyword, vector, hybrid, agentic +top_k = 10 # episodes to retrieve per question +eval_owner = "speaker_a" # which speaker's memory partition to query + +# --- Answer LLM --- +answer_model = "openai/gpt-4.1-mini" # LLM for generating answers from retrieved context +answer_temperature = 0.0 # deterministic answers +answer_max_tokens = 32768 # max output tokens per answer call +answer_timeout = 300.0 # per-request socket timeout (seconds) +answer_max_retries = 5 # retry budget (HTTP + empty-response retries) + +# --- Judge LLM --- +judge_model = "openai/gpt-4.1-mini" # LLM judge for correctness evaluation +judge_temperature = 0.0 # deterministic judging +judge_timeout = 300.0 # per-request socket timeout (seconds) +judge_max_retries = 5 # retry budget +judge_runs = 3 # majority vote across N independent judge runs + +# --- Concurrency --- +conversations_concurrency = 10 # how many conversations run at the same time +search_concurrency = 5 # concurrent /search requests per conversation +eval_concurrency = 20 # concurrent answer/judge LLM calls per conversation diff --git a/benchmarks/config_hybrid_judge4o.toml b/benchmarks/config_hybrid_judge4o.toml new file mode 100644 index 000000000..573e93b24 --- /dev/null +++ b/benchmarks/config_hybrid_judge4o.toml @@ -0,0 +1,33 @@ +# Benchmark configuration — single source of truth. +# +# All parameters aligned with the upstream evaluation reference so that +# benchmark numbers are directly comparable to published LoCoMo results. +# BenchmarkConfig reads this file on every run. + +# --- EverOS server --- +cascade_timeout = 7200 # max seconds to wait for cascade + OME to complete +batch_size = 25 # messages per /add request (everos caps at 500) + +# --- Search --- +methods = "hybrid" # comma-separated: keyword, vector, hybrid, agentic +top_k = 10 # episodes to retrieve per question +eval_owner = "speaker_a" # which speaker's memory partition to query + +# --- Answer LLM --- +answer_model = "openai/gpt-4.1-mini" # LLM for generating answers from retrieved context +answer_temperature = 0.0 # deterministic answers +answer_max_tokens = 32768 # max output tokens per answer call +answer_timeout = 300.0 # per-request socket timeout (seconds) +answer_max_retries = 5 # retry budget (HTTP + empty-response retries) + +# --- Judge LLM --- +judge_model = "openai/gpt-4o-mini" +judge_temperature = 0.0 # deterministic judging +judge_timeout = 300.0 # per-request socket timeout (seconds) +judge_max_retries = 5 # retry budget +judge_runs = 3 # majority vote across N independent judge runs + +# --- Concurrency --- +conversations_concurrency = 10 # how many conversations run at the same time +search_concurrency = 5 # concurrent /search requests per conversation +eval_concurrency = 20 # concurrent answer/judge LLM calls per conversation diff --git a/benchmarks/config_keyword.toml b/benchmarks/config_keyword.toml new file mode 100644 index 000000000..4b31197ab --- /dev/null +++ b/benchmarks/config_keyword.toml @@ -0,0 +1,33 @@ +# Benchmark configuration — single source of truth. +# +# All parameters aligned with the upstream evaluation reference so that +# benchmark numbers are directly comparable to published LoCoMo results. +# BenchmarkConfig reads this file on every run. + +# --- EverOS server --- +cascade_timeout = 7200 # max seconds to wait for cascade + OME to complete +batch_size = 25 # messages per /add request (everos caps at 500) + +# --- Search --- +methods = "keyword" # comma-separated: keyword, vector, hybrid, agentic +top_k = 10 # episodes to retrieve per question +eval_owner = "speaker_a" # which speaker's memory partition to query + +# --- Answer LLM --- +answer_model = "openai/gpt-4.1-mini" # LLM for generating answers from retrieved context +answer_temperature = 0.0 # deterministic answers +answer_max_tokens = 32768 # max output tokens per answer call +answer_timeout = 300.0 # per-request socket timeout (seconds) +answer_max_retries = 5 # retry budget (HTTP + empty-response retries) + +# --- Judge LLM --- +judge_model = "openai/gpt-4.1-mini" # LLM judge for correctness evaluation +judge_temperature = 0.0 # deterministic judging +judge_timeout = 300.0 # per-request socket timeout (seconds) +judge_max_retries = 5 # retry budget +judge_runs = 3 # majority vote across N independent judge runs + +# --- Concurrency --- +conversations_concurrency = 10 # how many conversations run at the same time +search_concurrency = 5 # concurrent /search requests per conversation +eval_concurrency = 20 # concurrent answer/judge LLM calls per conversation diff --git a/benchmarks/config_vector.toml b/benchmarks/config_vector.toml new file mode 100644 index 000000000..da7342cbf --- /dev/null +++ b/benchmarks/config_vector.toml @@ -0,0 +1,33 @@ +# Benchmark configuration — single source of truth. +# +# All parameters aligned with the upstream evaluation reference so that +# benchmark numbers are directly comparable to published LoCoMo results. +# BenchmarkConfig reads this file on every run. + +# --- EverOS server --- +cascade_timeout = 7200 # max seconds to wait for cascade + OME to complete +batch_size = 25 # messages per /add request (everos caps at 500) + +# --- Search --- +methods = "vector" # comma-separated: keyword, vector, hybrid, agentic +top_k = 10 # episodes to retrieve per question +eval_owner = "speaker_a" # which speaker's memory partition to query + +# --- Answer LLM --- +answer_model = "openai/gpt-4.1-mini" # LLM for generating answers from retrieved context +answer_temperature = 0.0 # deterministic answers +answer_max_tokens = 32768 # max output tokens per answer call +answer_timeout = 300.0 # per-request socket timeout (seconds) +answer_max_retries = 5 # retry budget (HTTP + empty-response retries) + +# --- Judge LLM --- +judge_model = "openai/gpt-4.1-mini" # LLM judge for correctness evaluation +judge_temperature = 0.0 # deterministic judging +judge_timeout = 300.0 # per-request socket timeout (seconds) +judge_max_retries = 5 # retry budget +judge_runs = 3 # majority vote across N independent judge runs + +# --- Concurrency --- +conversations_concurrency = 10 # how many conversations run at the same time +search_concurrency = 5 # concurrent /search requests per conversation +eval_concurrency = 20 # concurrent answer/judge LLM calls per conversation diff --git a/benchmarks/config_vector_judge4o.toml b/benchmarks/config_vector_judge4o.toml new file mode 100644 index 000000000..551958c76 --- /dev/null +++ b/benchmarks/config_vector_judge4o.toml @@ -0,0 +1,33 @@ +# Benchmark configuration — single source of truth. +# +# All parameters aligned with the upstream evaluation reference so that +# benchmark numbers are directly comparable to published LoCoMo results. +# BenchmarkConfig reads this file on every run. + +# --- EverOS server --- +cascade_timeout = 7200 # max seconds to wait for cascade + OME to complete +batch_size = 25 # messages per /add request (everos caps at 500) + +# --- Search --- +methods = "vector" # comma-separated: keyword, vector, hybrid, agentic +top_k = 10 # episodes to retrieve per question +eval_owner = "speaker_a" # which speaker's memory partition to query + +# --- Answer LLM --- +answer_model = "openai/gpt-4.1-mini" # LLM for generating answers from retrieved context +answer_temperature = 0.0 # deterministic answers +answer_max_tokens = 32768 # max output tokens per answer call +answer_timeout = 300.0 # per-request socket timeout (seconds) +answer_max_retries = 5 # retry budget (HTTP + empty-response retries) + +# --- Judge LLM --- +judge_model = "openai/gpt-4o-mini" +judge_temperature = 0.0 # deterministic judging +judge_timeout = 300.0 # per-request socket timeout (seconds) +judge_max_retries = 5 # retry budget +judge_runs = 3 # majority vote across N independent judge runs + +# --- Concurrency --- +conversations_concurrency = 10 # how many conversations run at the same time +search_concurrency = 5 # concurrent /search requests per conversation +eval_concurrency = 20 # concurrent answer/judge LLM calls per conversation diff --git a/pyproject.toml b/pyproject.toml index 26ecf6984..9ac898645 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -229,6 +229,7 @@ forbidden_modules = [ "everos.infra.persistence.sqlite.**", "everos.infra.persistence.index.**", "everos.infra.persistence.milvus.**", + "everos.infra.persistence.backends.**", ] # `forbidden` contracts run a *transitive closure* — any path from a # source module to a forbidden module fails the check, even when the @@ -255,12 +256,33 @@ ignore_imports = [ "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.mds", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.writers", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.readers", - "everos.infra.persistence.index -> everos.infra.persistence.index.lancedb", - "everos.infra.persistence.index -> everos.infra.persistence.index.milvus", "everos.infra.persistence.index -> everos.infra.persistence.index.predicate", "everos.infra.persistence.index -> everos.infra.persistence.index.protocols", "everos.infra.persistence.index -> everos.infra.persistence.index.router", "everos.infra.persistence.index -> everos.infra.persistence.index.schema", + "everos.infra.persistence.index -> everos.infra.persistence.backends.lancedb", + "everos.infra.persistence.index -> everos.infra.persistence.backends.milvus", +] + +[[tool.importlinter.contracts]] +# The facade and the router resolve a configured backend, so they import the +# adapters by design. The port definitions must not: the moment protocols or +# schema knows a concrete engine, "portable" stops being checkable and the +# next adapter inherits whatever leaked in. This is the rule that keeps the +# split from eroding -- moving files alone does not stop anyone moving them +# back. +name = "Derived-index ports know no concrete backend" +type = "forbidden" +source_modules = [ + "everos.infra.persistence.index.protocols", + "everos.infra.persistence.index.schema", + "everos.infra.persistence.predicate", +] +forbidden_modules = [ + "everos.infra.persistence.backends", + "everos.infra.persistence.lancedb", + "everos.infra.persistence.milvus", + "everos.core.persistence.lancedb", ] [[tool.importlinter.contracts]] diff --git a/src/everos/infra/persistence/backends/__init__.py b/src/everos/infra/persistence/backends/__init__.py new file mode 100644 index 000000000..7547c8ce3 --- /dev/null +++ b/src/everos/infra/persistence/backends/__init__.py @@ -0,0 +1,18 @@ +"""Concrete derived-index adapters. + +Each module here implements the ports in +:mod:`everos.infra.persistence.index.protocols` for one storage engine, and +owns everything specific to it: predicate rendering, physical schema, and the +connection lifecycle. + +They live outside :mod:`everos.infra.persistence.index` on purpose. That +package is the boundary outer layers depend on, and mixing adapters into it +made "which of these is the abstraction?" a question a reader had to answer by +opening files. Adding a third backend means adding a module here, not editing +the abstraction. + +Nothing outside ``persistence`` should import these directly — go through the +``index`` facade, which resolves the configured backend at call time. +""" + +from __future__ import annotations diff --git a/src/everos/infra/persistence/index/lancedb.py b/src/everos/infra/persistence/backends/lancedb.py similarity index 99% rename from src/everos/infra/persistence/index/lancedb.py rename to src/everos/infra/persistence/backends/lancedb.py index 2b284f571..6b7eadca2 100644 --- a/src/everos/infra/persistence/index/lancedb.py +++ b/src/everos/infra/persistence/backends/lancedb.py @@ -20,7 +20,7 @@ from everos.core.persistence import LanceRepoBase from everos.infra.persistence import lancedb as _lancedb -from .predicate import ( +from ..index.predicate import ( All, AnyOf, Comparison, diff --git a/src/everos/infra/persistence/index/milvus.py b/src/everos/infra/persistence/backends/milvus.py similarity index 97% rename from src/everos/infra/persistence/index/milvus.py rename to src/everos/infra/persistence/backends/milvus.py index f3e1e81a4..5291bd77e 100644 --- a/src/everos/infra/persistence/index/milvus.py +++ b/src/everos/infra/persistence/backends/milvus.py @@ -5,7 +5,7 @@ from types import ModuleType from typing import Any, ClassVar -from .protocols import IndexRepository +from ..index.protocols import IndexRepository class MilvusIndexBackend: diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py index 419df6b93..abf01382c 100644 --- a/src/everos/infra/persistence/index/__init__.py +++ b/src/everos/infra/persistence/index/__init__.py @@ -13,31 +13,31 @@ from everos.config import load_settings from everos.infra.persistence import lancedb as _lancedb -from .lancedb import ( +from ..backends.lancedb import ( agent_case_repo as _lance_agent_case_repo, ) -from .lancedb import ( +from ..backends.lancedb import ( agent_skill_repo as _lance_agent_skill_repo, ) -from .lancedb import ( +from ..backends.lancedb import ( atomic_fact_repo as _lance_atomic_fact_repo, ) -from .lancedb import ( +from ..backends.lancedb import ( episode_repo as _lance_episode_repo, ) -from .lancedb import ( +from ..backends.lancedb import ( foresight_repo as _lance_foresight_repo, ) -from .lancedb import ( +from ..backends.lancedb import ( knowledge_topic_repo as _lance_knowledge_topic_repo, ) -from .lancedb import ( +from ..backends.lancedb import ( lance_index_backend, ) -from .lancedb import ( +from ..backends.lancedb import ( user_profile_repo as _lance_user_profile_repo, ) -from .milvus import milvus_index_backend +from ..backends.milvus import milvus_index_backend from .predicate import ( All, AnyOf, diff --git a/tests/integration/test_reflection_integration.py b/tests/integration/test_reflection_integration.py index 401267ba7..6ab669505 100644 --- a/tests/integration/test_reflection_integration.py +++ b/tests/integration/test_reflection_integration.py @@ -34,7 +34,7 @@ ) from everos.core.persistence.lancedb import LanceDailyLogRepoBase, LanceRepoBase from everos.infra.ome.testing import FakeStrategyContext -from everos.infra.persistence.index.lancedb import LanceIndexRepository +from everos.infra.persistence.backends.lancedb import LanceIndexRepository from everos.infra.persistence.lancedb.tables.atomic_fact import AtomicFact from everos.infra.persistence.lancedb.tables.episode import Episode as LanceEpisode from everos.infra.persistence.markdown.writers.episode_writer import EpisodeWriter diff --git a/tests/unit/test_infra/test_index_contract.py b/tests/unit/test_infra/test_index_contract.py index 1e4cd5e79..9b58dd2ff 100644 --- a/tests/unit/test_infra/test_index_contract.py +++ b/tests/unit/test_infra/test_index_contract.py @@ -10,6 +10,11 @@ import pytest +from everos.infra.persistence.backends.lancedb import ( + lance_index_backend, + render_predicate, +) +from everos.infra.persistence.backends.milvus import milvus_index_backend from everos.infra.persistence.index import ( ALL_REPOS, Episode, @@ -21,8 +26,6 @@ eq, shutdown, ) -from everos.infra.persistence.index.lancedb import lance_index_backend, render_predicate -from everos.infra.persistence.index.milvus import milvus_index_backend from everos.infra.persistence.index.schema import IndexFieldKind, schema_for from everos.infra.persistence.lancedb import lancedb_manager diff --git a/tests/unit/test_memory/test_get/test_manager.py b/tests/unit/test_memory/test_get/test_manager.py index 6efa3d55c..ad8b78df7 100644 --- a/tests/unit/test_memory/test_get/test_manager.py +++ b/tests/unit/test_memory/test_get/test_manager.py @@ -254,7 +254,7 @@ async def test_episodic_memory_passes_where_and_sort_to_repo( filters=FilterNode.model_validate({"session_id": "sess_a"}), ) await mgr.get(req) - from everos.infra.persistence.index.lancedb import render_predicate + from everos.infra.persistence.backends.lancedb import render_predicate rendered = render_predicate(ep.last.where) assert "owner_id = 'u1'" in rendered diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index f2f16da2b..2936071dd 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -393,7 +393,7 @@ async def test_user_keyword_filters_compile_pinned_owner() -> None: ) await mgr.search(_user_req()) assert recaller.last_where is not None - from everos.infra.persistence.index.lancedb import render_predicate + from everos.infra.persistence.backends.lancedb import render_predicate rendered = render_predicate(recaller.last_where) assert "owner_id = 'alice'" in rendered diff --git a/tests/unit/test_memory/test_search/test_recall_or_semantics.py b/tests/unit/test_memory/test_search/test_recall_or_semantics.py index 3c55a3a4b..d2507d312 100644 --- a/tests/unit/test_memory/test_search/test_recall_or_semantics.py +++ b/tests/unit/test_memory/test_search/test_recall_or_semantics.py @@ -24,7 +24,7 @@ import pytest from everos.component.tokenizer import Tokenizer -from everos.infra.persistence.index.lancedb import build_or_query +from everos.infra.persistence.backends.lancedb import build_or_query from everos.infra.persistence.lancedb import ( Episode, ParentType, diff --git a/tests/unit/test_service/test_knowledge_search.py b/tests/unit/test_service/test_knowledge_search.py index 88bb7a0fb..83352615a 100644 --- a/tests/unit/test_service/test_knowledge_search.py +++ b/tests/unit/test_service/test_knowledge_search.py @@ -141,7 +141,7 @@ def test_basic_clause(self) -> None: def test_defaults(self) -> None: result = compile_knowledge_where("default", "default") - from everos.infra.persistence.index.lancedb import render_predicate + from everos.infra.persistence.backends.lancedb import render_predicate rendered = render_predicate(result) assert "app_id = 'default'" in rendered @@ -165,7 +165,7 @@ def test_rejects_empty_project_id(self) -> None: def test_accepts_valid_ids_with_special_chars(self) -> None: result = compile_knowledge_where("my_app.v2", "project-1") - from everos.infra.persistence.index.lancedb import render_predicate + from everos.infra.persistence.backends.lancedb import render_predicate rendered = render_predicate(result) assert "my_app.v2" in rendered @@ -173,7 +173,7 @@ def test_accepts_valid_ids_with_special_chars(self) -> None: def test_accepts_valid_ids_with_at_plus(self) -> None: result = compile_knowledge_where("app@org+v1", "proj_1") - from everos.infra.persistence.index.lancedb import render_predicate + from everos.infra.persistence.backends.lancedb import render_predicate rendered = render_predicate(result) assert "app@org+v1" in rendered From 8e576907ace10062f0a5d550ea98f1c87e5b4b32 Mon Sep 17 00:00:00 2001 From: zhanghui Date: Mon, 7 Sep 2026 13:36:29 +0800 Subject: [PATCH 20/27] fix(milvus): clamp search topK to the engine's result window Agentic search passes 100,000 as a child-pool sentinel and relies on the engine to clamp -- `_FACT_CHILD_CANDIDATES` says so in as many words, because LanceDB truncates silently at the row count. Milvus does not: its search topK must sit in [1, 16384] and anything larger comes back as `MilvusException code=1100`, so every agentic query returned a 500. Clamped in the adapter rather than lowering the sentinel. The limit is Milvus', and pushing it up into `memory/search` is the same mistake as `_MAX_FACT_RECALL_LIMIT`, which capped LanceDB's recall pool at a number only Zilliz needed. Callers keep one contract on both backends. Third instance of this shape: `list_by_owner_after_ts` overran the same 16,384 window, and the 1,024 topK ceiling had leaked into the domain layer. A second backend with hard limits surfaces them one call site at a time. Also fixes ten test call sites still passing SQL strings where the port now takes a Predicate. They date from before the AST migration and never ran -- two files carry `slow`/`live_llm` and are excluded by default, the rest short-circuit before reaching those lines. The product code was clean, which is why LOCOMO, the soak and CI all stayed green. Co-Authored-By: Claude Opus 5 --- .../infra/persistence/milvus/repository.py | 10 +++- tests/_consistency_assertions.py | 3 +- .../e2e/test_add_flush_agent_pipeline_e2e.py | 14 +++--- tests/integration/test_cascade_scenarios.py | 3 +- .../test_cli/test_backfill_phase3.py | 3 +- .../test_reflection_integration.py | 3 +- .../unit/test_infra/test_milvus/test_repo.py | 46 +++++++++++++++++++ 7 files changed, 69 insertions(+), 13 deletions(-) diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index 0e3f7cdb9..54774c10a 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -38,6 +38,12 @@ _DUMMY_VECTOR_FIELD = "_everos_dummy_vector" _DUMMY_VECTOR_DIMENSION = 2 +_SEARCH_TOPK_MAX = 16_384 +"""Milvus caps ``search`` topK at its result window and rejects anything +larger outright. LanceDB has no such ceiling, so callers pass generous +pool sizes -- agentic search deliberately passes a large sentinel and +expects the engine to clamp. Clamping here keeps that expectation true on +both backends instead of making every caller learn one engine's limit.""" _SPARSE_SUFFIX = "__sparse" _PRESENT_SUFFIX = "__present" @@ -606,7 +612,7 @@ async def sparse_search( data=[query], anns_field=_sparse_field(field), filter=self._expr(where), - limit=limit, + limit=min(limit, _SEARCH_TOPK_MAX), output_fields=self._output_fields(include_vectors=False), search_params={"metric_type": "BM25"}, ) @@ -647,7 +653,7 @@ async def dense_search( data=[list(vector)], anns_field=vector_field, filter=self._expr(all_of(where, present)), - limit=limit, + limit=min(limit, _SEARCH_TOPK_MAX), output_fields=self._output_fields(include_vectors=False), search_params={"metric_type": "COSINE"}, ) diff --git a/tests/_consistency_assertions.py b/tests/_consistency_assertions.py index 41f0879a4..9a8178a53 100644 --- a/tests/_consistency_assertions.py +++ b/tests/_consistency_assertions.py @@ -48,6 +48,7 @@ from everos.core.persistence import MarkdownReader from everos.core.persistence.markdown.entries import StructuredEntry +from everos.infra.persistence.index import eq from everos.infra.persistence.markdown import AgentSkillFrontmatter from everos.memory.cascade.handlers._common import content_sha256 from everos.memory.cascade.handlers._daily_log_base import BaseDailyLogHandler @@ -202,7 +203,7 @@ async def _check_daily_log( for entry in parsed.entries } lance_rows = await spec.index_repo.find_where( - f"md_path = '{_q(md_path)}'", limit=10_000 + eq("md_path", md_path), limit=10_000 ) lance_sha_by_id = {r.entry_id: r.content_sha256 for r in lance_rows} if md_sha_by_id != lance_sha_by_id: diff --git a/tests/e2e/test_add_flush_agent_pipeline_e2e.py b/tests/e2e/test_add_flush_agent_pipeline_e2e.py index 1491dc52f..c06c310b4 100644 --- a/tests/e2e/test_add_flush_agent_pipeline_e2e.py +++ b/tests/e2e/test_add_flush_agent_pipeline_e2e.py @@ -47,7 +47,7 @@ import everos.component.embedding.accessor as _embedding_accessor from everos.infra.ome.records import RunStatus -from everos.infra.persistence.index import agent_case_repo, agent_skill_repo +from everos.infra.persistence.index import agent_case_repo, agent_skill_repo, eq from everos.infra.persistence.markdown import AgentCaseDailyFrontmatter from everos.service.memorize import _get_engine @@ -188,9 +188,9 @@ async def test_agent_pipeline_e2e_mixed_tenancy( assert md_files, f"no agent_case md under {case_dir!s}" # 4.3 LanceDB agent_case rows per owner - pytest_cases = await agent_case_repo.find_where(f"owner_id = '{_AGENT_PYTEST}'") - sympy_cases = await agent_case_repo.find_where(f"owner_id = '{_AGENT_SYMPY}'") - django_cases = await agent_case_repo.find_where(f"owner_id = '{_AGENT_DJANGO}'") + pytest_cases = await agent_case_repo.find_where(eq("owner_id", _AGENT_PYTEST)) + sympy_cases = await agent_case_repo.find_where(eq("owner_id", _AGENT_SYMPY)) + django_cases = await agent_case_repo.find_where(eq("owner_id", _AGENT_DJANGO)) assert len(pytest_cases) >= 1, ( f"no agent_pytest rows in LanceDB (got {len(pytest_cases)})" @@ -230,9 +230,9 @@ async def test_agent_pipeline_e2e_mixed_tenancy( # SKILL.md (extract_agent_skill status "success", no retries); driving # 5 trajectories across 3 agents should clear an aggregate floor of 1 # even if any single agent's cluster is quality-gated to 0. - pytest_skills = await agent_skill_repo.find_where(f"owner_id = '{_AGENT_PYTEST}'") - sympy_skills = await agent_skill_repo.find_where(f"owner_id = '{_AGENT_SYMPY}'") - django_skills = await agent_skill_repo.find_where(f"owner_id = '{_AGENT_DJANGO}'") + pytest_skills = await agent_skill_repo.find_where(eq("owner_id", _AGENT_PYTEST)) + sympy_skills = await agent_skill_repo.find_where(eq("owner_id", _AGENT_SYMPY)) + django_skills = await agent_skill_repo.find_where(eq("owner_id", _AGENT_DJANGO)) total_skills = len(pytest_skills) + len(sympy_skills) + len(django_skills) assert total_skills >= 1, ( "agent-skill chain produced nothing — the strategy chain " diff --git a/tests/integration/test_cascade_scenarios.py b/tests/integration/test_cascade_scenarios.py index cda8fdb90..eebce3c05 100644 --- a/tests/integration/test_cascade_scenarios.py +++ b/tests/integration/test_cascade_scenarios.py @@ -34,6 +34,7 @@ from everos.component.embedding import EmbeddingProvider from everos.component.tokenizer import build_tokenizer from everos.core.persistence import MarkdownReader, MarkdownWriter, MemoryRoot +from everos.infra.persistence.index import eq from everos.infra.persistence.lancedb import ( atomic_fact_repo, dispose_connection, @@ -534,7 +535,7 @@ async def test_concurrent_writes_different_owners_no_bleed( for oid in owners: md_path = _atomic_fact_md_path(oid, bucket) - rows = await atomic_fact_repo.find_where(f"md_path = '{md_path}'", limit=10) + rows = await atomic_fact_repo.find_where(eq("md_path", md_path), limit=10) assert len(rows) == per_owner, ( f"{oid}: expected {per_owner} rows, got {len(rows)}" ) diff --git a/tests/integration/test_cli/test_backfill_phase3.py b/tests/integration/test_cli/test_backfill_phase3.py index e600cea7b..83512969b 100644 --- a/tests/integration/test_cli/test_backfill_phase3.py +++ b/tests/integration/test_cli/test_backfill_phase3.py @@ -42,6 +42,7 @@ from everos.component.utils.datetime import get_utc_now from everos.config import load_settings from everos.entrypoints.cli.commands._backfill_cmd import run_backfill +from everos.infra.persistence.index import eq from everos.infra.persistence.lancedb import ( AgentCase, agent_case_repo, @@ -251,7 +252,7 @@ async def test_skills_phase_grows_agent_skill_count( assert skills_after == skills_before + 1 assert "phase 3 complete" in out - rows = await agent_skill_repo.find_where("owner_id = 'agent1'", limit=10) + rows = await agent_skill_repo.find_where(eq("owner_id", "agent1"), limit=10) assert len(rows) == 1 assert rows[0].name == "summarise_doc" assert rows[0].cluster_id is not None diff --git a/tests/integration/test_reflection_integration.py b/tests/integration/test_reflection_integration.py index 6ab669505..47f43a36b 100644 --- a/tests/integration/test_reflection_integration.py +++ b/tests/integration/test_reflection_integration.py @@ -35,6 +35,7 @@ from everos.core.persistence.lancedb import LanceDailyLogRepoBase, LanceRepoBase from everos.infra.ome.testing import FakeStrategyContext from everos.infra.persistence.backends.lancedb import LanceIndexRepository +from everos.infra.persistence.index import all_of, eq from everos.infra.persistence.lancedb.tables.atomic_fact import AtomicFact from everos.infra.persistence.lancedb.tables.episode import Episode as LanceEpisode from everos.infra.persistence.markdown.writers.episode_writer import EpisodeWriter @@ -383,7 +384,7 @@ async def test_reflection_init_merges_cluster_episodes( # -- Verify: source episodes deprecated in LanceDB. for ep in [ep1, ep2, ep3]: rows = await ep_repo.find_where( - f"entry_id = '{ep.entry_id}' AND owner_id = '{owner_id}'" + all_of(eq("entry_id", ep.entry_id), eq("owner_id", owner_id)) ) assert len(rows) == 1, f"expected 1 row for {ep.entry_id}" assert rows[0].deprecated_by == merged_entry_id, ( diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index 7a676c372..5e5bb6749 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -602,3 +602,49 @@ def test_nullable_bm25_input_is_declared_not_null() -> None: # "evidence" is the same optional text without the analyzer attached, so # it stays nullable — the NOT NULL above is caused by BM25, nothing else. assert physical["evidence"].nullable is True + + +async def test_search_topk_is_clamped_to_the_milvus_window( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Milvus rejects a topK above its result window; LanceDB has no ceiling. + + Agentic search passes a large sentinel on purpose and expects the engine + to clamp -- a habit LanceDB permits and Milvus answers with + ``MilvusException code=1100``. The adapter absorbs the difference so no + caller has to learn one engine's limit. + """ + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + seen: dict[str, int] = {} + + class _FakeClient: + def has_collection(self, name: str) -> bool: + return True + + def describe_collection(self, name: str): # type: ignore[no-untyped-def] + return _describe_response(milvus_repo) + + def list_indexes(self, name: str): # type: ignore[no-untyped-def] + return list(_index_descriptions(_describe_response(milvus_repo))) + + def describe_index(self, name: str, index_name: str): # type: ignore[no-untyped-def] + return _index_descriptions(_describe_response(milvus_repo))[index_name] + + def search(self, *a, **kw): # type: ignore[no-untyped-def] + seen["limit"] = kw["limit"] + return [[]] + + async def _fake_get_client(): # type: ignore[no-untyped-def] + return _FakeClient() + + monkeypatch.setattr(repository, "get_client", _fake_get_client) + + await milvus_repo.dense_search([0.1] * 1024, None, limit=100_000) + assert seen["limit"] == repository._SEARCH_TOPK_MAX + + await milvus_repo.sparse_search(["hiking"], None, limit=100_000) + assert seen["limit"] == repository._SEARCH_TOPK_MAX + + # A limit inside the window is passed through untouched. + await milvus_repo.dense_search([0.1] * 1024, None, limit=25) + assert seen["limit"] == 25 From 0b183499c839db1c85b97edbaadb0f6366b79560 Mon Sep 17 00:00:00 2001 From: zhanghui Date: Mon, 7 Sep 2026 15:45:15 +0800 Subject: [PATCH 21/27] fix(predicate): reject an empty predicate group at construction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `all_of()` / `any_of()` already refused an empty result, and their docstrings give the reason: adapters render an empty group as an empty filter, which reads as "match every row". But `All` and `AnyOf` are exported, so `All(())` bypasses the factories entirely — and the two backends then disagree on it. LanceDB raises a raw SQL parse error; Milvus drops the filter and returns the whole table. An empty `AnyOf` means "match nothing", so the Milvus reading is inverted, and on `delete()` it is a silent table wipe. Move the guard onto the dataclasses so the input cannot be built. Nothing in `src/` constructs a group directly, so no production behaviour changes. --- src/everos/infra/persistence/predicate.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/everos/infra/persistence/predicate.py b/src/everos/infra/persistence/predicate.py index 0ca2da0a3..06f9cbb11 100644 --- a/src/everos/infra/persistence/predicate.py +++ b/src/everos/infra/persistence/predicate.py @@ -47,11 +47,31 @@ class IsNull(Predicate): class All(Predicate): children: tuple[Predicate, ...] + def __post_init__(self) -> None: + _reject_empty(self) + @dataclass(frozen=True) class AnyOf(Predicate): children: tuple[Predicate, ...] + def __post_init__(self) -> None: + _reject_empty(self) + + +def _reject_empty(group: All | AnyOf) -> None: + """Forbid an empty group at construction, not just in the factories. + + Adapters render an empty group as an empty filter, and the two backends + then disagree: LanceDB raises a raw SQL parse error, Milvus drops the + filter and matches every row. Either way the meaning is wrong — an empty + ``AnyOf`` means "match nothing" — and on :meth:`IndexRepository.delete` + the Milvus reading is a silent table wipe. ``All`` and ``AnyOf`` are public, + so guarding only :func:`all_of` / :func:`any_of` leaves the hole open. + """ + if not group.children: + raise ValueError(f"{type(group).__name__} requires at least one child") + def compare(field: str, operator: ComparisonOperator, value: Scalar) -> Predicate: return Comparison(field, operator, value) From 2ca2cd92559a11f28f1b496f8a96c2974bd18c2a Mon Sep 17 00:00:00 2001 From: zhanghui Date: Mon, 7 Sep 2026 15:45:23 +0800 Subject: [PATCH 22/27] fix(persistence): break the port/adapter import cycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The port facade wires both concrete backends at import time, while each adapter imported back into the port package — `backends/lancedb.py` from `..index.predicate`, `backends/milvus.py` from `..index.protocols`. Importing a submodule runs the package `__init__` first, so whichever side is imported first lands on a partially initialised module: ImportError: cannot import name 'agent_case_repo' from partially initialized module 'everos.infra.persistence.backends.lancedb' A whole-suite run hides it: some earlier test module always imports `index` first and primes the package. Running `tests/unit/test_infra/test_index_contract.py` on its own fails at collection — which is how a reviewer is most likely to run it. Neither back-reference is needed. `index.predicate` is a pure re-export of the canonical `..predicate`, so the adapter imports the canonical module; `protocols` is used only in a return annotation, so it moves under `TYPE_CHECKING`. --- src/everos/infra/persistence/backends/lancedb.py | 2 +- src/everos/infra/persistence/backends/milvus.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/everos/infra/persistence/backends/lancedb.py b/src/everos/infra/persistence/backends/lancedb.py index 6b7eadca2..784d013b4 100644 --- a/src/everos/infra/persistence/backends/lancedb.py +++ b/src/everos/infra/persistence/backends/lancedb.py @@ -20,7 +20,7 @@ from everos.core.persistence import LanceRepoBase from everos.infra.persistence import lancedb as _lancedb -from ..index.predicate import ( +from ..predicate import ( All, AnyOf, Comparison, diff --git a/src/everos/infra/persistence/backends/milvus.py b/src/everos/infra/persistence/backends/milvus.py index 5291bd77e..0a441a603 100644 --- a/src/everos/infra/persistence/backends/milvus.py +++ b/src/everos/infra/persistence/backends/milvus.py @@ -3,9 +3,10 @@ from __future__ import annotations from types import ModuleType -from typing import Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar -from ..index.protocols import IndexRepository +if TYPE_CHECKING: # importing the port package here would close an import cycle + from ..index.protocols import IndexRepository class MilvusIndexBackend: From ff6fba4d1ec74cba93d96a3c9e7d5d35ec536e61 Mon Sep 17 00:00:00 2001 From: zhanghui Date: Mon, 7 Sep 2026 15:45:30 +0800 Subject: [PATCH 23/27] test(integration): pass the low-level Lance repos a rendered filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three white-box probes were migrated to `Predicate` along with the port call sites, but they do not talk to the port: they hold a repository from `everos.infra.persistence.lancedb` (or a raw `_EpisodeRepo`) whose `find_where` still takes rendered LanceDB SQL. Each failed with `TypeError: argument 'predicate': 'All' object is not an instance of 'str'`. These files are LanceDB-pinned probes — they also reach for `get_table` and are not parameterised over the backend — and their sibling calls in the same file already pass f-string SQL. Restore that form so each file is internally consistent. --- tests/integration/test_cascade_scenarios.py | 3 +-- tests/integration/test_cli/test_backfill_phase3.py | 3 +-- tests/integration/test_reflection_integration.py | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_cascade_scenarios.py b/tests/integration/test_cascade_scenarios.py index eebce3c05..cda8fdb90 100644 --- a/tests/integration/test_cascade_scenarios.py +++ b/tests/integration/test_cascade_scenarios.py @@ -34,7 +34,6 @@ from everos.component.embedding import EmbeddingProvider from everos.component.tokenizer import build_tokenizer from everos.core.persistence import MarkdownReader, MarkdownWriter, MemoryRoot -from everos.infra.persistence.index import eq from everos.infra.persistence.lancedb import ( atomic_fact_repo, dispose_connection, @@ -535,7 +534,7 @@ async def test_concurrent_writes_different_owners_no_bleed( for oid in owners: md_path = _atomic_fact_md_path(oid, bucket) - rows = await atomic_fact_repo.find_where(eq("md_path", md_path), limit=10) + rows = await atomic_fact_repo.find_where(f"md_path = '{md_path}'", limit=10) assert len(rows) == per_owner, ( f"{oid}: expected {per_owner} rows, got {len(rows)}" ) diff --git a/tests/integration/test_cli/test_backfill_phase3.py b/tests/integration/test_cli/test_backfill_phase3.py index 83512969b..e600cea7b 100644 --- a/tests/integration/test_cli/test_backfill_phase3.py +++ b/tests/integration/test_cli/test_backfill_phase3.py @@ -42,7 +42,6 @@ from everos.component.utils.datetime import get_utc_now from everos.config import load_settings from everos.entrypoints.cli.commands._backfill_cmd import run_backfill -from everos.infra.persistence.index import eq from everos.infra.persistence.lancedb import ( AgentCase, agent_case_repo, @@ -252,7 +251,7 @@ async def test_skills_phase_grows_agent_skill_count( assert skills_after == skills_before + 1 assert "phase 3 complete" in out - rows = await agent_skill_repo.find_where(eq("owner_id", "agent1"), limit=10) + rows = await agent_skill_repo.find_where("owner_id = 'agent1'", limit=10) assert len(rows) == 1 assert rows[0].name == "summarise_doc" assert rows[0].cluster_id is not None diff --git a/tests/integration/test_reflection_integration.py b/tests/integration/test_reflection_integration.py index 47f43a36b..6ab669505 100644 --- a/tests/integration/test_reflection_integration.py +++ b/tests/integration/test_reflection_integration.py @@ -35,7 +35,6 @@ from everos.core.persistence.lancedb import LanceDailyLogRepoBase, LanceRepoBase from everos.infra.ome.testing import FakeStrategyContext from everos.infra.persistence.backends.lancedb import LanceIndexRepository -from everos.infra.persistence.index import all_of, eq from everos.infra.persistence.lancedb.tables.atomic_fact import AtomicFact from everos.infra.persistence.lancedb.tables.episode import Episode as LanceEpisode from everos.infra.persistence.markdown.writers.episode_writer import EpisodeWriter @@ -384,7 +383,7 @@ async def test_reflection_init_merges_cluster_episodes( # -- Verify: source episodes deprecated in LanceDB. for ep in [ep1, ep2, ep3]: rows = await ep_repo.find_where( - all_of(eq("entry_id", ep.entry_id), eq("owner_id", owner_id)) + f"entry_id = '{ep.entry_id}' AND owner_id = '{owner_id}'" ) assert len(rows) == 1, f"expected 1 row for {ep.entry_id}" assert rows[0].deprecated_by == merged_entry_id, ( From b894a7829ca9f9b98c51a7c27261e8a9135a5309 Mon Sep 17 00:00:00 2001 From: zhanghui Date: Mon, 7 Sep 2026 15:45:36 +0800 Subject: [PATCH 24/27] test(index): pin empty-group rejection and import-order independence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both guards the two preceding commits added were previously untested — the factories' own empty check had no coverage either. The import-order test runs in a subprocess: inside this process the package is already imported, so the cycle is invisible. Each of the two adapters and the port facade must import as the first everos import. Verified by mutation: reverting either fix turns the matching test red (the cycle one fails at collection, as it did in the field). --- tests/unit/test_infra/test_index_contract.py | 55 ++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/unit/test_infra/test_index_contract.py b/tests/unit/test_infra/test_index_contract.py index 9b58dd2ff..87dbc32ae 100644 --- a/tests/unit/test_infra/test_index_contract.py +++ b/tests/unit/test_infra/test_index_contract.py @@ -5,7 +5,10 @@ import datetime as dt import os import re +import subprocess +import sys import uuid +from collections.abc import Callable from pathlib import Path import pytest @@ -17,10 +20,14 @@ from everos.infra.persistence.backends.milvus import milvus_index_backend from everos.infra.persistence.index import ( ALL_REPOS, + All, + AnyOf, Episode, IndexBackend, IndexRepository, + Predicate, all_of, + any_of, drop_business_tables, episode_repo, eq, @@ -192,6 +199,54 @@ def test_lance_predicate_renderer_owns_escaping() -> None: assert "session_id = 'session'" in rendered +@pytest.mark.parametrize( + "module", + [ + "everos.infra.persistence.backends.lancedb", + "everos.infra.persistence.backends.milvus", + "everos.infra.persistence.index", + ], +) +def test_adapter_and_port_modules_import_in_any_order(module: str) -> None: + """Each module must import as the process's first everos import. + + The port facade wires the concrete backends at import time, so an adapter + reaching back into the ``index`` package closes a cycle: whichever side is + imported first fails on a partially initialised module. A subprocess is the + only honest check — inside this process the package is already imported, so + the cycle is invisible. + """ + proc = subprocess.run( + [sys.executable, "-c", f"import {module}"], + capture_output=True, + text=True, + ) + assert proc.returncode == 0, f"importing {module} first failed:\n{proc.stderr}" + + +@pytest.mark.parametrize("group", [All, AnyOf]) +def test_empty_predicate_group_is_rejected_at_construction( + group: type[Predicate], +) -> None: + """An empty group must never reach an adapter — the two disagree on it. + + Rendered empty, LanceDB raises a raw SQL parse error while Milvus drops + the filter and matches every row; on ``delete`` that reading is a silent + table wipe. ``All`` and ``AnyOf`` are exported, so the factories' own + guard is not enough — the classes have to refuse it too. + """ + with pytest.raises(ValueError, match="at least one child"): + group(()) + + +@pytest.mark.parametrize("factory", [all_of, any_of]) +def test_predicate_factories_reject_an_all_none_argument_list( + factory: Callable[..., Predicate], +) -> None: + with pytest.raises(ValueError, match="at least one non-None predicate"): + factory(None, None) + + async def test_repository_port_crud_count_and_uncapped_scan( index_backend: str, ) -> None: From 6e853d703b1d604fcc00895f0a53b6fdff859fbd Mon Sep 17 00:00:00 2001 From: zhanghui Date: Mon, 7 Sep 2026 16:47:01 +0800 Subject: [PATCH 25/27] test(e2e): revive the knowledge HTTP lifecycle test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test could not pass in any environment, and being marked `slow` + `live_llm` it is excluded from every CI run and from `make test` / `make integration`, so nothing said so. It is the only coverage of the knowledge_topic extraction chain. Three layers, each hiding the next: 1. It read `c["id"]` from GET /categories, but `CategoryDTO` exposes `category_id` — the field name in the committed, lint-checked `docs/openapi.json` since 1.1.0. The test was the stale side. 2. POST /documents then answered 422 PROVIDER_NOT_CONFIGURED for `embedding`: `tests/conftest.py` pins that capability unavailable for every test, and this module never opted back in the way `test_add_flush_agent_pipeline_e2e.py` does. 3. Same for `rerank`, which the knowledge write path also requires — the agent write path does not, which is why its own opt-in fixture covers embedding alone. Now passes on both index backends (2 passed, ~160s). --- tests/e2e/test_knowledge_e2e.py | 45 ++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test_knowledge_e2e.py b/tests/e2e/test_knowledge_e2e.py index 19dc8a5b2..88eff6f19 100644 --- a/tests/e2e/test_knowledge_e2e.py +++ b/tests/e2e/test_knowledge_e2e.py @@ -25,10 +25,14 @@ from __future__ import annotations import asyncio +from collections.abc import Iterator import httpx import pytest +import everos.component.embedding.accessor as _embedding_accessor +import everos.component.rerank.accessor as _rerank_accessor + # --------------------------------------------------------------------------- # Test document — 3 clear sections for predictable topic extraction. # --------------------------------------------------------------------------- @@ -64,6 +68,45 @@ # --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def _opt_in_real_embedding_and_rerank( + _reset_embedding_capability_singleton: None, + _reset_rerank_capability_singleton: None, +) -> Iterator[None]: + """Opt this module's tests into real embedding + rerank capabilities. + + ``tests/conftest.py``'s ``_reset_embedding_capability_singleton`` + autouse fixture pins the capability to unavailable for every test + (hermeticity). The knowledge write path requires it: ``POST + /documents`` rejects the request with ``PROVIDER_NOT_CONFIGURED`` + when ``get_embedding_capability()`` reports unavailable, so without + opting in here every test in this module fails at document creation + — no matter what credentials the environment carries. + + Requesting ``_reset_embedding_capability_singleton`` as a parameter + — rather than relying on fixture declaration order — makes pytest's + dependency graph guarantee this fixture's setup runs after it and + its teardown before it. + + Setting ``_capability = None`` (rather than constructing a + capability object) makes the accessor rebuild lazily from + ``load_settings()`` on next call, picking up the real credentials + ``tests/e2e/conftest.py`` loads at import time. + + Rerank needs the same treatment, and that is where this fixture + differs from its sibling in + ``test_add_flush_agent_pipeline_e2e.py``: nothing on the agent write + path touches rerank, so that module leaves it hermetic. The + knowledge write path does require it, and ``POST /documents`` + rejects the request the same way when it is unavailable. + """ + _embedding_accessor._capability = None + _rerank_accessor._capability = None + yield + _embedding_accessor._capability = None + _rerank_accessor._capability = None + + # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- @@ -88,7 +131,7 @@ async def test_knowledge_full_http_lifecycle( assert resp.status_code == 200 cats = resp.json()["data"]["categories"] assert len(cats) >= 10, f"Expected >=10 default categories, got {len(cats)}" - cat_ids = {c["id"] for c in cats} + cat_ids = {c["category_id"] for c in cats} assert "Sports" in cat_ids assert "Others" in cat_ids From f1768b83a1a8a06f8e54c842cc580fb9fdad0374 Mon Sep 17 00:00:00 2001 From: zhanghui Date: Mon, 7 Sep 2026 16:47:01 +0800 Subject: [PATCH 26/27] fix(deps): pin everalgo's transitive layer to what the suite resolves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pip install everos==1.2.3` answered 500 on POST /add for a default install. Reproduced from PyPI: `everos init`, start, /add → TypeError: DetectionResult.__new__() missing 1 required positional argument: 'should_wait' everalgo-boundary 0.3.0 added a third required field to the public `DetectionResult` NamedTuple; everalgo-agent-memory 0.4.0 still builds it with two. Both consumer packages declare the dependency as `>=0.2.0,<2.0.0`, so a resolver pairs 0.4.0 with 0.3.0 — while this repo's uv.lock holds 0.2.1, which is why 2141 unit, 215 integration and 112 e2e tests all stayed green against a broken release. Only `memorize.mode = "agent"` is affected, and that is the default: chat mode routes through everalgo.boundary's own detector, which is internally consistent. Verified by single-variable swap in one venv — 0.3.0 gives 500, 0.2.1 completes add → flush → cascade → get → search (all four methods). Pin boundary, core and clustering to the locked versions. clustering resolves to the same version today; it is pinned because a floating 0.x transitive dep is the hazard this incident is about, not because it is currently wrong. The upstream fix belongs in everalgo. --- pyproject.toml | 15 +++++++++++++++ uv.lock | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9ac898645..8f1aac26a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,21 @@ dependencies = [ "everalgo-agent-memory==0.4.0", "everalgo-rank==0.4.1", "everalgo-knowledge==0.1.1", + + # everalgo's own transitive layer, pinned to what this repo's uv.lock — + # and therefore the whole test suite — actually resolves. The packages + # above declare it as `>=0.2.0,<2.0.0`, treating a 0.x line as if a major + # bump were the only breaking one, so a fresh `pip install everos` picks + # up releases the suite has never run against. That is not theoretical: + # everalgo-boundary 0.3.0 added a third required field to the public + # `DetectionResult` NamedTuple while everalgo-agent-memory 0.4.0 still + # builds it with two, so `POST /add` answered 500 on a default + # (`memorize.mode = "agent"`) install of 1.2.3 while every local suite + # stayed green on the locked 0.2.1. Raise these deliberately, together + # with a run of the release smoke against a PyPI-resolved install. + "everalgo-boundary==0.2.1", + "everalgo-core==0.3.0", + "everalgo-clustering==0.2.1", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index 5d1c15895..d1644510a 100644 --- a/uv.lock +++ b/uv.lock @@ -588,6 +588,9 @@ dependencies = [ { name = "apscheduler" }, { name = "click" }, { name = "everalgo-agent-memory" }, + { name = "everalgo-boundary" }, + { name = "everalgo-clustering" }, + { name = "everalgo-core" }, { name = "everalgo-knowledge" }, { name = "everalgo-rank" }, { name = "everalgo-user-memory" }, @@ -647,6 +650,9 @@ requires-dist = [ { name = "apscheduler", specifier = ">=3.10.4,<4.0" }, { name = "click", specifier = ">=8.1" }, { name = "everalgo-agent-memory", specifier = "==0.4.0" }, + { name = "everalgo-boundary", specifier = "==0.2.1" }, + { name = "everalgo-clustering", specifier = "==0.2.1" }, + { name = "everalgo-core", specifier = "==0.3.0" }, { name = "everalgo-knowledge", specifier = "==0.1.1" }, { name = "everalgo-parser", extras = ["svg"], marker = "extra == 'multimodal'", specifier = ">=0.2.1" }, { name = "everalgo-rank", specifier = "==0.4.1" }, From ab24e708e0a611949482718abed593244c8a820e Mon Sep 17 00:00:00 2001 From: zhanghui Date: Mon, 7 Sep 2026 16:53:23 +0800 Subject: [PATCH 27/27] test: reset the partition-lock pool around every test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `extract_agent_skill` dead-lettered with `RuntimeError: is bound to a different event loop`, failing `test_agent_pipeline_e2e_mixed_tenancy[index=milvus]` on one run and passing on the next. `_partition_locks` caches one lock per (strategy, partition key) and never evicts — deliberate, and correct for a single-loop production process. Across tests it is not: pytest-asyncio gives each test its own loop, so the second test to touch a partition acquires a lock bound to a closed one. The parametrised suites hit it hardest, since both backends replay the same agent ids — which is why the failure landed on whichever parameter ran second and looked like a flake. Five unit modules already reset the pool through their own local fixtures; nothing did for integration or e2e, where the strategy actually runs. Move it next to its siblings in the root `_reset_settings_cache` fixture, which already clears the other process-global caches around every test. The five local fixtures are now redundant but harmless, so they stay for a separate cleanup. `test_partition_locks.py` drops its own reset — a second, file-local one would mask the contract — and pins it instead with an ordered dirty/assert-empty pair. Mutation-verified: reverting the fixture turns both that pair and the pre-existing FIFO test red. The agent pipeline e2e now passes on both backends in one process (2 passed, 191s), with no event-loop errors in the log. --- tests/conftest.py | 13 +++++++ .../test_strategies/test_partition_locks.py | 37 +++++++++++++++---- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 738a5ea13..b373d65ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,6 +49,7 @@ def _reset_settings_cache() -> Iterator[None]: from everos.component.utils import datetime as dt_module from everos.config import load_settings + from everos.memory import _partition_locks # ``configure_logging`` (called by some e2e fixtures / the CLI entry) # sets ``cache_logger_on_first_use=True``; once a logger is cached, @@ -56,13 +57,25 @@ def _reset_settings_cache() -> Iterator[None]: # which silently breaks log-assertion tests that run *after* it in the # same process. Reset structlog to defaults around every test so that # global config never leaks across the suite. + # ``_partition_locks`` caches one ``asyncio.Lock`` per (strategy, + # partition key) and never evicts it — correct for a single-loop + # production process, wrong across tests: pytest-asyncio gives each + # test its own event loop, so the second test to touch a partition + # gets a lock bound to a dead loop and the strategy dies with + # ``RuntimeError: ... is bound to a different event loop``. It hits + # the parametrised suites hardest, where both backends replay the + # same agent ids, and it surfaces as a dead-lettered OME run rather + # than a test error — reported as a flake on whichever parameter + # happens to run second. structlog.reset_defaults() load_settings.cache_clear() dt_module._display_tz.cache_clear() + _partition_locks._reset_for_tests() yield structlog.reset_defaults() load_settings.cache_clear() dt_module._display_tz.cache_clear() + _partition_locks._reset_for_tests() @pytest.fixture(autouse=True) diff --git a/tests/unit/test_memory/test_strategies/test_partition_locks.py b/tests/unit/test_memory/test_strategies/test_partition_locks.py index 949cf29a2..8e08db72d 100644 --- a/tests/unit/test_memory/test_strategies/test_partition_locks.py +++ b/tests/unit/test_memory/test_strategies/test_partition_locks.py @@ -10,18 +10,15 @@ import asyncio -import pytest - from everos.memory._partition_locks import ( + _pools, _reset_for_tests, get_partition_lock, ) - -@pytest.fixture(autouse=True) -def _isolate_locks() -> None: - """Each test gets a clean registry — no inherited holders / waiters.""" - _reset_for_tests() +# No local reset fixture: ``tests/conftest.py`` clears the pool around +# every test in the suite. A second, file-local reset would mask the +# very contract the two isolation tests at the bottom pin. def test_same_strategy_same_key_returns_identical_lock() -> None: @@ -124,3 +121,29 @@ async def waiter(tag: str, arrived: asyncio.Event) -> None: await asyncio.gather(task_holder, task_a, task_b) assert log == ["leave:holder", "enter:a", "enter:b"] + + +# ── Cross-test isolation ──────────────────────────────────────────────── +# +# The pool is a module global that never evicts, which is right for a +# single-loop production process and wrong across tests: pytest-asyncio +# hands each test its own event loop, so a lock carried over from an +# earlier test is bound to a loop that is already closed. Reusing it +# raises ``RuntimeError: ... is bound to a different event loop`` deep +# inside a strategy body, where it surfaces as a dead-lettered OME run +# rather than a test error. ``tests/conftest.py`` clears the pool around +# every test; these two pin that, in order — the first dirties the pool, +# the second asserts it did not survive. + + +def test_dirty_the_partition_lock_pool() -> None: + get_partition_lock("isolation_probe", "k") + assert "isolation_probe" in _pools + + +def test_partition_lock_pool_starts_empty_in_every_test() -> None: + assert _pools == {}, ( + "a previous test's locks survived into this one — the pool is not " + "being reset between tests, so a strategy can acquire a lock bound " + "to a closed event loop" + )