diff --git a/Cargo.lock b/Cargo.lock index 8b1112e..ce9b175 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1795,6 +1795,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.20", + "tinymemory-api", "uuid", ] @@ -1866,6 +1867,7 @@ dependencies = [ "tinycortex-api", "tinymemory", "tinymemory-api", + "tinymemory-sync", "tokio", "tracing", "url", @@ -1886,9 +1888,20 @@ dependencies = [ "sha2 0.10.9", "tinymemory", "tinymemory-api", + "tinymemory-conformance", "tokio", ] +[[package]] +name = "tinymemory-sync" +version = "0.1.0" +dependencies = [ + "chrono", + "log", + "serde_json", + "tracing", +] + [[package]] name = "tinymemory-tinycortex" version = "0.1.0" @@ -1902,6 +1915,7 @@ dependencies = [ "tinycortex", "tinymemory", "tinymemory-api", + "tinymemory-conformance", "tinymemory-core", "tokio", "uuid", diff --git a/Cargo.toml b/Cargo.toml index 2915028..fa90487 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] -members = [".", "api", "core", "adapters/tinycortex", "adapters/remote", "conformance"] -default-members = [".", "api", "core", "adapters/tinycortex", "adapters/remote", "conformance"] +# `sync` is the engine-neutral Composio normalisers (issue #18 §B3). +members = [".", "api", "core", "sync", "adapters/tinycortex", "adapters/remote", "conformance"] +default-members = [".", "api", "core", "sync", "adapters/tinycortex", "adapters/remote", "conformance"] # `vendor/` holds engine submodules (tinycortex, tinybus, tinyagents), each of # which is its own workspace with its own lockfile. Same exclusion # `vendor/tinycortex` uses for its own nested vendor directory. @@ -120,6 +121,19 @@ private_intra_doc_links = "warn" # so a host that already pins its own engine checkout unifies onto one copy # through its own patch table. These entries are what make a *standalone* build # of this workspace resolve them to the nested `vendor/` submodules. +# `tinycortex-api` takes this workspace's own contract crate by git, because +# neither crate is published (tinymemory#18 §A1). Without this entry cargo +# resolves the git copy *and* the path copy as two distinct crates, and +# `tinymemory_api::MemoryCategory` from one is not the same type as from the +# other — which is the exact duplication §A1 exists to delete, reintroduced by +# the fix for it. The error is loud rather than silent, but only at the seam. +# +# Patch tables apply from the workspace root being built, so this covers builds +# and tests here. A host that embeds this workspace needs the same entry, the +# same way it already patches tinycortex. +[patch."https://github.com/tinyhumansai/tinymemory"] +tinymemory-api = { path = "api" } + [patch.crates-io] tinycortex = { path = "vendor/tinycortex" } tinycortex-api = { path = "vendor/tinycortex/api" } diff --git a/adapters/remote/Cargo.toml b/adapters/remote/Cargo.toml index d78ff7e..3bb4da5 100644 --- a/adapters/remote/Cargo.toml +++ b/adapters/remote/Cargo.toml @@ -25,6 +25,9 @@ serde_json = "1" sha2 = "0.10" [dev-dependencies] +# The behavioural contract suite, run against these adapters over their own +# native doubles (issue #18 §E1, acceptance criterion 5). +tinymemory-conformance = { path = "../../conformance" } # Adapter tests run lightweight native-API doubles over a real TCP transport. axum = { version = "0.8", features = ["multipart"] } tokio = { version = "1", features = ["macros", "rt-multi-thread", "net"] } diff --git a/adapters/remote/src/conformance_test.rs b/adapters/remote/src/conformance_test.rs new file mode 100644 index 0000000..3694b63 --- /dev/null +++ b/adapters/remote/src/conformance_test.rs @@ -0,0 +1,497 @@ +//! The conformance suite, run against the hosted adapters. +//! +//! Issue #18's acceptance criterion 5: "the conformance suite passes for +//! TinyCortex and all three remote adapters". Until now it ran against the +//! in-memory reference driver and the null driver — both written alongside the +//! suite, so passing proved the assertions were self-consistent and little +//! else. +//! +//! These run the same `assert_provider` against the real adapters, over a real +//! TCP socket, against a double that speaks each vendor's own HTTP shapes and +//! **actually retains what it is sent**. That is the difference from +//! `failure_test`, whose doubles only need to misbehave: here the double has to +//! be a working backend, because the suite writes and reads back. +//! +//! What this proves is narrow and worth stating precisely. It is not that +//! Supermemory, Mem0 or Cognee uphold the contract — nobody here can prove that +//! about someone else's service. It is that **the adapter** does, given a +//! backend that answers its own documented shapes. A contract violation on the +//! adapter's side of the wire — a dropped taint, an export cursor that never +//! terminates, an upsert that duplicates — is caught here. + +#![allow(clippy::expect_used, clippy::panic)] + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use axum::extract::{Path, Query, State}; +use axum::routing::{delete, get, post, put}; +use axum::{Json, Router}; +use serde_json::{json, Value}; + +use crate::{mem0_provider, Mem0Memory}; + +/// A record as one of the vendor doubles holds it. +#[derive(Clone, Debug)] +struct Row { + id: String, + content: String, + metadata: Value, +} + +/// The doubles' shared store: `id -> Row`, plus a counter for fresh ids. +#[derive(Default, Debug)] +struct Backend { + rows: BTreeMap, + next: usize, +} + +impl Backend { + fn fresh_id(&mut self) -> String { + self.next += 1; + format!("rec-{}", self.next) + } +} + +type Store = Arc>; + +/// Serves `app` on an ephemeral port and returns its base URL. +async fn serve(app: Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let endpoint = format!("http://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve"); + }); + endpoint +} + +// ── Mem0's native shapes ───────────────────────────────────────────────────── +// +// Five routes, matching what `Mem0Dialect` issues: list, create, update, +// delete, search. The response envelopes (`results`, `memory`, `metadata`) are +// the ones its `decode` reads, so a shape drift on either side fails here +// rather than silently returning nothing. + +async fn mem0_list(State(store): State) -> Json { + let store = store.lock().expect("store lock"); + let results: Vec = store + .rows + .values() + .map(|r| { + json!({ + "id": r.id, + "memory": r.content, + "metadata": r.metadata, + "created_at": "1970-01-01T00:00:00Z", + }) + }) + .collect(); + Json(json!({ "results": results })) +} + +async fn mem0_create(State(store): State, Json(body): Json) -> Json { + let mut store = store.lock().expect("store lock"); + let id = store.fresh_id(); + let content = body["messages"][0]["content"] + .as_str() + .unwrap_or_default() + .to_owned(); + let metadata = body["metadata"].clone(); + store.rows.insert( + id.clone(), + Row { + id: id.clone(), + content, + metadata, + }, + ); + Json(json!({ "results": [{ "id": id }] })) +} + +async fn mem0_update( + State(store): State, + Path(id): Path, + Json(body): Json, +) -> Json { + let mut store = store.lock().expect("store lock"); + if let Some(row) = store.rows.get_mut(&id) { + if let Some(text) = body["text"].as_str() { + row.content = text.to_owned(); + } + if !body["metadata"].is_null() { + row.metadata = body["metadata"].clone(); + } + } + Json(json!({ "id": id })) +} + +async fn mem0_delete(State(store): State, Path(id): Path) -> Json { + store.lock().expect("store lock").rows.remove(&id); + Json(json!({ "deleted": true })) +} + +async fn mem0_search(State(store): State, Json(body): Json) -> Json { + // Substring matching is enough: the suite asserts that recall *narrows*, + // not that the backend ranks well. + let needle = body["query"].as_str().unwrap_or_default().to_lowercase(); + let limit = body["top_k"].as_u64().unwrap_or(100) as usize; + let store = store.lock().expect("store lock"); + let results: Vec = store + .rows + .values() + .filter(|r| r.content.to_lowercase().contains(&needle)) + .take(limit) + .map(|r| { + json!({ + "id": r.id, + "memory": r.content, + "metadata": r.metadata, + "score": 0.9, + }) + }) + .collect(); + Json(json!({ "results": results })) +} + +/// A Mem0 double that retains what it is sent. +async fn mem0_backend() -> String { + let store: Store = Arc::new(Mutex::new(Backend::default())); + let app = Router::new() + .route("/memories", get(mem0_list).post(mem0_create)) + .route("/memories/{id}", put(mem0_update).delete(mem0_delete)) + .route("/search", post(mem0_search)) + .with_state(store); + serve(app).await +} + +#[tokio::test] +async fn mem0_upholds_the_contract() { + let endpoint = mem0_backend().await; + let provider = mem0_provider(Mem0Memory::new(&endpoint, None).expect("client")); + tinymemory_conformance::assert_provider(Arc::new(provider)).await; +} + +/// The suite's write-path assertions only run when the driver retains, so a +/// double that silently dropped writes would let the whole run pass vacuously. +/// This pins that the Mem0 double is genuinely retaining. +#[tokio::test] +async fn the_mem0_double_actually_retains() { + let endpoint = mem0_backend().await; + let provider = mem0_provider(Mem0Memory::new(&endpoint, None).expect("client")); + assert!( + tinymemory_conformance::retains_writes(&provider).await, + "the Mem0 double must retain writes, or `assert_provider` skips every \ + assertion that matters and still reports success" + ); +} + +// ── Supermemory's native shapes ────────────────────────────────────────────── +// +// Container tags are Supermemory's namespace equivalent, and the adapter +// derives one per TinyMemory namespace. The double keeps a tag per row so the +// tag listing — which drives `entries()` — reflects what has actually been +// written, rather than a fixed set the adapter would then filter to nothing. + +/// The tag the adapter derives, as sent on create. +fn tag_of(row: &Row) -> String { + row.metadata + .get("tinymemory_namespace") + .and_then(Value::as_str) + .map(|ns| format!("tinymemory-{ns}")) + .unwrap_or_default() +} + +async fn sm_tags(State(store): State) -> Json { + let store = store.lock().expect("store lock"); + let mut tags: Vec = store.rows.values().map(tag_of).collect(); + tags.sort(); + tags.dedup(); + Json(Value::Array( + tags.into_iter() + .map(|t| json!({ "containerTag": t })) + .collect(), + )) +} + +async fn sm_list(State(store): State, Json(body): Json) -> Json { + // The adapter pages until a short page comes back, so a double that always + // returned a full page would spin. One page, then empty. + let page = body["page"].as_u64().unwrap_or(1); + let wanted = body["containerTags"][0].as_str().unwrap_or_default(); + let store = store.lock().expect("store lock"); + let entries: Vec = if page > 1 { + Vec::new() + } else { + store + .rows + .values() + .filter(|r| tag_of(r) == wanted) + .map(|r| { + json!({ + "id": r.id, + "content": r.content, + "metadata": r.metadata, + "createdAt": "1970-01-01T00:00:00Z", + "isLatest": true, + "isForgotten": false, + }) + }) + .collect() + }; + Json(json!({ "memoryEntries": entries })) +} + +async fn sm_create(State(store): State, Json(body): Json) -> Json { + let mut store = store.lock().expect("store lock"); + let id = store.fresh_id(); + let first = &body["memories"][0]; + store.rows.insert( + id.clone(), + Row { + id: id.clone(), + content: first["content"].as_str().unwrap_or_default().to_owned(), + metadata: first["metadata"].clone(), + }, + ); + Json(json!({ "memories": [{ "id": id }] })) +} + +async fn sm_update(State(store): State, Json(body): Json) -> Json { + let mut store = store.lock().expect("store lock"); + let id = body["id"].as_str().unwrap_or_default().to_owned(); + if let Some(row) = store.rows.get_mut(&id) { + if let Some(text) = body["newContent"].as_str() { + row.content = text.to_owned(); + } + if !body["metadata"].is_null() { + row.metadata = body["metadata"].clone(); + } + } + Json(json!({ "id": id })) +} + +async fn sm_delete(State(store): State, Json(body): Json) -> Json { + let id = body["id"].as_str().unwrap_or_default(); + store.lock().expect("store lock").rows.remove(id); + Json(json!({ "deleted": true })) +} + +async fn sm_search(State(store): State, Json(body): Json) -> Json { + let needle = body["q"] + .as_str() + .or_else(|| body["query"].as_str()) + .unwrap_or_default() + .to_lowercase(); + let limit = body["limit"].as_u64().unwrap_or(100) as usize; + let tag = body["containerTag"].as_str(); + let store = store.lock().expect("store lock"); + let results: Vec = store + .rows + .values() + .filter(|r| tag.is_none_or(|t| tag_of(r) == t)) + .filter(|r| r.content.to_lowercase().contains(&needle)) + .take(limit) + .map(|r| { + json!({ + "id": r.id, + "content": r.content, + "metadata": r.metadata, + "score": 0.9, + }) + }) + .collect(); + Json(json!({ "results": results })) +} + +async fn supermemory_backend() -> String { + let store: Store = Arc::new(Mutex::new(Backend::default())); + let app = Router::new() + .route("/v3/container-tags/list", get(sm_tags)) + .route("/v4/memories/list", post(sm_list)) + .route( + "/v4/memories", + post(sm_create).patch(sm_update).delete(sm_delete), + ) + .route("/v4/search", post(sm_search)) + .with_state(store); + serve(app).await +} + +#[tokio::test] +async fn supermemory_upholds_the_contract() { + let endpoint = supermemory_backend().await; + let provider = crate::supermemory_provider( + crate::SupermemoryMemory::new(&endpoint, None).expect("client"), + ); + tinymemory_conformance::assert_provider(Arc::new(provider)).await; +} + +#[tokio::test] +async fn the_supermemory_double_actually_retains() { + let endpoint = supermemory_backend().await; + let provider = crate::supermemory_provider( + crate::SupermemoryMemory::new(&endpoint, None).expect("client"), + ); + assert!( + tinymemory_conformance::retains_writes(&provider).await, + "the Supermemory double must retain writes, or the suite passes vacuously" + ); +} + +// ── Cognee's native shapes ─────────────────────────────────────────────────── +// +// The odd one out. Cognee has no per-record API: the adapter uploads each +// record as a JSON *file* into a per-namespace dataset, and reads it back +// through `/raw` — so the double stores the uploaded bytes verbatim and serves +// them unchanged. That is also why this double is the strictest of the three: +// the envelope it hands back is deserialised straight into `StoredEntry`, so a +// field the adapter fails to write is a parse failure here rather than a +// silently empty value. + +/// A dataset, keyed by the name the adapter derives from a namespace. +type Datasets = Arc>>>; + +async fn cg_datasets(State(sets): State) -> Json { + let sets = sets.lock().expect("store lock"); + Json(Value::Array( + sets.keys() + .map(|name| json!({ "id": name, "name": name })) + .collect(), + )) +} + +async fn cg_data(State(sets): State, Path(dataset): Path) -> Json { + let sets = sets.lock().expect("store lock"); + let ids: Vec = sets + .get(&dataset) + .map(|d| { + d.keys() + // `name` is required, and the adapter skips anything not + // ending `.tinymemory[.json]` — Cognee's own loader strips the + // extension, so both spellings are accepted. The data id here + // *is* the uploaded filename, which already carries it. + .map(|id| json!({ "id": id, "name": id })) + .collect() + }) + .unwrap_or_default(); + Json(Value::Array(ids)) +} + +async fn cg_raw( + State(sets): State, + Path((dataset, data_id)): Path<(String, String)>, +) -> String { + sets.lock() + .expect("store lock") + .get(&dataset) + .and_then(|d| d.get(&data_id)) + .cloned() + .unwrap_or_default() +} + +async fn cg_delete( + State(sets): State, + Path((dataset, data_id)): Path<(String, String)>, +) -> Json { + if let Some(d) = sets.lock().expect("store lock").get_mut(&dataset) { + d.remove(&data_id); + } + Json(json!({ "deleted": true })) +} + +/// Pulls the uploaded envelope and the dataset name out of a multipart body. +async fn multipart_parts(mut form: axum::extract::Multipart) -> (String, String, String) { + let (mut body, mut dataset, mut filename) = (String::new(), String::new(), String::new()); + while let Ok(Some(field)) = form.next_field().await { + match field.name().unwrap_or_default().to_owned().as_str() { + "datasetName" => dataset = field.text().await.unwrap_or_default(), + "data" | "file" | "files" => { + filename = field.file_name().unwrap_or_default().to_owned(); + body = field.text().await.unwrap_or_default(); + } + _ => { + let _ = field.bytes().await; + } + } + } + (body, dataset, filename) +} + +async fn cg_remember(State(sets): State, form: axum::extract::Multipart) -> Json { + let (body, dataset, filename) = multipart_parts(form).await; + let mut sets = sets.lock().expect("store lock"); + sets.entry(dataset).or_default().insert(filename, body); + Json(json!({ "status": "ok" })) +} + +async fn cg_update( + State(sets): State, + Query(q): Query>, + form: axum::extract::Multipart, +) -> Json { + let (body, _, _) = multipart_parts(form).await; + let dataset = q.get("dataset_id").cloned().unwrap_or_default(); + let data_id = q.get("data_id").cloned().unwrap_or_default(); + if let Some(d) = sets.lock().expect("store lock").get_mut(&dataset) { + d.insert(data_id, body); + } + Json(json!({ "status": "ok" })) +} + +async fn cg_recall(State(sets): State, Json(body): Json) -> Json { + let needle = body["query"].as_str().unwrap_or_default().to_lowercase(); + let limit = body["top_k"].as_u64().unwrap_or(100) as usize; + let wanted: Option> = body["datasets"].as_array().map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(str::to_owned)) + .collect() + }); + let sets = sets.lock().expect("store lock"); + let hits: Vec = sets + .iter() + .filter(|(name, _)| wanted.as_ref().is_none_or(|w| w.contains(name))) + .flat_map(|(_, d)| d.values()) + .filter(|raw| raw.to_lowercase().contains(&needle)) + .take(limit) + .map(|raw| json!({ "text": raw })) + .collect(); + Json(json!({ "results": hits })) +} + +async fn cognee_backend() -> String { + let sets: Datasets = Arc::new(Mutex::new(BTreeMap::new())); + let app = Router::new() + .route("/api/v1/datasets", get(cg_datasets)) + .route("/api/v1/datasets/{dataset}/data", get(cg_data)) + .route("/api/v1/datasets/{dataset}/data/{data_id}/raw", get(cg_raw)) + .route( + "/api/v1/datasets/{dataset}/data/{data_id}", + delete(cg_delete), + ) + .route("/api/v1/remember", post(cg_remember)) + .route("/api/v1/update", axum::routing::patch(cg_update)) + .route("/api/v1/recall", post(cg_recall)) + .with_state(sets); + serve(app).await +} + +#[tokio::test] +async fn cognee_upholds_the_contract() { + let endpoint = cognee_backend().await; + let provider = + crate::cognee_provider(crate::CogneeMemory::self_hosted(&endpoint, None).expect("client")); + tinymemory_conformance::assert_provider(Arc::new(provider)).await; +} + +#[tokio::test] +async fn the_cognee_double_actually_retains() { + let endpoint = cognee_backend().await; + let provider = + crate::cognee_provider(crate::CogneeMemory::self_hosted(&endpoint, None).expect("client")); + assert!( + tinymemory_conformance::retains_writes(&provider).await, + "the Cognee double must retain writes, or the suite passes vacuously" + ); +} diff --git a/adapters/remote/src/lib.rs b/adapters/remote/src/lib.rs index dba72ed..1b9d559 100644 --- a/adapters/remote/src/lib.rs +++ b/adapters/remote/src/lib.rs @@ -41,3 +41,6 @@ pub fn cognee_provider(memory: CogneeMemory) -> MemoryTraitProvider { #[cfg(test)] mod failure_test; + +#[cfg(test)] +mod conformance_test; diff --git a/adapters/tinycortex/Cargo.toml b/adapters/tinycortex/Cargo.toml index 57f28a1..73ba1e7 100644 --- a/adapters/tinycortex/Cargo.toml +++ b/adapters/tinycortex/Cargo.toml @@ -51,6 +51,9 @@ async-trait = "0.1" anyhow = "1" [dev-dependencies] +# The behavioural contract suite, run against this crate's drivers +# (issue #18 §E1, acceptance criterion 5). +tinymemory-conformance = { path = "../../conformance" } tokio = { version = "1", features = ["macros", "rt-multi-thread"] } [lints.rust] diff --git a/adapters/tinycortex/src/conformance_test.rs b/adapters/tinycortex/src/conformance_test.rs new file mode 100644 index 0000000..88f0ada --- /dev/null +++ b/adapters/tinycortex/src/conformance_test.rs @@ -0,0 +1,50 @@ +//! The conformance suite, run against the TinyCortex driver. +//! +//! The last name in issue #18's acceptance criterion 5, alongside the three +//! hosted adapters covered in `tinymemory-remote`. +//! +//! # Which TinyCortex driver +//! +//! This crate binds two, and they are conformance-tested differently. +//! +//! [`crate::provider`] composes the three mandatory families over any +//! `tinycortex::memory::Memory` backend. It needs nothing but the backend, so +//! the suite runs against it here with the engine's own `InMemoryMemoryStore`. +//! +//! [`crate::engine::TinycortexProvider`] serves all eighteen families, and +//! needs a `MemoryClient` — which needs the host's process-global seams +//! (`set_embedding_host` and friends) installed before it will open. A test +//! that installs a process global is order-dependent, which `AGENTS.md` rules +//! out, so covering it needs its own integration target that owns the global +//! for the whole binary. That is not written yet, and criterion 5 is not +//! complete until it is. +//! +//! Running against `InMemoryMemoryStore` rather than a SQLite workspace is +//! deliberate and is also the sharper test: it is the engine's simplest +//! `Memory`, so anything the suite catches is the *adapter's* behaviour rather +//! than the storage engine's. + +#![allow(clippy::expect_used, clippy::panic)] + +use std::sync::Arc; + +use tinycortex::memory::store::InMemoryMemoryStore; + +#[tokio::test] +async fn the_tinycortex_driver_upholds_the_contract() { + let driver = crate::provider(Arc::new(InMemoryMemoryStore::new())); + tinymemory_conformance::assert_provider(Arc::new(driver)).await; +} + +/// The suite skips every write-path assertion when a driver does not retain, so +/// a backend that silently dropped writes would let the run above pass having +/// asserted almost nothing. This pins that it does retain. +#[tokio::test] +async fn the_backend_actually_retains() { + let driver = crate::provider(Arc::new(InMemoryMemoryStore::new())); + assert!( + tinymemory_conformance::retains_writes(&driver).await, + "the engine's in-memory store must retain writes, or the suite above \ + reports success having run four assertions of eleven" + ); +} diff --git a/adapters/tinycortex/src/convert.rs b/adapters/tinycortex/src/convert.rs deleted file mode 100644 index 8d7d540..0000000 --- a/adapters/tinycortex/src/convert.rs +++ /dev/null @@ -1,197 +0,0 @@ -//! Value conversions between `tinycortex-api` and `tinymemory-api`. -//! -//! The two contracts describe the same values and, today, describe them -//! identically — `tinymemory-api` was moved out of `tinycortex-api`. They are -//! nonetheless distinct Rust types in distinct crates, so a value has to be -//! rebuilt to cross. -//! -//! ## Every conversion destructures exhaustively -//! -//! This is the whole discipline of this file, and the reason it is not written -//! with `..` or field-by-field assignment onto a `Default`. Two contracts that -//! are allowed to drift *will* drift: someone adds a field to one side, and a -//! lenient conversion silently drops it. A struct literal built from a full -//! destructuring pattern turns that into a compile error on the very next -//! build, naming the field. -//! -//! The same applies to the enums: each `match` lists every variant, so a new -//! category or a third taint level cannot fall into a catch-all arm and be -//! quietly downgraded. -//! -//! ## Taint conversion is the security-relevant one -//! -//! [`tinymemory_api::types::MemoryTaint`] records whether content came from outside. Mapping it -//! wrongly — or defaulting it on an unrecognised value — would let -//! externally-sourced content be treated as internal-trust content. Both sides -//! fail closed to `ExternalSync` when decoding an unknown persisted string, and -//! the mapping here is a total two-arm match with no default, so there is -//! nowhere for a wrong answer to come from. - -use tinycortex::memory::types as tc; -use tinymemory_api::types as tm; - -/// Converts a category to the TinyMemory contract's form. -#[must_use] -pub fn category_to_tinymemory(category: tc::MemoryCategory) -> tm::MemoryCategory { - match category { - tc::MemoryCategory::Core => tm::MemoryCategory::Core, - tc::MemoryCategory::Daily => tm::MemoryCategory::Daily, - tc::MemoryCategory::Conversation => tm::MemoryCategory::Conversation, - tc::MemoryCategory::Custom(name) => tm::MemoryCategory::Custom(name), - } -} - -/// Converts a category to the TinyCortex engine's form. -#[must_use] -pub fn category_to_tinycortex(category: tm::MemoryCategory) -> tc::MemoryCategory { - match category { - tm::MemoryCategory::Core => tc::MemoryCategory::Core, - tm::MemoryCategory::Daily => tc::MemoryCategory::Daily, - tm::MemoryCategory::Conversation => tc::MemoryCategory::Conversation, - tm::MemoryCategory::Custom(name) => tc::MemoryCategory::Custom(name), - } -} - -/// Converts provenance to the TinyMemory contract's form. -/// -/// A total match with no default arm: see the module docs on why this one may -/// not be lenient. -#[must_use] -pub fn taint_to_tinymemory(taint: tc::MemoryTaint) -> tm::MemoryTaint { - match taint { - tc::MemoryTaint::Internal => tm::MemoryTaint::Internal, - tc::MemoryTaint::ExternalSync => tm::MemoryTaint::ExternalSync, - } -} - -/// Converts provenance to the TinyCortex engine's form. -#[must_use] -pub fn taint_to_tinycortex(taint: tm::MemoryTaint) -> tc::MemoryTaint { - match taint { - tm::MemoryTaint::Internal => tc::MemoryTaint::Internal, - tm::MemoryTaint::ExternalSync => tc::MemoryTaint::ExternalSync, - } -} - -/// Converts an entry to the TinyMemory contract's form. -#[must_use] -pub fn entry_to_tinymemory(entry: tc::MemoryEntry) -> tm::MemoryEntry { - // Exhaustive destructuring: a field added to the engine's entry breaks this - // line rather than being dropped on the floor. - let tc::MemoryEntry { - id, - key, - content, - namespace, - category, - timestamp, - session_id, - score, - taint, - } = entry; - tm::MemoryEntry { - id, - key, - content, - namespace, - category: category_to_tinymemory(category), - timestamp, - session_id, - score, - taint: taint_to_tinymemory(taint), - } -} - -/// Converts a namespace summary to the TinyMemory contract's form. -#[must_use] -pub fn namespace_summary_to_tinymemory(summary: tc::NamespaceSummary) -> tm::NamespaceSummary { - let tc::NamespaceSummary { - namespace, - count, - last_updated, - } = summary; - tm::NamespaceSummary { - namespace, - count, - last_updated, - } -} - -/// The owned recall filters, in the engine's owned form. -/// -/// Returned owned rather than borrowed because the engine's `RecallOpts` -/// borrows its string fields, and a borrow of a value built inside a conversion -/// function cannot outlive the call. Callers keep this alive and borrow from it. -#[must_use] -pub fn recall_opts_to_tinycortex(opts: &tm::OwnedRecallOpts) -> tc::OwnedRecallOpts { - let tm::OwnedRecallOpts { - namespace, - category, - session_id, - min_score, - cross_session, - } = opts; - tc::OwnedRecallOpts { - namespace: namespace.clone(), - category: category.clone().map(category_to_tinycortex), - session_id: session_id.clone(), - min_score: *min_score, - cross_session: *cross_session, - } -} - -#[cfg(test)] -#[path = "convert_test.rs"] -mod test; - -// ── The reverse direction, for a driver whose contract is TinyMemory's ──────── -// -// Everything above converts engine values *into* the TinyMemory contract, which -// is what wrapping TinyCortex as a TinyMemory driver needs. A module-backed -// driver runs the other way: it speaks TinyMemory, and a host whose binding -// still speaks TinyCortex has to convert its answers back. -// -// Same discipline as above — exhaustive destructuring, total matches, no `..` -// and no `Default` — for the same reason: two contracts allowed to drift will. - -/// Converts an entry to the `TinyCortex` contract's form. -#[must_use] -pub fn entry_to_tinycortex(entry: tm::MemoryEntry) -> tc::MemoryEntry { - let tm::MemoryEntry { - id, - key, - content, - namespace, - category, - timestamp, - session_id, - score, - taint, - } = entry; - tc::MemoryEntry { - id, - key, - content, - namespace, - category: category_to_tinycortex(category), - timestamp, - session_id, - score, - taint: taint_to_tinycortex(taint), - } -} - -/// Converts a namespace summary to the `TinyCortex` contract's form. -#[must_use] -pub fn namespace_summary_to_tinycortex(summary: tm::NamespaceSummary) -> tc::NamespaceSummary { - let tm::NamespaceSummary { - namespace, - count, - last_updated, - } = summary; - tc::NamespaceSummary { - namespace, - count, - last_updated, - } -} diff --git a/adapters/tinycortex/src/convert_test.rs b/adapters/tinycortex/src/convert_test.rs deleted file mode 100644 index 75016ec..0000000 --- a/adapters/tinycortex/src/convert_test.rs +++ /dev/null @@ -1,170 +0,0 @@ -//! Conversion tests. -//! -//! Two contracts that are allowed to drift will drift. The exhaustive -//! destructuring in `convert.rs` catches a *new* field at compile time; these -//! tests catch a *mis-mapped* one, which the compiler cannot see because every -//! field on both sides has the same type. - -#![allow(clippy::expect_used, clippy::panic)] - -use super::*; - -/// Every category must survive a round trip, including the custom variant's -/// payload — a `Custom(String)` mapped onto the wrong arm would silently -/// re-file every custom-categorised memory. -#[test] -fn every_category_round_trips() { - let cases = [ - tm::MemoryCategory::Core, - tm::MemoryCategory::Daily, - tm::MemoryCategory::Conversation, - tm::MemoryCategory::Custom("project-notes".to_string()), - ]; - for category in cases { - let round_tripped = category_to_tinymemory(category_to_tinycortex(category.clone())); - assert_eq!(round_tripped, category); - } -} - -/// The wire form is the persisted form on both sides, so a conversion that -/// round-trips the Rust value but changes the string would still corrupt a -/// store. Comparing the rendered forms catches that. -#[test] -fn category_conversion_preserves_the_persisted_spelling() { - let cases = [ - tm::MemoryCategory::Core, - tm::MemoryCategory::Daily, - tm::MemoryCategory::Conversation, - tm::MemoryCategory::Custom("x".to_string()), - ]; - for category in cases { - let engine = category_to_tinycortex(category.clone()); - assert_eq!( - engine.to_string(), - category.to_string(), - "the two contracts must agree on the persisted spelling" - ); - } -} - -/// Provenance is the security-relevant conversion: mapping `ExternalSync` onto -/// `Internal` would upgrade the trust of externally-sourced content. -#[test] -fn every_taint_round_trips_and_keeps_its_db_spelling() { - for taint in [tm::MemoryTaint::Internal, tm::MemoryTaint::ExternalSync] { - let engine = taint_to_tinycortex(taint); - assert_eq!(taint_to_tinymemory(engine), taint); - assert_eq!( - engine.as_db_str(), - taint.as_db_str(), - "the two contracts must agree on the persisted spelling" - ); - } -} - -/// `ExternalSync` must never come back as `Internal`, stated as its own -/// assertion rather than left implicit in the round trip above. -#[test] -fn external_content_is_never_laundered_into_internal_trust() { - assert_eq!( - taint_to_tinymemory(taint_to_tinycortex(tm::MemoryTaint::ExternalSync)), - tm::MemoryTaint::ExternalSync - ); - assert_ne!( - taint_to_tinymemory(taint_to_tinycortex(tm::MemoryTaint::ExternalSync)), - tm::MemoryTaint::Internal - ); -} - -#[test] -fn an_entry_round_trips_every_field() { - let engine = tc::MemoryEntry { - id: "ns/key".to_string(), - key: "key".to_string(), - content: "body".to_string(), - namespace: Some("ns".to_string()), - category: tc::MemoryCategory::Custom("notes".to_string()), - timestamp: "2026-08-10T00:00:00Z".to_string(), - session_id: Some("s1".to_string()), - score: Some(0.75), - taint: tc::MemoryTaint::ExternalSync, - }; - - let converted = entry_to_tinymemory(engine.clone()); - - assert_eq!(converted.id, engine.id); - assert_eq!(converted.key, engine.key); - assert_eq!(converted.content, engine.content); - assert_eq!(converted.namespace, engine.namespace); - assert_eq!(converted.category.to_string(), engine.category.to_string()); - assert_eq!(converted.timestamp, engine.timestamp); - assert_eq!(converted.session_id, engine.session_id); - assert_eq!(converted.score, engine.score); - assert_eq!(converted.taint, tm::MemoryTaint::ExternalSync); -} - -/// A score of `None` must stay `None`. Defaulting it to `0.0` would make an -/// unranked entry look like a worst-ranked one. -#[test] -fn an_absent_score_stays_absent() { - let engine = tc::MemoryEntry { - id: "i".to_string(), - key: "k".to_string(), - content: "c".to_string(), - namespace: None, - category: tc::MemoryCategory::Core, - timestamp: "t".to_string(), - session_id: None, - score: None, - taint: tc::MemoryTaint::Internal, - }; - let converted = entry_to_tinymemory(engine); - assert!(converted.score.is_none()); - assert!(converted.namespace.is_none()); - assert!(converted.session_id.is_none()); -} - -#[test] -fn a_namespace_summary_round_trips_every_field() { - let engine = tc::NamespaceSummary { - namespace: "projects".to_string(), - count: 12, - last_updated: Some("2026-08-10T00:00:00Z".to_string()), - }; - let converted = namespace_summary_to_tinymemory(engine.clone()); - assert_eq!(converted.namespace, engine.namespace); - assert_eq!(converted.count, engine.count); - assert_eq!(converted.last_updated, engine.last_updated); -} - -/// Every recall filter must cross. A dropped `min_score` or `cross_session` -/// silently widens a query. -#[test] -fn every_recall_filter_crosses() { - let opts = tm::OwnedRecallOpts { - namespace: Some("ns".to_string()), - category: Some(tm::MemoryCategory::Daily), - session_id: Some("s1".to_string()), - min_score: Some(0.5), - cross_session: true, - }; - let engine = recall_opts_to_tinycortex(&opts); - assert_eq!(engine.namespace, opts.namespace); - assert_eq!( - engine.category.as_ref().map(ToString::to_string), - opts.category.as_ref().map(ToString::to_string) - ); - assert_eq!(engine.session_id, opts.session_id); - assert_eq!(engine.min_score, opts.min_score); - assert_eq!(engine.cross_session, opts.cross_session); -} - -#[test] -fn empty_recall_filters_stay_empty() { - let engine = recall_opts_to_tinycortex(&tm::OwnedRecallOpts::default()); - assert!(engine.namespace.is_none()); - assert!(engine.category.is_none()); - assert!(engine.session_id.is_none()); - assert!(engine.min_score.is_none()); - assert!(!engine.cross_session); -} diff --git a/adapters/tinycortex/src/lib.rs b/adapters/tinycortex/src/lib.rs index ea9ead8..fba0063 100644 --- a/adapters/tinycortex/src/lib.rs +++ b/adapters/tinycortex/src/lib.rs @@ -1,15 +1,18 @@ //! TinyCortex as a TinyMemory driver. //! //! This crate is the seam between the TinyCortex engine and the TinyMemory -//! contract. The two describe the same values but are distinct crates, so -//! something has to convert — and it is much better for that to be one small -//! audited crate than a conversion scattered across every call site in a host. +//! contract. //! -//! ## What is here +//! It used to carry a conversion layer as well. The two crates described the +//! same values under two names, so every call across the seam translated, and a +//! field added to one contract had to be added to the other and to the +//! conversion — three places, or the value was silently dropped. Since +//! issue #18 §A1 `tinycortex-api` re-exports `tinymemory-api` rather than +//! redefining it, so both sides name one type and `convert` is gone. What +//! remains is the trait shape: the engine's storage trait and the contract's +//! are still separate traits over the same values. //! -//! - [`convert`] — total, exhaustively-destructuring value conversions in both -//! directions. A field added to either contract becomes a compile error here -//! instead of a silently dropped value. +//! ## What is here //! - [`TinycortexMemory`] — wraps any TinyCortex [`tinycortex::memory::Memory`] //! backend as a TinyMemory //! [`Memory`](tinymemory_api::traits::Memory). @@ -45,7 +48,6 @@ //! [`engine::advertised_capabilities`] and not just the accessor — a build //! without the git-backed snapshot store must not claim a diff ledger. -pub mod convert; pub mod engine; mod memory; @@ -73,3 +75,6 @@ pub fn provider(memory: Arc) -> MemoryTraitProvi TINYCORTEX_DRIVER_ID, ) } + +#[cfg(test)] +mod conformance_test; diff --git a/adapters/tinycortex/src/memory.rs b/adapters/tinycortex/src/memory.rs index 02f7533..3235973 100644 --- a/adapters/tinycortex/src/memory.rs +++ b/adapters/tinycortex/src/memory.rs @@ -1,9 +1,12 @@ //! [`TinycortexMemory`] — a TinyCortex storage backend, seen through the //! TinyMemory contract's [`Memory`] trait. //! -//! Every method is a delegation plus a conversion. The two that are not purely -//! mechanical are called out below; both are cases where getting the -//! delegation "obviously right" would be wrong. +//! Every method is a plain delegation. It used to be a delegation *plus a +//! conversion*, because the engine's contract crate defined its own copies of +//! the memory value types; since tinymemory#18 §A1 `tinycortex-api` re-exports +//! this contract instead, so the two sides name one type and there is nothing +//! left to convert. The one method that is still not purely mechanical is +//! called out below. use std::sync::Arc; @@ -14,11 +17,6 @@ use tinymemory_api::types::{ MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts, }; -use crate::convert::{ - category_to_tinycortex, entry_to_tinymemory, namespace_summary_to_tinymemory, - recall_opts_to_tinycortex, taint_to_tinycortex, -}; - /// A TinyCortex backend exposed as a TinyMemory [`Memory`]. pub struct TinycortexMemory { inner: Arc, @@ -63,13 +61,7 @@ impl Memory for TinycortexMemory { session_id: Option<&str>, ) -> anyhow::Result<()> { self.inner - .store( - namespace, - key, - content, - category_to_tinycortex(category), - session_id, - ) + .store(namespace, key, content, category, session_id) .await } @@ -88,20 +80,13 @@ impl Memory for TinycortexMemory { taint: MemoryTaint, ) -> anyhow::Result<()> { self.inner - .store_with_taint( - namespace, - key, - content, - category_to_tinycortex(category), - session_id, - taint_to_tinycortex(taint), - ) + .store_with_taint(namespace, key, content, category, session_id, taint) .await } - /// The engine's `RecallOpts` borrows its string fields, so the owned - /// conversion has to outlive the borrow taken from it — hence the local - /// binding rather than a temporary in the call. + /// The engine's `RecallOpts` borrows its string fields, so the owned form + /// has to outlive the borrow taken from it — hence the local binding rather + /// than a temporary in the call. async fn recall( &self, query: &str, @@ -109,12 +94,7 @@ impl Memory for TinycortexMemory { opts: RecallOpts<'_>, ) -> anyhow::Result> { let owned = OwnedRecallOpts::from(opts); - let engine_owned = recall_opts_to_tinycortex(&owned); - let hits = self - .inner - .recall(query, limit, (&engine_owned).into()) - .await?; - Ok(hits.into_iter().map(entry_to_tinymemory).collect()) + Ok(self.inner.recall(query, limit, (&owned).into()).await?) } async fn recall_relevant_by_vector( @@ -130,11 +110,7 @@ impl Memory for TinycortexMemory { } async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - Ok(self - .inner - .get(namespace, key) - .await? - .map(entry_to_tinymemory)) + self.inner.get(namespace, key).await } async fn list( @@ -143,14 +119,7 @@ impl Memory for TinycortexMemory { category: Option<&MemoryCategory>, session_id: Option<&str>, ) -> anyhow::Result> { - // `category` is borrowed on both sides, so the converted value needs a - // binding to borrow from. - let engine_category = category.cloned().map(category_to_tinycortex); - let entries = self - .inner - .list(namespace, engine_category.as_ref(), session_id) - .await?; - Ok(entries.into_iter().map(entry_to_tinymemory).collect()) + self.inner.list(namespace, category, session_id).await } async fn forget(&self, namespace: &str, key: &str) -> anyhow::Result { @@ -158,13 +127,7 @@ impl Memory for TinycortexMemory { } async fn namespace_summaries(&self) -> anyhow::Result> { - Ok(self - .inner - .namespace_summaries() - .await? - .into_iter() - .map(namespace_summary_to_tinymemory) - .collect()) + self.inner.namespace_summaries().await } async fn count(&self) -> anyhow::Result { diff --git a/conformance/src/lib.rs b/conformance/src/lib.rs index 7bfca70..3e99c02 100644 --- a/conformance/src/lib.rs +++ b/conformance/src/lib.rs @@ -49,3 +49,11 @@ pub use suite::{ assert_store_get_round_trip, assert_taint_is_preserved, assert_upsert_replaces_rather_than_duplicates, }; +pub use suite::{ + // Exported alongside the assertions because a caller standing up its own + // backend double needs it: `assert_provider` skips every write-path + // assertion when the driver does not retain, so a double that silently + // dropped writes would let a whole run pass vacuously. Probing for that + // directly is how a caller proves its harness is real. + retains_writes, +}; diff --git a/conformance/src/suite/mod.rs b/conformance/src/suite/mod.rs index 194770c..1766887 100644 --- a/conformance/src/suite/mod.rs +++ b/conformance/src/suite/mod.rs @@ -601,8 +601,21 @@ pub async fn assert_awkward_content_round_trips(provider: &dyn MemoryProvider) { ("large", "x".repeat(64 * 1024)), ("newlines", "a\nb\r\nc\0d".to_string()), ]; + let mut accepted = 0usize; for (key, content) in &cases { - provider + // A driver may refuse a shape outright — `MemoryCore::store` documents + // `Invalid` "for caller input the driver rejects", and the TinyCortex + // engine uses that to refuse empty content. What a driver may *not* do + // is accept a value and hand back something else. + // + // The refusal is not yet required to be `Invalid` specifically. The + // engine's own error is flattened through `anyhow` before the mandatory + // composition sees it, so a validation refusal currently arrives as + // `Other` and is indistinguishable from a backend failure. That is the + // gap §A4 closes; when it does, this should tighten to require + // `MemoryError::Invalid` so a genuine backend failure stops passing + // here. + if provider .store( &ns, key, @@ -612,7 +625,11 @@ pub async fn assert_awkward_content_round_trips(provider: &dyn MemoryProvider) { MemoryTaint::Internal, ) .await - .unwrap_or_else(|e| panic!("{who}: store of `{key}` failed: {e}")); + .is_err() + { + continue; + } + accepted += 1; if let Some(got) = provider .get(&ns, key) .await @@ -621,6 +638,13 @@ pub async fn assert_awkward_content_round_trips(provider: &dyn MemoryProvider) { assert_eq!(&got.content, content, "{who}: `{key}` content was mangled"); } } + // Without this a driver that refused every shape would pass having stored + // nothing, which is the vacuous reading of "may refuse". + assert!( + accepted > 0, + "{who}: refused every content shape — unicode, empty, large and \ + newlines were all rejected, so this assertion proved nothing" + ); let keys: Vec<&str> = cases.iter().map(|(k, _)| *k).collect(); cleanup(provider, &ns, &keys).await; } diff --git a/core/Cargo.toml b/core/Cargo.toml index 628cafd..d876807 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -16,6 +16,11 @@ readme = "../README.md" # The contract. `tinymemory-core` implements and consumes it; the host seam # traits (config, event sink, embeddings, chat) live in `tinymemory_api::host`. tinymemory-api = { path = "../api" } +# Composio payload normalisers, extracted out of the engine (issue #18 §B3). +# They were reached through `tinycortex` until now, which meant a host binding a +# different engine could not have Composio sync despite none of this code +# caring which engine is bound. +tinymemory-sync = { path = "../sync" } tinymemory = { path = ".." } # The default embedded engine. `store/`, `tree/` and `sync/` drive it directly; diff --git a/core/src/conversations/blocking.rs b/core/src/conversations/blocking.rs index eda10aa..3e18309 100644 --- a/core/src/conversations/blocking.rs +++ b/core/src/conversations/blocking.rs @@ -1,7 +1,7 @@ //! Async wrappers that run the conversation store's **blocking** operations on //! tokio's blocking pool (#5156). //! -//! Every `tinycortex::memory::conversations` entry point is synchronous, and +//! Every `crate::engine::backend::conversations` entry point is synchronous, and //! each one takes the process-global `CONVERSATION_STORE_LOCK` — a //! `parking_lot::Mutex` — and then does fsync'd JSONL file IO while holding it. //! Calling one directly from an `async fn` therefore parks a tokio **worker** @@ -35,9 +35,9 @@ use std::path::PathBuf; -use tinycortex::memory::conversations as store; +use crate::engine::backend::conversations as store; -use tinycortex::memory::conversations::{ +use crate::engine::backend::conversations::{ ConversationMessage, ConversationMessagePatch, ConversationPurgeStats, ConversationStore, ConversationThread, CreateConversationThread, CrossThreadHit, }; diff --git a/core/src/conversations/mod.rs b/core/src/conversations/mod.rs index a6b5491..663dfb7 100644 --- a/core/src/conversations/mod.rs +++ b/core/src/conversations/mod.rs @@ -4,7 +4,7 @@ //! append-only in `threads.jsonl`; each thread's messages in a dedicated JSONL //! file). The store / inverted-index / tokenizer / types engine is the crate's //! (a byte-identical port, incl. the D1 rank-before-materialize fix), and -//! consumers name `tinycortex::memory::conversations` directly — this module no +//! consumers name `crate::engine::backend::conversations` directly — this module no //! longer re-exports that surface under a second path. //! //! Host-retained: diff --git a/core/src/diff/mod.rs b/core/src/diff/mod.rs index 31b3f13..e7adb6e 100644 --- a/core/src/diff/mod.rs +++ b/core/src/diff/mod.rs @@ -14,12 +14,12 @@ //! the ledger is a derived view used purely for change tracking. //! //! W7: the snapshot/diff/checkpoint/ledger engine is now -//! `tinycortex::memory::diff::DiffEngine` (a byte-identical port over the same +//! `crate::engine::backend::diff::DiffEngine` (a byte-identical port over the same //! `/memory_diff/repo` git layout). This module is a thin host shim: //! [`ops`] async-wraps the engine, [`source`] supplies the chunk-store item //! seam (`DiffEngine`'s `SnapshotItemSource`), and `rpc`/`schemas`/`tools` //! keep the RPC + agent surface. The wire types are the crate's, named directly -//! (`tinycortex::memory::diff::types`) rather than through a host re-export +//! (`crate::engine::backend::diff::types`) rather than through a host re-export //! module. //! //! Features: @@ -65,7 +65,7 @@ pub mod source; // `memory::diff::{types, source}` are serde-only wire types and stay compiled; // only the git-touching `ledger`/`DiffEngine` half sits behind `git-diff`. A // stub copy would be a second definition of one serde shape, free to drift. -pub use tinycortex::memory::diff::types::{ +pub use crate::engine::backend::diff::types::{ ChangeKind, Checkpoint, CrossSourceDiff, DiffResult, DiffSummary, ItemChange, Snapshot, SnapshotTrigger, }; diff --git a/core/src/diff/ops.rs b/core/src/diff/ops.rs index 8c50e8b..d56b401 100644 --- a/core/src/diff/ops.rs +++ b/core/src/diff/ops.rs @@ -1,5 +1,5 @@ //! Business logic for memory diff — thin host async wrappers over -//! `tinycortex::memory::diff::DiffEngine` (W7). +//! `crate::engine::backend::diff::DiffEngine` (W7). //! //! The snapshot/diff/checkpoint/ledger engine is the crate's; the git ledger it //! writes lives at the same `/memory_diff/repo` path with the same @@ -16,10 +16,10 @@ use tinymemory_api::host::test_support::TestHostConfig; use crate::sources::types::MemorySourceEntry; use crate::Config; -use tinycortex::memory::diff::{DiffEngine, SourceDescriptor}; +use crate::engine::backend::diff::{DiffEngine, SourceDescriptor}; use super::source::ChunkStoreItemSource; -use tinycortex::memory::diff::types::*; +use crate::engine::backend::diff::types::*; /// A crate [`SourceDescriptor`] from a host source entry. fn descriptor(source: &MemorySourceEntry) -> SourceDescriptor { @@ -101,7 +101,7 @@ pub async fn list_snapshots( let source_id = source_id.map(str::to_string); tokio::task::spawn_blocking(move || -> anyhow::Result> { - let ledger = tinycortex::memory::diff::Ledger::open(&workspace_dir)?; + let ledger = crate::engine::backend::diff::Ledger::open(&workspace_dir)?; ledger.list_snapshots(source_id.as_deref(), limit) }) .await @@ -310,7 +310,7 @@ pub async fn cleanup(config: &Config, older_than_days: u32) -> Result TestHostConfig { crate::test_seams::init(); diff --git a/core/src/diff/source.rs b/core/src/diff/source.rs index 2baff36..149ef34 100644 --- a/core/src/diff/source.rs +++ b/core/src/diff/source.rs @@ -1,6 +1,6 @@ //! The host implementation of the crate diff engine's chunk-source seam. //! -//! `tinycortex::memory::diff::DiffEngine` is generic over a +//! `crate::engine::backend::diff::DiffEngine` is generic over a //! [`SnapshotItemSource`]: during //! `take_snapshot` (directly, and transitively from `create_checkpoint` for any //! source lacking a baseline) it asks the source for a source's already-ingested @@ -22,7 +22,7 @@ use std::collections::HashMap; use std::sync::Arc; -use tinycortex::memory::diff::{extract_item_id, SnapshotItem, SnapshotItemSource}; +use crate::engine::backend::diff::{extract_item_id, SnapshotItem, SnapshotItemSource}; #[cfg(test)] use tinymemory_api::host::test_support::TestHostConfig; diff --git a/core/src/diff/stub.rs b/core/src/diff/stub.rs index f5658c8..fb49b35 100644 --- a/core/src/diff/stub.rs +++ b/core/src/diff/stub.rs @@ -26,7 +26,7 @@ use crate::sources::types::MemorySourceEntry; use crate::Config; -use tinycortex::memory::diff::types::{Checkpoint, CrossSourceDiff, Snapshot}; +use crate::engine::backend::diff::types::{Checkpoint, CrossSourceDiff, Snapshot}; /// The message every disabled entry point returns. /// diff --git a/core/src/engine/backend.rs b/core/src/engine/backend.rs new file mode 100644 index 0000000..c5485d7 --- /dev/null +++ b/core/src/engine/backend.rs @@ -0,0 +1,59 @@ +//! The engine's own memory surface, reached through one door. +//! +//! Issue #18 §C1 asks that nothing outside this module name the `tinycortex` +//! crate. Eighty-three files did, in two hundred and ninety-six places, which +//! made the engine's shape an ambient fact of the whole crate rather than a +//! dependency anyone had chosen — and made "what would a second engine have to +//! provide?" a question no one could answer without reading all of them. +//! +//! Re-exporting rather than wrapping is deliberate. A wrapper layer over three +//! hundred call sites would be a second surface to keep in step with the first, +//! which is the failure §A1 had just finished deleting. What this buys is not +//! insulation from the engine's API — the call sites still use it verbatim — +//! but a single place that *names* it, so the coupling is enumerable: this file +//! is the list of everything `tinymemory-core` needs an engine to provide. +//! +//! # Why this is not `MemoryProvider` +//! +//! §A3 proposes routing these call sites through `&dyn MemoryProvider` instead. +//! That is not possible, and the reason is worth recording where the next +//! reader will find it. Core does not *consume* the engine's memory API; it +//! shares the engine's SQLite database. Thirty-four files here hold a +//! `rusqlite::Transaction` or a `Connection`, and the entry points they call +//! take them: +//! +//! ```text +//! upsert_buffer_tx(tx: &Transaction<'_>, buf: &Buffer) -> Result<()> +//! shared_connection(config: &MemoryConfig) -> Result>> +//! ``` +//! +//! Serving those through the contract would put `rusqlite` in +//! `tinymemory-api`, which its own manifest forbids and CI now enforces. The +//! honest description is that core and the engine co-implement one store, and +//! separating them is a decomposition rather than a routing change. +//! +//! Nested under a module rather than re-exported flat because the seam already +//! has its own `ingest` and `sync` modules, which are host-side pieces and +//! not the engine's. + +// The engine's submodules. +pub use tinycortex::memory::{ + archivist, chunks, conversations, diff, graph, health, ingest, people, queue, retrieval, score, + sources, store, sync, tool_memory, tree, types, +}; + +// …and the items it re-exports at its own top level, which call sites reach for +// by the same short paths. Listed rather than globbed so this file stays the +// enumerable answer to "what does core need an engine to provide". +pub use tinycortex::memory::{ + GraphRelationRecord, InMemoryMemoryStore, MemoryCategory, MemoryConfig, MemoryEngineError, + MemoryEngineResult, MemoryEntry, MemoryId, MemoryInput, MemoryItemKind, MemoryKvRecord, + MemoryQuery, MemoryRecord, MemoryResult, MemoryStore, MemoryTaint, NamespaceDocumentInput, + NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, NamespaceSummary, + RecallOpts, RetrievalScoreBreakdown, SearchHit, StoreError, StoredMemoryDocument, + WeightProfile, GLOBAL_NAMESPACE, +}; + +// The storage trait itself. Since §A2 this is the contract's trait, not the +// engine's — the engine re-exports the same one. +pub use tinycortex::memory::Memory; diff --git a/core/src/tinycortex/chat.rs b/core/src/engine/chat.rs similarity index 100% rename from core/src/tinycortex/chat.rs rename to core/src/engine/chat.rs diff --git a/core/src/tinycortex/config.rs b/core/src/engine/config.rs similarity index 100% rename from core/src/tinycortex/config.rs rename to core/src/engine/config.rs diff --git a/core/src/tinycortex/embeddings.rs b/core/src/engine/embeddings.rs similarity index 100% rename from core/src/tinycortex/embeddings.rs rename to core/src/engine/embeddings.rs diff --git a/core/src/tinycortex/ingest.rs b/core/src/engine/ingest.rs similarity index 100% rename from core/src/tinycortex/ingest.rs rename to core/src/engine/ingest.rs diff --git a/core/src/tinycortex/mod.rs b/core/src/engine/mod.rs similarity index 99% rename from core/src/tinycortex/mod.rs rename to core/src/engine/mod.rs index 6c4f116..a6613e3 100644 --- a/core/src/tinycortex/mod.rs +++ b/core/src/engine/mod.rs @@ -46,6 +46,8 @@ mod seal; mod summariser; mod sync; +pub mod backend; + pub use chat::{build_chat_provider, SeamChatProvider}; pub use config::{engine_config, memory_config_from}; pub use embeddings::SeamEmbedder; diff --git a/core/src/tinycortex/parity.rs b/core/src/engine/parity.rs similarity index 100% rename from core/src/tinycortex/parity.rs rename to core/src/engine/parity.rs diff --git a/core/src/tinycortex/persona.rs b/core/src/engine/persona.rs similarity index 100% rename from core/src/tinycortex/persona.rs rename to core/src/engine/persona.rs diff --git a/core/src/tinycortex/queue_driver.rs b/core/src/engine/queue_driver.rs similarity index 100% rename from core/src/tinycortex/queue_driver.rs rename to core/src/engine/queue_driver.rs diff --git a/core/src/tinycortex/seal.rs b/core/src/engine/seal.rs similarity index 100% rename from core/src/tinycortex/seal.rs rename to core/src/engine/seal.rs diff --git a/core/src/tinycortex/summariser.rs b/core/src/engine/summariser.rs similarity index 100% rename from core/src/tinycortex/summariser.rs rename to core/src/engine/summariser.rs diff --git a/core/src/tinycortex/sync.rs b/core/src/engine/sync.rs similarity index 100% rename from core/src/tinycortex/sync.rs rename to core/src/engine/sync.rs diff --git a/core/src/ingest_pipeline.rs b/core/src/ingest_pipeline.rs index f3fcd2e..3d87d5c 100644 --- a/core/src/ingest_pipeline.rs +++ b/core/src/ingest_pipeline.rs @@ -2,16 +2,16 @@ use anyhow::Result; -use crate::store::chunks::store::RawRef; -use crate::Config; -use tinycortex::memory::ingest::canonicalize::{ +use crate::engine::backend::ingest::canonicalize::{ chat::{self, ChatBatch}, document::{self, DocumentInput}, email::{self, EmailThread}, CanonicalisedSource, }; +use crate::store::chunks::store::RawRef; +use crate::Config; -pub use tinycortex::memory::ingest::IngestSummary as IngestResult; +pub use crate::engine::backend::ingest::IngestSummary as IngestResult; pub async fn ingest_chat( config: &Config, @@ -22,8 +22,8 @@ pub async fn ingest_chat( ) -> Result { let canonical = chat::canonicalise(source_id, owner, &tags, batch.clone()).map_err(anyhow::Error::msg)?; - let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); - let result = tinycortex::memory::ingest::ingest_chat( + let (memory, sink, scoring) = crate::engine::ingest_context(config); + let result = crate::engine::backend::ingest::ingest_chat( &memory, source_id, owner, tags, batch, &sink, &scoring, ) .await?; @@ -40,8 +40,8 @@ pub async fn ingest_email( ) -> Result { let canonical = email::canonicalise(source_id, owner, &tags, thread.clone()).map_err(anyhow::Error::msg)?; - let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); - let result = tinycortex::memory::ingest::ingest_email( + let (memory, sink, scoring) = crate::engine::ingest_context(config); + let result = crate::engine::backend::ingest::ingest_email( &memory, source_id, owner, tags, thread, &sink, &scoring, ) .await?; @@ -59,8 +59,8 @@ pub async fn ingest_email_with_raw_refs( ) -> Result { let canonical = email::canonicalise(source_id, owner, &tags, thread.clone()).map_err(anyhow::Error::msg)?; - let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); - let result = tinycortex::memory::ingest::ingest_email_with_raw_refs( + let (memory, sink, scoring) = crate::engine::ingest_context(config); + let result = crate::engine::backend::ingest::ingest_email_with_raw_refs( &memory, source_id, owner, tags, thread, raw_refs, &sink, &scoring, ) .await?; @@ -101,8 +101,8 @@ pub async fn ingest_document_versioned( let canonical = document::canonicalise(source_id, owner, &tags, doc.clone(), path_scope.clone()) .map_err(anyhow::Error::msg)?; - let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); - let result = tinycortex::memory::ingest::ingest_document_versioned( + let (memory, sink, scoring) = crate::engine::ingest_context(config); + let result = crate::engine::backend::ingest::ingest_document_versioned( &memory, source_id, owner, tags, doc, path_scope, version_ms, &sink, &scoring, ) .await?; diff --git a/core/src/ingestion/mod.rs b/core/src/ingestion/mod.rs index 9ee1fb8..16b9714 100644 --- a/core/src/ingestion/mod.rs +++ b/core/src/ingestion/mod.rs @@ -14,12 +14,12 @@ pub mod queue; pub mod state; -pub use queue::{IngestionJob, IngestionQueue, DEFAULT_QUEUE_CAPACITY}; -pub use state::{IngestionState, IngestionStatusSnapshot}; -pub use tinycortex::memory::ingest::{ +pub use crate::engine::backend::ingest::{ ExtractedEntity, ExtractedRelation, ExtractionMode, MemoryIngestionConfig, MemoryIngestionRequest, MemoryIngestionResult, DEFAULT_MEMORY_EXTRACTION_MODEL, }; +pub use queue::{IngestionJob, IngestionQueue, DEFAULT_QUEUE_CAPACITY}; +pub use state::{IngestionState, IngestionStatusSnapshot}; use serde_json::json; @@ -35,7 +35,7 @@ impl UnifiedMemory { request: MemoryIngestionRequest, ) -> Result { let (enriched_input, mut extraction) = - tinycortex::memory::ingest::extract_enriched_document( + crate::engine::backend::ingest::extract_enriched_document( &request.document, &request.config, ); @@ -62,7 +62,7 @@ impl UnifiedMemory { config: &MemoryIngestionConfig, ) -> Result { let (_enriched, mut extraction) = - tinycortex::memory::ingest::extract_enriched_document(document, config); + crate::engine::backend::ingest::extract_enriched_document(document, config); let namespace = Self::sanitize_namespace(&document.namespace); self.upsert_graph_relations(&namespace, document_id, &extraction, config) diff --git a/core/src/lib.rs b/core/src/lib.rs index 1935dcd..4455dd7 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -39,6 +39,7 @@ pub mod conversations; pub mod diff; pub mod embedding_adapter; pub mod embedding_host; +pub mod engine; pub mod events; pub mod global; pub mod ingest_pipeline; @@ -63,7 +64,6 @@ pub mod test_env_lock; #[cfg(test)] pub(crate) mod test_seams; pub mod thread_context; -pub mod tinycortex; pub mod tool_memory; pub mod traits; pub mod tree; diff --git a/core/src/people/mod.rs b/core/src/people/mod.rs index feccd58..e3507e0 100644 --- a/core/src/people/mod.rs +++ b/core/src/people/mod.rs @@ -2,7 +2,7 @@ //! //! # Why this is a shim //! -//! The implementation moved down into [`tinycortex::memory::people`]. People is +//! The implementation moved down into [`crate::engine::backend::people`]. People is //! *storage*: a SQLite database of people, handle aliases and interactions, //! with its own migrations and its own workspace-keyed connection. Storage //! belongs to the engine, which is what lets the memory contract stay @@ -14,7 +14,7 @@ //! references to `people::types`, did not all have to move in the same change. //! //! This mirrors [`crate::store::chunks`], which has related the same way to -//! `tinycortex::memory::chunks` since the engine seam was drawn. +//! `crate::engine::backend::chunks` since the engine seam was drawn. //! //! # The address book rides two gates //! @@ -24,4 +24,6 @@ //! stub returns an empty contact list, so a refresh seeds nothing rather than //! failing. -pub use tinycortex::memory::people::{address_book, migrations, resolver, scorer, store, types}; +pub use crate::engine::backend::people::{ + address_book, migrations, resolver, scorer, store, types, +}; diff --git a/core/src/queue/ops.rs b/core/src/queue/ops.rs index 56468b1..4276d4f 100644 --- a/core/src/queue/ops.rs +++ b/core/src/queue/ops.rs @@ -7,14 +7,14 @@ /// Mark whether a re-embed backfill currently has pending work. pub fn set_backfill_in_progress(v: bool) { - tinycortex::memory::queue::set_backfill_in_progress(v); + crate::engine::backend::queue::set_backfill_in_progress(v); } /// True while a re-embed backfill chain still has rows to process. The /// #1365 absence-reasoning consumer checks this before treating an empty /// semantic-recall result as "no memory exists". pub fn backfill_in_progress() -> bool { - tinycortex::memory::queue::backfill_in_progress() + crate::engine::backend::queue::backfill_in_progress() } /// #1574 §4: ensure a re-embed backfill chain exists for the **current** @@ -30,9 +30,10 @@ pub fn backfill_in_progress() -> bool { /// covered space enqueues nothing. Errors are logged, never propagated — /// a failed enqueue must not fail the user's settings save. pub fn ensure_reembed_backfill(config: &crate::Config) { - let memory = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); - let delegates = crate::tinycortex::HostQueueDelegates::new(config.to_arc()); - if let Err(error) = tinycortex::memory::queue::ensure_reembed_backfill(&memory, &delegates) { + let memory = crate::engine::memory_config_from(config, config.workspace_dir().clone()); + let delegates = crate::engine::HostQueueDelegates::new(config.to_arc()); + if let Err(error) = crate::engine::backend::queue::ensure_reembed_backfill(&memory, &delegates) + { log::warn!("[memory::jobs] ensure_reembed_backfill failed: {error:#}"); } } diff --git a/core/src/queue/scheduler.rs b/core/src/queue/scheduler.rs index aa01e88..54435c9 100644 --- a/core/src/queue/scheduler.rs +++ b/core/src/queue/scheduler.rs @@ -43,8 +43,8 @@ pub fn start(config: Arc) { /// Unrecoverable failures stay parked — see /// [`store::requeue_transient_failed`]. fn retry_transient_failures(config: &Config) { - let memory = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); - match tinycortex::memory::queue::scheduler::self_heal(&memory) { + let memory = crate::engine::memory_config_from(config, config.workspace_dir().clone()); + match crate::engine::backend::queue::scheduler::self_heal(&memory) { Ok(0) => {} Ok(n) => { log::info!("[memory::jobs] periodic retry requeued {n} transient-failed job(s)"); @@ -72,8 +72,8 @@ fn retry_transient_failures(config: &Config) { /// `LabelStrategy` for every tree, which no production caller uses and which /// would apply one tree kind's labelling to all of them. pub fn enqueue_flush_stale_job(config: &Config) -> Result { - let memory = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); - match tinycortex::memory::queue::scheduler::enqueue_flush_stale(&memory) { + let memory = crate::engine::memory_config_from(config, config.workspace_dir().clone()); + match crate::engine::backend::queue::scheduler::enqueue_flush_stale(&memory) { Ok(Some(_)) => { super::worker::wake_workers(); Ok(true) diff --git a/core/src/queue/store.rs b/core/src/queue/store.rs index 2ad8d55..fd3c223 100644 --- a/core/src/queue/store.rs +++ b/core/src/queue/store.rs @@ -7,28 +7,28 @@ use crate::tree::health::PipelineFailure; use crate::Config; use super::types::{Job, JobFailure, JobStatus, NewJob}; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; -pub use tinycortex::memory::queue::DEFAULT_LOCK_DURATION_MS; +pub use crate::engine::backend::queue::DEFAULT_LOCK_DURATION_MS; pub fn enqueue(config: &Config, job: &NewJob) -> Result> { - tinycortex::memory::queue::enqueue(&engine_config(config), job) + crate::engine::backend::queue::enqueue(&engine_config(config), job) } pub fn enqueue_tx(tx: &Transaction<'_>, job: &NewJob) -> Result> { - tinycortex::memory::queue::enqueue_tx(tx, job) + crate::engine::backend::queue::enqueue_tx(tx, job) } pub fn claim_next(config: &Config, lock_duration_ms: i64) -> Result> { - tinycortex::memory::queue::claim_next(&engine_config(config), lock_duration_ms) + crate::engine::backend::queue::claim_next(&engine_config(config), lock_duration_ms) } pub fn mark_done(config: &Config, job: &Job) -> Result<()> { - tinycortex::memory::queue::mark_done(&engine_config(config), job) + crate::engine::backend::queue::mark_done(&engine_config(config), job) } pub fn mark_failed(config: &Config, job: &Job, error: &str) -> Result<()> { - tinycortex::memory::queue::mark_failed(&engine_config(config), job, error) + crate::engine::backend::queue::mark_failed(&engine_config(config), job, error) } pub fn mark_failed_typed( @@ -41,7 +41,7 @@ pub fn mark_failed_typed( code: failure.code.as_str(), class: failure.class.as_str(), }); - tinycortex::memory::queue::mark_failed_typed( + crate::engine::backend::queue::mark_failed_typed( &engine_config(config), job, error, @@ -50,41 +50,41 @@ pub fn mark_failed_typed( } pub fn mark_deferred(config: &Config, job: &Job, until_ms: i64, reason: &str) -> Result<()> { - tinycortex::memory::queue::mark_deferred(&engine_config(config), job, until_ms, reason) + crate::engine::backend::queue::mark_deferred(&engine_config(config), job, until_ms, reason) } pub fn recover_stale_locks(config: &Config) -> Result { - tinycortex::memory::queue::recover_stale_locks(&engine_config(config)) + crate::engine::backend::queue::recover_stale_locks(&engine_config(config)) } pub fn requeue_failed(config: &Config) -> Result { - tinycortex::memory::queue::requeue_failed(&engine_config(config)) + crate::engine::backend::queue::requeue_failed(&engine_config(config)) } pub fn requeue_transient_failed(config: &Config) -> Result { - tinycortex::memory::queue::requeue_transient_failed(&engine_config(config)) + crate::engine::backend::queue::requeue_transient_failed(&engine_config(config)) } pub fn release_running_locks(config: &Config) -> Result { - tinycortex::memory::queue::release_running_locks(&engine_config(config)) + crate::engine::backend::queue::release_running_locks(&engine_config(config)) } pub fn count_by_status(config: &Config, status: JobStatus) -> Result { - tinycortex::memory::queue::count_by_status(&engine_config(config), status) + crate::engine::backend::queue::count_by_status(&engine_config(config), status) } pub fn count_failed_unrecoverable(config: &Config) -> Result { - tinycortex::memory::queue::count_failed_unrecoverable(&engine_config(config)) + crate::engine::backend::queue::count_failed_unrecoverable(&engine_config(config)) } pub fn count_total(config: &Config) -> Result { - tinycortex::memory::queue::count_total(&engine_config(config)) + crate::engine::backend::queue::count_total(&engine_config(config)) } pub fn retry_all_failed(config: &Config) -> Result { - tinycortex::memory::queue::retry_all_failed(&engine_config(config)) + crate::engine::backend::queue::retry_all_failed(&engine_config(config)) } pub fn get_job(config: &Config, id: &str) -> Result> { - tinycortex::memory::queue::get_job(&engine_config(config), id) + crate::engine::backend::queue::get_job(&engine_config(config), id) } diff --git a/core/src/queue/types.rs b/core/src/queue/types.rs index fc87750..3b91b2d 100644 --- a/core/src/queue/types.rs +++ b/core/src/queue/types.rs @@ -1,6 +1,6 @@ //! Queue wire types owned by tinycortex. -pub use tinycortex::memory::queue::{ +pub use crate::engine::backend::queue::{ AppendBufferPayload, AppendTarget, ExtractChunkPayload, FlushStalePayload, Job, JobFailure, JobKind, JobOutcome, JobStatus, NewJob, NodeRef, ReembedBackfillPayload, SealDocumentPayload, SealPayload, diff --git a/core/src/queue/worker.rs b/core/src/queue/worker.rs index 811c540..ebb5b70 100644 --- a/core/src/queue/worker.rs +++ b/core/src/queue/worker.rs @@ -1,6 +1,6 @@ //! Worker pool: drives the crate queue engine (W4 flip). Each `run_once` -//! delegates claim → dispatch → settle to `tinycortex::memory::queue::run_once` -//! via [`crate::tinycortex::HostQueueDelegates`]; the legacy host +//! delegates claim → dispatch → settle to `crate::engine::backend::queue::run_once` +//! via [`crate::engine::HostQueueDelegates`]; the legacy host //! `handlers` engine that used to own dispatch was deleted at the flip. //! //! Concurrency control for LLM-bound work is delegated to @@ -288,9 +288,9 @@ pub async fn run_once(config: &Config) -> Result { // single-slot LLM gate serialises llm-bound jobs; the legacy per-job // local/cloud permit routing and the extract-batch coalescing are // intentionally dropped here (perf, not correctness — W4 follow-up). - let mc = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); - let delegates = crate::tinycortex::HostQueueDelegates::new(config.to_arc()); - tinycortex::memory::queue::run_once(&mc, &delegates).await + let mc = crate::engine::memory_config_from(config, config.workspace_dir().clone()); + let delegates = crate::engine::HostQueueDelegates::new(config.to_arc()); + crate::engine::backend::queue::run_once(&mc, &delegates).await } /// Classify whether an error is a transient I/O failure that should be diff --git a/core/src/sources/readers/conversation.rs b/core/src/sources/readers/conversation.rs index 7b18c40..2380821 100644 --- a/core/src/sources/readers/conversation.rs +++ b/core/src/sources/readers/conversation.rs @@ -19,10 +19,10 @@ impl SourceReader for ConversationReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - tinycortex::memory::sources::SourceReader::list_items( - &tinycortex::memory::sources::readers::conversation::ConversationReader, + crate::engine::backend::sources::SourceReader::list_items( + &crate::engine::backend::sources::readers::conversation::ConversationReader, source, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) @@ -34,11 +34,11 @@ impl SourceReader for ConversationReader { item_id: &str, config: &Config, ) -> Result { - tinycortex::memory::sources::SourceReader::read_item( - &tinycortex::memory::sources::readers::conversation::ConversationReader, + crate::engine::backend::sources::SourceReader::read_item( + &crate::engine::backend::sources::readers::conversation::ConversationReader, source, item_id, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) diff --git a/core/src/sources/readers/folder.rs b/core/src/sources/readers/folder.rs index 31922d5..56bbbfb 100644 --- a/core/src/sources/readers/folder.rs +++ b/core/src/sources/readers/folder.rs @@ -19,10 +19,10 @@ impl SourceReader for FolderReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - tinycortex::memory::sources::SourceReader::list_items( - &tinycortex::memory::sources::readers::folder::FolderReader, + crate::engine::backend::sources::SourceReader::list_items( + &crate::engine::backend::sources::readers::folder::FolderReader, source, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) @@ -34,11 +34,11 @@ impl SourceReader for FolderReader { item_id: &str, config: &Config, ) -> Result { - tinycortex::memory::sources::SourceReader::read_item( - &tinycortex::memory::sources::readers::folder::FolderReader, + crate::engine::backend::sources::SourceReader::read_item( + &crate::engine::backend::sources::readers::folder::FolderReader, source, item_id, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) diff --git a/core/src/sources/readers/github.rs b/core/src/sources/readers/github.rs index 91d464d..b35af29 100644 --- a/core/src/sources/readers/github.rs +++ b/core/src/sources/readers/github.rs @@ -12,7 +12,9 @@ use crate::sources::readers::SourceReader; use crate::sources::types::{MemorySourceEntry, SourceContent, SourceItem, SourceKind}; use crate::Config; -pub use tinycortex::memory::sources::readers::github::{repo_archive_source_id, repo_chunk_scope}; +pub use crate::engine::backend::sources::readers::github::{ + repo_archive_source_id, repo_chunk_scope, +}; pub struct GithubReader; @@ -27,10 +29,10 @@ impl SourceReader for GithubReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - tinycortex::memory::sources::SourceReader::list_items( - &tinycortex::memory::sources::readers::github::GithubReader, + crate::engine::backend::sources::SourceReader::list_items( + &crate::engine::backend::sources::readers::github::GithubReader, source, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) @@ -42,11 +44,11 @@ impl SourceReader for GithubReader { item_id: &str, config: &Config, ) -> Result { - tinycortex::memory::sources::SourceReader::read_item( - &tinycortex::memory::sources::readers::github::GithubReader, + crate::engine::backend::sources::SourceReader::read_item( + &crate::engine::backend::sources::readers::github::GithubReader, source, item_id, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) diff --git a/core/src/sources/readers/rss.rs b/core/src/sources/readers/rss.rs index 05a543a..d6213a2 100644 --- a/core/src/sources/readers/rss.rs +++ b/core/src/sources/readers/rss.rs @@ -12,13 +12,13 @@ use crate::Config; /// `read_item`, so constructing it per trait call would turn one sync into /// N+1 downloads. pub struct RssReader { - inner: tinycortex::memory::sources::readers::rss::RssReader, + inner: crate::engine::backend::sources::readers::rss::RssReader, } impl RssReader { pub fn new() -> Self { Self { - inner: tinycortex::memory::sources::readers::rss::RssReader::new(), + inner: crate::engine::backend::sources::readers::rss::RssReader::new(), } } } @@ -40,10 +40,10 @@ impl SourceReader for RssReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - tinycortex::memory::sources::SourceReader::list_items( + crate::engine::backend::sources::SourceReader::list_items( &self.inner, source, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) @@ -55,11 +55,11 @@ impl SourceReader for RssReader { item_id: &str, config: &Config, ) -> Result { - tinycortex::memory::sources::SourceReader::read_item( + crate::engine::backend::sources::SourceReader::read_item( &self.inner, source, item_id, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) diff --git a/core/src/sources/readers/web_page.rs b/core/src/sources/readers/web_page.rs index a2f693c..69723c8 100644 --- a/core/src/sources/readers/web_page.rs +++ b/core/src/sources/readers/web_page.rs @@ -19,10 +19,10 @@ impl SourceReader for WebPageReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - tinycortex::memory::sources::SourceReader::list_items( - &tinycortex::memory::sources::readers::web_page::WebPageReader, + crate::engine::backend::sources::SourceReader::list_items( + &crate::engine::backend::sources::readers::web_page::WebPageReader, source, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) @@ -34,11 +34,11 @@ impl SourceReader for WebPageReader { item_id: &str, config: &Config, ) -> Result { - tinycortex::memory::sources::SourceReader::read_item( - &tinycortex::memory::sources::readers::web_page::WebPageReader, + crate::engine::backend::sources::SourceReader::read_item( + &crate::engine::backend::sources::readers::web_page::WebPageReader, source, item_id, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) diff --git a/core/src/sources/registry.rs b/core/src/sources/registry.rs index a33f532..f5c90b9 100644 --- a/core/src/sources/registry.rs +++ b/core/src/sources/registry.rs @@ -5,7 +5,7 @@ use std::sync::OnceLock; use crate::config_loader as config_rpc; use crate::sources::types::{MemorySourceEntry, SourceKind}; -pub use tinycortex::memory::sources::{ +pub use crate::engine::backend::sources::{ memory_sync_defaults_for_toolkit, ComposioUpsertTarget, MemorySourcePatch, }; @@ -18,9 +18,9 @@ pub(crate) async fn memory_sources_write_guard() -> tokio::sync::MutexGuard<'sta .await } -async fn registry() -> Result { +async fn registry() -> Result { let config = config_rpc::load_config_with_timeout().await?; - Ok(tinycortex::memory::sources::SourceRegistry::new( + Ok(crate::engine::backend::sources::SourceRegistry::new( config.config_path(), )) } @@ -59,7 +59,7 @@ pub fn get_source_in( config: &crate::Config, id: &str, ) -> Result, String> { - tinycortex::memory::sources::SourceRegistry::new(config.config_path().clone()) + crate::engine::backend::sources::SourceRegistry::new(config.config_path().clone()) .get(id) .map_err(|error| error.to_string()) } diff --git a/core/src/sources/sync.rs b/core/src/sources/sync.rs index af362a0..134a6e8 100644 --- a/core/src/sources/sync.rs +++ b/core/src/sources/sync.rs @@ -89,7 +89,7 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu let mut composio_usage = ComposioUsage::default(); let outcome = match source.kind { SourceKind::Composio => { - match crate::tinycortex::run_source_pipeline(&source, &*config).await { + match crate::engine::run_source_pipeline(&source, &*config).await { Ok(outcome) => { composio_usage.actions_called = outcome.actions_called; composio_usage.cost_usd = outcome.provider_cost_usd; @@ -103,17 +103,17 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu } } SourceKind::Conversation | SourceKind::Folder => { - crate::tinycortex::run_source_pipeline(&source, &*config) + crate::engine::run_source_pipeline(&source, &*config) .await .map(|outcome| outcome.records_ingested as usize) .map_err(|error| error.to_string()) } - SourceKind::GithubRepo => crate::tinycortex::run_source_pipeline(&source, &*config) + SourceKind::GithubRepo => crate::engine::run_source_pipeline(&source, &*config) .await .map(|outcome| outcome.records_ingested as usize) .map_err(|error| error.to_string()), SourceKind::RssFeed | SourceKind::WebPage => { - crate::tinycortex::run_source_pipeline(&source, &*config) + crate::engine::run_source_pipeline(&source, &*config) .await .map(|outcome| outcome.records_ingested as usize) .map_err(|error| error.to_string()) @@ -142,7 +142,7 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu Some(&source.id), ); - use crate::tinycortex::{append_audit_entry, SyncAuditEntry}; + use crate::engine::{append_audit_entry, SyncAuditEntry}; append_audit_entry( &*config, &SyncAuditEntry { @@ -185,7 +185,7 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu } Err(error) => { // Audit failed syncs too. - use crate::tinycortex::{append_audit_entry, SyncAuditEntry}; + use crate::engine::{append_audit_entry, SyncAuditEntry}; append_audit_entry( &*config, &SyncAuditEntry { @@ -266,7 +266,7 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu /// Reconcile raw files that are not yet covered by tree summaries. pub(crate) async fn check_and_rebuild_tree(source: &MemorySourceEntry, config: &Config) { - use crate::tinycortex::{needs_rebuild, rebuild_tree_from_raw}; + use crate::engine::{needs_rebuild, rebuild_tree_from_raw}; for scope in derive_scopes(source, config) { if !needs_rebuild(config, &scope.tree_scope, &scope.archive_source_id) { diff --git a/core/src/sources/types.rs b/core/src/sources/types.rs index 6fbab9f..7588253 100644 --- a/core/src/sources/types.rs +++ b/core/src/sources/types.rs @@ -1,5 +1,5 @@ //! Stable host path for tinycortex-owned memory-source contracts. -pub use tinycortex::memory::sources::{ +pub use crate::engine::backend::sources::{ ContentType, MemorySourceEntry, SourceContent, SourceItem, SourceKind, }; diff --git a/core/src/store/chunks/connection.rs b/core/src/store/chunks/connection.rs index 7fe48ed..746fe6d 100644 --- a/core/src/store/chunks/connection.rs +++ b/core/src/store/chunks/connection.rs @@ -3,15 +3,15 @@ use anyhow::Result; use rusqlite::Connection; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; #[doc(hidden)] pub fn with_connection(config: &Config, f: impl FnOnce(&Connection) -> Result) -> Result { - tinycortex::memory::chunks::with_connection(&engine_config(config), f) + crate::engine::backend::chunks::with_connection(&engine_config(config), f) } pub(crate) fn recover_corrupt_db(config: &Config) -> Result { log::warn!("[memory:chunks] checking corrupt database recovery"); - tinycortex::memory::chunks::recover_corrupt_db(&engine_config(config)) + crate::engine::backend::chunks::recover_corrupt_db(&engine_config(config)) } diff --git a/core/src/store/chunks/embeddings.rs b/core/src/store/chunks/embeddings.rs index 812852a..07d6a6f 100644 --- a/core/src/store/chunks/embeddings.rs +++ b/core/src/store/chunks/embeddings.rs @@ -5,15 +5,15 @@ use std::collections::HashMap; use anyhow::Result; use rusqlite::{Connection, Transaction}; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; pub(crate) fn tree_active_signature(config: &Config) -> String { - tinycortex::memory::chunks::tree_active_signature(&engine_config(config)) + crate::engine::backend::chunks::tree_active_signature(&engine_config(config)) } pub fn set_chunk_embedding(config: &Config, id: &str, embedding: &[f32]) -> Result<()> { - tinycortex::memory::chunks::set_chunk_embedding(&engine_config(config), id, embedding) + crate::engine::backend::chunks::set_chunk_embedding(&engine_config(config), id, embedding) } pub fn set_chunk_embedding_for_signature( @@ -22,7 +22,7 @@ pub fn set_chunk_embedding_for_signature( signature: &str, embedding: &[f32], ) -> Result<()> { - tinycortex::memory::chunks::set_chunk_embedding_for_signature( + crate::engine::backend::chunks::set_chunk_embedding_for_signature( &engine_config(config), id, signature, @@ -34,7 +34,7 @@ pub(crate) fn has_uncovered_reembed_work( conn: &Connection, signature: &str, ) -> rusqlite::Result { - tinycortex::memory::chunks::has_uncovered_reembed_work(conn, signature) + crate::engine::backend::chunks::has_uncovered_reembed_work(conn, signature) } pub fn mark_chunk_reembed_skipped( @@ -43,7 +43,7 @@ pub fn mark_chunk_reembed_skipped( signature: &str, reason: &str, ) -> Result<()> { - tinycortex::memory::chunks::mark_chunk_reembed_skipped( + crate::engine::backend::chunks::mark_chunk_reembed_skipped( &engine_config(config), id, signature, @@ -52,11 +52,15 @@ pub fn mark_chunk_reembed_skipped( } pub fn clear_chunk_reembed_skipped(config: &Config, id: &str, signature: &str) -> Result<()> { - tinycortex::memory::chunks::clear_chunk_reembed_skipped(&engine_config(config), id, signature) + crate::engine::backend::chunks::clear_chunk_reembed_skipped( + &engine_config(config), + id, + signature, + ) } pub fn clear_reembed_skipped_for_signature(config: &Config, signature: &str) -> Result { - tinycortex::memory::chunks::clear_reembed_skipped_for_signature( + crate::engine::backend::chunks::clear_reembed_skipped_for_signature( &engine_config(config), signature, ) @@ -68,7 +72,9 @@ pub(crate) fn set_chunk_embedding_for_signature_tx( signature: &str, embedding: &[f32], ) -> Result<()> { - tinycortex::memory::chunks::set_chunk_embedding_for_signature_tx(tx, id, signature, embedding) + crate::engine::backend::chunks::set_chunk_embedding_for_signature_tx( + tx, id, signature, embedding, + ) } pub fn get_chunk_embedding_for_signature( @@ -76,7 +82,7 @@ pub fn get_chunk_embedding_for_signature( id: &str, signature: &str, ) -> Result>> { - tinycortex::memory::chunks::get_chunk_embedding_for_signature( + crate::engine::backend::chunks::get_chunk_embedding_for_signature( &engine_config(config), id, signature, @@ -84,7 +90,7 @@ pub fn get_chunk_embedding_for_signature( } pub fn get_chunk_embedding(config: &Config, id: &str) -> Result>> { - tinycortex::memory::chunks::get_chunk_embedding(&engine_config(config), id) + crate::engine::backend::chunks::get_chunk_embedding(&engine_config(config), id) } pub fn get_chunk_embeddings_for_signature_batch( @@ -92,7 +98,7 @@ pub fn get_chunk_embeddings_for_signature_batch( ids: &[String], signature: &str, ) -> Result>> { - tinycortex::memory::chunks::get_chunk_embeddings_for_signature_batch( + crate::engine::backend::chunks::get_chunk_embeddings_for_signature_batch( &engine_config(config), ids, signature, @@ -103,5 +109,5 @@ pub fn get_chunk_embeddings_batch( config: &Config, ids: &[String], ) -> Result>> { - tinycortex::memory::chunks::get_chunk_embeddings_batch(&engine_config(config), ids) + crate::engine::backend::chunks::get_chunk_embeddings_batch(&engine_config(config), ids) } diff --git a/core/src/store/chunks/mod.rs b/core/src/store/chunks/mod.rs index 9b6fb29..4ba07f1 100644 --- a/core/src/store/chunks/mod.rs +++ b/core/src/store/chunks/mod.rs @@ -20,7 +20,7 @@ pub mod semantic; pub mod store; pub mod types; +pub use crate::engine::backend::chunks::{chunk_markdown, ChunkerInput, ChunkerOptions}; pub use semantic::chunk_markdown as chunk_semantic; pub use store::*; -pub use tinycortex::memory::chunks::{chunk_markdown, ChunkerInput, ChunkerOptions}; pub use types::*; diff --git a/core/src/store/chunks/raw_refs.rs b/core/src/store/chunks/raw_refs.rs index bef5c76..c750ece 100644 --- a/core/src/store/chunks/raw_refs.rs +++ b/core/src/store/chunks/raw_refs.rs @@ -6,7 +6,7 @@ //! directly instead of going through the SQL preview path. //! //! **W3 sub-store flip:** these operations now delegate to -//! [`tinycortex::memory::chunks`] (ported from this exact module — identical SQL +//! [`crate::engine::backend::chunks`] (ported from this exact module — identical SQL //! against the same `mem_tree_chunks` / `mem_tree_summaries` tables in the shared //! `chunks.db` the crate now owns). The host signatures are preserved so the ~4 //! external callers (`content::read`, memory_sync gmail/slack ingest, rebuild) @@ -15,27 +15,27 @@ use anyhow::Result; use rusqlite::Transaction; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; // `RawRef` is re-exported from the crate (identical fields + serde derives), so // every `chunks::RawRef { path, start, end }` construction site keeps compiling. -pub use tinycortex::memory::chunks::RawRef; +pub use crate::engine::backend::chunks::RawRef; /// Stash a list of [`RawRef`] entries on a chunk row. Replaces any previous /// value. pub fn set_chunk_raw_refs(config: &Config, chunk_id: &str, refs: &[RawRef]) -> Result<()> { - tinycortex::memory::chunks::set_chunk_raw_refs(&engine_config(config), chunk_id, refs) + crate::engine::backend::chunks::set_chunk_raw_refs(&engine_config(config), chunk_id, refs) } /// Stash raw archive pointers on a chunk row inside a caller-owned transaction. pub fn set_chunk_raw_refs_tx(tx: &Transaction<'_>, chunk_id: &str, refs: &[RawRef]) -> Result<()> { - tinycortex::memory::chunks::set_chunk_raw_refs_tx(tx, chunk_id, refs) + crate::engine::backend::chunks::set_chunk_raw_refs_tx(tx, chunk_id, refs) } /// Return the raw-archive pointers stored in SQLite for `chunk_id`, or `None`. pub fn get_chunk_raw_refs(config: &Config, chunk_id: &str) -> Result>> { - tinycortex::memory::chunks::get_chunk_raw_refs(&engine_config(config), chunk_id) + crate::engine::backend::chunks::get_chunk_raw_refs(&engine_config(config), chunk_id) } /// Collect every raw-archive path referenced by any chunk row, restricted to @@ -44,7 +44,7 @@ pub fn list_chunk_raw_ref_paths_with_prefix( config: &Config, rel_prefix: &str, ) -> Result> { - tinycortex::memory::chunks::list_chunk_raw_ref_paths_with_prefix( + crate::engine::backend::chunks::list_chunk_raw_ref_paths_with_prefix( &engine_config(config), rel_prefix, ) @@ -55,12 +55,12 @@ pub fn get_chunk_content_pointers( config: &Config, chunk_id: &str, ) -> Result> { - tinycortex::memory::chunks::get_chunk_content_pointers(&engine_config(config), chunk_id) + crate::engine::backend::chunks::get_chunk_content_pointers(&engine_config(config), chunk_id) } /// Return the `content_path` stored in SQLite for `chunk_id`, if any. pub fn get_chunk_content_path(config: &Config, chunk_id: &str) -> Result> { - tinycortex::memory::chunks::get_chunk_content_path(&engine_config(config), chunk_id) + crate::engine::backend::chunks::get_chunk_content_path(&engine_config(config), chunk_id) } /// Return both `content_path` and `content_sha256` stored in SQLite for `summary_id`. @@ -68,10 +68,10 @@ pub fn get_summary_content_pointers( config: &Config, summary_id: &str, ) -> Result> { - tinycortex::memory::chunks::get_summary_content_pointers(&engine_config(config), summary_id) + crate::engine::backend::chunks::get_summary_content_pointers(&engine_config(config), summary_id) } /// List all summary rows that have a non-NULL `content_path`. pub fn list_summaries_with_content_path(config: &Config) -> Result> { - tinycortex::memory::chunks::list_summaries_with_content_path(&engine_config(config)) + crate::engine::backend::chunks::list_summaries_with_content_path(&engine_config(config)) } diff --git a/core/src/store/chunks/semantic.rs b/core/src/store/chunks/semantic.rs index a624762..b682e85 100644 --- a/core/src/store/chunks/semantic.rs +++ b/core/src/store/chunks/semantic.rs @@ -1,7 +1,7 @@ //! Compatibility exports for tinycortex's semantic Markdown chunker. -pub use tinycortex::memory::chunks::SemanticChunk as Chunk; +pub use crate::engine::backend::chunks::SemanticChunk as Chunk; pub fn chunk_markdown(text: &str, max_tokens: usize) -> Vec { - tinycortex::memory::chunks::chunk_semantic(text, max_tokens) + crate::engine::backend::chunks::chunk_semantic(text, max_tokens) } diff --git a/core/src/store/chunks/store.rs b/core/src/store/chunks/store.rs index f2f7557..47cb55c 100644 --- a/core/src/store/chunks/store.rs +++ b/core/src/store/chunks/store.rs @@ -5,34 +5,38 @@ use std::collections::HashMap; use anyhow::Result; use rusqlite::Transaction; +use crate::engine::engine_config; use crate::store::chunks::types::{Chunk, SourceKind}; use crate::store::content::StagedChunk; -use crate::tinycortex::engine_config; use crate::Config; -pub use tinycortex::memory::chunks::{ +pub use crate::engine::backend::chunks::{ ListChunksQuery, RawRef, CHUNK_STATUS_ADMITTED, CHUNK_STATUS_BUFFERED, CHUNK_STATUS_DROPPED, CHUNK_STATUS_PENDING_EXTRACTION, CHUNK_STATUS_SEALED, RAW_FILE_GATE_KIND, }; pub fn upsert_chunks(config: &Config, chunks: &[Chunk]) -> Result { - tinycortex::memory::chunks::upsert_chunks(&engine_config(config), chunks) + crate::engine::backend::chunks::upsert_chunks(&engine_config(config), chunks) } pub fn upsert_chunks_tx(tx: &Transaction<'_>, chunks: &[Chunk]) -> Result { - tinycortex::memory::chunks::upsert_chunks_tx(tx, chunks) + crate::engine::backend::chunks::upsert_chunks_tx(tx, chunks) } pub fn upsert_staged_chunks_tx(tx: &Transaction<'_>, chunks: &[StagedChunk]) -> Result { - tinycortex::memory::chunks::upsert_staged_chunks_tx(tx, chunks) + crate::engine::backend::chunks::upsert_staged_chunks_tx(tx, chunks) } pub fn update_chunk_content_sha256(config: &Config, id: &str, sha256: &str) -> Result<()> { - tinycortex::memory::chunks::update_chunk_content_sha256(&engine_config(config), id, sha256) + crate::engine::backend::chunks::update_chunk_content_sha256(&engine_config(config), id, sha256) } pub fn update_summary_content_sha256(config: &Config, id: &str, sha256: &str) -> Result<()> { - tinycortex::memory::chunks::update_summary_content_sha256(&engine_config(config), id, sha256) + crate::engine::backend::chunks::update_summary_content_sha256( + &engine_config(config), + id, + sha256, + ) } pub fn list_source_ids_with_prefix( @@ -40,31 +44,35 @@ pub fn list_source_ids_with_prefix( kind: SourceKind, prefix: &str, ) -> Result> { - tinycortex::memory::chunks::list_source_ids_with_prefix(&engine_config(config), kind, prefix) + crate::engine::backend::chunks::list_source_ids_with_prefix( + &engine_config(config), + kind, + prefix, + ) } pub fn get_chunk(config: &Config, id: &str) -> Result> { - tinycortex::memory::chunks::get_chunk(&engine_config(config), id) + crate::engine::backend::chunks::get_chunk(&engine_config(config), id) } pub fn get_chunks_batch(config: &Config, ids: &[String]) -> Result> { - tinycortex::memory::chunks::get_chunks_batch(&engine_config(config), ids) + crate::engine::backend::chunks::get_chunks_batch(&engine_config(config), ids) } pub fn list_chunks(config: &Config, query: &ListChunksQuery) -> Result> { - tinycortex::memory::chunks::list_chunks(&engine_config(config), query) + crate::engine::backend::chunks::list_chunks(&engine_config(config), query) } pub fn count_chunks(config: &Config) -> Result { - tinycortex::memory::chunks::count_chunks(&engine_config(config)) + crate::engine::backend::chunks::count_chunks(&engine_config(config)) } pub fn extraction_coverage(config: &Config) -> Result { - tinycortex::memory::chunks::extraction_coverage(&engine_config(config)) + crate::engine::backend::chunks::extraction_coverage(&engine_config(config)) } pub fn set_chunk_lifecycle_status(config: &Config, id: &str, status: &str) -> Result<()> { - tinycortex::memory::chunks::set_chunk_lifecycle_status(&engine_config(config), id, status) + crate::engine::backend::chunks::set_chunk_lifecycle_status(&engine_config(config), id, status) } pub(crate) fn set_chunk_lifecycle_status_tx( @@ -72,23 +80,23 @@ pub(crate) fn set_chunk_lifecycle_status_tx( id: &str, status: &str, ) -> Result<()> { - tinycortex::memory::chunks::set_chunk_lifecycle_status_tx(tx, id, status) + crate::engine::backend::chunks::set_chunk_lifecycle_status_tx(tx, id, status) } pub fn get_chunk_lifecycle_status(config: &Config, id: &str) -> Result> { - tinycortex::memory::chunks::get_chunk_lifecycle_status(&engine_config(config), id) + crate::engine::backend::chunks::get_chunk_lifecycle_status(&engine_config(config), id) } pub fn get_chunk_lifecycle_status_tx(tx: &Transaction<'_>, id: &str) -> Result> { - tinycortex::memory::chunks::get_chunk_lifecycle_status_tx(tx, id) + crate::engine::backend::chunks::get_chunk_lifecycle_status_tx(tx, id) } pub fn count_chunks_by_lifecycle_status(config: &Config, status: &str) -> Result { - tinycortex::memory::chunks::count_chunks_by_lifecycle_status(&engine_config(config), status) + crate::engine::backend::chunks::count_chunks_by_lifecycle_status(&engine_config(config), status) } pub fn is_source_ingested(config: &Config, kind: SourceKind, id: &str) -> Result { - tinycortex::memory::chunks::is_source_ingested(&engine_config(config), kind, id) + crate::engine::backend::chunks::is_source_ingested(&engine_config(config), kind, id) } pub fn claim_source_ingest_tx( @@ -97,23 +105,26 @@ pub fn claim_source_ingest_tx( id: &str, now_ms: i64, ) -> Result { - tinycortex::memory::chunks::claim_source_ingest_tx(tx, kind, id, now_ms) + crate::engine::backend::chunks::claim_source_ingest_tx(tx, kind, id, now_ms) } pub fn mark_raw_paths_ingested(config: &Config, paths: &[String]) -> Result { - tinycortex::memory::chunks::mark_raw_paths_ingested(&engine_config(config), paths) + crate::engine::backend::chunks::mark_raw_paths_ingested(&engine_config(config), paths) } pub fn filter_raw_paths_not_ingested(config: &Config, paths: &[String]) -> Result> { - tinycortex::memory::chunks::filter_raw_paths_not_ingested(&engine_config(config), paths) + crate::engine::backend::chunks::filter_raw_paths_not_ingested(&engine_config(config), paths) } pub fn count_raw_paths_ingested_with_prefix(config: &Config, prefix: &str) -> Result { - tinycortex::memory::chunks::count_raw_paths_ingested_with_prefix(&engine_config(config), prefix) + crate::engine::backend::chunks::count_raw_paths_ingested_with_prefix( + &engine_config(config), + prefix, + ) } pub fn delete_chunks_by_source(config: &Config, kind: SourceKind, id: &str) -> Result { - tinycortex::memory::chunks::delete_chunks_by_source(&engine_config(config), kind, id) + crate::engine::backend::chunks::delete_chunks_by_source(&engine_config(config), kind, id) } pub fn delete_chunks_by_source_prefix( @@ -121,15 +132,19 @@ pub fn delete_chunks_by_source_prefix( kind: SourceKind, prefix: &str, ) -> Result { - tinycortex::memory::chunks::delete_chunks_by_source_prefix(&engine_config(config), kind, prefix) + crate::engine::backend::chunks::delete_chunks_by_source_prefix( + &engine_config(config), + kind, + prefix, + ) } pub fn delete_chunks_by_owner(config: &Config, kind: SourceKind, owner: &str) -> Result { - tinycortex::memory::chunks::delete_chunks_by_owner(&engine_config(config), kind, owner) + crate::engine::backend::chunks::delete_chunks_by_owner(&engine_config(config), kind, owner) } pub fn delete_orphaned_source_tree(config: &Config, kind: SourceKind, id: &str) -> Result { - tinycortex::memory::chunks::delete_orphaned_source_tree(&engine_config(config), kind, id) + crate::engine::backend::chunks::delete_orphaned_source_tree(&engine_config(config), kind, id) } #[path = "connection.rs"] diff --git a/core/src/store/chunks/types.rs b/core/src/store/chunks/types.rs index bd080fc..8c25440 100644 --- a/core/src/store/chunks/types.rs +++ b/core/src/store/chunks/types.rs @@ -12,13 +12,13 @@ //! **W3 type cutover:** these types + chunk-id/token helpers are now //! **re-exported from the `tinycortex` crate** (ported from this exact module — //! identical fields, derives, serde wire form, and `chunk_id` derivation, all -//! pinned by `tinycortex::memory::chunks::types_tests`). Re-exporting keeps one source of truth and lets +//! pinned by `crate::engine::backend::chunks::types_tests`). Re-exporting keeps one source of truth and lets //! the chunk store operations delegate to the crate without host↔crate type //! conversions. `DataSource` moved with the ingest cutover and is re-exported //! here alongside the chunk types. `StagedChunk` remains host-owned in //! `memory_store::content`. -pub use tinycortex::memory::chunks::{ +pub use crate::engine::backend::chunks::{ approx_token_count, chunk_id, conservative_token_estimate, truncate_to_conservative_tokens, Chunk, DataSource, Metadata, SourceKind, SourceRef, }; diff --git a/core/src/store/client.rs b/core/src/store/client.rs index 24cd438..992946e 100644 --- a/core/src/store/client.rs +++ b/core/src/store/client.rs @@ -77,8 +77,7 @@ impl MemoryClient { /// Typed access to the profile/facet tables. /// - /// **Not guarded.** The profile tables have no capability family in the - /// thirteen-family `tinycortex_api` contract, so these reads and writes + /// **Not guarded.** These reads and writes /// still run beneath `crate::guard::MemoryGuard`'s /// seven steps. What this buys is confinement, not policy: the SQL is in /// the memory family and the compiler keeps it there. diff --git a/core/src/store/content/mod.rs b/core/src/store/content/mod.rs index 52ec477..6da0a48 100644 --- a/core/src/store/content/mod.rs +++ b/core/src/store/content/mod.rs @@ -17,13 +17,13 @@ pub mod read; pub mod tags; -pub use tinycortex::memory::chunks::StagedChunk; +pub use crate::engine::backend::chunks::StagedChunk; /// The git-backed wiki content format. Re-exported only when `memory-git` is /// on: it lives behind tinycortex's `wiki-git` feature, which the gate carries /// along with `git-diff` and the libgit2 cohort. #[cfg(feature = "memory-git")] -pub use tinycortex::memory::store::content::wiki_git; -pub use tinycortex::memory::store::content::{ +pub use crate::engine::backend::store::content::wiki_git; +pub use crate::engine::backend::store::content::{ atomic, compose, obsidian, obsidian_registry, paths, raw, stage_chunks, StagedSummary, SummaryComposeInput, SummaryTreeKind, }; diff --git a/core/src/store/content/read.rs b/core/src/store/content/read.rs index d15ee13..f745e20 100644 --- a/core/src/store/content/read.rs +++ b/core/src/store/content/read.rs @@ -1,15 +1,15 @@ //! Product Config adapters over tinycortex content readers. -use crate::tinycortex::engine_config; +use crate::engine::engine_config; -pub use tinycortex::memory::store::content::{ +pub use crate::engine::backend::store::content::{ read_chunk_file, read_summary_file, verify_chunk_file, verify_summary_file, ChunkFileContents, VerifyResult, }; pub fn read_chunk_body(config: &crate::Config, chunk_id: &str) -> anyhow::Result { - tinycortex::memory::store::content::read_chunk_body(&engine_config(config), chunk_id) + crate::engine::backend::store::content::read_chunk_body(&engine_config(config), chunk_id) } pub fn read_summary_body(config: &crate::Config, summary_id: &str) -> anyhow::Result { - tinycortex::memory::store::content::read_summary_body(&engine_config(config), summary_id) + crate::engine::backend::store::content::read_summary_body(&engine_config(config), summary_id) } diff --git a/core/src/store/content/tags.rs b/core/src/store/content/tags.rs index 92413b1..87ab188 100644 --- a/core/src/store/content/tags.rs +++ b/core/src/store/content/tags.rs @@ -13,7 +13,7 @@ use crate::store::content::compose::{ use crate::tree::score::store::list_entity_ids_for_node; use crate::Config; -pub use tinycortex::memory::store::content::tags::{ +pub use crate::engine::backend::store::content::tags::{ entity_tag, slugify_tag_kind, slugify_tag_value, update_chunk_tags, }; diff --git a/core/src/store/entities.rs b/core/src/store/entities.rs index 47024f4..5cd375a 100644 --- a/core/src/store/entities.rs +++ b/core/src/store/entities.rs @@ -2,13 +2,13 @@ use std::sync::Arc; -use anyhow::Result; -use tinycortex::memory::store::entity_index::{ +use crate::engine::backend::store::entity_index::{ CanonicalEntity, EntityIndex, EntityKind, SelfIdentity, }; +use anyhow::Result; +use crate::engine::memory_config_from; use crate::sync::composio::providers::profile::{is_self_identity_any_toolkit, IdentityKind}; -use crate::tinycortex::memory_config_from; use crate::Config; /// Aggregate entity-index row for capability providers. @@ -32,7 +32,7 @@ pub fn namespace_entities( limit: usize, ) -> Result> { let memory = memory_config_from(config, config.workspace_dir().clone()); - let connection = tinycortex::memory::chunks::shared_connection(&memory)?; + let connection = crate::engine::backend::chunks::shared_connection(&memory)?; let guard = connection.lock(); let pattern = query.map(|value| format!("%{}%", value.to_ascii_lowercase())); let mut statement = guard.prepare( @@ -69,7 +69,7 @@ pub fn namespace_entity_edges( limit: usize, ) -> Result> { let memory = memory_config_from(config, config.workspace_dir().clone()); - let connection = tinycortex::memory::chunks::shared_connection(&memory)?; + let connection = crate::engine::backend::chunks::shared_connection(&memory)?; let guard = connection.lock(); let mut statement = guard.prepare( "SELECT b.entity_id, COUNT(*) @@ -97,7 +97,7 @@ pub fn namespace_entity_edges( Ok(rows) } -pub use tinycortex::memory::store::entity_index::EntityHit; +pub use crate::engine::backend::store::entity_index::EntityHit; #[derive(Debug)] struct HostSelfIdentity; @@ -115,7 +115,7 @@ impl SelfIdentity for HostSelfIdentity { fn index(config: &Config) -> Result { let memory = memory_config_from(config, config.workspace_dir().clone()); - let connection = tinycortex::memory::chunks::shared_connection(&memory)?; + let connection = crate::engine::backend::chunks::shared_connection(&memory)?; EntityIndex::from_shared_connection(connection, Arc::new(HostSelfIdentity)) } @@ -169,7 +169,7 @@ pub fn count_entity_index(config: &Config) -> Result { /// Most frequently observed entities, with recency as the tie-breaker. pub fn top_entities(config: &Config, limit: usize) -> Result> { let memory = memory_config_from(config, config.workspace_dir().clone()); - let connection = tinycortex::memory::chunks::shared_connection(&memory)?; + let connection = crate::engine::backend::chunks::shared_connection(&memory)?; let guard = connection.lock(); let mut statement = guard.prepare( "SELECT entity_id, entity_kind, MAX(surface), COUNT(*) @@ -209,7 +209,7 @@ mod tests { fn insert_entity(config: &Config, tree: &str, entity: &str, node: &str, surface: &str) { let memory = memory_config_from(config, config.workspace_dir().clone()); - let connection = tinycortex::memory::chunks::shared_connection(&memory).expect("db"); + let connection = crate::engine::backend::chunks::shared_connection(&memory).expect("db"); connection .lock() .execute( diff --git a/core/src/store/kv.rs b/core/src/store/kv.rs index 21f9827..625c7af 100644 --- a/core/src/store/kv.rs +++ b/core/src/store/kv.rs @@ -8,7 +8,7 @@ //! again. Canonicalizing here is a no-op for the write path (the transform is //! idempotent and identical) and makes the read path symmetric. -use tinycortex::memory::store::kv::KvStore; +use crate::engine::backend::store::kv::KvStore; use crate::store::namespace_store::UnifiedMemory; use crate::store::safety::canonical_identifier; @@ -92,7 +92,9 @@ impl UnifiedMemory { } } -fn convert_records(records: Vec) -> Vec { +fn convert_records( + records: Vec, +) -> Vec { records .into_iter() .map(|record| MemoryKvRecord { diff --git a/core/src/store/namespace_store/segments.rs b/core/src/store/namespace_store/segments.rs index 66744b6..3af779b 100644 --- a/core/src/store/namespace_store/segments.rs +++ b/core/src/store/namespace_store/segments.rs @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS conversation_segments ( status TEXT NOT NULL DEFAULT 'open', created_at REAL NOT NULL, updated_at REAL NOT NULL, - -- Per-session sequence numbers from tinycortex::memory::archivist::store, populated + -- Per-session sequence numbers from crate::engine::backend::archivist::store, populated -- alongside start_episodic_id / end_episodic_id during the FTS5 -> md -- migration. Once STM recall switches its segment-span dedup to use -- (session_id, seq) the legacy episodic_id columns can be dropped. diff --git a/core/src/store/profile_store.rs b/core/src/store/profile_store.rs index 9441d9a..0eb734f 100644 --- a/core/src/store/profile_store.rs +++ b/core/src/store/profile_store.rs @@ -8,8 +8,7 @@ //! [`super::namespace_store::profile`], both inside `crate`; //! callers outside the family hold this handle and never a `Connection`. //! -//! **This is not a guard win.** The profile/facet tables have no capability -//! family in the `tinycortex_api` contract, so reads and writes through this +//! **This is not a guard win.** Reads and writes through this //! type still run beneath `crate::guard::MemoryGuard`'s //! seven policy steps: no tier check, no source-scope predicate, no taint //! stamping, no redaction, no budget, no audit event. What changed is the shape diff --git a/core/src/store/safety/mod.rs b/core/src/store/safety/mod.rs index bf48ad4..fe021a0 100644 --- a/core/src/store/safety/mod.rs +++ b/core/src/store/safety/mod.rs @@ -1,5 +1,5 @@ //! Secret-detection and redaction for memory writes — thin host shim over -//! `tinycortex::memory::store::safety` (W3). +//! `crate::engine::backend::store::safety` (W3). //! //! The conservative secret + PII scrubbers (`has_likely_secret`, //! `has_likely_pii`, `sanitize_text`, `sanitize_json`) + the @@ -15,7 +15,7 @@ pub mod pii; use crate::store::types::NamespaceDocumentInput; -pub use tinycortex::memory::store::safety::{ +pub use crate::engine::backend::store::safety::{ has_likely_pii, has_likely_secret, sanitize_json, sanitize_text, SanitizationReport, Sanitized, }; diff --git a/core/src/store/safety/pii.rs b/core/src/store/safety/pii.rs index 9a9a74b..8acff9f 100644 --- a/core/src/store/safety/pii.rs +++ b/core/src/store/safety/pii.rs @@ -1,8 +1,8 @@ //! Personal-PII detection — thin host re-export of the crate scrubber (W3). //! //! The full multilingual national-ID PII module (checksum-gated patterns + -//! Unicode normalization) now lives in `tinycortex::memory::store::safety::pii`; +//! Unicode normalization) now lives in `crate::engine::backend::store::safety::pii`; //! content scrubbing runs inside the crate `sanitize_text`. Host consumers keep //! their `safety::pii::has_likely_pii` import path. -pub use tinycortex::memory::store::safety::pii::{has_likely_pii, redact_pii}; +pub use crate::engine::backend::store::safety::pii::{has_likely_pii, redact_pii}; diff --git a/core/src/store/trees/hotness.rs b/core/src/store/trees/hotness.rs index 5fc3ab4..5ab594a 100644 --- a/core/src/store/trees/hotness.rs +++ b/core/src/store/trees/hotness.rs @@ -2,29 +2,29 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::store::trees::types::HotnessCounters; -use crate::tinycortex::engine_config; use crate::Config; pub fn get(config: &Config, entity_id: &str) -> Result> { - tinycortex::memory::tree::store::hotness::get(&engine_config(config), entity_id) + crate::engine::backend::tree::store::hotness::get(&engine_config(config), entity_id) } pub fn get_or_fresh(config: &Config, entity_id: &str) -> Result { - tinycortex::memory::tree::store::hotness::get_or_fresh(&engine_config(config), entity_id) + crate::engine::backend::tree::store::hotness::get_or_fresh(&engine_config(config), entity_id) } pub fn upsert(config: &Config, counters: &HotnessCounters) -> Result<()> { - tinycortex::memory::tree::store::hotness::upsert(&engine_config(config), counters) + crate::engine::backend::tree::store::hotness::upsert(&engine_config(config), counters) } pub fn distinct_sources_for(config: &Config, entity_id: &str) -> Result { - tinycortex::memory::tree::store::hotness::distinct_sources_for( + crate::engine::backend::tree::store::hotness::distinct_sources_for( &engine_config(config), entity_id, ) } pub fn count(config: &Config) -> Result { - tinycortex::memory::tree::store::hotness::count(&engine_config(config)) + crate::engine::backend::tree::store::hotness::count(&engine_config(config)) } diff --git a/core/src/store/trees/registry.rs b/core/src/store/trees/registry.rs index 3ce8e1b..32b596b 100644 --- a/core/src/store/trees/registry.rs +++ b/core/src/store/trees/registry.rs @@ -2,15 +2,15 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::store::trees::types::{Tree, TreeKind}; -use crate::tinycortex::engine_config; use crate::Config; pub fn list_trees_by_kind(config: &Config, kind: TreeKind) -> Result> { - tinycortex::memory::tree::store::list_trees_by_kind(&engine_config(config), kind) + crate::engine::backend::tree::store::list_trees_by_kind(&engine_config(config), kind) } pub fn archive_tree(config: &Config, tree_id: &str) -> Result<()> { log::debug!("[memory:trees] archive tree_id={tree_id}"); - tinycortex::memory::tree::store::archive_tree(&engine_config(config), tree_id) + crate::engine::backend::tree::store::archive_tree(&engine_config(config), tree_id) } diff --git a/core/src/store/trees/store.rs b/core/src/store/trees/store.rs index 2508757..0bd52e2 100644 --- a/core/src/store/trees/store.rs +++ b/core/src/store/trees/store.rs @@ -6,29 +6,29 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use rusqlite::{Connection, Transaction}; +use crate::engine::engine_config; use crate::store::content::StagedSummary; use crate::store::trees::types::{Buffer, SummaryNode, Tree, TreeKind}; -use crate::tinycortex::engine_config; use crate::Config; pub fn insert_tree(config: &Config, tree: &Tree) -> Result<()> { - tinycortex::memory::tree::store::insert_tree(&engine_config(config), tree) + crate::engine::backend::tree::store::insert_tree(&engine_config(config), tree) } pub fn get_tree_by_scope(config: &Config, kind: TreeKind, scope: &str) -> Result> { - tinycortex::memory::tree::store::get_tree_by_scope(&engine_config(config), kind, scope) + crate::engine::backend::tree::store::get_tree_by_scope(&engine_config(config), kind, scope) } pub fn get_tree(config: &Config, id: &str) -> Result> { - tinycortex::memory::tree::store::get_tree(&engine_config(config), id) + crate::engine::backend::tree::store::get_tree(&engine_config(config), id) } pub fn get_trees_batch(config: &Config, ids: &[String]) -> Result> { - tinycortex::memory::tree::store::get_trees_batch(&engine_config(config), ids) + crate::engine::backend::tree::store::get_trees_batch(&engine_config(config), ids) } pub fn list_trees_by_kind(config: &Config, kind: TreeKind) -> Result> { - tinycortex::memory::tree::store::list_trees_by_kind(&engine_config(config), kind) + crate::engine::backend::tree::store::list_trees_by_kind(&engine_config(config), kind) } pub fn update_tree_after_seal_tx( @@ -38,7 +38,7 @@ pub fn update_tree_after_seal_tx( max_level: u32, sealed_at: DateTime, ) -> Result<()> { - tinycortex::memory::tree::store::update_tree_after_seal_tx( + crate::engine::backend::tree::store::update_tree_after_seal_tx( tx, tree_id, root_id, max_level, sealed_at, ) } @@ -49,7 +49,7 @@ pub fn insert_summary_tx( staged: Option<&StagedSummary>, model_signature: &str, ) -> Result<()> { - tinycortex::memory::tree::store::insert_staged_summary_tx(tx, node, staged, model_signature) + crate::engine::backend::tree::store::insert_staged_summary_tx(tx, node, staged, model_signature) } pub fn set_summary_embedding( @@ -57,7 +57,7 @@ pub fn set_summary_embedding( summary_id: &str, embedding: &[f32], ) -> Result { - tinycortex::memory::tree::store::set_summary_embedding( + crate::engine::backend::tree::store::set_summary_embedding( &engine_config(config), summary_id, embedding, @@ -66,7 +66,7 @@ pub fn set_summary_embedding( } pub fn get_summary_embedding(config: &Config, summary_id: &str) -> Result>> { - tinycortex::memory::tree::store::get_summary_embedding(&engine_config(config), summary_id) + crate::engine::backend::tree::store::get_summary_embedding(&engine_config(config), summary_id) } pub fn set_summary_embedding_for_signature( @@ -75,7 +75,7 @@ pub fn set_summary_embedding_for_signature( signature: &str, embedding: &[f32], ) -> Result<()> { - tinycortex::memory::tree::store::set_summary_embedding_for_signature( + crate::engine::backend::tree::store::set_summary_embedding_for_signature( &engine_config(config), summary_id, signature, @@ -89,7 +89,7 @@ pub fn mark_summary_reembed_skipped( signature: &str, reason: &str, ) -> Result<()> { - tinycortex::memory::chunks::mark_summary_reembed_skipped( + crate::engine::backend::chunks::mark_summary_reembed_skipped( &engine_config(config), summary_id, signature, @@ -102,7 +102,7 @@ pub fn clear_summary_reembed_skipped( summary_id: &str, signature: &str, ) -> Result<()> { - tinycortex::memory::chunks::clear_summary_reembed_skipped( + crate::engine::backend::chunks::clear_summary_reembed_skipped( &engine_config(config), summary_id, signature, @@ -115,7 +115,7 @@ pub(crate) fn set_summary_embedding_for_signature_tx( signature: &str, embedding: &[f32], ) -> Result<()> { - tinycortex::memory::chunks::set_summary_embedding_for_signature_tx( + crate::engine::backend::chunks::set_summary_embedding_for_signature_tx( tx, summary_id, signature, embedding, ) } @@ -125,7 +125,7 @@ pub fn get_summary_embedding_for_signature( summary_id: &str, signature: &str, ) -> Result>> { - tinycortex::memory::tree::store::get_summary_embedding_for_signature( + crate::engine::backend::tree::store::get_summary_embedding_for_signature( &engine_config(config), summary_id, signature, @@ -137,7 +137,7 @@ pub fn get_summary_embeddings_for_signature_batch( ids: &[String], signature: &str, ) -> Result>> { - tinycortex::memory::tree::store::get_summary_embeddings_for_signature_batch( + crate::engine::backend::tree::store::get_summary_embeddings_for_signature_batch( &engine_config(config), ids, signature, @@ -148,18 +148,18 @@ pub fn get_summary_embeddings_batch( config: &Config, ids: &[String], ) -> Result>> { - tinycortex::memory::tree::store::get_summary_embeddings_batch(&engine_config(config), ids) + crate::engine::backend::tree::store::get_summary_embeddings_batch(&engine_config(config), ids) } pub fn get_summary(config: &Config, id: &str) -> Result> { - tinycortex::memory::tree::store::get_summary(&engine_config(config), id) + crate::engine::backend::tree::store::get_summary(&engine_config(config), id) } pub fn get_summaries_batch( config: &Config, ids: &[String], ) -> Result> { - tinycortex::memory::tree::store::get_summaries_batch(&engine_config(config), ids) + crate::engine::backend::tree::store::get_summaries_batch(&engine_config(config), ids) } pub fn list_summaries_at_level( @@ -167,7 +167,11 @@ pub fn list_summaries_at_level( tree_id: &str, level: u32, ) -> Result> { - tinycortex::memory::tree::store::list_summaries_at_level(&engine_config(config), tree_id, level) + crate::engine::backend::tree::store::list_summaries_at_level( + &engine_config(config), + tree_id, + level, + ) } pub fn list_summaries_in_window( @@ -176,7 +180,7 @@ pub fn list_summaries_in_window( since_ms: i64, until_ms: i64, ) -> Result> { - tinycortex::memory::tree::store::list_summaries_in_window( + crate::engine::backend::tree::store::list_summaries_in_window( &engine_config(config), tree_id, since_ms, @@ -185,21 +189,21 @@ pub fn list_summaries_in_window( } pub fn count_summaries(config: &Config, tree_id: &str) -> Result { - tinycortex::memory::tree::store::count_summaries(&engine_config(config), tree_id) + crate::engine::backend::tree::store::count_summaries(&engine_config(config), tree_id) } pub fn get_buffer(config: &Config, tree_id: &str, level: u32) -> Result { - tinycortex::memory::tree::store::get_buffer(&engine_config(config), tree_id, level) + crate::engine::backend::tree::store::get_buffer(&engine_config(config), tree_id, level) } pub(crate) fn get_buffer_conn(conn: &Connection, tree_id: &str, level: u32) -> Result { - tinycortex::memory::tree::store::get_buffer_conn(conn, tree_id, level) + crate::engine::backend::tree::store::get_buffer_conn(conn, tree_id, level) } pub fn upsert_buffer_tx(tx: &Transaction<'_>, buffer: &Buffer) -> Result<()> { - tinycortex::memory::tree::store::upsert_buffer_tx(tx, buffer) + crate::engine::backend::tree::store::upsert_buffer_tx(tx, buffer) } pub fn list_stale_buffers(config: &Config, older_than: DateTime) -> Result> { - tinycortex::memory::tree::store::list_stale_buffers(&engine_config(config), older_than) + crate::engine::backend::tree::store::list_stale_buffers(&engine_config(config), older_than) } diff --git a/core/src/store/trees/store_tests.rs b/core/src/store/trees/store_tests.rs index d470951..741fe4d 100644 --- a/core/src/store/trees/store_tests.rs +++ b/core/src/store/trees/store_tests.rs @@ -297,7 +297,7 @@ fn buffer_upsert_and_clear() { with_connection(&cfg, |conn| { let tx = conn.unchecked_transaction()?; - tinycortex::memory::tree::store::clear_buffer_tx(&tx, "tree-1", 0)?; + crate::engine::backend::tree::store::clear_buffer_tx(&tx, "tree-1", 0)?; tx.commit()?; Ok(()) }) diff --git a/core/src/store/trees/types.rs b/core/src/store/trees/types.rs index 114fb97..b0cf40f 100644 --- a/core/src/store/trees/types.rs +++ b/core/src/store/trees/types.rs @@ -1,6 +1,6 @@ //! Compatibility exports for tinycortex summary-tree persistence types. -pub use tinycortex::memory::tree::store::{ +pub use crate::engine::backend::tree::store::{ Buffer, EntityIndexStats, HotnessCounters, SummaryNode, Tree, TreeKind, TreeStatus, DEFAULT_FLUSH_AGE_SECS, INPUT_TOKEN_BUDGET, OUTPUT_TOKEN_BUDGET, SUMMARY_FANOUT, TOPIC_ARCHIVE_THRESHOLD, TOPIC_CREATION_THRESHOLD, TOPIC_RECHECK_EVERY, diff --git a/core/src/store/types.rs b/core/src/store/types.rs index d72101e..4cc1ead 100644 --- a/core/src/store/types.rs +++ b/core/src/store/types.rs @@ -1,9 +1,9 @@ //! Stable host path for tinycortex-owned namespace memory contracts. -pub use tinycortex::memory::{ +pub use crate::engine::backend::{ GraphRelationRecord, MemoryItemKind, MemoryKvRecord, NamespaceDocumentInput, NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown, StoredMemoryDocument, }; -pub(crate) use tinycortex::memory::types::GLOBAL_NAMESPACE; +pub(crate) use crate::engine::backend::types::GLOBAL_NAMESPACE; diff --git a/core/src/sync/composio/mod.rs b/core/src/sync/composio/mod.rs index 37e2721..b3df5eb 100644 --- a/core/src/sync/composio/mod.rs +++ b/core/src/sync/composio/mod.rs @@ -161,12 +161,8 @@ pub async fn run_connection_sync( .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_millis() as u64; - match crate::tinycortex::run_composio_connection( - &target.toolkit, - &target.connection_id, - &*config, - ) - .await + match crate::engine::run_composio_connection(&target.toolkit, &target.connection_id, &*config) + .await { Ok(outcome) => { let usage = ComposioUsage { diff --git a/core/src/sync/composio/periodic.rs b/core/src/sync/composio/periodic.rs index 22cfa71..400e43b 100644 --- a/core/src/sync/composio/periodic.rs +++ b/core/src/sync/composio/periodic.rs @@ -58,7 +58,7 @@ use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; use super::providers::{get_provider, ComposioUsage}; use crate::composio_host; -use crate::tinycortex::{append_audit_entry, try_read_audit_log, SyncAuditEntry}; +use crate::engine::{append_audit_entry, try_read_audit_log, SyncAuditEntry}; use chrono::{DateTime, Utc}; /// How often the scheduler wakes up to look for due syncs. Independent @@ -555,7 +555,7 @@ pub(crate) async fn run_one_tick() -> Result<(), String> { "[composio:periodic] firing sync" ); let sync_started = Instant::now(); - let result = crate::tinycortex::run_source_pipeline(&source, &*config).await; + let result = crate::engine::run_source_pipeline(&source, &*config).await; let duration_ms = sync_started.elapsed().as_millis() as u64; match result { diff --git a/core/src/sync/composio/providers/clickup/mod.rs b/core/src/sync/composio/providers/clickup/mod.rs index bbf3908..f38484a 100644 --- a/core/src/sync/composio/providers/clickup/mod.rs +++ b/core/src/sync/composio/providers/clickup/mod.rs @@ -6,7 +6,8 @@ //! re-learning a new shape: //! //! - `provider.rs` — `impl ComposioProvider for ClickUpProvider` -//! - `normalization` — payload-shape helpers, now `tinycortex::…::normalize::clickup` +//! - `normalization` — payload-shape helpers, now reached through +//! `crate::engine::engine` (issue #18 §C1) //! - `ingest.rs` — memory_tree document ingest (issue #2885) //! - `tools.rs` — `CLICKUP_CURATED` whitelist of Composio actions //! - `tests.rs` — unit tests for the helpers + trait metadata @@ -16,7 +17,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use tinycortex::memory::sync::composio::providers::normalize::clickup as normalization; +use tinymemory_sync::clickup as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/github/mod.rs b/core/src/sync/composio/providers/github/mod.rs index 691b6e2..90633f3 100644 --- a/core/src/sync/composio/providers/github/mod.rs +++ b/core/src/sync/composio/providers/github/mod.rs @@ -6,7 +6,8 @@ //! re-learning a new shape: //! //! - `provider.rs` — `impl ComposioProvider for GitHubProvider` -//! - `normalization` — payload-shape helpers, now `tinycortex::…::normalize::github` +//! - `normalization` — payload-shape helpers, now reached through +//! `crate::engine::engine` (issue #18 §C1) //! - `tools.rs` — `GITHUB_CURATED` whitelist of Composio actions //! - `tests.rs` — unit tests for the helpers + trait metadata //! @@ -15,7 +16,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use tinycortex::memory::sync::composio::providers::normalize::github as normalization; +use tinymemory_sync::github as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/gmail/mod.rs b/core/src/sync/composio/providers/gmail/mod.rs index fcdc23e..df067ec 100644 --- a/core/src/sync/composio/providers/gmail/mod.rs +++ b/core/src/sync/composio/providers/gmail/mod.rs @@ -1,7 +1,7 @@ // The Gmail post-processor moved to tinycortex (a pure Value transform, i.e. // driver-side). Aliased under the old module name so the single call site in // `provider.rs` stays unchanged. -use tinycortex::memory::sync::composio::providers::normalize::gmail_post_process as post_process; +use tinymemory_sync::gmail_post_process as post_process; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/gmail/provider.rs b/core/src/sync/composio/providers/gmail/provider.rs index 07b7f0c..a46617e 100644 --- a/core/src/sync/composio/providers/gmail/provider.rs +++ b/core/src/sync/composio/providers/gmail/provider.rs @@ -152,12 +152,9 @@ impl ComposioProvider for GmailProvider { let Some(connection_id) = ctx.connection_id.as_deref() else { return Err("[composio:gmail] trigger missing connection_id".to_string()); }; - if let Err(e) = crate::tinycortex::run_composio_connection( - "gmail", - connection_id, - ctx.config.as_ref(), - ) - .await + if let Err(e) = + crate::engine::run_composio_connection("gmail", connection_id, ctx.config.as_ref()) + .await { tracing::warn!( error = %e, @@ -171,5 +168,5 @@ impl ComposioProvider for GmailProvider { // Message fetching (the `GMAIL_FETCH_EMAILS` action, the search query, the // `max_items` cap math and the `sync_depth_days` `after:` floor) is owned -// by `tinycortex::memory::sync::GmailSyncPipeline`. What stays here is the +// by `crate::engine::backend::sync::GmailSyncPipeline`. What stays here is the // host-side provider surface: profile lookup and trigger dispatch. diff --git a/core/src/sync/composio/providers/gmail/tests.rs b/core/src/sync/composio/providers/gmail/tests.rs index 70b5e6b..17a1825 100644 --- a/core/src/sync/composio/providers/gmail/tests.rs +++ b/core/src/sync/composio/providers/gmail/tests.rs @@ -1,7 +1,7 @@ //! Host-owned Gmail provider surface tests. //! //! Pagination, cursor, envelope parsing, and ingest behavior are owned and -//! tested by `tinycortex::memory::sync::GmailSyncPipeline`. +//! tested by `crate::engine::backend::sync::GmailSyncPipeline`. use super::GmailProvider; use crate::sync::composio::providers::ComposioProvider; diff --git a/core/src/sync/composio/providers/helpers.rs b/core/src/sync/composio/providers/helpers.rs index 6e1e2ba..e0358e2 100644 --- a/core/src/sync/composio/providers/helpers.rs +++ b/core/src/sync/composio/providers/helpers.rs @@ -1,11 +1,11 @@ //! Shared helpers for Composio provider implementations. //! //! `pick_str` used to live here. It is a provider payload normaliser, so it -//! moved to `tinycortex::memory::sync::composio::providers::normalize::helpers` +//! moved to `tinymemory_sync::helpers` //! and is re-exported from this module's parent. The helpers that remain are //! request-building rather than normalisation, and stay host-side. -use tinycortex::memory::sync::composio::providers::normalize::helpers::pick_str; +use tinymemory_sync::helpers::pick_str; /// Shallow-merge an `extra` JSON object into a (mutable) action-args /// object. Only object-typed extras are merged; non-object `extra` diff --git a/core/src/sync/composio/providers/linear/mod.rs b/core/src/sync/composio/providers/linear/mod.rs index 27ff7b7..7ccce87 100644 --- a/core/src/sync/composio/providers/linear/mod.rs +++ b/core/src/sync/composio/providers/linear/mod.rs @@ -6,7 +6,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use tinycortex::memory::sync::composio::providers::normalize::linear as normalization; +use tinymemory_sync::linear as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/mod.rs b/core/src/sync/composio/providers/mod.rs index 0a86355..9637d30 100644 --- a/core/src/sync/composio/providers/mod.rs +++ b/core/src/sync/composio/providers/mod.rs @@ -285,7 +285,7 @@ pub use registry::{ all_providers, get_provider, init_default_providers, register_provider, ProviderArc, }; pub use scope_lookup::{curated_scope_for, toolkit_has_scope}; -pub(crate) use tinycortex::memory::sync::composio::providers::normalize::helpers::pick_str; +pub(crate) use tinymemory_sync::helpers::pick_str; pub use tool_scope::{classify_unknown, find_curated, toolkit_from_slug, CuratedTool, ToolScope}; pub use traits::{resolve_sync_interval_secs, sync_interval_env_var, ComposioProvider}; pub use types::{ diff --git a/core/src/sync/composio/providers/notion/mod.rs b/core/src/sync/composio/providers/notion/mod.rs index 5613ed0..138fe60 100644 --- a/core/src/sync/composio/providers/notion/mod.rs +++ b/core/src/sync/composio/providers/notion/mod.rs @@ -1,7 +1,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use tinycortex::memory::sync::composio::providers::normalize::notion as normalization; +use tinymemory_sync::notion as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/notion/provider.rs b/core/src/sync/composio/providers/notion/provider.rs index 63990fd..d02cc3e 100644 --- a/core/src/sync/composio/providers/notion/provider.rs +++ b/core/src/sync/composio/providers/notion/provider.rs @@ -279,7 +279,7 @@ impl ComposioProvider for NotionProvider { return Err("[composio:notion] trigger missing connection_id".to_string()); }; if let Err(e) = - crate::tinycortex::run_composio_connection("notion", connection_id, ctx.config.as_ref()) + crate::engine::run_composio_connection("notion", connection_id, ctx.config.as_ref()) .await { tracing::warn!( diff --git a/core/src/sync/composio/providers/slack/mod.rs b/core/src/sync/composio/providers/slack/mod.rs index 034492e..39b745a 100644 --- a/core/src/sync/composio/providers/slack/mod.rs +++ b/core/src/sync/composio/providers/slack/mod.rs @@ -10,7 +10,7 @@ // driver-side). Re-exported under the old module name — `pub`, not a plain // `use`, because `tests/raw_coverage/memory_threads_raw_coverage_e2e.rs` // imports this path directly. -pub use tinycortex::memory::sync::composio::providers::normalize::slack_post_process as post_process; +pub use tinymemory_sync::slack_post_process as post_process; pub mod types; mod provider; diff --git a/core/src/sync/composio/providers/slack/provider.rs b/core/src/sync/composio/providers/slack/provider.rs index 5d22208..3ed5270 100644 --- a/core/src/sync/composio/providers/slack/provider.rs +++ b/core/src/sync/composio/providers/slack/provider.rs @@ -233,12 +233,9 @@ impl ComposioProvider for SlackProvider { let Some(connection_id) = ctx.connection_id.as_deref() else { return Err("[composio:slack] trigger missing connection_id".to_string()); }; - if let Err(e) = crate::tinycortex::run_composio_connection( - "slack", - connection_id, - ctx.config.as_ref(), - ) - .await + if let Err(e) = + crate::engine::run_composio_connection("slack", connection_id, ctx.config.as_ref()) + .await { tracing::warn!( error = %e, @@ -262,13 +259,10 @@ pub async fn run_backfill_via_search( .as_deref() .ok_or_else(|| "[composio:slack] search backfill missing connection_id".to_string())?; let started_at_ms = now_ms(); - let outcome = crate::tinycortex::run_slack_search_backfill( - connection_id, - backfill_days, - ctx.config.as_ref(), - ) - .await - .map_err(|error| error.to_string())?; + let outcome = + crate::engine::run_slack_search_backfill(connection_id, backfill_days, ctx.config.as_ref()) + .await + .map_err(|error| error.to_string())?; Ok(SyncOutcome { toolkit: "slack".into(), connection_id: Some(connection_id.into()), diff --git a/core/src/sync/composio/providers/sync_state.rs b/core/src/sync/composio/providers/sync_state.rs index da33c44..4e33f91 100644 --- a/core/src/sync/composio/providers/sync_state.rs +++ b/core/src/sync/composio/providers/sync_state.rs @@ -1,9 +1,9 @@ //! Compatibility exports for sync state now owned by tinycortex. -pub use tinycortex::memory::sync::state::DEFAULT_DAILY_REQUEST_LIMIT; -pub use tinycortex::memory::sync::{DailyBudget, SyncState}; +pub use crate::engine::backend::sync::state::DEFAULT_DAILY_REQUEST_LIMIT; +pub use crate::engine::backend::sync::{DailyBudget, SyncState}; -pub const KV_NAMESPACE: &str = crate::tinycortex::HOST_SYNC_STATE_NAMESPACE; +pub const KV_NAMESPACE: &str = crate::engine::HOST_SYNC_STATE_NAMESPACE; pub fn extract_item_id(item: &serde_json::Value, paths: &[&str]) -> Option { paths.iter().find_map(|path| { diff --git a/core/src/sync/composio/providers/traits.rs b/core/src/sync/composio/providers/traits.rs index 8585eb2..b9a0e30 100644 --- a/core/src/sync/composio/providers/traits.rs +++ b/core/src/sync/composio/providers/traits.rs @@ -59,7 +59,7 @@ pub trait ComposioProvider: Send + Sync { ) })?; let started_at_ms = now_ms(); - let outcome = crate::tinycortex::run_composio_connection_with_budgets( + let outcome = crate::engine::run_composio_connection_with_budgets( self.toolkit_slug(), connection_id, ctx.config.as_ref(), diff --git a/core/src/sync/sync_status/mod.rs b/core/src/sync/sync_status/mod.rs index 019bbd6..11c5b83 100644 --- a/core/src/sync/sync_status/mod.rs +++ b/core/src/sync/sync_status/mod.rs @@ -13,4 +13,4 @@ //! * `openhuman.memory_sync_status_list` — handler in `rpc` //! * Controller registration via `schemas::all_registered_controllers` -pub use tinycortex::memory::sync::{FreshnessLabel, MemorySyncStatus}; +pub use crate::engine::backend::sync::{FreshnessLabel, MemorySyncStatus}; diff --git a/core/src/sync/workspace/periodic.rs b/core/src/sync/workspace/periodic.rs index a1d145e..dbc3a0b 100644 --- a/core/src/sync/workspace/periodic.rs +++ b/core/src/sync/workspace/periodic.rs @@ -33,13 +33,13 @@ use chrono::{DateTime, Utc}; use tokio::time::interval; use crate::config_loader as config_rpc; +use crate::engine::{try_read_audit_log, SyncAuditEntry}; use crate::scheduler_gate::resume_notify; use crate::sources::sync::sync_source; use crate::sources::types::{MemorySourceEntry, SourceKind}; use crate::sync::composio::periodic::{ connection_is_due, effective_interval_secs, periodic_pause_reason, }; -use crate::tinycortex::{try_read_audit_log, SyncAuditEntry}; use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; /// How often the scheduler wakes up to look for due syncs. Matches the diff --git a/core/src/sync/workspace/watcher.rs b/core/src/sync/workspace/watcher.rs index 27bdaf8..c43e529 100644 --- a/core/src/sync/workspace/watcher.rs +++ b/core/src/sync/workspace/watcher.rs @@ -56,7 +56,7 @@ use tokio::sync::mpsc; use crate::Config; use crate::config_loader as config_rpc; use crate::ingest_pipeline::ingest_document_with_scope; -use tinycortex::memory::ingest::canonicalize::document::DocumentInput; +use crate::engine::backend::ingest::canonicalize::document::DocumentInput; use crate::sync::workspace::watcher::state::WatcherStateStore; use crate::scheduler_gate::current_policy; use crate::scheduler_gate::PauseReason; diff --git a/core/src/tool_memory/mod.rs b/core/src/tool_memory/mod.rs index 0bbf8fa..6314279 100644 --- a/core/src/tool_memory/mod.rs +++ b/core/src/tool_memory/mod.rs @@ -18,9 +18,9 @@ //! //! ## Components //! -//! - [`tinycortex::memory::tool_memory::types`] owns [`ToolMemoryRule`], +//! - [`crate::engine::backend::tool_memory::types`] owns [`ToolMemoryRule`], //! [`ToolMemoryPriority`], and [`ToolMemorySource`]. -//! - [`tinycortex::memory::tool_memory::store`] owns [`ToolMemoryStore`], the +//! - [`crate::engine::backend::tool_memory::store`] owns [`ToolMemoryStore`], the //! put/list/delete/prompt API built on top of an `Arc`. //! - `capture` — `ToolMemoryCaptureHook`, the post-turn //! `PostTurnHook` that records user edicts and repeated tool @@ -37,8 +37,8 @@ mod store; #[cfg(any(test, feature = "test-support"))] pub mod test_helpers; -pub use store::tool_memory_store; -pub use tinycortex::memory::tool_memory::{ +pub use crate::engine::backend::tool_memory::{ store::{ToolMemoryStore, TOOL_MEMORY_PROMPT_CAP}, types::{tool_memory_namespace, ToolMemoryPriority, ToolMemoryRule, ToolMemorySource}, }; +pub use store::tool_memory_store; diff --git a/core/src/tool_memory/store.rs b/core/src/tool_memory/store.rs index 3446fba..857a497 100644 --- a/core/src/tool_memory/store.rs +++ b/core/src/tool_memory/store.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use crate::Memory; -use tinycortex::memory::tool_memory::store::ToolMemoryStore; +use crate::engine::backend::tool_memory::store::ToolMemoryStore; /// Build the crate-owned store over OpenHuman's shared memory object. pub fn tool_memory_store(memory: Arc) -> ToolMemoryStore { diff --git a/core/src/traits.rs b/core/src/traits.rs index 022bfb0..d57f618 100644 --- a/core/src/traits.rs +++ b/core/src/traits.rs @@ -1,11 +1,16 @@ //! Core traits and data structures for the OpenHuman memory system. //! //! This module defines the foundational `Memory` trait that all storage backends -//! must implement. The standard memory value types (`MemoryEntry`, +//! must implement. The trait and the standard memory value types (`MemoryEntry`, //! `MemoryCategory`, `MemoryTaint`, `RecallOpts`, `NamespaceSummary`) are -//! **re-exported from the `tinycortex` crate** (migration W2, spec §0.5): the -//! crate is the single source of truth for these wire-compatible types, and the -//! 30+ host consumers keep their `memory::traits::…` import paths unchanged. +//! **re-exported from `tinymemory-api`**, the engine-neutral contract. +//! +//! They used to come from the `tinycortex` crate — from the *engine*, in other +//! words, which meant `tinymemory_core::MemoryEntry` was the engine's type and +//! not the contract's, and a second engine could not have been bound without +//! translating (issue #18 §A1/§A2). Naming the contract directly is what makes +//! this crate engine-neutral at the type level; the 30+ host consumers keep +//! their `memory::traits::…` import paths unchanged either way. //! //! `MemoryTaint` is security-critical provenance — it fails closed to //! `ExternalSync` for unknown/corrupt values so the subconscious gate refuses @@ -13,19 +18,19 @@ //! byte-identical to the former host definition before re-exporting; the tests //! below are the host-side seam that pins that contract on the crate type. //! -//! The `Memory` trait is also re-exported from `tinycortex`; backend-specific -//! resources such as SQLite connections are carried explicitly by factories -//! instead of being exposed through the storage abstraction. +//! Backend-specific resources such as SQLite connections are carried explicitly +//! by factories instead of being exposed through the storage abstraction. -// ── Value types: re-exported from the crate (W2 type-unification, spec §0.5) ── +// ── The contract's trait and value types ───────────────────────────────────── // -// These were formerly defined here. They are now the crate's types verbatim -// (identical fields, derives, serde attrs, and — for `MemoryTaint` — the same -// fail-closed `from_db_str`). Re-exporting keeps one source of truth while every -// `use crate::traits::{MemoryEntry, …}` site compiles unchanged. -pub use tinycortex::memory::{ - Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts, -}; +// Named directly rather than reached through the engine's re-export. Since §A1 the +// engine re-exports this same contract, so the two spellings resolve to one type +// either way — but going through the engine to reach an engine-neutral contract +// is what §1.1 of issue #18 calls out, and it is what would have to be undone +// before a second engine could be bound. +pub use tinymemory_api::recall::RecallOpts; +pub use tinymemory_api::traits::Memory; +pub use tinymemory_api::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary}; #[cfg(test)] mod tests { diff --git a/core/src/tree/graph/bfs.rs b/core/src/tree/graph/bfs.rs index 444d068..51c379e 100644 --- a/core/src/tree/graph/bfs.rs +++ b/core/src/tree/graph/bfs.rs @@ -4,15 +4,15 @@ use anyhow::Result; use crate::Config; -pub use tinycortex::memory::graph::PairDistance; +pub use crate::engine::backend::graph::PairDistance; pub fn pair_distances( config: &Config, entity_ids: &[String], max_h: u32, ) -> Result> { - tinycortex::memory::graph::pair_distances( - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + crate::engine::backend::graph::pair_distances( + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), entity_ids, max_h, ) diff --git a/core/src/tree/graph/store.rs b/core/src/tree/graph/store.rs index 2408300..69094ed 100644 --- a/core/src/tree/graph/store.rs +++ b/core/src/tree/graph/store.rs @@ -3,17 +3,17 @@ use anyhow::Result; use rusqlite::Transaction; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; -pub use tinycortex::memory::graph::pairs_from_entities; +pub use crate::engine::backend::graph::pairs_from_entities; pub fn upsert_edges_tx( transaction: &Transaction<'_>, pairs: &[(String, String)], timestamp_ms: i64, ) -> Result { - tinycortex::memory::graph::upsert_edges_tx(transaction, pairs, timestamp_ms) + crate::engine::backend::graph::upsert_edges_tx(transaction, pairs, timestamp_ms) } pub fn upsert_edges( @@ -21,20 +21,20 @@ pub fn upsert_edges( pairs: &[(String, String)], timestamp_ms: i64, ) -> Result { - tinycortex::memory::graph::upsert_edges(&engine_config(config), pairs, timestamp_ms) + crate::engine::backend::graph::upsert_edges(&engine_config(config), pairs, timestamp_ms) } pub fn neighbors(config: &Config, entity_id: &str) -> Result> { - tinycortex::memory::graph::edge_neighbors(&engine_config(config), entity_id) + crate::engine::backend::graph::edge_neighbors(&engine_config(config), entity_id) } pub fn clear_edges_for_entities_tx( transaction: &Transaction<'_>, entity_ids: &[String], ) -> Result { - tinycortex::memory::graph::clear_edges_for_entities_tx(transaction, entity_ids) + crate::engine::backend::graph::clear_edges_for_entities_tx(transaction, entity_ids) } pub fn count_edges(config: &Config) -> Result { - tinycortex::memory::graph::count_edges(&engine_config(config)) + crate::engine::backend::graph::count_edges(&engine_config(config)) } diff --git a/core/src/tree/health/mod.rs b/core/src/tree/health/mod.rs index 87bbbc8..2bfb1a8 100644 --- a/core/src/tree/health/mod.rs +++ b/core/src/tree/health/mod.rs @@ -3,7 +3,7 @@ //! The **taxonomy itself** — [`FailureCode`], [`FailureClass`], //! [`PipelineFailure`], [`DegradedState`], and the `classify_embed_error` //! classifier — now lives in the engine crate at -//! `tinycortex::memory::health`, and is re-exported below so every existing +//! `crate::engine::backend::health`, and is re-exported below so every existing //! `memory::tree::health::…` path keeps resolving. It moved because a build //! whose only driver was a third-party external backend would have no use for //! the engine's private failure vocabulary. @@ -27,7 +27,7 @@ pub(crate) use user_error::publish_local_model_unavailable_user_error; /// The failure taxonomy proper. Re-exported (rather than re-declared) so the /// ~30 `crate::tree::health::{…}` call sites across the host /// are unaffected by the move, and so there is exactly one definition. -pub use tinycortex::memory::health::{ +pub use crate::engine::backend::health::{ classify_embed_error, classify_embed_error_str, DegradedState, FailureClass, FailureCode, PipelineFailure, }; diff --git a/core/src/tree/ingest.rs b/core/src/tree/ingest.rs index 805c60e..edb2067 100644 --- a/core/src/tree/ingest.rs +++ b/core/src/tree/ingest.rs @@ -4,13 +4,13 @@ use anyhow::Context; use anyhow::Result; +use crate::engine::{memory_config_from, HostSummariser}; #[cfg(feature = "memory-git")] use crate::store::content::wiki_git::{SummaryCommitBatch, SummaryCommitEntry}; use crate::store::trees::types::Tree; -use crate::tinycortex::{memory_config_from, HostSummariser}; use crate::Config; -pub use tinycortex::memory::tree::{SummaryIngestInput, SummaryIngestOutcome}; +pub use crate::engine::backend::tree::{SummaryIngestInput, SummaryIngestOutcome}; pub async fn ingest_summary( config: &Config, @@ -27,7 +27,7 @@ pub async fn ingest_summary( log::warn!("[memory_tree::ingest] obsidian defaults failed: {error:#}"); } - let outcome = tinycortex::memory::tree::ingest_summary( + let outcome = crate::engine::backend::tree::ingest_summary( &memory_config_from(config, config.workspace_dir().clone()), tree, input.clone(), diff --git a/core/src/tree/mod.rs b/core/src/tree/mod.rs index 61e0ab4..3e26f8a 100644 --- a/core/src/tree/mod.rs +++ b/core/src/tree/mod.rs @@ -21,7 +21,7 @@ pub mod tree; pub mod tree_runtime; // Tree I/O contracts are engine-owned. -pub use tinycortex::memory::tree::{ +pub use crate::engine::backend::tree::{ TreeLabelStrategy, TreeLeafPayload, TreeReadHit, TreeReadRequest, TreeReadResult, TreeWriteOutcome, TreeWriteRequest, }; diff --git a/core/src/tree/retrieval/benchmarks.rs b/core/src/tree/retrieval/benchmarks.rs index f59338d..35ee2a9 100644 --- a/core/src/tree/retrieval/benchmarks.rs +++ b/core/src/tree/retrieval/benchmarks.rs @@ -25,12 +25,12 @@ use tempfile::TempDir; #[cfg(test)] use tinymemory_api::host::test_support::TestHostConfig; +use crate::engine::backend::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; use crate::ingest_pipeline::ingest_chat; use crate::queue::testing::drain_until_idle; use crate::store::chunks::types::SourceKind; use crate::tree::retrieval::{fetch_leaves, query_source, search_entities}; use crate::Config; -use tinycortex::memory::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; /// Shared test config — disables embedding for deterministic inert behaviour. fn bench_config() -> (TempDir, TestHostConfig) { diff --git a/core/src/tree/retrieval/cover.rs b/core/src/tree/retrieval/cover.rs index d5b7b09..2f196b8 100644 --- a/core/src/tree/retrieval/cover.rs +++ b/core/src/tree/retrieval/cover.rs @@ -1,8 +1,8 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::source_scope::current_source_scope; use crate::store::chunks::types::SourceKind; -use crate::tinycortex::engine_config; use crate::tree::retrieval::types::QueryResponse; use crate::Config; @@ -62,7 +62,7 @@ pub async fn cover_window_scoped( source_kind.map(|k| k.as_str()), limit ); - let mut response = tinycortex::memory::retrieval::cover_window_scoped( + let mut response = crate::engine::backend::retrieval::cover_window_scoped( &engine_config(config), since_ms, until_ms, diff --git a/core/src/tree/retrieval/drill_down.rs b/core/src/tree/retrieval/drill_down.rs index d74f428..7e58af9 100644 --- a/core/src/tree/retrieval/drill_down.rs +++ b/core/src/tree/retrieval/drill_down.rs @@ -1,7 +1,7 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::source_scope::current_source_scope; -use crate::tinycortex::engine_config; use crate::tree::retrieval::engine::EmbedderBridge; use crate::tree::retrieval::types::RetrievalHit; use crate::tree::score::embed::{build_embedder_from_config, InertEmbedder}; @@ -60,7 +60,7 @@ pub async fn drill_down_scoped( // limiting before the retain below would cap the result set with rows that // are about to be discarded. let engine_limit = scope.as_ref().map(|_| None).unwrap_or(limit); - let mut hits = tinycortex::memory::retrieval::drill_down( + let mut hits = crate::engine::backend::retrieval::drill_down( &engine_config(config), node_id, max_depth, diff --git a/core/src/tree/retrieval/engine.rs b/core/src/tree/retrieval/engine.rs index a23bcfe..5c541c5 100644 --- a/core/src/tree/retrieval/engine.rs +++ b/core/src/tree/retrieval/engine.rs @@ -6,7 +6,7 @@ use crate::tree::score::embed::Embedder as HostEmbedder; pub(super) struct EmbedderBridge<'a>(pub &'a dyn HostEmbedder); #[async_trait] -impl tinycortex::memory::score::embed::Embedder for EmbedderBridge<'_> { +impl crate::engine::backend::score::embed::Embedder for EmbedderBridge<'_> { fn name(&self) -> &'static str { self.0.name() } diff --git a/core/src/tree/retrieval/fast.rs b/core/src/tree/retrieval/fast.rs index 926e8a6..f8df280 100644 --- a/core/src/tree/retrieval/fast.rs +++ b/core/src/tree/retrieval/fast.rs @@ -2,15 +2,15 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::source_scope::current_source_scope; -use crate::tinycortex::engine_config; use crate::tree::nlp; use crate::tree::retrieval::engine::EmbedderBridge; use crate::tree::retrieval::types::QueryResponse; use crate::tree::score::embed::build_embedder_from_config; use crate::Config; -pub use tinycortex::memory::retrieval::FastRetrieveOptions; +pub use crate::engine::backend::retrieval::FastRetrieveOptions; /// Deterministic graph-walk retrieval using the **ambient** source scope. /// @@ -49,7 +49,7 @@ pub async fn fast_retrieve_scoped( options.max_hops ); let embedder = build_embedder_from_config(config)?; - tinycortex::memory::retrieval::fast_retrieve( + crate::engine::backend::retrieval::fast_retrieve( &engine_config(config), query, &entity_ids, diff --git a/core/src/tree/retrieval/fetch.rs b/core/src/tree/retrieval/fetch.rs index d45c4dd..5fc2502 100644 --- a/core/src/tree/retrieval/fetch.rs +++ b/core/src/tree/retrieval/fetch.rs @@ -1,13 +1,13 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::source_scope::chunk_source_allowed_in; use crate::source_scope::current_source_scope; use crate::store::chunks::store::get_chunks_batch; -use crate::tinycortex::engine_config; use crate::tree::retrieval::types::RetrievalHit; use crate::Config; -pub use tinycortex::memory::retrieval::MAX_BATCH; +pub use crate::engine::backend::retrieval::MAX_BATCH; /// Fetch leaf chunks by id, using the **ambient** scope. /// @@ -47,5 +47,5 @@ pub async fn fetch_leaves_scoped( } else { chunk_ids.to_vec() }; - tinycortex::memory::retrieval::fetch_leaves(&engine_config(config), &permitted_ids) + crate::engine::backend::retrieval::fetch_leaves(&engine_config(config), &permitted_ids) } diff --git a/core/src/tree/retrieval/integration_tests.rs b/core/src/tree/retrieval/integration_tests.rs index 9e41c8f..f449aa8 100644 --- a/core/src/tree/retrieval/integration_tests.rs +++ b/core/src/tree/retrieval/integration_tests.rs @@ -18,10 +18,10 @@ use tinymemory_api::host::MemoryHostConfig; #[cfg(test)] use tinymemory_api::host::test_support::TestHostConfig; +use crate::engine::backend::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; use crate::ingest_pipeline::ingest_chat; use crate::store::chunks::types::SourceKind; use crate::tree::retrieval::{drill_down, fetch_leaves, query_source, search_entities}; -use tinycortex::memory::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; fn test_config() -> (TempDir, TestHostConfig) { crate::test_seams::init(); diff --git a/core/src/tree/retrieval/search.rs b/core/src/tree/retrieval/search.rs index 9b379d8..2f0c69e 100644 --- a/core/src/tree/retrieval/search.rs +++ b/core/src/tree/retrieval/search.rs @@ -1,6 +1,6 @@ use anyhow::Result; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::tree::retrieval::types::EntityMatch; use crate::tree::score::extract::EntityKind; use crate::Config; @@ -17,7 +17,7 @@ pub async fn search_entities( kinds.as_ref().map_or(0, Vec::len), limit ); - tinycortex::memory::retrieval::search_entities( + crate::engine::backend::retrieval::search_entities( &engine_config(config), query, kinds.as_deref(), diff --git a/core/src/tree/retrieval/source.rs b/core/src/tree/retrieval/source.rs index dbc0f62..95145a5 100644 --- a/core/src/tree/retrieval/source.rs +++ b/core/src/tree/retrieval/source.rs @@ -1,8 +1,8 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::source_scope::current_source_scope; use crate::store::chunks::types::SourceKind; -use crate::tinycortex::engine_config; use crate::tree::retrieval::engine::EmbedderBridge; use crate::tree::retrieval::types::QueryResponse; use crate::tree::score::embed::build_embedder_from_config; @@ -90,7 +90,7 @@ pub async fn query_source_scoped( let mut response = if let Some(query) = semantic_query { let embedder = build_embedder_from_config(config)?; let bridge = EmbedderBridge(embedder.as_ref()); - tinycortex::memory::retrieval::query_source( + crate::engine::backend::retrieval::query_source( &engine_config(config), source_id, source_kind, @@ -101,13 +101,13 @@ pub async fn query_source_scoped( ) .await? } else { - tinycortex::memory::retrieval::query_source( + crate::engine::backend::retrieval::query_source( &engine_config(config), source_id, source_kind, time_window_days, None, - &tinycortex::memory::score::embed::InertEmbedder::new(), + &crate::engine::backend::score::embed::InertEmbedder::new(), usize::MAX, ) .await? diff --git a/core/src/tree/retrieval/source_scope_tests.rs b/core/src/tree/retrieval/source_scope_tests.rs index a36c4ea..92fd07c 100644 --- a/core/src/tree/retrieval/source_scope_tests.rs +++ b/core/src/tree/retrieval/source_scope_tests.rs @@ -15,7 +15,7 @@ //! (`tinycortex` `retrieval::{fetch,drill_down}`), so on leaves this is //! strictly narrower than predicate 1. `source.rs` / `cover.rs` additionally //! carry a *pre-filter* short circuit on the explicit `source_id` argument. -//! 3. `tinycortex::memory::chunks::store_list::append_source_scope` — a SQL +//! 3. `crate::engine::backend::chunks::store_list::append_source_scope` — a SQL //! predicate applied BEFORE `LIMIT`. Reached via `cover_window_scoped` and //! the raw `list_chunks` callers. It admits `mem_src:src-abc:` (empty item //! id), diverging from predicate 1. diff --git a/core/src/tree/retrieval/types.rs b/core/src/tree/retrieval/types.rs index 91b48fa..83e866e 100644 --- a/core/src/tree/retrieval/types.rs +++ b/core/src/tree/retrieval/types.rs @@ -1,6 +1,6 @@ //! Stable host path for tinycortex-owned retrieval wire types and converters. -pub use tinycortex::memory::retrieval::{ +pub use crate::engine::backend::retrieval::{ hit_from_chunk, hit_from_summary, hit_from_summary_with_tree, leaf_tree_placeholder, EntityMatch, NodeKind, QueryResponse, RetrievalHit, }; diff --git a/core/src/tree/score/extract/mod.rs b/core/src/tree/score/extract/mod.rs index 4768ebf..adc9c5d 100644 --- a/core/src/tree/score/extract/mod.rs +++ b/core/src/tree/score/extract/mod.rs @@ -5,23 +5,21 @@ use std::sync::Arc; use crate::Config; use async_trait::async_trait; -pub use tinycortex::memory::score::extract::{ +pub use crate::engine::backend::score::extract::{ ChatPrompt, ChatProvider, CompositeExtractor, EntityExtractor, EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic, LlmExtractorConfig, RegexEntityExtractor, }; pub mod regex { - pub use tinycortex::memory::score::extract::regex::extract; + pub use crate::engine::backend::score::extract::regex::extract; } -pub struct LlmEntityExtractor(tinycortex::memory::score::extract::LlmEntityExtractor); +pub struct LlmEntityExtractor(crate::engine::backend::score::extract::LlmEntityExtractor); impl LlmEntityExtractor { pub fn new(config: LlmExtractorConfig, provider: Arc) -> Self { - let provider = Arc::new(crate::tinycortex::SeamChatProvider::new(provider)); - Self(tinycortex::memory::score::extract::LlmEntityExtractor::new( - config, provider, - )) + let provider = Arc::new(crate::engine::SeamChatProvider::new(provider)); + Self(crate::engine::backend::score::extract::LlmEntityExtractor::new(config, provider)) } } diff --git a/core/src/tree/score/mod.rs b/core/src/tree/score/mod.rs index 136aeeb..240cf50 100644 --- a/core/src/tree/score/mod.rs +++ b/core/src/tree/score/mod.rs @@ -6,20 +6,20 @@ pub mod store; use std::sync::Arc; -pub use anyhow::Result; -pub use tinycortex::memory::score::{ +pub use crate::engine::backend::score::{ persist_score_tx, score_chunk, score_chunks, score_chunks_fast, ScoreResult, ScoringConfig, DEFAULT_DEFINITE_DROP, DEFAULT_DEFINITE_KEEP, DEFAULT_DROP_THRESHOLD, PRIORITY_BOOST, PRIORITY_TAG, }; -pub use tinycortex::memory::score::{resolver, signals}; +pub use crate::engine::backend::score::{resolver, signals}; +pub use anyhow::Result; /// Build crate scoring policy from product inference routing. pub fn scoring_config_from(config: &crate::Config) -> ScoringConfig { let (provider, model) = match crate::chat::build_chat_runtime(config) { Ok((provider, model)) => ( - Arc::new(crate::tinycortex::SeamChatProvider::new(provider)) - as Arc, + Arc::new(crate::engine::SeamChatProvider::new(provider)) + as Arc, model, ), Err(error) => { @@ -29,8 +29,8 @@ pub fn scoring_config_from(config: &crate::Config) -> ScoringConfig { return ScoringConfig::default_regex_only(); } }; - let extractor = tinycortex::memory::score::extract::LlmEntityExtractor::new( - tinycortex::memory::score::extract::LlmExtractorConfig { + let extractor = crate::engine::backend::score::extract::LlmEntityExtractor::new( + crate::engine::backend::score::extract::LlmExtractorConfig { model, output_language: config.output_language().map(str::to_string), ..Default::default() diff --git a/core/src/tree/score/store.rs b/core/src/tree/score/store.rs index 0a757e8..6f74abc 100644 --- a/core/src/tree/score/store.rs +++ b/core/src/tree/score/store.rs @@ -4,21 +4,21 @@ use std::collections::HashMap; use anyhow::Result; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; -pub use tinycortex::memory::score::store::{EntityHit, ScoreRow}; +pub use crate::engine::backend::score::store::{EntityHit, ScoreRow}; pub fn upsert_score(config: &Config, row: &ScoreRow) -> Result<()> { - tinycortex::memory::score::store::upsert_score(&engine_config(config), row) + crate::engine::backend::score::store::upsert_score(&engine_config(config), row) } pub fn get_score(config: &Config, chunk_id: &str) -> Result> { - tinycortex::memory::score::store::get_score(&engine_config(config), chunk_id) + crate::engine::backend::score::store::get_score(&engine_config(config), chunk_id) } pub fn get_scores_batch(config: &Config, chunk_ids: &[String]) -> Result> { - tinycortex::memory::score::store::get_scores_batch(&engine_config(config), chunk_ids) + crate::engine::backend::score::store::get_scores_batch(&engine_config(config), chunk_ids) } pub use crate::store::entities::{ @@ -27,7 +27,7 @@ pub use crate::store::entities::{ pub fn index_entity( config: &Config, - entity: &tinycortex::memory::score::resolver::CanonicalEntity, + entity: &crate::engine::backend::score::resolver::CanonicalEntity, node_id: &str, node_kind: &str, timestamp_ms: i64, @@ -39,13 +39,13 @@ pub fn index_entity( pub fn index_entities( config: &Config, - entities: &[tinycortex::memory::score::resolver::CanonicalEntity], + entities: &[crate::engine::backend::score::resolver::CanonicalEntity], node_id: &str, node_kind: &str, timestamp_ms: i64, tree_id: Option<&str>, ) -> Result { - let entities: Vec = entities + let entities: Vec = entities .iter() .map(to_store_entity) .collect::>()?; @@ -60,11 +60,11 @@ pub fn index_entities( } fn to_store_entity( - entity: &tinycortex::memory::score::resolver::CanonicalEntity, -) -> Result { - Ok(tinycortex::memory::store::CanonicalEntity { + entity: &crate::engine::backend::score::resolver::CanonicalEntity, +) -> Result { + Ok(crate::engine::backend::store::CanonicalEntity { canonical_id: entity.canonical_id.clone(), - kind: tinycortex::memory::store::EntityKind::parse(entity.kind.as_str()) + kind: crate::engine::backend::store::EntityKind::parse(entity.kind.as_str()) .map_err(anyhow::Error::msg)?, surface: entity.surface.clone(), span_start: entity.span_start, @@ -74,5 +74,5 @@ fn to_store_entity( } pub fn count_scores(config: &Config) -> Result { - tinycortex::memory::score::store::count_scores(&engine_config(config)) + crate::engine::backend::score::store::count_scores(&engine_config(config)) } diff --git a/core/src/tree/summarise.rs b/core/src/tree/summarise.rs index fc69fdd..0fe5db0 100644 --- a/core/src/tree/summarise.rs +++ b/core/src/tree/summarise.rs @@ -5,7 +5,7 @@ use anyhow::{Context, Result}; use crate::chat::{build_chat_provider, ChatPrompt}; use crate::Config; -pub use tinycortex::memory::tree::{SummaryContext, SummaryInput}; +pub use crate::engine::backend::tree::{SummaryContext, SummaryInput}; /// Compatibility result carrying provider usage alongside the crate-owned /// summary output fields. @@ -25,9 +25,11 @@ pub async fn summarise( inputs: &[SummaryInput], context: &SummaryContext<'_>, ) -> Result { - let Some(prepared) = - tinycortex::memory::tree::prepare_summary_prompt(inputs, context, config.output_language()) - else { + let Some(prepared) = crate::engine::backend::tree::prepare_summary_prompt( + inputs, + context, + config.output_language(), + ) else { return Ok(SummaryOutput::default()); }; let provider = @@ -50,7 +52,7 @@ pub async fn summarise( .await .with_context(|| format!("memory_tree::summarise: provider={}", provider.name()))?; let output = - tinycortex::memory::tree::finish_provider_summary(&text, prepared.effective_budget); + crate::engine::backend::tree::finish_provider_summary(&text, prepared.effective_budget); let input_tokens = usage.as_ref().map_or(0, |usage| usage.input_tokens); let output_tokens = usage.as_ref().map_or(0, |usage| usage.output_tokens); let charged_amount_usd = usage @@ -75,7 +77,7 @@ pub async fn summarise( } pub fn fallback_summary(inputs: &[SummaryInput], budget: u32) -> SummaryOutput { - let output = tinycortex::memory::tree::fallback_summary(inputs, budget); + let output = crate::engine::backend::tree::fallback_summary(inputs, budget); SummaryOutput { content: output.content, token_count: output.token_count, diff --git a/core/src/tree/tree/bucket_seal.rs b/core/src/tree/tree/bucket_seal.rs index 6903832..ca2fd63 100644 --- a/core/src/tree/tree/bucket_seal.rs +++ b/core/src/tree/tree/bucket_seal.rs @@ -3,11 +3,11 @@ use anyhow::Result; use chrono::{DateTime, Utc}; +use crate::engine::engine_config; use crate::store::trees::types::{Buffer, Tree}; -use crate::tinycortex::engine_config; use crate::Config; -pub use tinycortex::memory::tree::{LabelStrategy, LeafRef, MERGE_LEVEL_BASE}; +pub use crate::engine::backend::tree::{LabelStrategy, LeafRef, MERGE_LEVEL_BASE}; pub async fn append_leaf( config: &Config, @@ -23,11 +23,11 @@ pub async fn append_leaf( leaf.token_count as i64, leaf.timestamp, )?; - crate::tinycortex::cascade_tree(config, tree, 0, false, strategy).await + crate::engine::cascade_tree(config, tree, 0, false, strategy).await } pub fn append_leaf_deferred(config: &Config, tree: &Tree, leaf: &LeafRef) -> Result { - tinycortex::memory::tree::append_leaf_deferred(&engine_config(config), tree, leaf) + crate::engine::backend::tree::append_leaf_deferred(&engine_config(config), tree, leaf) } pub fn append_to_buffer( @@ -38,7 +38,7 @@ pub fn append_to_buffer( token_delta: i64, item_ts: DateTime, ) -> Result<()> { - tinycortex::memory::tree::append_to_buffer( + crate::engine::backend::tree::append_to_buffer( &engine_config(config), tree_id, level, @@ -55,7 +55,7 @@ pub async fn cascade_all_from( force_now: Option>, strategy: &LabelStrategy, ) -> Result> { - crate::tinycortex::cascade_tree(config, tree, start_level, force_now.is_some(), strategy).await + crate::engine::cascade_tree(config, tree, start_level, force_now.is_some(), strategy).await } pub async fn seal_document_subtree( @@ -66,7 +66,7 @@ pub async fn seal_document_subtree( chunk_ids: &[String], strategy: &LabelStrategy, ) -> Result { - crate::tinycortex::seal_document_subtree(config, tree, doc_id, version_ms, chunk_ids, strategy) + crate::engine::seal_document_subtree(config, tree, doc_id, version_ms, chunk_ids, strategy) .await } @@ -77,5 +77,5 @@ pub async fn seal_one_level( strategy: &LabelStrategy, enqueue_follow_ups: bool, ) -> Result { - crate::tinycortex::seal_tree_level(config, tree, buffer, strategy, enqueue_follow_ups).await + crate::engine::seal_tree_level(config, tree, buffer, strategy, enqueue_follow_ups).await } diff --git a/core/src/tree/tree/factory.rs b/core/src/tree/tree/factory.rs index fbb738e..2ff6414 100644 --- a/core/src/tree/tree/factory.rs +++ b/core/src/tree/tree/factory.rs @@ -21,36 +21,36 @@ use crate::tree::tree::flush::force_flush_tree; use crate::tree::tree::registry::get_or_create_tree; use crate::Config; -pub use tinycortex::memory::tree::{TreeProfile, GLOBAL_SCOPE}; +pub use crate::engine::backend::tree::{TreeProfile, GLOBAL_SCOPE}; /// Factory/config object for one tree instance. #[derive(Debug, Clone)] pub struct TreeFactory<'a> { - inner: tinycortex::memory::tree::TreeFactory<'a>, + inner: crate::engine::backend::tree::TreeFactory<'a>, } impl<'a> TreeFactory<'a> { pub fn source(scope: impl Into>) -> Self { Self { - inner: tinycortex::memory::tree::TreeFactory::source(scope), + inner: crate::engine::backend::tree::TreeFactory::source(scope), } } pub fn topic(scope: impl Into>) -> Self { Self { - inner: tinycortex::memory::tree::TreeFactory::topic(scope), + inner: crate::engine::backend::tree::TreeFactory::topic(scope), } } pub fn global() -> Self { Self { - inner: tinycortex::memory::tree::TreeFactory::global(), + inner: crate::engine::backend::tree::TreeFactory::global(), } } pub fn from_tree(tree: &'a Tree) -> Self { Self { - inner: tinycortex::memory::tree::TreeFactory::from_tree(tree), + inner: crate::engine::backend::tree::TreeFactory::from_tree(tree), } } diff --git a/core/src/tree/tree/flush.rs b/core/src/tree/tree/flush.rs index 9c781e5..df74586 100644 --- a/core/src/tree/tree/flush.rs +++ b/core/src/tree/tree/flush.rs @@ -12,7 +12,7 @@ pub async fn flush_stale_buffers( max_age: Duration, strategy: &LabelStrategy, ) -> Result { - crate::tinycortex::flush_stale_tree_buffers(config, max_age, strategy).await + crate::engine::flush_stale_tree_buffers(config, max_age, strategy).await } pub async fn flush_stale_buffers_default( diff --git a/core/src/tree/tree_runtime/engine.rs b/core/src/tree/tree_runtime/engine.rs index ca3ec68..a67ee26 100644 --- a/core/src/tree/tree_runtime/engine.rs +++ b/core/src/tree/tree_runtime/engine.rs @@ -2,16 +2,16 @@ use std::sync::Arc; +use crate::engine::backend::tree::runtime::{ + NodeLevel, RuntimeObserver, Summariser, TreeNode, TreeStatus, +}; use anyhow::{Context, Result}; use async_trait::async_trait; use chrono::{DateTime, Timelike, Utc}; use tinyagents::harness::message::Message; use tinyagents::harness::model::{ChatModel, ModelRequest}; -use tinycortex::memory::tree::runtime::{ - NodeLevel, RuntimeObserver, Summariser, TreeNode, TreeStatus, -}; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; const SUMMARIZATION_TEMP: f64 = 0.3; @@ -83,7 +83,7 @@ pub async fn run_summarization( ts: DateTime, ) -> Result> { log::debug!("[tree_summarizer] tinycortex run namespace={namespace}"); - let result = tinycortex::memory::tree::runtime::run_summarization_observed( + let result = crate::engine::backend::tree::runtime::run_summarization_observed( &engine_config(config), &ChatSummariser(provider), namespace, @@ -105,7 +105,7 @@ pub async fn rebuild_tree( namespace: &str, ) -> Result { log::debug!("[tree_summarizer] tinycortex rebuild namespace={namespace}"); - tinycortex::memory::tree::runtime::rebuild_tree_observed( + crate::engine::backend::tree::runtime::rebuild_tree_observed( &engine_config(config), &ChatSummariser(provider), namespace, @@ -134,8 +134,9 @@ pub async fn run_hourly_loop(config: Arc, provider: Arc PathBuf { - tinycortex::memory::tree::runtime::store::tree_dir(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::tree_dir(&engine_config(config), namespace) } pub fn buffer_dir(config: &Config, namespace: &str) -> PathBuf { - tinycortex::memory::tree::runtime::store::buffer_dir(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::buffer_dir(&engine_config(config), namespace) } pub fn node_file_path(config: &Config, namespace: &str, node_id: &str) -> PathBuf { - tinycortex::memory::tree::runtime::store::node_file_path( + crate::engine::backend::tree::runtime::store::node_file_path( &engine_config(config), namespace, node_id, ) } -pub use tinycortex::memory::tree::runtime::store::{validate_namespace, validate_node_id}; +pub use crate::engine::backend::tree::runtime::store::{validate_namespace, validate_node_id}; pub fn write_node(config: &Config, node: &TreeNode) -> Result<()> { - tinycortex::memory::tree::runtime::store::write_node(&engine_config(config), node) + crate::engine::backend::tree::runtime::store::write_node(&engine_config(config), node) } pub fn read_node(config: &Config, namespace: &str, node_id: &str) -> Result> { - tinycortex::memory::tree::runtime::store::read_node(&engine_config(config), namespace, node_id) + crate::engine::backend::tree::runtime::store::read_node( + &engine_config(config), + namespace, + node_id, + ) } pub fn read_children(config: &Config, namespace: &str, parent_id: &str) -> Result> { - tinycortex::memory::tree::runtime::store::read_children( + crate::engine::backend::tree::runtime::store::read_children( &engine_config(config), namespace, parent_id, @@ -45,7 +49,7 @@ pub fn read_children(config: &Config, namespace: &str, parent_id: &str) -> Resul } pub fn read_ancestors(config: &Config, namespace: &str, node_id: &str) -> Result> { - tinycortex::memory::tree::runtime::store::read_ancestors( + crate::engine::backend::tree::runtime::store::read_ancestors( &engine_config(config), namespace, node_id, @@ -53,11 +57,11 @@ pub fn read_ancestors(config: &Config, namespace: &str, node_id: &str) -> Result } pub fn count_nodes(config: &Config, namespace: &str) -> Result { - tinycortex::memory::tree::runtime::store::count_nodes(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::count_nodes(&engine_config(config), namespace) } pub fn get_tree_status(config: &Config, namespace: &str) -> Result { - tinycortex::memory::tree::runtime::store::get_tree_status(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::get_tree_status(&engine_config(config), namespace) } pub fn collect_root_summaries_with_caps( @@ -65,7 +69,7 @@ pub fn collect_root_summaries_with_caps( per_namespace_cap: usize, total_cap: usize, ) -> Vec<(String, String, DateTime)> { - tinycortex::memory::tree::runtime::store::collect_root_summaries_with_caps( + crate::engine::backend::tree::runtime::store::collect_root_summaries_with_caps( workspace_dir, per_namespace_cap, total_cap, @@ -73,11 +77,11 @@ pub fn collect_root_summaries_with_caps( } pub fn list_namespaces_with_root(config: &Config) -> Result> { - tinycortex::memory::tree::runtime::store::list_namespaces_with_root(&engine_config(config)) + crate::engine::backend::tree::runtime::store::list_namespaces_with_root(&engine_config(config)) } pub fn delete_tree(config: &Config, namespace: &str) -> Result { - tinycortex::memory::tree::runtime::store::delete_tree(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::delete_tree(&engine_config(config), namespace) } pub fn buffer_write( @@ -87,7 +91,7 @@ pub fn buffer_write( ts: &DateTime, metadata: Option<&Value>, ) -> Result { - tinycortex::memory::tree::runtime::store::buffer_write( + crate::engine::backend::tree::runtime::store::buffer_write( &engine_config(config), namespace, content, @@ -97,11 +101,11 @@ pub fn buffer_write( } pub fn buffer_read(config: &Config, namespace: &str) -> Result> { - tinycortex::memory::tree::runtime::store::buffer_read(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::buffer_read(&engine_config(config), namespace) } pub fn buffer_delete(config: &Config, namespace: &str, filenames: &[String]) -> Result<()> { - tinycortex::memory::tree::runtime::store::buffer_delete( + crate::engine::backend::tree::runtime::store::buffer_delete( &engine_config(config), namespace, filenames, @@ -109,9 +113,9 @@ pub fn buffer_delete(config: &Config, namespace: &str, filenames: &[String]) -> } pub fn buffer_drain(config: &Config, namespace: &str) -> Result> { - tinycortex::memory::tree::runtime::store::buffer_drain(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::buffer_drain(&engine_config(config), namespace) } pub fn parse_node_markdown_pub(raw: &str, namespace: &str, node_id: &str) -> Result { - tinycortex::memory::tree::runtime::store::parse_node_markdown_pub(raw, namespace, node_id) + crate::engine::backend::tree::runtime::store::parse_node_markdown_pub(raw, namespace, node_id) } diff --git a/crates/tinymemory-module/Cargo.lock b/crates/tinymemory-module/Cargo.lock index e5599eb..e9d5751 100644 --- a/crates/tinymemory-module/Cargo.lock +++ b/crates/tinymemory-module/Cargo.lock @@ -1935,6 +1935,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.20", + "tinymemory-api", "uuid", ] @@ -1991,6 +1992,7 @@ dependencies = [ "tinycortex-api", "tinymemory", "tinymemory-api", + "tinymemory-sync", "tokio", "tracing", "url", @@ -2021,6 +2023,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "tinymemory-sync" +version = "0.1.0" +dependencies = [ + "chrono", + "log", + "serde_json", + "tracing", +] + [[package]] name = "tinymemory-tinycortex" version = "0.1.0" diff --git a/crates/tinymemory-module/Cargo.toml b/crates/tinymemory-module/Cargo.toml index adbc845..ced3f1e 100644 --- a/crates/tinymemory-module/Cargo.toml +++ b/crates/tinymemory-module/Cargo.toml @@ -79,6 +79,14 @@ tempfile = "3" # which is unusual but correct: these are the same submodules the parent builds # against, and pointing somewhere else would compile the module against a # different engine than the workspace it ships from. +# `tinycortex-api` takes the contract by git (issue #18 §A1). This crate is its +# own workspace root, and a patch table only applies from the root being built, +# so the entry has to be repeated here — the parent workspace's identical entry +# does not reach it. Without this, cargo resolves the git copy alongside the +# path copy and `MemoryTaint` from one is not the same type as from the other. +[patch."https://github.com/tinyhumansai/tinymemory"] +tinymemory-api = { path = "../../api" } + [patch.crates-io] tinycortex = { path = "../../vendor/tinycortex" } tinycortex-api = { path = "../../vendor/tinycortex/api" } diff --git a/crates/tinymemory-module/tests/module_e2e.rs b/crates/tinymemory-module/tests/module_e2e.rs index 79c1344..7c24f77 100644 --- a/crates/tinymemory-module/tests/module_e2e.rs +++ b/crates/tinymemory-module/tests/module_e2e.rs @@ -611,3 +611,118 @@ async fn the_manifest_declares_every_method_the_module_serves() { declared.difference(&expected).collect::>() ); } + +#[tokio::test] +#[ignore = "drives a real dlopen'ed module; must be the only such test in the process — see the module docs"] +async fn what_is_written_lands_in_the_workspace_it_was_given() { + // Issue #18 §E5 asks for a shutdown/restart cycle. It cannot be written + // here, and the reason is structural rather than an omission: TinyBus never + // unloads a library, so a second admission in the same process is refused — + // `ModuleRefused { reason: "module initialization failed" }` — and every + // test in this file must be the only one in its process for the same + // reason. A genuine restart needs a second *process* against a shared + // workspace, which is a change to the CI loop rather than a test. + // + // What is assertable in one process is the property that restart would be + // checking: that a write goes to the durable workspace the host supplied, + // and not somewhere that disappears. A module that stored into a temporary + // directory of its own, or in memory, passes every other test in this file + // — each one stores and reads back inside a single admission, so the + // difference never shows. It would show on the user's next launch. + let workspace = tempfile::tempdir().expect("tempdir"); + + // Nothing has been asked of it yet. + let before = std::fs::read_dir(workspace.path()) + .expect("workspace readable") + .count(); + + let (client, _host, _task) = admit_module(workspace.path()).await; + proxy(&client) + .call::<()>( + "Store", + ( + "e2e", + "durable", + "written to the host's workspace", + MemoryCategory::Core, + Option::::None, + MemoryTaint::default(), + ), + ) + .await + .expect("Store"); + + let entry: Option = proxy(&client) + .call("Get", ("e2e", "durable")) + .await + .expect("Get"); + assert_eq!( + entry.expect("just stored").content, + "written to the host's workspace" + ); + + // The assertion that a same-admission round trip cannot make: the bytes are + // in the directory the host named. + let after = std::fs::read_dir(workspace.path()) + .expect("workspace readable") + .count(); + assert!( + after > before, + "the module answered correctly but wrote nothing into the workspace it \ + was given — a store that does not land here does not survive a restart" + ); +} + +#[tokio::test] +#[ignore = "drives a real dlopen'ed module; must be the only such test in the process — see the module docs"] +async fn every_declared_method_is_actually_routed() { + // Issue #18 §E5 asks the E2E to cover every family the module advertises. + // It advertises `Capabilities::all()` — eighteen families — and the tests + // above exercise three of them. + // + // Rather than eighteen bespoke round trips, this asserts the property that + // makes the advertisement honest at this layer: every method the manifest + // declares is actually *reachable*. `the_manifest_declares_every_method_the + // _module_serves` compares two lists and would pass for a method that is + // declared, routed, and answers "unknown member" — which is the bus-level + // version of a capability set that overstates its accessors. + // + // Each method is called with no arguments, so most fail. That is fine and is + // the point: what is asserted is the *kind* of failure. A method that is + // wired rejects the arguments; a method that is not wired rejects the + // member, and those carry different wire names. + let workspace = tempfile::tempdir().expect("tempdir"); + let (client, _host, _task) = admit_module(workspace.path()).await; + + // Establish the shape of a genuine not-routed refusal from this very build, + // rather than hard-coding tinybus's spelling of it. The two outcomes are + // distinct names — `ai.tinyhumans.tinybus.Error.UnknownMethod` for a member + // that is not there, `…Error.BadArguments` for one that is and did not like + // the empty argument list — which is what makes this test discriminate + // rather than pass vacuously. + let unrouted = proxy(&client) + .call::("NoSuchMethodAtAll", ()) + .await + .expect_err("an unknown member must be refused") + .wire_name() + .to_string(); + + let mut missing = Vec::new(); + for method in EXPECTED_METHODS { + // `Shutdown` would stop the module and strand every later iteration. + if *method == "Shutdown" { + continue; + } + let outcome = proxy(&client).call::(method, ()).await; + if let Err(error) = outcome { + if error.wire_name() == unrouted { + missing.push(*method); + } + } + } + + assert!( + missing.is_empty(), + "declared in the manifest but not routed: {missing:?}" + ); +} diff --git a/sync/Cargo.toml b/sync/Cargo.toml new file mode 100644 index 0000000..223e667 --- /dev/null +++ b/sync/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "tinymemory-sync" +publish = false +version = "0.1.0" +edition = "2021" +rust-version = "1.96" +license = "MIT" +repository = "https://github.com/tinyhumansai/tinymemory" +description = "Engine-neutral Composio payload normalisers for TinyMemory" + +# The whole dependency list, and it is the point of the crate. These are pure +# `Value -> Value` transforms: no engine, no storage, no network, no async +# runtime. A dependency added here should have to argue for itself against that +# sentence (issue #18 §B3). +[dependencies] +serde_json = "1" +# Two logging facades, neither an implementation, both carried over from the +# engine layout this crate was extracted from: `gmail_post_process` traces +# through `tracing`, `slack_post_process` through `log`. Preserved rather than +# unified, because §B3 is a *move* and swapping a facade would change where a +# host's log lines surface — a behaviour change hiding inside a relocation. +# Worth reconciling in its own change. +tracing = "0.1" +log = "0.4" +# RFC 2822/3339 date handling for Gmail `Date:` headers. +# +# `clock` is on, and it is the one place this crate is not a pure function of +# its input: `format_email_local_time` renders in `chrono::Local`, so it reads +# the host's timezone. That is deliberate upstream — the agent presents local +# times without doing UTC arithmetic, and the raw UTC field is preserved +# alongside — but it means "pure `Value -> Value`" is true of every normaliser +# here except that one. Better said out loud than discovered by someone whose +# output moved when they changed TZ. +chrono = { version = "0.4", features = ["clock"] } + +[lints.rust] +unsafe_code = "forbid" +missing_docs = "warn" +unreachable_pub = "warn" + +[lints.clippy] +all = { level = "warn", priority = -1 } +unwrap_used = "warn" +expect_used = "warn" +panic = "warn" diff --git a/sync/src/clickup.rs b/sync/src/clickup.rs new file mode 100644 index 0000000..ae340b6 --- /dev/null +++ b/sync/src/clickup.rs @@ -0,0 +1,133 @@ +//! ClickUp host normalization helpers — result extraction, task-title extraction, +//! and time utilities. +//! +//! ClickUp's REST API (and therefore Composio's wrapping of it) returns +//! task lists in a small handful of shapes depending on which endpoint +//! is called. The functions here walk the union of common shapes so the +//! provider doesn't have to branch per Composio envelope variant. + +use serde_json::Value; + +use super::helpers::pick_str; + +/// Walk the Composio response envelope for ClickUp task list results. +/// +/// ClickUp's "filtered team tasks" endpoint returns `{ "tasks": [...] }` +/// at the top level; Composio re-wraps the upstream payload under +/// `data` or `data.data` depending on the action. We probe each shape +/// in order and return the first array we find. +pub fn extract_tasks(data: &Value) -> Vec { + let candidates = [ + data.pointer("/data/tasks"), + data.pointer("/tasks"), + data.pointer("/data/data/tasks"), + data.pointer("/data/results"), + data.pointer("/results"), + data.pointer("/data/items"), + data.pointer("/items"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + return arr.clone(); + } + } + Vec::new() +} + +/// Extract a human-readable title from a ClickUp task object. +/// +/// ClickUp tasks store the name at `name` (or `data.name` after Composio +/// envelope wrapping). When the name is missing we fall back to the +/// task ID so chunks remain identifiable. +pub fn extract_task_name(task: &Value) -> Option { + pick_str(task, &["name", "data.name", "title", "data.title"]) +} + +/// Extract a stable cursor timestamp (milliseconds since epoch as a +/// string) from a ClickUp task object. +/// +/// The ClickUp API returns `date_updated` as a stringified epoch ms +/// (e.g. `"1733412345678"`); we keep it as a string so lexicographic +/// comparison against the stored cursor remains valid as long as the +/// length doesn't change (it won't until year 33658). +pub fn extract_task_updated(task: &Value) -> Option { + pick_str( + task, + &[ + "date_updated", + "data.date_updated", + "updated_at", + "data.updated_at", + "dateUpdated", + "data.dateUpdated", + ], + ) +} + +/// Current wall-clock time in milliseconds since the UNIX epoch. +pub fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Extract the authorized user's numeric ID from the +/// `CLICKUP_GET_AUTHORIZED_USER` response. +/// +/// Composio wraps the upstream `{"user": {"id": …}}` shape; this walker +/// is defensive against both raw and wrapped payloads. Returns the ID +/// as a string because `CLICKUP_GET_FILTERED_TEAM_TASKS` accepts the +/// `assignees` filter as a string array. +pub fn extract_user_id(data: &Value) -> Option { + let candidates = [ + data.pointer("/user/id"), + data.pointer("/data/user/id"), + data.pointer("/id"), + data.pointer("/data/id"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(n) = cand.as_u64() { + return Some(n.to_string()); + } + if let Some(n) = cand.as_i64() { + return Some(n.to_string()); + } + if let Some(s) = cand.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + +/// Extract a list of workspace (team) IDs from the +/// `CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES` response. +/// +/// ClickUp returns `{"teams": [{"id": "...", "name": "..."}, …]}`. We +/// keep the IDs as strings — `CLICKUP_GET_FILTERED_TEAM_TASKS` requires +/// a `team_id` (string) argument. +pub fn extract_workspace_ids(data: &Value) -> Vec { + let candidates = [ + data.pointer("/teams"), + data.pointer("/data/teams"), + data.pointer("/workspaces"), + data.pointer("/data/workspaces"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + return arr + .iter() + .filter_map(|t| pick_str(t, &["id", "team_id", "workspace_id"])) + .collect(); + } + } + Vec::new() +} + +#[cfg(test)] +#[path = "clickup_tests.rs"] +mod tests; diff --git a/sync/src/clickup_tests.rs b/sync/src/clickup_tests.rs new file mode 100644 index 0000000..99b4667 --- /dev/null +++ b/sync/src/clickup_tests.rs @@ -0,0 +1,101 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +#[test] +fn extract_tasks_from_data_tasks() { + let data = json!({ "data": { "tasks": [{"id": "t1"}] } }); + assert_eq!(extract_tasks(&data).len(), 1); +} + +#[test] +fn extract_tasks_from_top_level_tasks() { + let data = json!({ "tasks": [{"id": "a"}, {"id": "b"}] }); + assert_eq!(extract_tasks(&data).len(), 2); +} + +#[test] +fn extract_tasks_empty_when_missing() { + let data = json!({ "foo": "bar" }); + assert!(extract_tasks(&data).is_empty()); +} + +#[test] +fn extract_task_name_from_top_level() { + let task = json!({ "id": "t1", "name": "Build feature X" }); + assert_eq!(extract_task_name(&task), Some("Build feature X".into())); +} + +#[test] +fn extract_task_name_falls_back_to_data_name() { + let task = json!({ "data": { "name": "Wrapped" } }); + assert_eq!(extract_task_name(&task), Some("Wrapped".into())); +} + +#[test] +fn extract_task_name_none_when_missing() { + let task = json!({ "id": "t1" }); + assert!(extract_task_name(&task).is_none()); +} + +#[test] +fn extract_task_updated_handles_string_form() { + let task = json!({ "date_updated": "1733412345678" }); + assert_eq!( + extract_task_updated(&task), + Some("1733412345678".to_string()) + ); +} + +#[test] +fn extract_task_updated_handles_nested_data() { + let task = json!({ "data": { "dateUpdated": "1700000000000" } }); + assert_eq!( + extract_task_updated(&task), + Some("1700000000000".to_string()) + ); +} + +#[test] +fn extract_user_id_handles_numeric_id() { + let data = json!({ "user": { "id": 12345 } }); + assert_eq!(extract_user_id(&data), Some("12345".to_string())); +} + +#[test] +fn extract_user_id_handles_wrapped_payload() { + let data = json!({ "data": { "user": { "id": "777" } } }); + assert_eq!(extract_user_id(&data), Some("777".to_string())); +} + +#[test] +fn extract_user_id_none_when_missing() { + let data = json!({ "foo": "bar" }); + assert!(extract_user_id(&data).is_none()); +} + +#[test] +fn extract_workspace_ids_from_teams_array() { + let data = json!({ + "teams": [ + { "id": "ws1", "name": "Personal" }, + { "id": "ws2", "name": "Acme" }, + ] + }); + assert_eq!(extract_workspace_ids(&data), vec!["ws1", "ws2"]); +} + +#[test] +fn extract_workspace_ids_empty_when_no_teams() { + let data = json!({ "foo": "bar" }); + assert!(extract_workspace_ids(&data).is_empty()); +} + +#[test] +fn now_ms_returns_nonzero() { + assert!(now_ms() > 0); +} diff --git a/sync/src/github.rs b/sync/src/github.rs new file mode 100644 index 0000000..393e93e --- /dev/null +++ b/sync/src/github.rs @@ -0,0 +1,130 @@ +//! GitHub host normalization helpers — result extraction, identity helpers, and time utilities. +//! +//! GitHub's REST API (proxied through Composio) returns search results and +//! authenticated-user payloads in a small number of shapes. The functions here +//! walk the union of common Composio envelope variants so the provider stays +//! clean and branch-free. + +use serde_json::Value; + +use super::helpers::pick_str; + +/// Walk the Composio response envelope for GitHub search issue results. +/// +/// `GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS` wraps GitHub's `GET /search/issues` response, which +/// returns `{"total_count": N, "items": [...]}`. Composio may re-wrap this under +/// `data` or `data.data`; we probe each shape in order. +pub fn extract_issues(data: &Value) -> Vec { + let candidates = [ + data.pointer("/data/items"), + data.pointer("/items"), + data.pointer("/data/data/items"), + data.pointer("/data/results"), + data.pointer("/results"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + return arr.clone(); + } + } + Vec::new() +} + +/// Extract a stable, globally unique identifier for a GitHub issue or PR. +/// +/// GitHub's internal `id` field is a large integer unique across all issues +/// and PRs on github.com. We convert it to a string for use as a sync key. +/// Falls back to composing from `html_url` path if `id` is absent. +pub fn extract_issue_id(issue: &Value) -> Option { + // Primary: numeric internal GitHub ID. + if let Some(id) = issue.get("id").or_else(|| issue.pointer("/data/id")) { + if let Some(n) = id.as_u64() { + return Some(n.to_string()); + } + if let Some(s) = id.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + // Fallback: parse owner/repo/number from html_url path segments. + // URL shape: https://github.com/{owner}/{repo}/issues/{number} + if let Some(url) = pick_str(issue, &["html_url", "data.html_url", "url", "data.url"]) { + if let Some(slug) = github_url_to_slug(&url) { + return Some(slug); + } + } + None +} + +/// Build a human-readable document title for a GitHub issue/PR. +/// +/// Format: `GitHub: {owner}/{repo}#{number}: {title}`. +/// Falls back to just the title or a placeholder when fields are missing. +pub fn extract_issue_title(issue: &Value) -> Option { + let title = pick_str(issue, &["title", "data.title"])?; + + // Best-effort: extract owner/repo#N from html_url for the prefix. + let prefix = pick_str(issue, &["html_url", "data.html_url"]) + .and_then(|url| github_url_to_slug(&url)) + .unwrap_or_default(); + + if prefix.is_empty() { + Some(title) + } else { + Some(format!("GitHub: {prefix}: {title}")) + } +} + +/// Parse `https://github.com/{owner}/{repo}/issues/{number}` (or `/pull/`) +/// into `"{owner}/{repo}#{number}"`. Returns `None` for unrecognised shapes. +fn github_url_to_slug(url: &str) -> Option { + let segs: Vec<&str> = url.trim_end_matches('/').split('/').collect(); + // Minimum: ["https:", "", "github.com", owner, repo, "issues", number] + if segs.len() >= 7 { + let number = segs[segs.len() - 1]; + let _kind = segs[segs.len() - 2]; // "issues" or "pull" — ignored + let repo = segs[segs.len() - 3]; + let owner = segs[segs.len() - 4]; + if !owner.is_empty() && !repo.is_empty() && !number.is_empty() { + return Some(format!("{owner}/{repo}#{number}")); + } + } + None +} + +/// Extract the `updated_at` ISO 8601 timestamp from a GitHub issue. +/// +/// GitHub returns `updated_at` as `"2024-05-21T15:30:00Z"`. ISO 8601 strings +/// sort lexicographically, so we use them directly as the sync cursor. +pub fn extract_issue_updated_at(issue: &Value) -> Option { + pick_str( + issue, + &[ + "updated_at", + "data.updated_at", + "updatedAt", + "data.updatedAt", + ], + ) +} + +/// Extract the authenticated user's login handle from a +/// `GITHUB_GET_THE_AUTHENTICATED_USER` response. +pub fn extract_user_login(data: &Value) -> Option { + pick_str(data, &["login", "data.login"]) +} + +/// Current wall-clock time in milliseconds since the UNIX epoch. +pub fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[cfg(test)] +#[path = "github_tests.rs"] +mod tests; diff --git a/sync/src/github_tests.rs b/sync/src/github_tests.rs new file mode 100644 index 0000000..17aa138 --- /dev/null +++ b/sync/src/github_tests.rs @@ -0,0 +1,123 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +#[test] +fn extract_issues_from_data_items() { + let data = json!({ "data": { "items": [{"id": 1}] } }); + assert_eq!(extract_issues(&data).len(), 1); +} + +#[test] +fn extract_issues_from_top_level_items() { + let data = json!({ "items": [{"id": 1}, {"id": 2}] }); + assert_eq!(extract_issues(&data).len(), 2); +} + +#[test] +fn extract_issues_empty_when_missing() { + let data = json!({ "foo": "bar" }); + assert!(extract_issues(&data).is_empty()); +} + +#[test] +fn extract_issue_id_from_numeric_field() { + let issue = json!({ "id": 123456789u64, "title": "Fix bug" }); + assert_eq!(extract_issue_id(&issue), Some("123456789".to_string())); +} + +#[test] +fn extract_issue_id_from_wrapped_data() { + let issue = json!({ "data": { "id": 99u64 } }); + assert_eq!(extract_issue_id(&issue), Some("99".to_string())); +} + +#[test] +fn extract_issue_id_falls_back_to_html_url() { + let issue = json!({ + "html_url": "https://github.com/owner/repo/issues/42" + }); + assert_eq!(extract_issue_id(&issue), Some("owner/repo#42".to_string())); +} + +#[test] +fn extract_issue_id_none_when_missing() { + let issue = json!({ "title": "No ID here" }); + assert!(extract_issue_id(&issue).is_none()); +} + +#[test] +fn extract_issue_title_builds_prefixed_title() { + let issue = json!({ + "id": 1u64, + "title": "Fix race condition", + "html_url": "https://github.com/acme/core/issues/99" + }); + assert_eq!( + extract_issue_title(&issue), + Some("GitHub: acme/core#99: Fix race condition".to_string()) + ); +} + +#[test] +fn extract_issue_title_returns_raw_title_when_no_url() { + let issue = json!({ "title": "Bare title" }); + assert_eq!(extract_issue_title(&issue), Some("Bare title".to_string())); +} + +#[test] +fn extract_issue_title_none_when_missing() { + let issue = json!({ "id": 1u64 }); + assert!(extract_issue_title(&issue).is_none()); +} + +#[test] +fn extract_issue_updated_at_from_top_level() { + let issue = json!({ "updated_at": "2024-05-21T15:30:00Z" }); + assert_eq!( + extract_issue_updated_at(&issue), + Some("2024-05-21T15:30:00Z".to_string()) + ); +} + +#[test] +fn extract_issue_updated_at_from_data_wrapper() { + let issue = json!({ "data": { "updated_at": "2023-01-01T00:00:00Z" } }); + assert_eq!( + extract_issue_updated_at(&issue), + Some("2023-01-01T00:00:00Z".to_string()) + ); +} + +#[test] +fn extract_issue_updated_at_none_when_missing() { + let issue = json!({ "id": 1u64 }); + assert!(extract_issue_updated_at(&issue).is_none()); +} + +#[test] +fn extract_user_login_from_top_level() { + let data = json!({ "login": "octocat" }); + assert_eq!(extract_user_login(&data), Some("octocat".to_string())); +} + +#[test] +fn extract_user_login_from_data_wrapper() { + let data = json!({ "data": { "login": "monalisa" } }); + assert_eq!(extract_user_login(&data), Some("monalisa".to_string())); +} + +#[test] +fn extract_user_login_none_when_missing() { + let data = json!({ "id": 1u64 }); + assert!(extract_user_login(&data).is_none()); +} + +#[test] +fn now_ms_returns_nonzero() { + assert!(now_ms() > 0); +} diff --git a/sync/src/gmail_post_process.rs b/sync/src/gmail_post_process.rs new file mode 100644 index 0000000..327479a --- /dev/null +++ b/sync/src/gmail_post_process.rs @@ -0,0 +1,503 @@ +//! Gmail-specific post-processing of Composio action responses. +//! +//! The upstream `GMAIL_FETCH_EMAILS` payload is extremely verbose +//! (full MIME tree under `payload.parts[]`, 50+ `Received:` headers, +//! display-layer noise the model never uses). This module rewrites +//! it into a slim envelope per message: +//! +//! ```json +//! { +//! "messages": [ +//! { +//! "id": "…", +//! "threadId": "…", +//! "subject": "…", +//! "from": "…", +//! "to": "…", +//! "date": "…", +//! "labels": ["INBOX", "UNREAD"], +//! "markdown": "…body…", +//! "attachments": [ { "filename": "...", "mimeType": "..." } ] +//! } +//! ], +//! "nextPageToken": "…", +//! "resultSizeEstimate": 201 +//! } +//! ``` +//! +//! ## Body source +//! +//! Composio's backend ships a +//! `markdownFormatted` field on the response envelope — one string +//! per tool call, pre-rendered with HTML stripped, URLs shortened, +//! footers removed, whitespace normalised. We split it per message +//! along `\n---\n` boundaries (with `## ` heading fallbacks) and +//! pin each slice to the corresponding entry in `messages[]` via +//! [`apply_response_level_markdown`]. The reshape's +//! `extract_markdown_body` then prefers that pinned field over +//! falling back to the upstream `messageText`. +//! +//! No in-house HTML→markdown conversion lives here anymore — the +//! backend does the cleaning. If `markdownFormatted` is absent for +//! a given response we fall through to whatever plain text the +//! upstream provided in `messageText`. +//! +//! Callers that need the raw Composio shape can pass `raw_html: +//! true` (or `rawHtml: true`) in the action arguments — this +//! short-circuits the reshape entirely. +//! +//! Only `GMAIL_FETCH_EMAILS` is reshaped today; other Gmail action +//! responses are passed through unchanged. When we add envelopes for +//! more slugs they should live in this file, branched from +//! [`post_process`]. + +use serde_json::{json, Map, Value}; + +/// Entry point called from `GmailProvider::post_process_action_result`. +/// +/// Dispatches on the Composio action slug. Unknown Gmail slugs fall +/// through to a no-op. +pub fn post_process(slug: &str, arguments: Option<&Value>, data: &mut Value) { + if is_raw_html_flag_set(arguments) { + tracing::debug!( + slug, + "[composio:gmail][post-process] raw_html flag set, passing through" + ); + return; + } + if slug == "GMAIL_FETCH_EMAILS" { + reshape_fetch_emails(data) + } +} + +/// Stash per-message slices of the response-level `markdownFormatted` +/// onto the corresponding entries inside `data.messages[]`. +/// +/// The Composio backend (tinyhumansai/backend#683) ships ONE +/// `markdownFormatted` string per tool call covering all messages — +/// already URL-shortened, footer-stripped, and whitespace-normalised. +/// To get per-email files in the raw archive we split that string +/// along section boundaries (`## ` headings or `---` rules) and pin +/// each slice to the message at the same index. `extract_markdown_body` +/// then prefers `msg.markdownFormatted` over re-decoding the MIME +/// tree. +/// +/// **Must be called BEFORE [`post_process`]** because `post_process` +/// reshapes `data` into the slim envelope; once `messages[]` carries +/// our slim shape the upstream message ordering is already locked in +/// but we may have lost original ordering signals if any. +/// +/// No-op when the slice count doesn't match `messages.len()` — we +/// can't safely align segments to messages without an exact match, +/// so we let `extract_markdown_body` fall through to its MIME path. +pub fn apply_response_level_markdown(data: &mut Value, top_md: &str) { + let trimmed = top_md.trim(); + if trimmed.is_empty() { + return; + } + // Presence is checked immutably first, then fetched mutably. The original + // form re-fetched with `unwrap()` after a mutable probe, which is sound but + // relies on the reader to see why; this crate forbids `unwrap`, and the + // immutable probe expresses the same reasoning to the compiler. + let container = if data.get("messages").is_some() { + data + } else if data.get("data").and_then(Value::as_object).is_some() { + match data.get_mut("data") { + Some(inner) => inner, + None => return, + } + } else { + tracing::debug!( + "[composio:gmail][post-process] apply_response_level_markdown: \ + no messages container in response — skipping" + ); + return; + }; + let Some(messages) = container.get_mut("messages").and_then(|v| v.as_array_mut()) else { + return; + }; + let count = messages.len(); + if count == 0 { + return; + } + // Clone hints out of the messages array so the slice borrows + // don't conflict with the upcoming `messages.iter_mut()` mutation. + let hints: Vec = messages.clone(); + let Some(slices) = split_response_markdown_per_message_with_hint(trimmed, count, Some(&hints)) + else { + tracing::debug!( + messages = count, + md_len = trimmed.len(), + "[composio:gmail][post-process] could not split response-level markdownFormatted \ + into {count} slices — falling back to per-message MIME decode" + ); + return; + }; + for (msg, slice) in messages.iter_mut().zip(slices) { + if let Some(obj) = msg.as_object_mut() { + obj.insert("markdownFormatted".to_string(), Value::String(slice)); + } + } + tracing::debug!( + messages = count, + "[composio:gmail][post-process] stashed per-message markdownFormatted slices" + ); +} + +/// Split a top-level `markdownFormatted` string into per-message +/// segments. Returns `Some(slices)` only when the split yields +/// exactly `expected_count` entries — otherwise the format isn't one +/// of the patterns we know about and we let the caller fall back. +/// +/// Primary boundary is the `\n---\n` horizontal rule the backend +/// emits between messages (confirmed against real +/// `GMAIL_FETCH_EMAILS` output). H2/H3 headings are kept as +/// fallbacks for older renderings. The preamble (`# Inbox (N +/// messages)`-style intro, if present) is dropped — we accept +/// either `expected` parts (no preamble) or `expected + 1` +/// (preamble + N messages). +/// +/// `messages_hint` is the slim message array from the same response +/// — when present we use the per-message `subject` field to verify +/// each segment really does belong to the message at the same index. +/// Mismatches force a fallback so we never write a wrong-message body +/// to the raw archive. +pub fn split_response_markdown_per_message(md: &str, expected_count: usize) -> Option> { + split_response_markdown_per_message_with_hint(md, expected_count, None) +} + +/// Split a response-level markdown blob into one slice per message. +/// +/// `hint` carries the message ids in response order, which is what makes the +/// split reliable: the blob's own section headings are backend-rendered and +/// have changed shape between versions, so matching on them alone silently +/// mis-attributed bodies. +pub fn split_response_markdown_per_message_with_hint( + md: &str, + expected_count: usize, + messages_hint: Option<&[Value]>, +) -> Option> { + if expected_count == 0 { + return None; + } + if expected_count == 1 { + return Some(vec![md.to_string()]); + } + + // Boundary patterns to try, in priority order. `\n---\n` is the + // confirmed marker; the heading variants stay as belt-and-braces + // for older / variant backend renderings. + let candidates: &[(&str, &str)] = &[ + ("\n---\n", "---\n"), + ("\n\n## ", "## "), + ("\n\n### ", "### "), + ("\n\n# ", "# "), + ("\n***\n", "***\n"), + ]; + + for (sep, prefix) in candidates { + let parts: Vec<&str> = md.split(sep).collect(); + let (drop_preamble, prepend_first) = if parts.len() == expected_count { + (false, false) // no preamble; first segment had no prefix + } else if parts.len() == expected_count + 1 { + (true, true) // preamble dropped; every kept segment had a prefix + } else { + continue; + }; + let segments: Vec = parts + .into_iter() + .skip(if drop_preamble { 1 } else { 0 }) + .enumerate() + .map(|(i, s)| { + if i == 0 && !prepend_first { + s.to_string() + } else { + format!("{prefix}{s}") + } + }) + .collect(); + + // Validate alignment against the JSON message array: every + // segment whose corresponding message has a non-empty subject + // must mention that subject somewhere in its body. If a single + // pair fails, we treat the split as unreliable and try the + // next pattern. Empty / null subjects skip validation (e.g. + // notification mails where the subject is ""). + if let Some(hints) = messages_hint { + if !validate_segments_against_hints(&segments, hints) { + tracing::debug!( + expected = expected_count, + sep = sep, + "[composio:gmail][post-process] split candidate failed subject check" + ); + continue; + } + } + return Some(segments); + } + None +} + +/// True if every (segment, message) pair where the message has a +/// non-empty subject contains that subject somewhere in the segment +/// (case-insensitive substring match — a defensive heuristic, not a +/// strict equality check, since the backend may format subjects +/// inside markdown links or with surrounding decoration). +fn validate_segments_against_hints(segments: &[String], hints: &[Value]) -> bool { + if segments.len() != hints.len() { + return false; + } + for (seg, hint) in segments.iter().zip(hints.iter()) { + let subject = hint + .get("subject") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim(); + if subject.is_empty() { + continue; + } + if !seg + .to_ascii_lowercase() + .contains(&subject.to_ascii_lowercase()) + { + return false; + } + } + true +} + +/// Returns true when the caller explicitly set `raw_html: true` (or the +/// camelCase `rawHtml: true`) in the `arguments` object. +fn is_raw_html_flag_set(arguments: Option<&Value>) -> bool { + let Some(obj) = arguments.and_then(|v| v.as_object()) else { + return false; + }; + obj.get("raw_html") + .or_else(|| obj.get("rawHtml")) + .and_then(|v| v.as_bool()) + .unwrap_or(false) +} + +/// Rewrite a `GMAIL_FETCH_EMAILS` `data` object in place into the slim +/// envelope documented at the module level. +/// +/// The Composio response can be shaped either as `{ messages, nextPageToken, ... }` +/// directly, or wrapped one level deeper under `{ data: { messages: … } }` +/// depending on backend version; we handle both. +fn reshape_fetch_emails(data: &mut Value) { + // Unwrap an optional `data:` envelope so downstream logic only has + // to deal with one shape. + let container = if data.get("messages").is_some() { + data + } else if data.get("data").and_then(Value::as_object).is_some() { + match data.get_mut("data") { + Some(inner) => inner, + None => return, + } + } else { + return; + }; + + let Some(obj) = container.as_object_mut() else { + return; + }; + + let raw_messages = obj + .remove("messages") + .and_then(|v| match v { + Value::Array(arr) => Some(arr), + _ => None, + }) + .unwrap_or_default(); + let next_page_token = obj.remove("nextPageToken").unwrap_or(Value::Null); + let result_size_estimate = obj.remove("resultSizeEstimate").unwrap_or(Value::Null); + + let messages: Vec = raw_messages.into_iter().map(reshape_message).collect(); + + let mut envelope = Map::new(); + envelope.insert("messages".into(), Value::Array(messages)); + if !next_page_token.is_null() { + envelope.insert("nextPageToken".into(), next_page_token); + } + if !result_size_estimate.is_null() { + envelope.insert("resultSizeEstimate".into(), result_size_estimate); + } + + *container = Value::Object(envelope); +} + +/// Parse an RFC 3339 or RFC 2822 date string into a UTC `DateTime`. +pub fn parse_email_date(date_str: &str) -> Option> { + date_str + .parse::>() + .or_else(|_| { + chrono::DateTime::parse_from_rfc2822(date_str).map(|d| d.with_timezone(&chrono::Utc)) + }) + .ok() +} + +const EMAIL_LOCAL_TIME_FMT: &str = "%Y-%m-%d %I:%M %p %:z"; + +/// Format a UTC `DateTime` in the given timezone. Returns `None` when the +/// formatted result is identical to the UTC rendering (no-op for UTC hosts). +pub fn format_at_tz( + utc: chrono::DateTime, + tz: &Tz, +) -> Option +where + Tz::Offset: std::fmt::Display, +{ + let local_dt = utc.with_timezone(tz); + let formatted = local_dt.format(EMAIL_LOCAL_TIME_FMT).to_string(); + + let utc_formatted = utc.format(EMAIL_LOCAL_TIME_FMT).to_string(); + if formatted == utc_formatted { + return None; + } + Some(formatted) +} + +/// Convert a UTC email timestamp string to a human-readable local-time string. +/// +/// Accepts RFC 3339 (`"2026-05-31T10:33:00Z"`) or RFC 2822 +/// (`"Sat, 31 May 2026 10:33:00 +0000"`) input. Returns a formatted string +/// in the host's local timezone, e.g. `"2026-05-31 05:33 AM -05:00"`, +/// so the agent can present local times without UTC arithmetic. +/// +/// The raw `date` field is always preserved alongside this field so +/// internal sorting, deduplication, and debugging remain UTC-based. +/// +/// Returns `None` when the input cannot be parsed or the output format +/// would be identical to the UTC input (no-op for UTC hosts). +pub fn format_email_local_time(date_str: &str) -> Option { + let utc = parse_email_date(date_str)?; + format_at_tz(utc, &chrono::Local) +} + +/// Map one raw Composio message object to its slim counterpart. +/// +/// Body source picked by [`extract_markdown_body`]: +/// 1. The per-message `markdownFormatted` slice pinned by +/// [`apply_response_level_markdown`] (preferred — backend-rendered). +/// 2. The upstream `messageText` plaintext (fallback). +/// 3. Empty string. +fn reshape_message(raw: Value) -> Value { + let Value::Object(obj) = raw else { + return raw; + }; + + let id = obj.get("messageId").cloned().unwrap_or(Value::Null); + let thread_id = obj.get("threadId").cloned().unwrap_or(Value::Null); + let subject = obj.get("subject").cloned().unwrap_or(Value::Null); + let sender = obj.get("sender").cloned().unwrap_or(Value::Null); + let to = obj.get("to").cloned().unwrap_or(Value::Null); + let date = obj + .get("messageTimestamp") + .cloned() + .or_else(|| pick_header(&obj, "Date")) + .unwrap_or(Value::Null); + let labels = obj + .get("labelIds") + .cloned() + .unwrap_or_else(|| Value::Array(Vec::new())); + let list_unsubscribe = pick_header(&obj, "List-Unsubscribe").unwrap_or(Value::Null); + + let markdown = extract_markdown_body(&obj); + let attachments = extract_attachments(&obj); + + // Compute a local-time representation of the UTC `date` so the agent + // presents times in the user's timezone rather than quoting raw UTC. + let date_local = date.as_str().and_then(format_email_local_time); + + let mut out = Map::new(); + out.insert("id".into(), id); + out.insert("threadId".into(), thread_id); + out.insert("subject".into(), subject); + out.insert("from".into(), sender); + out.insert("to".into(), to); + out.insert("date".into(), date); + if let Some(local) = date_local { + out.insert("date_local".into(), Value::String(local)); + } + out.insert("labels".into(), labels); + if !list_unsubscribe.is_null() { + out.insert("list_unsubscribe".into(), list_unsubscribe); + } + out.insert("markdown".into(), Value::String(markdown)); + if !attachments.is_empty() { + out.insert("attachments".into(), Value::Array(attachments)); + } + Value::Object(out) +} + +/// Find a header value by (case-insensitive) name in the Composio +/// `payload.headers[]` array. Returns `Some(Value::String)` on hit. +fn pick_header(msg: &Map, name: &str) -> Option { + let headers = msg.get("payload")?.get("headers")?.as_array()?; + for h in headers { + let hn = h.get("name").and_then(|v| v.as_str()).unwrap_or(""); + if hn.eq_ignore_ascii_case(name) { + if let Some(v) = h.get("value").and_then(|v| v.as_str()) { + return Some(Value::String(v.to_string())); + } + } + } + None +} + +/// Pick a body for the slim envelope. +/// +/// We trust the Composio backend's pre-rendered `markdownFormatted` +/// (set per-message by [`apply_response_level_markdown`] from the +/// response-level field). When that's absent we fall back to the +/// upstream's plain-text `messageText` verbatim — no in-house +/// HTML→markdown decoding lives here anymore. The backend already +/// strips HTML, shortens URLs, and normalises whitespace; running +/// our own pipeline on top duplicated work and corrupted some +/// renderings. +fn extract_markdown_body(msg: &Map) -> String { + if let Some(formatted) = msg + .get("markdownFormatted") + .or_else(|| msg.get("markdown_formatted")) + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + { + return formatted.to_string(); + } + if let Some(text) = msg + .get("messageText") + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + { + return text.to_string(); + } + String::new() +} + +/// Pull a minimal attachments descriptor from the Composio +/// `attachmentList` array. +fn extract_attachments(msg: &Map) -> Vec { + if let Some(list) = msg.get("attachmentList").and_then(|v| v.as_array()) { + return list + .iter() + .filter_map(|a| { + let filename = a.get("filename").and_then(|v| v.as_str())?; + if filename.is_empty() { + return None; + } + let mime = a + .get("mimeType") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + Some(json!({ "filename": filename, "mimeType": mime })) + }) + .collect(); + } + Vec::new() +} + +#[cfg(test)] +#[path = "gmail_post_process_tests.rs"] +mod tests; diff --git a/sync/src/gmail_post_process_tests.rs b/sync/src/gmail_post_process_tests.rs new file mode 100644 index 0000000..69eda63 --- /dev/null +++ b/sync/src/gmail_post_process_tests.rs @@ -0,0 +1,359 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +fn fixture_with_backend_markdown() -> Value { + json!({ + "messages": [ + { + "messageId": "m1", + "threadId": "t1", + "subject": "Hello", + "sender": "a@x.com", + "to": "b@y.com", + "messageTimestamp": "2026-04-17T12:00:00Z", + "labelIds": ["INBOX", "UNREAD"], + // Pre-rendered slice (set by `apply_response_level_markdown` + // in production; inline here for the reshape test). + "markdownFormatted": "# Hello\n\nbody copy", + "messageText": "fallback should not be used", + "display_url": "ignore-me", + "preview": { "body": "Hi plain", "subject": "Hello" }, + "attachmentList": [ + { "filename": "report.pdf", "mimeType": "application/pdf", "size": 12345 }, + { "filename": "", "mimeType": "text/html" } + ], + "payload": {} + } + ], + "nextPageToken": "tok-1", + "resultSizeEstimate": 42 + }) +} + +#[test] +fn reshape_emits_slim_envelope() { + let mut v = fixture_with_backend_markdown(); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + + assert_eq!(v["nextPageToken"], "tok-1"); + assert_eq!(v["resultSizeEstimate"], 42); + + let msgs = v["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + let m = &msgs[0]; + + assert_eq!(m["id"], "m1"); + assert_eq!(m["threadId"], "t1"); + assert_eq!(m["subject"], "Hello"); + assert_eq!(m["from"], "a@x.com"); + assert_eq!(m["to"], "b@y.com"); + assert_eq!(m["date"], "2026-04-17T12:00:00Z"); + assert_eq!(m["labels"], json!(["INBOX", "UNREAD"])); + + let md = m["markdown"].as_str().unwrap(); + assert_eq!(md, "# Hello\n\nbody copy"); + + // Noise fields removed. + assert!(m.get("display_url").is_none()); + assert!(m.get("preview").is_none()); + assert!(m.get("payload").is_none()); + assert!(m.get("messageText").is_none()); + + // Attachments: empty filename entry is filtered. + let atts = m["attachments"].as_array().unwrap(); + assert_eq!(atts.len(), 1); + assert_eq!(atts[0]["filename"], "report.pdf"); + assert_eq!(atts[0]["mimeType"], "application/pdf"); +} + +#[test] +fn raw_html_flag_passes_through_unchanged() { + let mut v = fixture_with_backend_markdown(); + let original = v.clone(); + let args = json!({ "raw_html": true }); + post_process("GMAIL_FETCH_EMAILS", Some(&args), &mut v); + assert_eq!( + v, original, + "raw_html=true must preserve the Composio shape" + ); +} + +#[test] +fn camel_case_raw_html_also_recognized() { + let mut v = fixture_with_backend_markdown(); + let original = v.clone(); + let args = json!({ "rawHtml": true }); + post_process("GMAIL_FETCH_EMAILS", Some(&args), &mut v); + assert_eq!(v, original); +} + +#[test] +fn falls_back_to_message_text_when_no_backend_markdown() { + let mut v = json!({ + "messages": [{ + "messageId": "m1", + "threadId": "t1", + "subject": "s", + "sender": "a@x.com", + "to": "b@y.com", + "messageTimestamp": "2026-04-17", + "labelIds": [], + "messageText": " plain body text ", + "payload": {} + }], + "nextPageToken": null + }); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + let md = v["messages"][0]["markdown"].as_str().unwrap(); + assert_eq!(md, "plain body text"); + assert!(v.get("nextPageToken").is_none(), "null tokens dropped"); +} + +#[test] +fn unwraps_data_envelope() { + let mut v = json!({ + "data": { + "messages": [{ + "messageId": "m1", + "threadId": "t1", + "subject": "s", + "sender": "a@x.com", + "to": "b@y.com", + "messageTimestamp": "2026-04-17", + "labelIds": [], + "messageText": "body", + "payload": {} + }] + } + }); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + // Reshape writes into `data` in place. + let msgs = v["data"]["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["markdown"], "body"); +} + +#[test] +fn non_fetch_slug_is_noop() { + let mut v = json!({ "messages": [{ "messageId": "m1", "messageText": "x" }] }); + let original = v.clone(); + post_process("GMAIL_SEND_EMAIL", None, &mut v); + assert_eq!(v, original); +} + +#[test] +fn prefers_backend_markdown_formatted_when_present() { + // Composio backend (tinyhumansai/backend#683 +) ships + // `markdownFormatted` already URL-shortened + footer-stripped + // per message (after `apply_response_level_markdown` slices the + // response-level field). When present, our post-processor must + // use it verbatim instead of falling back to `messageText`. + let mut v = json!({ + "messages": [{ + "messageId": "m1", + "threadId": "t1", + "subject": "s", + "sender": "a@x.com", + "to": "b@y.com", + "messageTimestamp": "2026-04-17", + "labelIds": [], + "markdownFormatted": "# Already nice\n\nShort URL: https://gh.io/abc", + "messageText": "fallback should not be used", + "payload": {} + }] + }); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + let md = v["messages"][0]["markdown"].as_str().unwrap(); + assert_eq!(md, "# Already nice\n\nShort URL: https://gh.io/abc"); +} + +#[test] +fn empty_markdown_formatted_falls_through_to_message_text() { + let mut v = json!({ + "messages": [{ + "messageId": "m1", + "threadId": "t1", + "subject": "s", + "sender": "a@x.com", + "to": "b@y.com", + "messageTimestamp": "2026-04-17", + "labelIds": [], + "markdownFormatted": " \n \n", + "messageText": "real body", + "payload": {} + }] + }); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + let md = v["messages"][0]["markdown"].as_str().unwrap(); + assert!(md.contains("real body")); +} + +// ── split_response_markdown_per_message ───────────────────────────────── + +#[test] +fn split_response_markdown_uses_horizontal_rule_marker() { + // The confirmed backend marker is `\n---\n`. Three messages → + // expect three slices when there's no preamble. + let md = "## Alice's update\n\nbody A with https://gh.io/abc\n---\n## Bob's reply\n\nbody B\n---\n## Carol\n\nbody C"; + let slices = super::split_response_markdown_per_message(md, 3).unwrap(); + assert_eq!(slices.len(), 3); + assert!(slices[0].contains("Alice's update")); + assert!(slices[1].contains("Bob's reply")); + assert!(slices[2].contains("Carol")); + // The `---\n` prefix is preserved on every-but-the-first segment + // so the section break survives the round-trip. + assert!(slices[1].starts_with("---\n")); + assert!(slices[2].starts_with("---\n")); +} + +#[test] +fn split_response_markdown_drops_preamble() { + // When a preamble like `# Inbox` precedes the first marker, we + // see N+1 parts after split — the preamble must be dropped. + let md = "# Inbox (2 messages)\n---\n## A\n\nbody A\n---\n## B\n\nbody B"; + let slices = super::split_response_markdown_per_message(md, 2).unwrap(); + assert_eq!(slices.len(), 2); + assert!(slices[0].contains("body A")); + assert!(slices[1].contains("body B")); + // Both segments should carry the prefix when preamble was dropped. + assert!(slices[0].starts_with("---\n")); + assert!(slices[1].starts_with("---\n")); +} + +#[test] +fn split_response_markdown_falls_back_to_h2_marker() { + // No `---` rules — backend used h2 headings as boundaries. + let md = "## Alice\n\nbody A\n\n## Bob\n\nbody B"; + let slices = super::split_response_markdown_per_message(md, 2).unwrap(); + assert_eq!(slices.len(), 2); + assert!(slices[0].contains("body A")); + assert!(slices[1].contains("body B")); +} + +#[test] +fn split_response_markdown_returns_none_on_count_mismatch() { + let md = "## only one section here"; + assert!(super::split_response_markdown_per_message(md, 3).is_none()); +} + +#[test] +fn split_response_markdown_single_message_returns_whole_input() { + let md = "## solo\n\nthe whole body"; + let slices = super::split_response_markdown_per_message(md, 1).unwrap(); + assert_eq!(slices, vec![md.to_string()]); +} + +#[test] +fn split_with_hint_rejects_when_subjects_dont_match() { + let md = "## Foo\nbody1\n---\n## Bar\nbody2"; + let hints = vec![ + json!({"subject": "Completely different subject A"}), + json!({"subject": "Completely different subject B"}), + ]; + let out = super::split_response_markdown_per_message_with_hint(md, 2, Some(&hints)); + assert!(out.is_none(), "subject mismatch must force fallback"); +} + +#[test] +fn split_with_hint_accepts_when_subjects_match() { + let md = "## Welcome to Gmail\nbody1\n---\n## Your invoice\nbody2"; + let hints = vec![ + json!({"subject": "Welcome to Gmail"}), + json!({"subject": "Your invoice"}), + ]; + let slices = super::split_response_markdown_per_message_with_hint(md, 2, Some(&hints)).unwrap(); + assert_eq!(slices.len(), 2); + assert!(slices[0].contains("Welcome to Gmail")); + assert!(slices[1].contains("Your invoice")); +} + +#[test] +fn split_with_hint_skips_messages_with_blank_subject() { + let md = "## A\nbody1\n---\n## B\nbody2"; + let hints = vec![json!({"subject": "A"}), json!({"subject": ""})]; + let slices = super::split_response_markdown_per_message_with_hint(md, 2, Some(&hints)).unwrap(); + assert_eq!(slices.len(), 2); +} + +// ── format_email_local_time ────────────────────────────────────────────────── + +#[test] +fn format_email_local_time_returns_none_for_unparseable_date() { + assert!(super::format_email_local_time("not-a-date").is_none()); + assert!(super::format_email_local_time("").is_none()); +} + +#[test] +fn format_email_local_time_preserves_utc_raw_date_in_reshape() { + let mut v = json!({ + "messages": [{ + "messageId": "m1", + "threadId": "t1", + "subject": "Test", + "sender": "a@example.com", + "to": "b@example.com", + "messageTimestamp": "2026-05-31T10:33:00Z", + "labelIds": [], + "messageText": "body", + "payload": {} + }] + }); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + let msg = &v["messages"][0]; + assert_eq!(msg["date"], "2026-05-31T10:33:00Z"); +} + +#[test] +fn parse_email_date_accepts_rfc3339_and_rfc2822() { + assert!(super::parse_email_date("2026-05-31T10:33:00Z").is_some()); + assert!(super::parse_email_date("Sun, 31 May 2026 10:33:00 +0000").is_some()); + assert!(super::parse_email_date("not-a-date").is_none()); +} + +#[test] +fn format_at_tz_deterministic_with_fixed_offset() { + use chrono::FixedOffset; + + let utc = super::parse_email_date("2026-05-31T10:33:00Z").unwrap(); + + let est = FixedOffset::west_opt(5 * 3600).unwrap(); + let result = super::format_at_tz(utc, &est).unwrap(); + assert_eq!(result, "2026-05-31 05:33 AM -05:00"); + + let ist = FixedOffset::east_opt(5 * 3600 + 1800).unwrap(); + let result = super::format_at_tz(utc, &ist).unwrap(); + assert_eq!(result, "2026-05-31 04:03 PM +05:30"); +} + +#[test] +fn format_at_tz_returns_none_for_utc() { + let utc = super::parse_email_date("2026-05-31T10:33:00Z").unwrap(); + let utc_tz = chrono::FixedOffset::east_opt(0).unwrap(); + assert!(super::format_at_tz(utc, &utc_tz).is_none()); +} + +#[test] +fn apply_response_level_markdown_stashes_per_message_field() { + let mut data = json!({ + "messages": [ + {"messageId": "m1", "subject": "Hello"}, + {"messageId": "m2", "subject": "World"}, + ] + }); + let top_md = "## Hello\nbody A — link https://gh.io/abc\n---\n## World\nbody B"; + super::apply_response_level_markdown(&mut data, top_md); + let m1 = data["messages"][0]["markdownFormatted"].as_str().unwrap(); + let m2 = data["messages"][1]["markdownFormatted"].as_str().unwrap(); + assert!(m1.contains("Hello")); + assert!( + m1.contains("https://gh.io/abc"), + "shortened URL must survive" + ); + assert!(m2.contains("World")); + assert!(!m1.contains("World"), "no cross-message bleed"); +} diff --git a/sync/src/helpers.rs b/sync/src/helpers.rs new file mode 100644 index 0000000..101239e --- /dev/null +++ b/sync/src/helpers.rs @@ -0,0 +1,50 @@ +//! Shared helpers for the provider normalisers in this module. + +/// Walk a JSON object using a list of dotted-path candidates and return the +/// first non-empty **string** match. +/// +/// # This is deliberately NOT `super::super::common::pick_str` +/// +/// The crate carries two `pick_str` functions with the same name and +/// genuinely different behaviour. Do not "deduplicate" them: +/// +/// | | this one (`normalize::helpers`) | `common::pick_str` | +/// |---|---|---| +/// | traversal | `Value::get` per `.`-separated segment — objects only | `Value::pointer` — also indexes into arrays | +/// | non-string leaf | rejected, returns `None` | `Number` is coerced via `to_string()` | +/// +/// The number case is the one that bites. A payload whose `id` is `42` +/// rather than `"42"` yields `None` here and `Some("42")` there, which +/// silently changes what a normaliser emits as a document id. The callers of +/// this function were written against the reject-non-strings behaviour and +/// have a test pinning it (`pick_str_rejects_non_string_values` below, and +/// the host-side mirror of it). +pub fn pick_str(value: &serde_json::Value, paths: &[&str]) -> Option { + for path in paths { + let mut cur = value; + let mut ok = true; + for segment in path.split('.') { + match cur.get(segment) { + Some(next) => cur = next, + None => { + ok = false; + break; + } + } + } + if !ok { + continue; + } + if let Some(s) = cur.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + +#[cfg(test)] +#[path = "helpers_tests.rs"] +mod tests; diff --git a/sync/src/helpers_tests.rs b/sync/src/helpers_tests.rs new file mode 100644 index 0000000..b6410b8 --- /dev/null +++ b/sync/src/helpers_tests.rs @@ -0,0 +1,40 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +#[test] +fn pick_str_finds_first_non_empty_match() { + let v = json!({"data": {"user": {"name": "Ada", "email": "ada@example.com"}}}); + assert_eq!( + pick_str(&v, &["data.user.name", "data.user.email"]), + Some("Ada".into()) + ); + assert_eq!( + pick_str(&v, &["data.missing", "data.user.email"]), + Some("ada@example.com".into()) + ); + assert_eq!(pick_str(&v, &["nope.nope"]), None); +} + +#[test] +fn pick_str_respects_path_order() { + let v = json!({"a": "first", "b": "second"}); + assert_eq!(pick_str(&v, &["a", "b"]), Some("first".into())); + assert_eq!(pick_str(&v, &["b", "a"]), Some("second".into())); +} + +/// The drift guard for the divergence documented on [`pick_str`]. If this +/// ever starts returning `Some("42")`, someone has re-pointed the +/// normalisers at `common::pick_str` and changed their output. +#[test] +fn pick_str_rejects_non_string_values() { + let v = json!({"count": 42, "flag": true, "empty": "", "whitespace": " "}); + assert_eq!(pick_str(&v, &["count"]), None); + assert_eq!(pick_str(&v, &["flag"]), None); + assert_eq!(pick_str(&v, &["empty"]), None); + assert_eq!(pick_str(&v, &["whitespace"]), None); +} diff --git a/sync/src/lib.rs b/sync/src/lib.rs new file mode 100644 index 0000000..4eb3ba6 --- /dev/null +++ b/sync/src/lib.rs @@ -0,0 +1,38 @@ +//! Composio provider payload normalisers, engine-neutral by construction. +//! +//! Issue #18 §B3: "Payload normalisers … are pure `Value → Value` transforms +//! with no engine dependency. Move them back into a `tinymemory-sync` crate … +//! so a non-TinyCortex engine gets Composio sync for free." +//! +//! They lived inside the TinyCortex engine, and `tinymemory-core` reached into +//! it to use them — which meant a host binding a *different* memory engine +//! could not have Composio sync at all, despite none of this code caring which +//! engine is bound. Nothing here reads a database, opens a socket, or names an +//! engine type; the dependency list is `serde_json`, two logging facades, and +//! `chrono`. +//! +//! One caveat on "pure", because it is load-bearing and easy to miss. +//! [`gmail_post_process::format_email_local_time`] renders in `chrono::Local`, +//! so it reads the host's timezone — every other normaliser here is a function +//! of its input alone. The raw UTC field is preserved alongside it, so sorting +//! and deduplication stay UTC-based; what varies by host is only the +//! presentation string. +//! +//! These are pure `serde_json::Value` → `Value` transforms: given a raw +//! Composio action response, pull out the fields that make up a task, an +//! issue, a page or a message. They hold no credentials, touch no network, +//! and make no scheduling decisions — provider-specific normalisation is +//! driver-side by definition (see the host's `docs/specs/kernel.md` §4). +//! + +pub mod clickup; +pub mod github; +pub mod helpers; +pub mod linear; +pub mod notion; + +// The `_post_process` suffix is kept from the engine layout it came from, where +// `slack.rs` and `github.rs` one directory up already held those names. Renaming +// on the way out would have made this a rename *and* a move in one diff. +pub mod gmail_post_process; +pub mod slack_post_process; diff --git a/sync/src/linear.rs b/sync/src/linear.rs new file mode 100644 index 0000000..843f98a --- /dev/null +++ b/sync/src/linear.rs @@ -0,0 +1,157 @@ +//! Linear host normalization helpers — result extraction, issue-title extraction, +//! viewer identity, cursor extraction, and time utilities. +//! +//! Linear's GraphQL API (and therefore Composio's wrapping of it) returns +//! connection-style lists (`{ nodes: [...], pageInfo: {...} }`) at the top +//! level or nested under `data`. The functions here walk the union of +//! common shapes so the provider does not have to branch per Composio +//! envelope variant. + +use serde_json::Value; + +use super::helpers::pick_str; + +/// Walk the Composio response envelope for Linear issue list results. +/// +/// Linear's list endpoints return `{ nodes: [...] }` or +/// `{ issues: { nodes: [...] } }` shapes; Composio may re-wrap the +/// upstream payload under `data` or `data.data`. We probe each shape +/// in order and return the first array we find. +pub fn extract_issues(data: &Value) -> Vec { + let candidates = [ + data.pointer("/data/nodes"), + data.pointer("/nodes"), + data.pointer("/data/issues/nodes"), + data.pointer("/issues/nodes"), + data.pointer("/data/data/nodes"), + data.pointer("/data/data/issues/nodes"), + data.pointer("/data/results"), + data.pointer("/results"), + data.pointer("/data/items"), + data.pointer("/items"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + return arr.clone(); + } + } + Vec::new() +} + +/// Extract a human-readable title from a Linear issue object. +/// +/// Linear issues store the name at `title` (or `data.title` after +/// Composio envelope wrapping). Falls back to `name` / `identifier` +/// so the chunk remains identifiable even for unusual response shapes. +pub fn extract_issue_title(issue: &Value) -> Option { + pick_str( + issue, + &[ + "title", + "data.title", + "name", + "data.name", + "identifier", + "data.identifier", + ], + ) +} + +/// Extract a stable cursor timestamp from a Linear issue object. +/// +/// Linear uses ISO-8601 strings for timestamps (`updatedAt`). We keep +/// the value as a string so lexicographic comparison against the stored +/// cursor is valid. +pub fn extract_issue_updated(issue: &Value) -> Option { + pick_str( + issue, + &[ + "updatedAt", + "data.updatedAt", + "updated_at", + "data.updated_at", + ], + ) +} + +/// Extract the viewer (authenticated user) object from a +/// `LINEAR_LIST_LINEAR_USERS { isMe: true }` response. +/// +/// Linear's GraphQL viewer endpoint returns `{ nodes: [{ id, email, … }] }`. +/// Composio may wrap this under `data` or `data.data`. We probe each +/// shape and return the first element of the nodes array, falling back +/// to the payload itself if it looks like a direct user object (has +/// `id` or `email`). +pub fn extract_viewer(data: &Value) -> Option { + let array_candidates = [ + data.pointer("/data/nodes"), + data.pointer("/nodes"), + data.pointer("/data/data/nodes"), + data.pointer("/data/users/nodes"), + ]; + for cand in array_candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + if let Some(first) = arr.first() { + return Some(first.clone()); + } + } + } + // Fallback: if the payload itself looks like a user object, return it. + if data.get("id").is_some() || data.get("email").is_some() { + return Some(data.clone()); + } + None +} + +/// Extract the viewer's ID string from a `LINEAR_LIST_LINEAR_USERS` +/// response. Returns `None` if the payload does not contain a +/// recognizable user ID. +pub fn extract_viewer_id(data: &Value) -> Option { + let viewer = extract_viewer(data)?; + pick_str(&viewer, &["id", "data.id"]) +} + +/// Extract a pagination cursor from a Linear connection `pageInfo` block. +/// +/// Returns `Some(endCursor)` only when `hasNextPage` is `true`; +/// `None` when the last page has been reached or when the envelope does +/// not carry `pageInfo` at all. +pub fn extract_pagination_cursor(data: &Value) -> Option { + // Mirrors the `extract_issues` envelope shapes, so every shape that can + // carry a node list can also carry its `pageInfo` cursor. + let page_info_candidates = [ + data.pointer("/data/pageInfo"), + data.pointer("/pageInfo"), + data.pointer("/data/data/pageInfo"), + data.pointer("/data/issues/pageInfo"), + data.pointer("/data/data/issues/pageInfo"), + ]; + for cand in page_info_candidates.into_iter().flatten() { + let has_next = cand + .get("hasNextPage") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + if has_next { + if let Some(cursor) = cand.get("endCursor").and_then(|v| v.as_str()) { + let trimmed = cursor.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + } + None +} + +/// Current wall-clock time in milliseconds since the UNIX epoch. +pub fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[cfg(test)] +#[path = "linear_tests.rs"] +mod tests; diff --git a/sync/src/linear_tests.rs b/sync/src/linear_tests.rs new file mode 100644 index 0000000..78b5bf1 --- /dev/null +++ b/sync/src/linear_tests.rs @@ -0,0 +1,189 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +// ── extract_issues ─────────────────────────────────────────────── + +#[test] +fn extract_issues_from_data_nodes() { + let data = json!({ "data": { "nodes": [{"id": "i1"}, {"id": "i2"}] } }); + assert_eq!(extract_issues(&data).len(), 2); +} + +#[test] +fn extract_issues_from_top_level_nodes() { + let data = json!({ "nodes": [{"id": "i3"}] }); + assert_eq!(extract_issues(&data).len(), 1); +} + +#[test] +fn extract_issues_from_data_issues_nodes() { + let data = + json!({ "data": { "issues": { "nodes": [{"id": "i4"}, {"id": "i5"}, {"id": "i6"}] } } }); + assert_eq!(extract_issues(&data).len(), 3); +} + +#[test] +fn extract_issues_from_top_level_issues_nodes() { + let data = json!({ "issues": { "nodes": [{"id": "i7"}] } }); + assert_eq!(extract_issues(&data).len(), 1); +} + +#[test] +fn extract_issues_from_doubly_nested_issues_nodes() { + let data = + json!({ "data": { "data": { "issues": { "nodes": [{"id": "i8"}, {"id": "i9"}] } } } }); + assert_eq!(extract_issues(&data).len(), 2); +} + +#[test] +fn extract_issues_from_results() { + let data = json!({ "results": [{"id": "i7"}] }); + assert_eq!(extract_issues(&data).len(), 1); +} + +#[test] +fn extract_issues_empty_when_missing() { + let data = json!({ "foo": "bar" }); + assert!(extract_issues(&data).is_empty()); +} + +// ── extract_issue_title ────────────────────────────────────────── + +#[test] +fn extract_issue_title_from_title_field() { + let issue = json!({ "id": "i1", "title": "Fix the login bug" }); + assert_eq!( + extract_issue_title(&issue), + Some("Fix the login bug".into()) + ); +} + +#[test] +fn extract_issue_title_falls_back_to_wrapped_data() { + let issue = json!({ "data": { "title": "Wrapped issue" } }); + assert_eq!(extract_issue_title(&issue), Some("Wrapped issue".into())); +} + +#[test] +fn extract_issue_title_falls_back_to_identifier() { + let issue = json!({ "identifier": "ENG-42" }); + assert_eq!(extract_issue_title(&issue), Some("ENG-42".into())); +} + +// ── extract_issue_updated ──────────────────────────────────────── + +#[test] +fn extract_issue_updated_from_updated_at() { + let issue = json!({ "updatedAt": "2026-03-01T12:00:00.000Z" }); + assert_eq!( + extract_issue_updated(&issue), + Some("2026-03-01T12:00:00.000Z".to_string()) + ); +} + +#[test] +fn extract_issue_updated_falls_back_to_snake_case() { + let issue = json!({ "data": { "updated_at": "2026-01-15T08:30:00.000Z" } }); + assert_eq!( + extract_issue_updated(&issue), + Some("2026-01-15T08:30:00.000Z".to_string()) + ); +} + +// ── extract_viewer ─────────────────────────────────────────────── + +#[test] +fn extract_viewer_from_data_nodes() { + let data = json!({ "data": { "nodes": [{ "id": "usr_1", "email": "a@b.com" }] } }); + let v = extract_viewer(&data).expect("should find viewer"); + assert_eq!(v["id"], "usr_1"); +} + +#[test] +fn extract_viewer_from_top_level_nodes() { + let data = json!({ "nodes": [{ "id": "usr_2" }] }); + let v = extract_viewer(&data).expect("should find viewer"); + assert_eq!(v["id"], "usr_2"); +} + +#[test] +fn extract_viewer_fallback_direct_object() { + let data = json!({ "id": "usr_direct", "name": "Direct User" }); + let v = extract_viewer(&data).expect("should return direct object"); + assert_eq!(v["id"], "usr_direct"); +} + +#[test] +fn extract_viewer_returns_none_when_absent() { + let data = json!({ "foo": "bar" }); + assert!(extract_viewer(&data).is_none()); +} + +// ── extract_pagination_cursor ──────────────────────────────────── + +#[test] +fn extract_pagination_cursor_returns_cursor_when_has_next_page() { + let data = json!({ + "data": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "cursor_abc" + } + } + }); + assert_eq!( + extract_pagination_cursor(&data), + Some("cursor_abc".to_string()) + ); +} + +#[test] +fn extract_pagination_cursor_returns_none_when_last_page() { + let data = json!({ + "pageInfo": { + "hasNextPage": false, + "endCursor": "cursor_xyz" + } + }); + assert!(extract_pagination_cursor(&data).is_none()); +} + +#[test] +fn extract_pagination_cursor_from_doubly_nested_issues() { + // The same `data.data.issues` shape `extract_issues` reads must also + // expose its pageInfo cursor, or a doubly-nested payload never pages. + let data = json!({ + "data": { + "data": { + "issues": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "cursor_issue_2" + } + } + } + } + }); + assert_eq!( + extract_pagination_cursor(&data), + Some("cursor_issue_2".to_string()) + ); +} + +#[test] +fn extract_pagination_cursor_returns_none_when_absent() { + let data = json!({ "nodes": [{"id": "i1"}] }); + assert!(extract_pagination_cursor(&data).is_none()); +} + +// ── now_ms ─────────────────────────────────────────────────────── + +#[test] +fn now_ms_returns_nonzero() { + assert!(now_ms() > 0); +} diff --git a/sync/src/notion.rs b/sync/src/notion.rs new file mode 100644 index 0000000..f13c17f --- /dev/null +++ b/sync/src/notion.rs @@ -0,0 +1,120 @@ +//! Notion host normalization helpers — result extraction, pagination cursor, +//! page title extraction, and time utilities. + +use serde_json::Value; + +use super::helpers::pick_str; + +/// Walk the Composio response envelope for Notion page results. +pub fn extract_results(data: &Value) -> Vec { + let candidates = [ + data.pointer("/data/results"), + data.pointer("/results"), + data.pointer("/data/data/results"), + data.pointer("/data/items"), + data.pointer("/items"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + return arr.clone(); + } + } + Vec::new() +} + +/// Extract the rendered page body markdown from a `NOTION_GET_PAGE_MARKDOWN` +/// response. Composio wraps action output in varying envelope shapes, so we +/// try the common locations tolerantly and return the first non-empty string. +/// Returns `None` if no markdown field is found (caller falls back to the +/// metadata-only body and logs the raw shape for diagnosis). +pub fn extract_page_markdown(data: &Value) -> Option { + const PATHS: &[&str] = &[ + "/markdown", + "/data/markdown", + "/data/response_data/markdown", + "/response_data/markdown", + "/data/content", + "/content", + "/data/markdown_content", + "/markdown_content", + "/text", + "/data/text", + ]; + for p in PATHS { + if let Some(s) = data.pointer(p).and_then(Value::as_str) { + if !s.trim().is_empty() { + return Some(s.to_string()); + } + } + } + None +} + +/// Extract the Notion pagination cursor (for `start_cursor` on the +/// next request). +pub fn extract_notion_cursor(data: &Value) -> Option { + let candidates = [ + data.pointer("/data/next_cursor"), + data.pointer("/next_cursor"), + data.pointer("/data/data/next_cursor"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(s) = cand.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + +/// Try to extract a human-readable title from a Notion page object. +/// +/// Notion pages store the title in `properties.title` or +/// `properties.Name.title[0].plain_text`. We try several shapes. +pub fn extract_page_title(page: &Value) -> Option { + // Try the common `properties.title.title[0].plain_text` shape. + let props = page + .get("properties") + .or_else(|| page.get("data")?.get("properties")); + if let Some(props) = props { + // Walk all properties looking for a "title" type field. + if let Some(obj) = props.as_object() { + for (_key, val) in obj { + if val.get("type").and_then(Value::as_str) == Some("title") { + if let Some(arr) = val.get("title").and_then(Value::as_array) { + let text: String = arr + .iter() + .filter_map(|t| t.get("plain_text").and_then(Value::as_str)) + .collect::>() + .join(""); + if !text.is_empty() { + return Some(text); + } + } + } + } + } + } + + // Fallback: top-level "title" field (some Composio shapes). + pick_str(page, &["title", "data.title", "name", "data.name"]) +} + +/// Milliseconds since the Unix epoch. +/// +/// The one clock read in this crate. Notion payloads carry no ingestion +/// timestamp, so the normaliser stamps one; everything else here is a function +/// of its input alone. +pub fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[cfg(test)] +#[path = "notion_tests.rs"] +mod tests; diff --git a/sync/src/notion_tests.rs b/sync/src/notion_tests.rs new file mode 100644 index 0000000..84b46e1 --- /dev/null +++ b/sync/src/notion_tests.rs @@ -0,0 +1,142 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +#[test] +fn extract_results_from_data_results() { + let data = json!({"data": {"results": [{"id": "page1"}]}}); + let results = extract_results(&data); + assert_eq!(results.len(), 1); +} + +#[test] +fn extract_page_markdown_reads_top_level_field() { + // Matches the live GET_PAGE_MARKDOWN envelope observed empirically: + // {id, markdown, object, request_id, truncated, unknown_block_ids}. + let data = json!({ + "id": "p1", + "markdown": "# Heading\n\nbody text", + "object": "page", + "truncated": false, + }); + assert_eq!( + extract_page_markdown(&data).as_deref(), + Some("# Heading\n\nbody text") + ); +} + +#[test] +fn extract_page_markdown_reads_nested_envelope() { + let data = json!({ "data": { "markdown": "nested body" } }); + assert_eq!(extract_page_markdown(&data).as_deref(), Some("nested body")); +} + +#[test] +fn extract_page_markdown_none_for_empty_or_missing() { + // Empty markdown (a DB row with no body blocks) → None → metadata-only. + assert_eq!(extract_page_markdown(&json!({ "markdown": "" })), None); + assert_eq!(extract_page_markdown(&json!({ "markdown": " " })), None); + // No markdown field at all → None. + assert_eq!(extract_page_markdown(&json!({ "id": "p1" })), None); +} + +#[test] +fn extract_results_from_top_level() { + let data = json!({"results": [{"id": "a"}, {"id": "b"}]}); + let results = extract_results(&data); + assert_eq!(results.len(), 2); +} + +#[test] +fn extract_results_from_data_items() { + let data = json!({"data": {"items": [{"id": "x"}]}}); + let results = extract_results(&data); + assert_eq!(results.len(), 1); +} + +#[test] +fn extract_results_empty_when_no_match() { + let data = json!({"foo": "bar"}); + assert!(extract_results(&data).is_empty()); +} + +#[test] +fn extract_notion_cursor_from_data() { + let data = json!({"data": {"next_cursor": "cur123"}}); + assert_eq!(extract_notion_cursor(&data), Some("cur123".into())); +} + +#[test] +fn extract_notion_cursor_from_top_level() { + let data = json!({"next_cursor": "abc"}); + assert_eq!(extract_notion_cursor(&data), Some("abc".into())); +} + +#[test] +fn extract_notion_cursor_none_when_empty() { + let data = json!({"data": {"next_cursor": " "}}); + assert_eq!(extract_notion_cursor(&data), None); +} + +#[test] +fn extract_notion_cursor_none_when_missing() { + assert_eq!(extract_notion_cursor(&json!({})), None); +} + +#[test] +fn extract_page_title_from_properties_title_type() { + let page = json!({ + "properties": { + "Name": { + "type": "title", + "title": [{"plain_text": "Hello"}, {"plain_text": " World"}] + } + } + }); + assert_eq!(extract_page_title(&page), Some("Hello World".into())); +} + +#[test] +fn extract_page_title_from_nested_data_properties() { + let page = json!({ + "data": { + "properties": { + "Title": { + "type": "title", + "title": [{"plain_text": "My Page"}] + } + } + } + }); + assert_eq!(extract_page_title(&page), Some("My Page".into())); +} + +#[test] +fn extract_page_title_fallback_to_top_level_title() { + let page = json!({"title": "Fallback Title"}); + assert_eq!(extract_page_title(&page), Some("Fallback Title".into())); +} + +#[test] +fn extract_page_title_none_when_empty() { + let page = json!({"properties": {"Name": {"type": "title", "title": []}}}); + // Empty title array means no text + assert!( + extract_page_title(&page).is_none() || extract_page_title(&page) == Some(String::new()) + ); +} + +#[test] +fn extract_page_title_none_when_no_title_field() { + let page = json!({"id": "123"}); + assert!(extract_page_title(&page).is_none()); +} + +#[test] +fn now_ms_returns_nonzero() { + assert!(now_ms() > 0); +} diff --git a/sync/src/slack_post_process.rs b/sync/src/slack_post_process.rs new file mode 100644 index 0000000..28ce30b --- /dev/null +++ b/sync/src/slack_post_process.rs @@ -0,0 +1,323 @@ +//! Slack-specific post-processing of Composio action responses. +//! +//! Composio's Slack responses are verbose API envelopes. This module +//! rewrites each supported action's response into a slim, stable shape +//! that the ingest pipeline and enrichers can consume without walking +//! Composio's unstable nested envelopes. +//! +//! ## Supported slugs +//! +//! - `SLACK_FETCH_CONVERSATION_HISTORY` — reshapes into top-level +//! `messages[]` with `{ ts, user, text, thread_ts, channel_id }`. +//! Empty-text messages are dropped. `channel_id` is absent here (it's +//! in the request, not the response); the caller injects it via the +//! enricher in the host's `SlackSyncPipeline`. +//! +//! - `SLACK_LIST_CONVERSATIONS` — reshapes into top-level `channels[]` +//! with `{ id, name, is_private }` per channel. Entries with an empty +//! id are dropped. +//! +//! - `SLACK_SEARCH_MESSAGES` — reshapes `messages.matches[]` (possibly +//! nested) into top-level `messages[]` with `{ ts, user, text, +//! thread_ts, channel_id }`. `channel_id` is pulled from each match's +//! `channel.id` field. `paging.pages` is preserved at top-level for +//! caller pagination. +//! +//! ## Design note: user-id resolution is NOT here +//! +//! `SlackUsers` is a per-sync cache built from a separate API call — +//! not a function of any individual response. Resolving user ids +//! happens in the host's `SlackSyncPipeline` (the enricher layer), keeping +//! this module purely data-shape–oriented. +//! This matches Gmail's pattern of "post_process is data-only". +//! +//! Unknown slugs are silently no-ops so new Composio actions don't +//! break the provider. + +use serde_json::{Map, Value}; + +/// Entry point called from `SlackProvider::post_process_action_result`. +/// +/// Dispatches on the Composio action slug and rewrites `data` in place. +/// Unknown slugs are silently ignored. +pub fn post_process(slug: &str, _arguments: Option<&Value>, data: &mut Value) { + log::debug!("[composio:slack][post-process] slug={slug}"); + match slug { + "SLACK_FETCH_CONVERSATION_HISTORY" => reshape_fetch_history(data), + "SLACK_LIST_CONVERSATIONS" => reshape_list_conversations(data), + "SLACK_SEARCH_MESSAGES" => reshape_search_messages(data), + _ => { + log::debug!("[composio:slack][post-process] unknown slug={slug}, passing through"); + } + } +} + +// ─── SLACK_FETCH_CONVERSATION_HISTORY ────────────────────────────────────── + +/// Rewrite a `SLACK_FETCH_CONVERSATION_HISTORY` response in place. +/// +/// Walks possible nested envelopes (`/data/messages`, `/messages`, +/// `/data/data/messages`) to find the raw messages array, drops messages +/// with empty `text`, and emits a slim `{ ts, user, text, thread_ts }` +/// shape under a top-level `messages[]` key. The consumed nested array is +/// removed from the payload so the raw verbose rows don't linger alongside +/// the slim copy. The caller injects `channel_id` via +/// [`super::sync::extract_messages`]. +fn reshape_fetch_history(data: &mut Value) { + let arr = take_array( + data, + &["/data/messages", "/messages", "/data/data/messages"], + 0, + ); + let slim: Vec = arr.into_iter().filter_map(slim_history_message).collect(); + let obj = ensure_object(data); + obj.insert("messages".to_string(), Value::Array(slim)); + log::debug!("[composio:slack][post-process] SLACK_FETCH_CONVERSATION_HISTORY reshaped"); +} + +fn slim_history_message(raw: Value) -> Option { + let text = raw + .get("text") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim(); + if text.is_empty() { + return None; + } + let mut out = Map::new(); + // `ts` is required: without it a caller can neither cursor nor archive. + out.insert("ts".into(), raw.get("ts")?.clone()); + if let Some(user) = raw.get("user").or_else(|| raw.get("bot_id")) { + out.insert("user".into(), user.clone()); + } + out.insert("text".into(), Value::String(text.to_string())); + if let Some(thread_ts) = raw.get("thread_ts") { + out.insert("thread_ts".into(), thread_ts.clone()); + } + if let Some(permalink) = raw.get("permalink") { + out.insert("permalink".into(), permalink.clone()); + } + Some(Value::Object(out)) +} + +/// Find the first array at any of `candidates`, remove that field (plus +/// `envelope_depth` ancestor object envelopes) from `data`, and return the +/// array. Removing the consumed nested payload keeps the reshaped output from +/// carrying duplicate raw rows. +fn take_array(data: &mut Value, candidates: &[&str], envelope_depth: usize) -> Vec { + for path in candidates { + let arr = match data.pointer(path).and_then(|v| v.as_array().cloned()) { + Some(a) => a, + None => continue, + }; + let mut remove_path = path.to_string(); + for _ in 0..envelope_depth { + remove_path = match remove_path.rsplit_once('/') { + Some((parent, _)) => parent.to_string(), + None => break, + }; + } + remove_nested(data, &remove_path); + return arr; + } + Vec::new() +} + +/// Remove the field at `path` from `data`, pruning any ancestor object that +/// the removal left empty so a consumed `data` envelope disappears entirely +/// instead of lingering as `{}`. +fn remove_nested(data: &mut Value, path: &str) { + let segments: Vec<&str> = path + .trim_start_matches('/') + .split('/') + .filter(|s| !s.is_empty()) + .collect(); + if segments.is_empty() { + return; + } + + // Remove the leaf field. + let mut current = &mut *data; + for seg in &segments[..segments.len() - 1] { + current = match current.get_mut(*seg) { + Some(next) => next, + None => return, + }; + } + if let Value::Object(map) = current { + map.remove(segments[segments.len() - 1]); + } + + // Prune empty object ancestors, deepest first. + for depth in (0..segments.len().saturating_sub(1)).rev() { + // Re-walk to the object at `segments[..=depth]`. + let mut ancestor = &mut *data; + for seg in &segments[..=depth] { + ancestor = match ancestor.get_mut(*seg) { + Some(next) => next, + None => return, + }; + } + if !matches!(ancestor, Value::Object(m) if m.is_empty()) { + break; + } + // Remove it from its parent (`segments[..depth]`). For `depth == 0` + // the parent is the top-level object, so an emptied `data` envelope + // key disappears entirely. + let mut parent = &mut *data; + for seg in &segments[..depth] { + parent = match parent.get_mut(*seg) { + Some(next) => next, + None => return, + }; + } + if let Value::Object(map) = parent { + map.remove(segments[depth]); + } + } +} + +// ─── SLACK_LIST_CONVERSATIONS ─────────────────────────────────────────────── + +/// Rewrite a `SLACK_LIST_CONVERSATIONS` response in place. +/// +/// Reshapes into a top-level `channels[]` with `{ id, name, is_private }` +/// per channel; entries with an empty id are dropped. +fn reshape_list_conversations(data: &mut Value) { + let arr = take_array( + data, + &[ + "/data/channels", + "/channels", + "/data/data/channels", + "/data/conversations", + "/conversations", + ], + 0, + ); + + let slim: Vec = arr.into_iter().filter_map(slim_channel).collect(); + let obj = ensure_object(data); + obj.insert("channels".to_string(), Value::Array(slim)); + log::debug!("[composio:slack][post-process] SLACK_LIST_CONVERSATIONS reshaped"); +} + +fn slim_channel(raw: Value) -> Option { + let id = raw.get("id").and_then(|v| v.as_str()).unwrap_or("").trim(); + if id.is_empty() { + return None; + } + let name = raw + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or(id) + .trim(); + let is_private = raw + .get("is_private") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + Some(Value::Object({ + let mut m = Map::new(); + m.insert("id".into(), Value::String(id.to_string())); + m.insert("name".into(), Value::String(name.to_string())); + m.insert("is_private".into(), Value::Bool(is_private)); + m + })) +} + +// ─── SLACK_SEARCH_MESSAGES ────────────────────────────────────────────────── + +/// Rewrite a `SLACK_SEARCH_MESSAGES` response in place. +/// +/// Reshapes `messages.matches[]` (possibly nested under one or two +/// `data` envelopes) into top-level `messages[]`. `channel_id` is pulled +/// from each match's `channel.id` field. `paging.pages` is preserved at +/// top-level under `pages` for the caller to drive pagination. +fn reshape_search_messages(data: &mut Value) { + // Preserve paging info before mutating data (take_array below removes the + // envelope that carries it). + let pages = [ + data.pointer("/data/messages/paging/pages"), + data.pointer("/messages/paging/pages"), + data.pointer("/data/data/messages/paging/pages"), + ] + .into_iter() + .flatten() + .find_map(|v| v.as_u64()) + .unwrap_or(1); + + // Envelope depth 1 removes the `messages` object (matches + paging) that + // held the consumed rows, not just the `matches` array. + let arr = take_array( + data, + &[ + "/data/messages/matches", + "/messages/matches", + "/data/data/messages/matches", + ], + 1, + ); + + let slim: Vec = arr.into_iter().filter_map(slim_search_match).collect(); + let obj = ensure_object(data); + obj.insert("messages".to_string(), Value::Array(slim)); + obj.insert("pages".to_string(), Value::Number(pages.into())); + log::debug!("[composio:slack][post-process] SLACK_SEARCH_MESSAGES reshaped"); +} + +fn slim_search_match(raw: Value) -> Option { + let text = raw + .get("text") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim(); + if text.is_empty() { + return None; + } + let ts = raw.get("ts")?; + let channel_id = raw + .pointer("/channel/id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim(); + + let mut out = Map::new(); + out.insert("ts".into(), ts.clone()); + if let Some(user) = raw.get("user").or_else(|| raw.get("bot_id")) { + out.insert("user".into(), user.clone()); + } + out.insert("text".into(), Value::String(text.to_string())); + if let Some(thread_ts) = raw.get("thread_ts") { + out.insert("thread_ts".into(), thread_ts.clone()); + } + if !channel_id.is_empty() { + out.insert("channel_id".into(), Value::String(channel_id.to_string())); + } + if let Some(permalink) = raw.get("permalink") { + out.insert("permalink".into(), permalink.clone()); + } + Some(Value::Object(out)) +} + +// ─── Helpers ──────────────────────────────────────────────────────────────── + +/// Ensure `data` is a JSON object, replacing it with an empty object if +/// not. Returns a mutable ref to the inner map. +// Scoped rather than blanket, for the case `AGENTS.md` names: "genuinely +// unreachable states — where `expect` must carry a message explaining the +// invariant." The line below assigns `Value::Object` whenever `data` is not +// one, so the read-back cannot fail; the compiler cannot see that across the +// assignment. The two `unwrap`s this crate inherited elsewhere were removed +// rather than allowed. +#[allow(clippy::expect_used)] +fn ensure_object(data: &mut Value) -> &mut Map { + if !data.is_object() { + *data = Value::Object(Map::new()); + } + data.as_object_mut() + .expect("assigned Value::Object immediately above when data was not one") +} + +#[cfg(test)] +#[path = "slack_post_process_tests.rs"] +mod tests; diff --git a/sync/src/slack_post_process_tests.rs b/sync/src/slack_post_process_tests.rs new file mode 100644 index 0000000..aeec63a --- /dev/null +++ b/sync/src/slack_post_process_tests.rs @@ -0,0 +1,262 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +// ─── SLACK_FETCH_CONVERSATION_HISTORY ───────────────────────────────────── + +#[test] +fn history_reshapes_top_level_messages() { + let mut data = json!({ + "messages": [ + { "ts": "1714003200.000100", "user": "U1", "text": "hello" }, + { "ts": "1714003300.000200", "user": "U2", "text": "world", "thread_ts": "1714003200.0" }, + { "ts": "1714003400.000300", "user": "U3", "text": " " }, // dropped: empty text + ], + "response_metadata": { "next_cursor": "abc" } + }); + post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); + + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 2, "empty-text message must be dropped"); + assert_eq!(msgs[0]["ts"], "1714003200.000100"); + assert_eq!(msgs[0]["user"], "U1"); + assert_eq!(msgs[0]["text"], "hello"); + assert!(msgs[0].get("thread_ts").is_none()); + assert_eq!(msgs[1]["thread_ts"], "1714003200.0"); +} + +#[test] +fn history_reshapes_nested_data_envelope() { + let mut data = json!({ + "data": { + "messages": [ + { "ts": "1714003200.0", "user": "U1", "text": "hi" } + ] + } + }); + post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["text"], "hi"); +} + +#[test] +fn history_reshapes_doubly_nested_envelope() { + let mut data = json!({ + "data": { + "data": { + "messages": [ + { "ts": "1714003200.0", "user": "U1", "text": "deep" } + ] + } + } + }); + post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["text"], "deep"); +} + +#[test] +fn history_drops_message_without_ts() { + let mut data = json!({ + "messages": [ + { "user": "U1", "text": "no timestamp" }, + { "ts": "1714003200.0", "user": "U2", "text": "has ts" }, + ] + }); + post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["text"], "has ts"); +} + +#[test] +fn history_removes_nested_envelope_after_reshape() { + let mut data = json!({ + "data": { + "messages": [ + { "ts": "1714003200.0", "user": "U1", "text": "hi" } + ] + } + }); + post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); + + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["text"], "hi"); + assert!( + data.pointer("/data").is_none(), + "consumed `data.messages` envelope must be removed, got: {data}" + ); +} + +// ─── SLACK_LIST_CONVERSATIONS ───────────────────────────────────────────── + +#[test] +fn list_conversations_reshapes_channels() { + let mut data = json!({ + "data": { + "channels": [ + { "id": "C1", "name": "eng", "is_private": false, "extra": "noise" }, + { "id": "G1", "name": "ops", "is_private": true }, + { "id": "", "name": "empty-id" }, // dropped + ] + } + }); + post_process("SLACK_LIST_CONVERSATIONS", None, &mut data); + let channels = data["channels"].as_array().unwrap(); + assert_eq!(channels.len(), 2, "empty-id entry must be dropped"); + assert_eq!(channels[0]["id"], "C1"); + assert_eq!(channels[0]["name"], "eng"); + assert_eq!(channels[0]["is_private"], false); + assert!( + channels[0].get("extra").is_none(), + "noise fields must be removed" + ); + assert_eq!(channels[1]["id"], "G1"); + assert_eq!(channels[1]["is_private"], true); +} + +#[test] +fn list_conversations_falls_back_to_conversations_key() { + let mut data = json!({ + "conversations": [ + { "id": "C2", "name": "dev", "is_private": false } + ] + }); + post_process("SLACK_LIST_CONVERSATIONS", None, &mut data); + let channels = data["channels"].as_array().unwrap(); + assert_eq!(channels.len(), 1); + assert_eq!(channels[0]["id"], "C2"); + assert!( + data.pointer("/conversations").is_none(), + "consumed `conversations` field must be removed" + ); +} + +// ─── SLACK_SEARCH_MESSAGES ──────────────────────────────────────────────── + +#[test] +fn search_messages_reshapes_matches() { + let mut data = json!({ + "messages": { + "matches": [ + { + "ts": "1714003200.0", + "user": "U1", + "text": "hello from search", + "channel": { "id": "C1" } + }, + { + "ts": "1714003300.0", + "user": "U2", + "text": " ", // dropped: whitespace only + "channel": { "id": "C1" } + }, + ], + "paging": { "pages": 3 } + } + }); + post_process("SLACK_SEARCH_MESSAGES", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1, "empty-text match must be dropped"); + assert_eq!(msgs[0]["ts"], "1714003200.0"); + assert_eq!(msgs[0]["text"], "hello from search"); + assert_eq!(msgs[0]["channel_id"], "C1"); + assert_eq!(data["pages"], 3, "paging.pages must be preserved"); +} + +#[test] +fn search_messages_nested_data_envelope() { + let mut data = json!({ + "data": { + "messages": { + "matches": [ + { "ts": "1714003200.0", "user": "U1", "text": "nested", "channel": { "id": "C2" } } + ], + "paging": { "pages": 1 } + } + } + }); + post_process("SLACK_SEARCH_MESSAGES", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["channel_id"], "C2"); + assert_eq!(data["pages"], 1_u64); +} + +#[test] +fn search_messages_no_matches_emits_empty_array() { + let mut data = json!({ "messages": { "matches": [] } }); + post_process("SLACK_SEARCH_MESSAGES", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert!(msgs.is_empty()); +} + +#[test] +fn search_messages_removes_nested_envelope_after_reshape() { + let mut data = json!({ + "data": { + "messages": { + "matches": [ + { "ts": "1714003200.0", "user": "U1", "text": "nested", "channel": { "id": "C2" } } + ], + "paging": { "pages": 1 } + } + } + }); + post_process("SLACK_SEARCH_MESSAGES", None, &mut data); + + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["channel_id"], "C2"); + assert_eq!(data["pages"], 1_u64); + assert!( + data.pointer("/data").is_none(), + "consumed `data.messages` envelope must be removed, got: {data}" + ); +} + +#[test] +fn search_messages_doubly_nested_paging_preserved() { + let mut data = json!({ + "data": { + "data": { + "messages": { + "matches": [ + { "ts": "1714003200.0", "user": "U1", "text": "deep", "channel": { "id": "C3" } } + ], + "paging": { "pages": 4 } + } + } + } + }); + post_process("SLACK_SEARCH_MESSAGES", None, &mut data); + + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["text"], "deep"); + assert_eq!( + data["pages"], 4_u64, + "doubly-nested paging must be preserved" + ); + assert!( + data.pointer("/data").is_none(), + "consumed `data.data.messages` envelope must be removed, got: {data}" + ); +} + +// ─── Unknown slug ───────────────────────────────────────────────────────── + +#[test] +fn unknown_slug_is_noop() { + let mut data = json!({ "foo": "bar" }); + let original = data.clone(); + post_process("SLACK_SEND_MESSAGE", None, &mut data); + assert_eq!(data, original, "unknown slug must not mutate data"); +} diff --git a/vendor/tinycortex b/vendor/tinycortex index 5fdeac9..8401346 160000 --- a/vendor/tinycortex +++ b/vendor/tinycortex @@ -1 +1 @@ -Subproject commit 5fdeac984c09d2dac65b61e92fd27e2c92ce1e6b +Subproject commit 8401346b574cacb1dc0cf6b36bc608ff5ef9f6f5