Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions crates/uffs-client/src/connect.rs
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,34 @@ impl UffsClient {
Ok(response)
}

/// Ask the daemon which drives the given search would promote
/// (the dispatch promote-plan, bloom pre-check included) —
/// `warm_plan` RPC, daemon v0.6.38+.
///
/// Send the SAME params the search itself will carry; the daemon
/// resolves them through its own filter pipeline, so the answer is
/// bit-exact with what dispatching that search would promote. An
/// empty `needs_promote` means the query serves without any
/// re-warm. Older daemons answer with a method-not-found
/// [`crate::error::ClientError::DaemonError`]
/// (code [`crate::protocol::ERR_METHOD_NOT_FOUND`]); callers
/// gate-keeping on this should fall back to a tier-based check.
///
/// # Errors
///
/// Returns a `ClientError` on connection, protocol, or timeout
/// failure, and the method-not-found daemon error described above.
pub async fn warm_plan(
&mut self,
params: &SearchParams,
) -> Result<crate::protocol::response::WarmPlanResponse, crate::error::ClientError> {
let value = serde_json::to_value(params)
.map_err(|err| crate::error::ClientError::Protocol(err.to_string()))?;
let result = self.send_request("warm_plan", Some(value)).await?;
serde_json::from_value(result)
.map_err(|err| crate::error::ClientError::Protocol(err.to_string()))
}

/// List loaded drives.
///
/// # Errors
Expand Down
2 changes: 1 addition & 1 deletion crates/uffs-client/src/protocol/response.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize};
pub use super::response_journal::{ChangedSinceParams, ChangedSinceResponse, JournalChange};
pub use super::response_status::{
DaemonPaths, DaemonStatus, DriveInfo, DriveMemoryInfo, DrivesResponse, LiveUpdateInfo,
ShardTier, StatsResponse, StatusResponse,
ShardTier, StatsResponse, StatusResponse, WarmPlanResponse,
};
pub use super::response_tiering::{
DEFAULT_PRELOAD_PIN_MINUTES, DriveTierStatus, ForgetParams, ForgetResponse, HibernateParams,
Expand Down
22 changes: 22 additions & 0 deletions crates/uffs-client/src/protocol/response_status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,28 @@ pub struct DrivesResponse {
pub drives: Vec<DriveInfo>,
}

/// Response for the `warm_plan` method (v0.6.38+).
///
/// Carries the drives the daemon's dispatch would actually promote for
/// the given search params — Parked/Cold shards in scope whose bloom
/// cannot prove them irrelevant to the query's resolved extension
/// filter.
///
/// This is the daemon-authoritative answer external cold-index gates
/// must consume instead of inferring readiness from `DriveInfo::tier`
/// alone: a Parked drive whose bloom proves "no matching extensions"
/// serves the query instantly with zero rows and MUST NOT be gated or
/// force-warmed (field 2026-08-24: the MCP gate's tier-only view
/// reported such a drive "warming — retry" and paged its body back
/// into RAM for a query the bloom had already answered).
#[derive(Debug, Serialize, Deserialize)]
pub struct WarmPlanResponse {
/// Drives the search would promote, in registry order. Empty
/// means the query dispatches without any re-warm — the caller's
/// gate must pass.
pub needs_promote: Vec<uffs_mft::platform::DriveLetter>,
}

/// Memory-tiering state of a single shard, as surfaced over the wire.
///
/// Mirrors the daemon-internal `ShardState` enum (`crates/uffs-daemon/
Expand Down
26 changes: 25 additions & 1 deletion crates/uffs-core/src/compact.rs
Original file line number Diff line number Diff line change
Expand Up @@ -267,9 +267,33 @@ impl DriveCompactIndex {
/// mutation — then resets `delta = None` so subsequent searches take the
/// zero-overhead base fast path.
///
/// Fold any pending delta overlay into fresh bases so this index
/// is safe to serialize — the boundary every save path must cross.
///
/// A delta-carrying index is UNSERIALIZABLE as-is: the patch path
/// appends created records (the header's record count grows) while
/// `children` / `trigram` / `ext_index` stay the Arc-shared bases,
/// so the writer emits a children CSR sized for the OLD record
/// count and the reader — which sizes the section from the header
/// — mis-walks every later section (field signature: `bloom
/// k_hashes out of range`, quarantining an otherwise-healthy cache
/// on the next start; winbox M/C/D/S 2026-08-24, and the 2026-08
/// "drive C for ten days" incident). Even an aligned save would
/// persist a trigram base missing the new records — a silent
/// substring-search hole after reload. No-op when `delta` is
/// already `None`, so calling it unconditionally before a save is
/// free in the steady state.
pub fn fold_delta_for_save(&mut self) {
if self.delta.is_some() {
self.compact_base();
}
}

/// O(total records); the per-apply path drives toward this running only
/// occasionally (every [`TRIGRAM_COMPACT_THRESHOLD`] touched records) or
/// before serialization (the on-disk cache is always delta-free).
/// before serialization (the on-disk cache is always delta-free —
/// enforced by [`Self::fold_delta_for_save`] +
/// `save_compact_cache_background`'s refusal guard).
pub(crate) fn compact_base(&mut self) {
self.trigram = Arc::new(TrigramIndex::build(&self.records, &self.names, self.fold));
self.ext_index = Arc::new(ExtensionIndex::build(&self.records));
Expand Down
44 changes: 39 additions & 5 deletions crates/uffs-core/src/compact_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -707,20 +707,39 @@ fn parse_compact_body(
return Err("truncated trigram header");
}
let tkc = read_u32(data, pe) as usize;
let (trigram_loaded, after_tri) = if version >= 6 && tkc > 0 {
let (trigram_loaded, after_tri) = if version >= 6 {
// The v6+ writer ALWAYS emits the full trigram CSR — keys,
// offsets, and the values count — even for an empty index:
// `TrigramIndex` maintains the CSR invariant `offsets.len()
// == keys.len() + 1` in memory, so an empty index still
// serialises its `[0]` offsets entry plus `vc = 0`. The old
// `tkc > 0` fast-path consumed only the 4-byte key count in
// the empty case, leaving those 8 bytes unread and shifting
// every later section (ext names, bloom, trie, frs) — the
// `bloom k_hashes out of range` corruption class that
// quarantined otherwise-valid caches (2026-08 "drive C for
// ten days"; 2026-08-24 winbox C/D/S quarantine storm).
let (ks, ke, oe) = (pe + 4, pe + 4 + tkc * 8, pe + 4 + tkc * 8 + (tkc + 1) * 4);
if data.len() < oe + 4 {
return Err("truncated trigram CSR");
}
let tk: Vec<u64> = aligned_vec_from_bytes(data.get(ks..ke).ok_or("trigram keys")?);
let to: Vec<u32> = aligned_vec_from_bytes(data.get(ke..oe).ok_or("trigram offsets")?);
let vc = read_u32(data, oe) as usize;
let ve = oe + 4 + vc * 4;
if data.len() < ve {
return Err("truncated trigram values");
}
let tv: Vec<u32> = aligned_vec_from_bytes(data.get(oe + 4..ve).ok_or("trigram values")?);
(Some(TrigramIndex::from_csr(tk, to, tv)), ve)
if tkc > 0 {
let tk: Vec<u64> = aligned_vec_from_bytes(data.get(ks..ke).ok_or("trigram keys")?);
let to: Vec<u32> = aligned_vec_from_bytes(data.get(ke..oe).ok_or("trigram offsets")?);
let tv: Vec<u32> =
aligned_vec_from_bytes(data.get(oe + 4..ve).ok_or("trigram values")?);
(Some(TrigramIndex::from_csr(tk, to, tv)), ve)
} else {
// Empty on disk: report `None` so the assembler rebuilds
// from records — adopting the empty index verbatim would
// silently break substring search on a non-empty drive.
(None, ve)
}
} else {
(None, pe + 4)
};
Expand Down Expand Up @@ -1018,6 +1037,21 @@ pub fn save_compact_cache(index: &DriveCompactIndex) -> io::Result<()> {
/// # Errors
/// Returns an error only if compression or directory creation fails.
pub fn save_compact_cache_background(index: &DriveCompactIndex) -> io::Result<()> {
// Boundary enforcement of the "on-disk cache is always delta-free"
// invariant: a delta-carrying index serialises a children CSR sized
// for the pre-patch record count while the header carries the
// post-patch count, mis-aligning every later section for the reader
// (see `DriveCompactIndex::fold_delta_for_save` for the full
// mechanism + field incidents). Callers fold the delta first; this
// guard turns any future path that forgets into a loud error
// instead of a poisoned cache that quarantines on the next start.
if index.delta.is_some() {
return Err(io::Error::other(
"refusing to persist a delta-carrying compact index: fold the delta first \
(DriveCompactIndex::fold_delta_for_save) — its base sections are stale \
against the patched records and would serialize misaligned",
));
}
let profile = std::env::var_os("UFFS_CACHE_PROFILE").is_some();
let t_compress = Instant::now();

Expand Down
21 changes: 10 additions & 11 deletions crates/uffs-core/src/compact_cache/parked.rs
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,17 @@ fn find_v9_filters_offset(data: &[u8]) -> Result<(usize, u64), &'static str> {
return Err("truncated trigram header");
}

// Trigram CSR (always present for v >= 6).
// Trigram CSR (always present for v >= 6, INCLUDING when empty:
// the writer serialises the in-memory CSR invariant `offsets.len()
// == keys.len() + 1`, so an empty index still emits its `[0]`
// offsets entry plus a zero values count). The old `key_count >
// 0` fast-path treated the count word as a bare "no trigram"
// sentinel — a shape no writer ever produced — and left 8 bytes
// unconsumed, shifting the ext-names/bloom/trie reads that follow
// (the `bloom k_hashes out of range` corruption class; see
// `parse_compact_body` for the full incident note).
let trigram_key_count = read_u32(data, postings_end) as usize;
let after_trigram = if trigram_key_count > 0 {
let after_trigram = {
let keys_end = postings_end
.checked_add(4)
.and_then(|x| x.checked_add(trigram_key_count.checked_mul(8)?))
Expand All @@ -222,15 +230,6 @@ fn find_v9_filters_offset(data: &[u8]) -> Result<(usize, u64), &'static str> {
return Err("truncated trigram values");
}
values_end
} else {
// v >= 6 with `trigram_key_count == 0` is the legacy "no
// trigram on disk" sentinel; v9 always emits a real
// trigram, but a malformed v9 buffer could land here.
// The 4-byte sentinel header has been consumed; advance
// past it so the next section follows correctly.
postings_end
.checked_add(4)
.ok_or("after-trigram-sentinel overflow")?
};

// Skip the v7+ ext_names table without allocating. v9 always
Expand Down
8 changes: 8 additions & 0 deletions crates/uffs-core/src/compact_cache/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -701,3 +701,11 @@ fn load_compact_cache_at_quarantines_nothing_itself() {
}
assert!(path.exists(), "inner loader must not touch the file");
}

// The cache-poisoning regression cluster (empty-trigram shape, the
// delta-carrying misalignment, and the save-boundary refusal) lives in
// a nested sibling file so this module stays under the workspace
// 800-LOC ceiling. `#[path]` keeps it a child of `tests`, so it can
// use this module's `make_test_index` fixture directly.
#[path = "tests_cache_poison.rs"]
mod cache_poison;
173 changes: 173 additions & 0 deletions crates/uffs-core/src/compact_cache/tests_cache_poison.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
// SPDX-License-Identifier: MPL-2.0
// Copyright (c) 2025-2026 SKY, LLC.

//! Cache-poisoning regression cluster: shapes a serialized compact
//! cache must survive (empty trigram CSR) or must never assume
//! (delta-carrying base sections), plus the save-boundary refusal.
//! Split from [`super`] (the `tests` module) for the 800-LOC policy;
//! shares its [`super::make_test_index`] fixture.

use super::super::*;
use super::make_test_index;

/// Regression: a cache saved with an EMPTY trigram index must still
/// round-trip every later section intact.
///
/// The writer always emits the full trigram CSR — offsets carries
/// `keys.len() + 1` entries, so an empty index still writes its `[0]`
/// offsets entry plus the values count. The reader's old `tkc > 0`
/// fast-path skipped only the 4-byte key count in the empty case,
/// leaving 8 bytes unconsumed and shifting the ext-names / bloom /
/// trie / frs sections — surfacing as `bloom k_hashes out of range`
/// and a quarantined cache (2026-08 "drive C for ten days" incident;
/// 2026-08-24 winbox C/D/S quarantine storm).
#[test]
fn empty_trigram_round_trips_without_shifting_later_sections() {
let mut index = make_test_index();
index.trigram = Arc::new(TrigramIndex::empty());
// Give the later sections real content so a shift is detectable.
index.ext_names = vec![Box::from("rs"), Box::from("toml")];
index.bloom = Some(index.build_bloom());
index.path_trie = Some(index.build_path_trie());

let serialized = serialize_compact(&index);
let (loaded, _tri_ms) = deserialize_compact(&serialized, uffs_mft::platform::DriveLetter::T)
.unwrap_or_else(|err| panic!("empty-trigram cache must deserialize, got: {err}"));

assert_eq!(
loaded.ext_names,
vec![Box::from("rs"), Box::from("toml")],
"ext_names must survive an empty trigram section"
);
let bloom = loaded.bloom.as_ref().expect("bloom section must load");
assert!(
bloom.contains(b"rs"),
"the loaded bloom must answer for the drive's real extensions"
);
assert_eq!(
loaded.frs_to_compact, index.frs_to_compact,
"frs_to_compact must survive an empty trigram section"
);
// The empty on-disk trigram is rebuilt from records (an empty
// index over a non-empty drive would silently break substring
// search); records are non-empty here, so the rebuild is real.
assert!(
loaded.trigram.posting_count() > 0,
"trigram must be rebuilt from records, not adopted empty"
);
}

/// The parked-tier loader walks the same section chain with its own
/// arithmetic (`find_v9_filters_offset`) — pin that it, too, lands on
/// the real bloom when the trigram section is empty (its old
/// `key_count > 0` branch believed in a bare-sentinel shape no writer
/// ever produced, shifting the filter reads by 8 bytes).
#[test]
fn parked_loader_finds_filters_past_an_empty_trigram() {
let mut index = make_test_index();
index.trigram = Arc::new(TrigramIndex::empty());
index.ext_names = vec![Box::from("rs"), Box::from("toml")];
index.bloom = Some(index.build_bloom());
index.path_trie = Some(index.build_path_trie());

let serialized = serialize_compact(&index);
let parked = deserialize_parked_body(&serialized, uffs_mft::platform::DriveLetter::T)
.unwrap_or_else(|err| panic!("parked load must survive an empty trigram, got: {err}"));
assert!(
parked.bloom.contains(b"rs"),
"the parked bloom must answer for the drive's real extensions"
);
}

/// Regression: a DELTA-CARRYING index must never serialize — its
/// Arc-shared base sections are stale against the patched records.
///
/// A create appends a record (the header's `rc` grows) while
/// `children` stays the base CSR with `rc_base + 1` offsets; the
/// reader sizes the children section from the header, so every later
/// section is read shifted and the cache quarantines on the next
/// start (field: `bloom k_hashes out of range`, winbox M/C/D/S
/// 2026-08-24 — caches written BY the then-current daemon between two
/// restarts). [`DriveCompactIndex::fold_delta_for_save`] is the
/// repair the save path applies; pin both halves.
#[test]
fn delta_carrying_index_misaligns_and_fold_repairs_it() {
let mut index = make_test_index();
index.bloom = Some(index.build_bloom());
index.path_trie = Some(index.build_path_trie());

// Simulate the surgical-patch state: one created record appended
// (name "qux" appended to the blob), bases left Arc-shared/stale,
// delta overlay armed.
let mut records: Vec<CompactRecord> = index.records.as_slice().to_vec();
let mut names: Vec<u8> = index.names.as_slice().to_vec();
let name_offset = u32::try_from(names.len()).unwrap_or(u32::MAX);
names.extend_from_slice(b"qux");
records.push(CompactRecord {
name_offset,
parent_idx: 0,
name_len: 3,
name_first_byte: b'q',
..CompactRecord::default()
});
index.records = ColumnStorage::from_vec(records);
index.names = ColumnStorage::from_vec(names);
index.delta = Some(crate::compact::IndexDelta::default());

// The poisoned bytes can NEVER round-trip faithfully: the children
// CSR on disk was written for the 3 base records, so the created
// record's parent→child edge does not exist in the file (the delta
// that held it is not serialised), and the header-driven reader
// walks the shifted sections — on big real-world drives that
// surfaces as a parse error and a quarantined cache ("bloom
// k_hashes out of range"); on this small fixture the shifted bytes
// happen to parse, which is the SILENT face of the same bug.
// Either face proves the shape must never reach disk.
let poisoned = serialize_compact(&index);
match deserialize_compact(&poisoned, uffs_mft::platform::DriveLetter::T) {
Err(_parse_error) => {} // the quarantine face
Ok((corrupt, _tri_ms)) => {
let mut root_children: Vec<u32> = Vec::new();
corrupt.for_each_child(0, |child| root_children.push(child));
assert!(
!root_children.contains(&3),
"the created record's children edge was never written — a load that \
reports it would falsify the poison analysis"
);
}
}

// The repair the save path applies: fold the delta, then serialize.
index.fold_delta_for_save();
assert!(index.delta.is_none(), "fold must clear the overlay");
let healthy = serialize_compact(&index);
let (loaded, _tri_ms) = deserialize_compact(&healthy, uffs_mft::platform::DriveLetter::T)
.unwrap_or_else(|err| panic!("folded index must round-trip, got: {err}"));
assert_eq!(loaded.records.len(), 4, "the created record must persist");
let mut root_children: Vec<u32> = Vec::new();
loaded.for_each_child(0, |child| root_children.push(child));
assert!(
root_children.contains(&3),
"the folded children CSR must carry the created record's edge"
);
assert!(
loaded.bloom.is_some(),
"the bloom section must land intact after the fold"
);
}

/// The save boundary refuses a delta-carrying index outright — the
/// second line of defence behind the save path's fold, so any future
/// writer path that forgets to fold fails loudly instead of writing a
/// poisoned cache.
#[test]
fn background_save_refuses_a_delta_carrying_index() {
let mut index = make_test_index();
index.delta = Some(crate::compact::IndexDelta::default());
let err = save_compact_cache_background(&index)
.expect_err("a delta-carrying index must be refused at the save boundary");
assert!(
err.to_string().contains("delta"),
"the refusal must name the delta invariant: {err}"
);
}
Loading
Loading