From 53d07f5a12fbf4d93dade946be04943b27fa1ecb Mon Sep 17 00:00:00 2001 From: "Sakamoto, Kazunori" Date: Wed, 7 Oct 2026 11:39:05 +0900 Subject: [PATCH 1/4] feat: list keys in the order their current values were written Reading every value of a 1.17 GB store in the order `keys` returned took 64 s, because the order was that of a hash map and most reads decoded a group again; in write order it takes 0.5 s. The README also tells callers to share one store per directory in a process: stores that write at the same time each leave a segment behind. Co-authored-by: WillBooster (Claude Code) --- README.md | 6 ++++++ python/commitkv/_native.pyi | 4 +++- rust/crates/commitkv/src/store.rs | 23 ++++++++++++++++------- rust/crates/commitkv/tests/store.rs | 15 +++++++++++++++ src/index.ts | 4 ++++ 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d369699..d5610c4 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,12 @@ store.refresh(); // pick up what other stores wrote to the directory writing (the first `put`, and the first after a failed one); records that other stores write stay invisible until one of those happens. Call `refresh` after replacing segments under an open store, for example by checking out another branch. +- Stores that write to a directory at the same time each append to a segment of their own, and + the next run continues only one of them. Open one store per directory in a process and share + it; a store per use leaves a small segment behind on every run, and a small segment + compresses worse because its values have few earlier ones to refer to. +- `keys` lists the keys in the order their current values were written. Read every value in + that order: a scattered order decodes the surrounding values again for most of them. - Among records of a key that stores wrote without seeing each other's, which one wins is unspecified, but every reader picks the same one. - `max_segment_bytes` / `maxSegmentBytes` must be between 1,024 and 100,000,000; the upper diff --git a/python/commitkv/_native.pyi b/python/commitkv/_native.pyi index cfbaea0..6ece6d1 100644 --- a/python/commitkv/_native.pyi +++ b/python/commitkv/_native.pyi @@ -25,6 +25,8 @@ class Store: def __contains__(self, key: str | bytes) -> bool: ... def __len__(self) -> int: ... - def keys(self) -> list[bytes]: ... + def keys(self) -> list[bytes]: + """Lists the keys in the order their current values were written; reading every value + is fastest in this order.""" def refresh(self) -> None: """Picks up records that other stores wrote to the directory.""" diff --git a/rust/crates/commitkv/src/store.rs b/rust/crates/commitkv/src/store.rs index 2f32f2b..0869815 100644 --- a/rust/crates/commitkv/src/store.rs +++ b/rust/crates/commitkv/src/store.rs @@ -214,9 +214,13 @@ impl Store { self.len() == 0 } + /// The keys in the order their current records were written, so reading every value in + /// this order decodes each group once. pub fn keys(&self) -> Vec> { let state = self.read_state(); - state.index.keys().map(|key| key.to_vec()).collect() + let mut entries: Vec<_> = state.index.iter().collect(); + entries.sort_unstable_by_key(|(_, location)| state.position(location)); + entries.into_iter().map(|(key, _)| key.to_vec()).collect() } /// Picks up records that other stores wrote to the directory since the last call, and @@ -611,6 +615,12 @@ impl State { index } + /// Orders records by age: segment names order by creation time and offsets order within a + /// segment. + fn position(&self, location: &Location) -> (&str, u64) { + (&self.segments[location.segment].name, location.end) + } + /// Adds a record that follows the scanned part of its segment; `false` when the record /// cannot be placed, which leaves the state unchanged. fn apply(&mut self, segment_index: usize, record: &Record) -> bool { @@ -648,12 +658,11 @@ impl State { segment.scanned = location.end; segment.group_raw = group_raw_end; segment.raw_total = raw_total; - // The newest record of a key wins: segment names order by creation time and offsets - // order within a segment. - let newer = self.index.get(record.key).is_none_or(|current| { - (&self.segments[current.segment].name, current.end) - < (&self.segments[segment_index].name, location.end) - }); + // The newest record of a key wins. + let newer = self + .index + .get(record.key) + .is_none_or(|current| self.position(current) < self.position(&location)); if newer { self.index.insert(record.key.into(), location); } diff --git a/rust/crates/commitkv/tests/store.rs b/rust/crates/commitkv/tests/store.rs index 53152f2..a529807 100644 --- a/rust/crates/commitkv/tests/store.rs +++ b/rust/crates/commitkv/tests/store.rs @@ -55,6 +55,21 @@ fn values_survive_reopening_and_the_latest_value_of_a_key_wins() { assert!(keys.binary_search(&b"large".to_vec()).is_ok()); } +#[test] +fn keys_are_listed_in_the_order_their_current_values_were_written() { + let dir = tempfile::tempdir().unwrap(); + { + let store = Store::open(dir.path(), small_segments()).unwrap(); + for i in 0..400 { + store.put(&key(i), &noise(i as u64, 1000)).unwrap(); + } + store.put(&key(3), b"replaced").unwrap(); + } + assert!(segment_paths(dir.path()).len() > 1); + let expected: Vec> = (0..400).filter(|&i| i != 3).chain([3]).map(key).collect(); + assert_eq!(open(dir.path()).keys(), expected); +} + #[test] fn no_segment_exceeds_the_limit() { let dir = tempfile::tempdir().unwrap(); diff --git a/src/index.ts b/src/index.ts index 9e6e2ae..181800f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -57,6 +57,10 @@ export class Store { return this.#native.size; } + /** + * Lists the keys in the order their current values were written; reading every value is + * fastest in this order. + */ keys(): Buffer[] { return this.#native.keys(); } From 96e61c584d1f70363f559c351190312648573bdc Mon Sep 17 00:00:00 2001 From: "Sakamoto, Kazunori" Date: Wed, 7 Oct 2026 11:44:51 +0900 Subject: [PATCH 2/4] docs: describe the order of keys as that of the segments Co-authored-by: WillBooster (Claude Code) --- README.md | 6 ++++-- python/commitkv/_native.pyi | 5 +++-- rust/crates/commitkv/src/store.rs | 5 +++-- rust/crates/commitkv/tests/store.rs | 2 +- src/index.ts | 4 ++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d5610c4..20f6c43 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,10 @@ store.refresh(); // pick up what other stores wrote to the directory the next run continues only one of them. Open one store per directory in a process and share it; a store per use leaves a small segment behind on every run, and a small segment compresses worse because its values have few earlier ones to refer to. -- `keys` lists the keys in the order their current values were written. Read every value in - that order: a scattered order decodes the surrounding values again for most of them. +- `keys` lists the keys in the order their current values lie in the segments, oldest segment + first. One store's writes are listed in the order it made them; writes that several stores + made at the same time are grouped by store, not interleaved by time. Read every value in that + order: a scattered order decodes the surrounding values again for most of them. - Among records of a key that stores wrote without seeing each other's, which one wins is unspecified, but every reader picks the same one. - `max_segment_bytes` / `maxSegmentBytes` must be between 1,024 and 100,000,000; the upper diff --git a/python/commitkv/_native.pyi b/python/commitkv/_native.pyi index 6ece6d1..384c111 100644 --- a/python/commitkv/_native.pyi +++ b/python/commitkv/_native.pyi @@ -26,7 +26,8 @@ class Store: def __contains__(self, key: str | bytes) -> bool: ... def __len__(self) -> int: ... def keys(self) -> list[bytes]: - """Lists the keys in the order their current values were written; reading every value - is fastest in this order.""" + """Lists the keys in the order their current values lie in the segments, oldest segment + first, which is the order one store wrote them; reading every value is fastest in this + order.""" def refresh(self) -> None: """Picks up records that other stores wrote to the directory.""" diff --git a/rust/crates/commitkv/src/store.rs b/rust/crates/commitkv/src/store.rs index 0869815..8dfcb5d 100644 --- a/rust/crates/commitkv/src/store.rs +++ b/rust/crates/commitkv/src/store.rs @@ -214,8 +214,9 @@ impl Store { self.len() == 0 } - /// The keys in the order their current records were written, so reading every value in - /// this order decodes each group once. + /// The keys in the order their current records lie in the segments, oldest segment first, + /// so reading every value in this order decodes each group once. This is the order one + /// store wrote them; stores that write at the same time fill segments of their own. pub fn keys(&self) -> Vec> { let state = self.read_state(); let mut entries: Vec<_> = state.index.iter().collect(); diff --git a/rust/crates/commitkv/tests/store.rs b/rust/crates/commitkv/tests/store.rs index a529807..4754b7b 100644 --- a/rust/crates/commitkv/tests/store.rs +++ b/rust/crates/commitkv/tests/store.rs @@ -56,7 +56,7 @@ fn values_survive_reopening_and_the_latest_value_of_a_key_wins() { } #[test] -fn keys_are_listed_in_the_order_their_current_values_were_written() { +fn keys_are_listed_in_the_order_a_store_wrote_their_current_values() { let dir = tempfile::tempdir().unwrap(); { let store = Store::open(dir.path(), small_segments()).unwrap(); diff --git a/src/index.ts b/src/index.ts index 181800f..9878a2d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -58,8 +58,8 @@ export class Store { } /** - * Lists the keys in the order their current values were written; reading every value is - * fastest in this order. + * Lists the keys in the order their current values lie in the segments, oldest segment first, + * which is the order one store wrote them; reading every value is fastest in this order. */ keys(): Buffer[] { return this.#native.keys(); From 3062b8de4dd65ee48f1a84f3c88bcbce90079108 Mon Sep 17 00:00:00 2001 From: "Sakamoto, Kazunori" Date: Wed, 7 Oct 2026 11:49:39 +0900 Subject: [PATCH 3/4] docs: say that keys of several stores are grouped by segment Co-authored-by: WillBooster (Claude Code) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 20f6c43..7af423f 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,9 @@ store.refresh(); // pick up what other stores wrote to the directory it; a store per use leaves a small segment behind on every run, and a small segment compresses worse because its values have few earlier ones to refer to. - `keys` lists the keys in the order their current values lie in the segments, oldest segment - first. One store's writes are listed in the order it made them; writes that several stores - made at the same time are grouped by store, not interleaved by time. Read every value in that - order: a scattered order decodes the surrounding values again for most of them. + first. One store's writes are listed in the order it made them; the writes of several stores + are grouped by segment, so their order does not tell which was made first. Read every value + in that order: a scattered order decodes the surrounding values again for most of them. - Among records of a key that stores wrote without seeing each other's, which one wins is unspecified, but every reader picks the same one. - `max_segment_bytes` / `maxSegmentBytes` must be between 1,024 and 100,000,000; the upper From 154b217de2301282e6196c02a8306b0dc6108cc2 Mon Sep 17 00:00:00 2001 From: "Sakamoto, Kazunori" Date: Wed, 7 Oct 2026 11:53:28 +0900 Subject: [PATCH 4/4] docs: say that the next run continues at most one concurrent segment Co-authored-by: WillBooster (Claude Code) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7af423f..b9e2a70 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ store.refresh(); // pick up what other stores wrote to the directory stay invisible until one of those happens. Call `refresh` after replacing segments under an open store, for example by checking out another branch. - Stores that write to a directory at the same time each append to a segment of their own, and - the next run continues only one of them. Open one store per directory in a process and share + the next run continues at most one of them. Open one store per directory in a process and share it; a store per use leaves a small segment behind on every run, and a small segment compresses worse because its values have few earlier ones to refer to. - `keys` lists the keys in the order their current values lie in the segments, oldest segment