fix: absolute TTL, coarse ttl resolution, and jitter for caches

src/internal/auth/jwks/manager.ts

@@ -4,6 +4,7 @@ import {
   type CacheLookupOptions,
   createLruCache,
   DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS,
+  DEFAULT_CACHE_TTL_JITTER_RATIO,
   TENANT_JWKS_CACHE_NAME,
 } from '@internal/cache'
 import { createInvalidatableSingleFlightByKey } from '@internal/concurrency'
@@ -23,7 +24,7 @@ export const TENANT_JWKS_CACHE_TTL_MS = 1000 * 60 * 60 // 1h
 const tenantJwksConfigCache = createLruCache<string, JwksConfig>(TENANT_JWKS_CACHE_NAME, {
   max: TENANT_JWKS_CACHE_MAX_ITEMS,
   ttl: TENANT_JWKS_CACHE_TTL_MS,
-  updateAgeOnGet: true,
+  ttlJitterRatio: DEFAULT_CACHE_TTL_JITTER_RATIO,
   allowStale: false,
   purgeStaleIntervalMs: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS,
 })

src/internal/cache/lru.test.ts

@@ -1,4 +1,4 @@
-import { createLruCache } from '@internal/cache'
+import { createLruCache, DEFAULT_CACHE_TTL_RESOLUTION_MS } from '@internal/cache'
 import { vi } from 'vitest'
 
 describe('lru cache wrapper', () => {
@@ -11,10 +11,57 @@ describe('lru cache wrapper', () => {
     vi.useRealTimers()
   })
 
+  test('defaults ttl clock resolution for caches with a ttl', () => {
+    let now = 1
+    const perf = { now: vi.fn(() => now) }
+    const cache = createLruCache<string, { bytes: number }>({
+      max: 2,
+      ttl: DEFAULT_CACHE_TTL_RESOLUTION_MS * 2,
+      perf,
+    })
+
+    cache.set('entry', { bytes: 1 })
+    expect(cache.get('entry')).toEqual({ bytes: 1 })
+    expect(perf.now).toHaveBeenCalledTimes(2)
+
+    now += DEFAULT_CACHE_TTL_RESOLUTION_MS - 1
+    vi.advanceTimersByTime(DEFAULT_CACHE_TTL_RESOLUTION_MS - 1)
+
+    expect(cache.get('entry')).toEqual({ bytes: 1 })
+    expect(perf.now).toHaveBeenCalledTimes(2)
+
+    now += 1
+    vi.advanceTimersByTime(1)
+
+    expect(cache.get('entry')).toEqual({ bytes: 1 })
+    expect(perf.now).toHaveBeenCalledTimes(3)
+  })
+
+  test('respects an explicit ttl clock resolution override', () => {
+    let now = 1
+    const perf = { now: vi.fn(() => now) }
+    const cache = createLruCache<string, { bytes: number }>({
+      max: 2,
+      ttl: 10,
+      ttlResolution: 0,
+      perf,
+    })
+
+    cache.set('entry', { bytes: 1 })
+    expect(cache.get('entry')).toEqual({ bytes: 1 })
+
+    now = 12
+
+    expect(cache.get('entry')).toBeUndefined()
+    expect(perf.now).toHaveBeenCalledTimes(3)
+    expect(vi.getTimerCount()).toBe(0)
+  })
+
   test('plain get returns hits misses and stale values according to allowStale', () => {
     const staleCache = createLruCache<string, { bytes: number }>({
       max: 2,
       ttl: 10,
+      ttlResolution: 0,
       allowStale: true,
       perf: {
         now: () => Date.now(),
@@ -33,6 +80,7 @@ describe('lru cache wrapper', () => {
     const expiringCache = createLruCache<string, { bytes: number }>({
       max: 2,
       ttl: 10,
+      ttlResolution: 0,
       allowStale: false,
       perf: {
         now: () => Date.now(),
@@ -50,6 +98,7 @@ describe('lru cache wrapper', () => {
     const cache = createLruCache<string, { bytes: number }>({
       max: 2,
       ttl: 10,
+      ttlResolution: 0,
       purgeStaleIntervalMs: 20,
       perf: {
         now: () => Date.now(),
@@ -70,6 +119,7 @@ describe('lru cache wrapper', () => {
     const cache = createLruCache<string, { bytes: number }>({
       max: 2,
       ttl: 15,
+      ttlResolution: 0,
       perf: {
         now: () => Date.now(),
       },
@@ -94,10 +144,66 @@ describe('lru cache wrapper', () => {
     expect(cache.getStats()).toEqual({ entries: 0 })
   })
 
+  test('expires entries at an absolute ttl even when read continuously', () => {
+    const cache = createLruCache<string, { bytes: number }>({
+      max: 2,
+      ttl: 10,
+      ttlResolution: 0,
+      allowStale: false,
+      perf: {
+        now: () => Date.now(),
+      },
+    })
+
+    cache.set('entry', { bytes: 1 })
+
+    vi.advanceTimersByTime(4)
+    expect(cache.get('entry')).toEqual({ bytes: 1 })
+
+    vi.advanceTimersByTime(4)
+    expect(cache.get('entry')).toEqual({ bytes: 1 })
+
+    vi.advanceTimersByTime(3)
+    expect(cache.get('entry')).toBeUndefined()
+  })
+
+  test('ttl jitter shortens the effective ttl by up to the configured ratio', () => {
+    const random = vi.spyOn(Math, 'random')
+    const cache = createLruCache<string, { bytes: number }>({
+      max: 2,
+      ttl: 10,
+      ttlResolution: 0,
+      ttlJitterRatio: 0.5,
+      perf: {
+        now: () => Date.now(),
+      },
+    })
+
+    random.mockReturnValue(1)
+    cache.set('full-jitter', { bytes: 1 })
+    random.mockReturnValue(0)
+    cache.set('no-jitter', { bytes: 1 })
+
+    vi.advanceTimersByTime(6)
+    expect(cache.get('full-jitter')).toBeUndefined()
+    expect(cache.get('no-jitter')).toEqual({ bytes: 1 })
+
+    vi.advanceTimersByTime(5)
+    expect(cache.get('no-jitter')).toBeUndefined()
+    random.mockRestore()
+  })
+
+  test('rejects a jitter ratio outside [0, 1)', () => {
+    expect(() => createLruCache({ max: 2, ttl: 10, ttlJitterRatio: 1 })).toThrow()
+    expect(() => createLruCache({ max: 2, ttl: 10, ttlJitterRatio: -0.1 })).toThrow()
+    expect(() => createLruCache({ max: 2, ttl: 10, ttlJitterRatio: NaN })).toThrow()
+  })
+
   test('clears the stale purge timer on dispose', () => {
     const cache = createLruCache<string, { bytes: number }>({
       max: 2,
       ttl: 10,
+      ttlResolution: 0,
       purgeStaleIntervalMs: 20,
       perf: {
         now: () => Date.now(),

src/internal/cache/lru.ts

@@ -7,18 +7,42 @@ export type LruCacheSetOptions<K extends {}, V extends {}> = BaseLruCache.SetOpt
 
 export type LruCacheOptions<K extends {}, V extends {}> = BaseLruCache.Options<K, V, unknown> & {
   purgeStaleIntervalMs?: number
+  ttlJitterRatio?: number
 }
 
 export const DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS = 1000 * 60 // 1 minute
+export const DEFAULT_CACHE_TTL_RESOLUTION_MS = 1000 * 30 // 30 seconds
+export const DEFAULT_CACHE_TTL_JITTER_RATIO = 0.1
+
+function withDefaultTtlResolution<K extends {}, V extends {}>(
+  options: LruCacheOptions<K, V>
+): LruCacheOptions<K, V> {
+  if (!options.ttl || options.ttlResolution !== undefined) {
+    return options
+  }
+
+  return {
+    ...options,
+    ttlResolution: DEFAULT_CACHE_TTL_RESOLUTION_MS,
+  }
+}
 
 export class LruCache<K extends {}, V extends {}>
   implements DisposableCache<K, V, LruCacheSetOptions<K, V>>
 {
   private readonly cache: BaseLruCache<K, V>
   private readonly purgeStaleTimer?: ReturnType<typeof setInterval>
+  private readonly ttlJitterRatio?: number
+  private readonly defaultTtl?: number
 
   constructor(options: LruCacheOptions<K, V>) {
-    const { purgeStaleIntervalMs, ...cacheOptions } = options
+    const { purgeStaleIntervalMs, ttlJitterRatio, ...cacheOptions } = options
+
+    if (ttlJitterRatio !== undefined && !(ttlJitterRatio >= 0 && ttlJitterRatio < 1)) {
+      throw new Error(`ttlJitterRatio must be in [0, 1), got ${ttlJitterRatio}`)
+    }
+    this.ttlJitterRatio = ttlJitterRatio
+    this.defaultTtl = cacheOptions.ttl
 
     this.cache = new BaseLruCache<K, V>({
       ...cacheOptions,
@@ -37,6 +61,13 @@ export class LruCache<K extends {}, V extends {}>
   }
 
   set(key: K, value: V, options?: LruCacheSetOptions<K, V>): void {
+    const ttl = options?.ttl ?? this.defaultTtl
+    if (this.ttlJitterRatio && ttl && Number.isFinite(ttl)) {
+      options = {
+        ...options,
+        ttl: Math.max(1, Math.round(ttl * (1 - this.ttlJitterRatio * Math.random()))),
+      }
+    }
     this.cache.set(key, value, options)
   }
 
@@ -73,11 +104,11 @@ export function createLruCache<K extends {}, V extends {}>(
   maybeOptions?: LruCacheOptions<K, V>
 ) {
   if (typeof nameOrOptions !== 'string') {
-    return new LruCache(nameOrOptions)
+    return new LruCache(withDefaultTtlResolution(nameOrOptions))
   }
 
   const cacheName = nameOrOptions
-  const options = maybeOptions as LruCacheOptions<K, V>
+  const options = withDefaultTtlResolution(maybeOptions as LruCacheOptions<K, V>)
   const cache = new LruCache<K, V>({
     ...options,
     disposeAfter: withCacheEvictionMetrics(cacheName, options.disposeAfter),

src/internal/cache/monitoring.test.ts

@@ -81,6 +81,7 @@ describe('cache telemetry helpers', () => {
     const cache = createLruCache(TENANT_CONFIG_CACHE_NAME, {
       max: 2,
       ttl: 10,
+      ttlResolution: 0,
       allowStale: true,
       perf: {
         now: () => Date.now(),
@@ -157,6 +158,7 @@ describe('cache telemetry helpers', () => {
     const cache = createLruCache(TENANT_CONFIG_CACHE_NAME, {
       max: 2,
       ttl: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS - 1,
+      ttlResolution: 0,
       purgeStaleIntervalMs: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS,
       perf: {
         now: () => Date.now(),
@@ -195,6 +197,7 @@ describe('cache telemetry helpers', () => {
     const cache = createLruCache(TENANT_CONFIG_CACHE_NAME, {
       max: 2,
       ttl: 10,
+      ttlResolution: 0,
       perf: {
         now: () => Date.now(),
       },

src/internal/database/tenant.ts

@@ -2,6 +2,7 @@ import {
   CACHE_LOOKUP_WITHOUT_METRICS,
   createLruCache,
   DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS,
+  DEFAULT_CACHE_TTL_JITTER_RATIO,
   TENANT_CONFIG_CACHE_NAME,
 } from '@internal/cache'
 import { lastLocalMigrationName } from '@internal/database/migrations/files'
@@ -92,7 +93,7 @@ export const TENANT_CONFIG_CACHE_TTL_MS = 1000 * 60 * 60 // 1h
 const tenantConfigCache = createLruCache<string, TenantConfig>(TENANT_CONFIG_CACHE_NAME, {
   max: TENANT_CONFIG_CACHE_MAX_ITEMS,
   ttl: TENANT_CONFIG_CACHE_TTL_MS,
-  updateAgeOnGet: true,
+  ttlJitterRatio: DEFAULT_CACHE_TTL_JITTER_RATIO,
   allowStale: false,
   purgeStaleIntervalMs: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS,
 })

src/storage/protocols/s3/credentials/manager.ts

@@ -5,6 +5,7 @@ import {
   type CacheLookupOptions,
   createLruCache,
   DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS,
+  DEFAULT_CACHE_TTL_JITTER_RATIO,
   TENANT_S3_CREDENTIALS_CACHE_NAME,
 } from '@internal/cache'
 import { createInvalidatableSingleFlightByKey } from '@internal/concurrency'
@@ -25,7 +26,7 @@ const tenantS3CredentialsCache = createLruCache<string, S3Credentials>(
   {
     max: TENANT_S3_CREDENTIALS_CACHE_MAX_ITEMS,
     ttl: TENANT_S3_CREDENTIALS_CACHE_TTL_MS,
-    updateAgeOnGet: true,
+    ttlJitterRatio: DEFAULT_CACHE_TTL_JITTER_RATIO,
     allowStale: false,
     purgeStaleIntervalMs: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS,
   }

src/storage/protocols/vector/adapter/pgvector/metric-cache.test.ts

@@ -1,23 +1,53 @@
+import { vi } from 'vitest'
 import { createMetricCache } from './metric-cache'
 
 describe('pgvector metric cache', () => {
-  it('renews the TTL on hot reads', () => {
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('expires entries at an absolute ttl even when read continuously', () => {
     let now = 1
     const cache = createMetricCache({
       ttl: 10,
+      ttlJitterRatio: 0,
       ttlResolution: 0,
       perf: { now: () => now },
     })
 
     cache.set('index', 'euclidean')
-    now = 10
+    now = 5
     expect(cache.get('index')).toBe('euclidean')
 
-    now = 12
+    now = 9
     expect(cache.get('index')).toBe('euclidean')
 
-    now = 23
+    now = 12
     expect(cache.get('index')).toBeUndefined()
     cache.dispose()
   })
+
+  it('applies per-set ttl jitter', () => {
+    const random = vi.spyOn(Math, 'random')
+    let now = 1
+    const cache = createMetricCache({
+      ttl: 10,
+      ttlJitterRatio: 0.5,
+      ttlResolution: 0,
+      perf: { now: () => now },
+    })
+
+    random.mockReturnValue(1)
+    cache.set('full-jitter', 'euclidean')
+    random.mockReturnValue(0)
+    cache.set('no-jitter', 'cosine')
+
+    now = 7
+    expect(cache.get('full-jitter')).toBeUndefined()
+    expect(cache.get('no-jitter')).toBe('cosine')
+
+    now = 12
+    expect(cache.get('no-jitter')).toBeUndefined()
+    cache.dispose()
+  })
 })

src/storage/protocols/vector/adapter/pgvector/metric-cache.ts

@@ -2,6 +2,7 @@ import type { DistanceMetric } from '@aws-sdk/client-s3vectors'
 import {
   createLruCache,
   DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS,
+  DEFAULT_CACHE_TTL_JITTER_RATIO,
   PGVECTOR_METRIC_CACHE_NAME,
 } from '@internal/cache'
 import type { Perf } from 'lru-cache'
@@ -11,16 +12,18 @@ const METRIC_CACHE_MAX = 1_000
 
 interface MetricCacheOptions {
   ttl?: number
+  ttlJitterRatio?: number
   ttlResolution?: number
   perf?: Perf
 }
 
 export function createMetricCache(options: MetricCacheOptions = {}) {
   return createLruCache<string, DistanceMetric>(PGVECTOR_METRIC_CACHE_NAME, {
     ttl: options.ttl ?? METRIC_CACHE_TTL_MS,
+    ttlJitterRatio: options.ttlJitterRatio ?? DEFAULT_CACHE_TTL_JITTER_RATIO,
     ttlResolution: options.ttlResolution,
     max: METRIC_CACHE_MAX,
-    updateAgeOnGet: true,
+    allowStale: false,
     purgeStaleIntervalMs: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS,
     perf: options.perf,
   })