From 343e763d211a42bd939ea070fc3343c476bffd2b Mon Sep 17 00:00:00 2001 From: ferhat elmas Date: Thu, 13 Aug 2026 09:42:12 +0200 Subject: [PATCH] fix: absolute TTL, coarse ttl resolution, and jitter for caches Signed-off-by: ferhat elmas --- src/internal/auth/jwks/manager.ts | 3 +- src/internal/cache/lru.test.ts | 108 +++++++++++++++++- src/internal/cache/lru.ts | 37 +++++- src/internal/cache/monitoring.test.ts | 3 + src/internal/database/tenant.ts | 3 +- .../protocols/s3/credentials/manager.ts | 3 +- .../adapter/pgvector/metric-cache.test.ts | 38 +++++- .../vector/adapter/pgvector/metric-cache.ts | 5 +- src/test/tenant.test.ts | 38 +++++- 9 files changed, 224 insertions(+), 14 deletions(-) diff --git a/src/internal/auth/jwks/manager.ts b/src/internal/auth/jwks/manager.ts index 0aec38146..c709f1733 100644 --- a/src/internal/auth/jwks/manager.ts +++ b/src/internal/auth/jwks/manager.ts @@ -4,6 +4,7 @@ import { type CacheLookupOptions, createLruCache, DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS, + DEFAULT_CACHE_TTL_JITTER_RATIO, TENANT_JWKS_CACHE_NAME, } from '@internal/cache' import { createInvalidatableSingleFlightByKey } from '@internal/concurrency' @@ -23,7 +24,7 @@ export const TENANT_JWKS_CACHE_TTL_MS = 1000 * 60 * 60 // 1h const tenantJwksConfigCache = createLruCache(TENANT_JWKS_CACHE_NAME, { max: TENANT_JWKS_CACHE_MAX_ITEMS, ttl: TENANT_JWKS_CACHE_TTL_MS, - updateAgeOnGet: true, + ttlJitterRatio: DEFAULT_CACHE_TTL_JITTER_RATIO, allowStale: false, purgeStaleIntervalMs: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS, }) diff --git a/src/internal/cache/lru.test.ts b/src/internal/cache/lru.test.ts index 4ea1a640e..254e1fe74 100644 --- a/src/internal/cache/lru.test.ts +++ b/src/internal/cache/lru.test.ts @@ -1,4 +1,4 @@ -import { createLruCache } from '@internal/cache' +import { createLruCache, DEFAULT_CACHE_TTL_RESOLUTION_MS } from '@internal/cache' import { vi } from 'vitest' describe('lru cache wrapper', () => { @@ -11,10 +11,57 @@ describe('lru cache wrapper', () => { vi.useRealTimers() }) + test('defaults ttl clock resolution for caches with a ttl', () => { + let now = 1 + const perf = { now: vi.fn(() => now) } + const cache = createLruCache({ + max: 2, + ttl: DEFAULT_CACHE_TTL_RESOLUTION_MS * 2, + perf, + }) + + cache.set('entry', { bytes: 1 }) + expect(cache.get('entry')).toEqual({ bytes: 1 }) + expect(perf.now).toHaveBeenCalledTimes(2) + + now += DEFAULT_CACHE_TTL_RESOLUTION_MS - 1 + vi.advanceTimersByTime(DEFAULT_CACHE_TTL_RESOLUTION_MS - 1) + + expect(cache.get('entry')).toEqual({ bytes: 1 }) + expect(perf.now).toHaveBeenCalledTimes(2) + + now += 1 + vi.advanceTimersByTime(1) + + expect(cache.get('entry')).toEqual({ bytes: 1 }) + expect(perf.now).toHaveBeenCalledTimes(3) + }) + + test('respects an explicit ttl clock resolution override', () => { + let now = 1 + const perf = { now: vi.fn(() => now) } + const cache = createLruCache({ + max: 2, + ttl: 10, + ttlResolution: 0, + perf, + }) + + cache.set('entry', { bytes: 1 }) + expect(cache.get('entry')).toEqual({ bytes: 1 }) + + now = 12 + + expect(cache.get('entry')).toBeUndefined() + expect(perf.now).toHaveBeenCalledTimes(3) + expect(vi.getTimerCount()).toBe(0) + }) + test('plain get returns hits misses and stale values according to allowStale', () => { const staleCache = createLruCache({ max: 2, ttl: 10, + ttlResolution: 0, allowStale: true, perf: { now: () => Date.now(), @@ -33,6 +80,7 @@ describe('lru cache wrapper', () => { const expiringCache = createLruCache({ max: 2, ttl: 10, + ttlResolution: 0, allowStale: false, perf: { now: () => Date.now(), @@ -50,6 +98,7 @@ describe('lru cache wrapper', () => { const cache = createLruCache({ max: 2, ttl: 10, + ttlResolution: 0, purgeStaleIntervalMs: 20, perf: { now: () => Date.now(), @@ -70,6 +119,7 @@ describe('lru cache wrapper', () => { const cache = createLruCache({ max: 2, ttl: 15, + ttlResolution: 0, perf: { now: () => Date.now(), }, @@ -94,10 +144,66 @@ describe('lru cache wrapper', () => { expect(cache.getStats()).toEqual({ entries: 0 }) }) + test('expires entries at an absolute ttl even when read continuously', () => { + const cache = createLruCache({ + max: 2, + ttl: 10, + ttlResolution: 0, + allowStale: false, + perf: { + now: () => Date.now(), + }, + }) + + cache.set('entry', { bytes: 1 }) + + vi.advanceTimersByTime(4) + expect(cache.get('entry')).toEqual({ bytes: 1 }) + + vi.advanceTimersByTime(4) + expect(cache.get('entry')).toEqual({ bytes: 1 }) + + vi.advanceTimersByTime(3) + expect(cache.get('entry')).toBeUndefined() + }) + + test('ttl jitter shortens the effective ttl by up to the configured ratio', () => { + const random = vi.spyOn(Math, 'random') + const cache = createLruCache({ + max: 2, + ttl: 10, + ttlResolution: 0, + ttlJitterRatio: 0.5, + perf: { + now: () => Date.now(), + }, + }) + + random.mockReturnValue(1) + cache.set('full-jitter', { bytes: 1 }) + random.mockReturnValue(0) + cache.set('no-jitter', { bytes: 1 }) + + vi.advanceTimersByTime(6) + expect(cache.get('full-jitter')).toBeUndefined() + expect(cache.get('no-jitter')).toEqual({ bytes: 1 }) + + vi.advanceTimersByTime(5) + expect(cache.get('no-jitter')).toBeUndefined() + random.mockRestore() + }) + + test('rejects a jitter ratio outside [0, 1)', () => { + expect(() => createLruCache({ max: 2, ttl: 10, ttlJitterRatio: 1 })).toThrow() + expect(() => createLruCache({ max: 2, ttl: 10, ttlJitterRatio: -0.1 })).toThrow() + expect(() => createLruCache({ max: 2, ttl: 10, ttlJitterRatio: NaN })).toThrow() + }) + test('clears the stale purge timer on dispose', () => { const cache = createLruCache({ max: 2, ttl: 10, + ttlResolution: 0, purgeStaleIntervalMs: 20, perf: { now: () => Date.now(), diff --git a/src/internal/cache/lru.ts b/src/internal/cache/lru.ts index b4889357b..33f52268a 100644 --- a/src/internal/cache/lru.ts +++ b/src/internal/cache/lru.ts @@ -7,18 +7,42 @@ export type LruCacheSetOptions = BaseLruCache.SetOpt export type LruCacheOptions = BaseLruCache.Options & { purgeStaleIntervalMs?: number + ttlJitterRatio?: number } export const DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS = 1000 * 60 // 1 minute +export const DEFAULT_CACHE_TTL_RESOLUTION_MS = 1000 * 30 // 30 seconds +export const DEFAULT_CACHE_TTL_JITTER_RATIO = 0.1 + +function withDefaultTtlResolution( + options: LruCacheOptions +): LruCacheOptions { + if (!options.ttl || options.ttlResolution !== undefined) { + return options + } + + return { + ...options, + ttlResolution: DEFAULT_CACHE_TTL_RESOLUTION_MS, + } +} export class LruCache implements DisposableCache> { private readonly cache: BaseLruCache private readonly purgeStaleTimer?: ReturnType + private readonly ttlJitterRatio?: number + private readonly defaultTtl?: number constructor(options: LruCacheOptions) { - const { purgeStaleIntervalMs, ...cacheOptions } = options + const { purgeStaleIntervalMs, ttlJitterRatio, ...cacheOptions } = options + + if (ttlJitterRatio !== undefined && !(ttlJitterRatio >= 0 && ttlJitterRatio < 1)) { + throw new Error(`ttlJitterRatio must be in [0, 1), got ${ttlJitterRatio}`) + } + this.ttlJitterRatio = ttlJitterRatio + this.defaultTtl = cacheOptions.ttl this.cache = new BaseLruCache({ ...cacheOptions, @@ -37,6 +61,13 @@ export class LruCache } set(key: K, value: V, options?: LruCacheSetOptions): void { + const ttl = options?.ttl ?? this.defaultTtl + if (this.ttlJitterRatio && ttl && Number.isFinite(ttl)) { + options = { + ...options, + ttl: Math.max(1, Math.round(ttl * (1 - this.ttlJitterRatio * Math.random()))), + } + } this.cache.set(key, value, options) } @@ -73,11 +104,11 @@ export function createLruCache( maybeOptions?: LruCacheOptions ) { if (typeof nameOrOptions !== 'string') { - return new LruCache(nameOrOptions) + return new LruCache(withDefaultTtlResolution(nameOrOptions)) } const cacheName = nameOrOptions - const options = maybeOptions as LruCacheOptions + const options = withDefaultTtlResolution(maybeOptions as LruCacheOptions) const cache = new LruCache({ ...options, disposeAfter: withCacheEvictionMetrics(cacheName, options.disposeAfter), diff --git a/src/internal/cache/monitoring.test.ts b/src/internal/cache/monitoring.test.ts index 7d0acc9ab..f054c4d6e 100644 --- a/src/internal/cache/monitoring.test.ts +++ b/src/internal/cache/monitoring.test.ts @@ -81,6 +81,7 @@ describe('cache telemetry helpers', () => { const cache = createLruCache(TENANT_CONFIG_CACHE_NAME, { max: 2, ttl: 10, + ttlResolution: 0, allowStale: true, perf: { now: () => Date.now(), @@ -157,6 +158,7 @@ describe('cache telemetry helpers', () => { const cache = createLruCache(TENANT_CONFIG_CACHE_NAME, { max: 2, ttl: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS - 1, + ttlResolution: 0, purgeStaleIntervalMs: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS, perf: { now: () => Date.now(), @@ -195,6 +197,7 @@ describe('cache telemetry helpers', () => { const cache = createLruCache(TENANT_CONFIG_CACHE_NAME, { max: 2, ttl: 10, + ttlResolution: 0, perf: { now: () => Date.now(), }, diff --git a/src/internal/database/tenant.ts b/src/internal/database/tenant.ts index 3a5f100d0..512424a44 100644 --- a/src/internal/database/tenant.ts +++ b/src/internal/database/tenant.ts @@ -2,6 +2,7 @@ import { CACHE_LOOKUP_WITHOUT_METRICS, createLruCache, DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS, + DEFAULT_CACHE_TTL_JITTER_RATIO, TENANT_CONFIG_CACHE_NAME, } from '@internal/cache' import { lastLocalMigrationName } from '@internal/database/migrations/files' @@ -92,7 +93,7 @@ export const TENANT_CONFIG_CACHE_TTL_MS = 1000 * 60 * 60 // 1h const tenantConfigCache = createLruCache(TENANT_CONFIG_CACHE_NAME, { max: TENANT_CONFIG_CACHE_MAX_ITEMS, ttl: TENANT_CONFIG_CACHE_TTL_MS, - updateAgeOnGet: true, + ttlJitterRatio: DEFAULT_CACHE_TTL_JITTER_RATIO, allowStale: false, purgeStaleIntervalMs: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS, }) diff --git a/src/storage/protocols/s3/credentials/manager.ts b/src/storage/protocols/s3/credentials/manager.ts index 95cad75e1..bd0f1767c 100644 --- a/src/storage/protocols/s3/credentials/manager.ts +++ b/src/storage/protocols/s3/credentials/manager.ts @@ -5,6 +5,7 @@ import { type CacheLookupOptions, createLruCache, DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS, + DEFAULT_CACHE_TTL_JITTER_RATIO, TENANT_S3_CREDENTIALS_CACHE_NAME, } from '@internal/cache' import { createInvalidatableSingleFlightByKey } from '@internal/concurrency' @@ -25,7 +26,7 @@ const tenantS3CredentialsCache = createLruCache( { max: TENANT_S3_CREDENTIALS_CACHE_MAX_ITEMS, ttl: TENANT_S3_CREDENTIALS_CACHE_TTL_MS, - updateAgeOnGet: true, + ttlJitterRatio: DEFAULT_CACHE_TTL_JITTER_RATIO, allowStale: false, purgeStaleIntervalMs: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS, } diff --git a/src/storage/protocols/vector/adapter/pgvector/metric-cache.test.ts b/src/storage/protocols/vector/adapter/pgvector/metric-cache.test.ts index afa213746..321694e71 100644 --- a/src/storage/protocols/vector/adapter/pgvector/metric-cache.test.ts +++ b/src/storage/protocols/vector/adapter/pgvector/metric-cache.test.ts @@ -1,23 +1,53 @@ +import { vi } from 'vitest' import { createMetricCache } from './metric-cache' describe('pgvector metric cache', () => { - it('renews the TTL on hot reads', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('expires entries at an absolute ttl even when read continuously', () => { let now = 1 const cache = createMetricCache({ ttl: 10, + ttlJitterRatio: 0, ttlResolution: 0, perf: { now: () => now }, }) cache.set('index', 'euclidean') - now = 10 + now = 5 expect(cache.get('index')).toBe('euclidean') - now = 12 + now = 9 expect(cache.get('index')).toBe('euclidean') - now = 23 + now = 12 expect(cache.get('index')).toBeUndefined() cache.dispose() }) + + it('applies per-set ttl jitter', () => { + const random = vi.spyOn(Math, 'random') + let now = 1 + const cache = createMetricCache({ + ttl: 10, + ttlJitterRatio: 0.5, + ttlResolution: 0, + perf: { now: () => now }, + }) + + random.mockReturnValue(1) + cache.set('full-jitter', 'euclidean') + random.mockReturnValue(0) + cache.set('no-jitter', 'cosine') + + now = 7 + expect(cache.get('full-jitter')).toBeUndefined() + expect(cache.get('no-jitter')).toBe('cosine') + + now = 12 + expect(cache.get('no-jitter')).toBeUndefined() + cache.dispose() + }) }) diff --git a/src/storage/protocols/vector/adapter/pgvector/metric-cache.ts b/src/storage/protocols/vector/adapter/pgvector/metric-cache.ts index 74fff760b..bdc66f96b 100644 --- a/src/storage/protocols/vector/adapter/pgvector/metric-cache.ts +++ b/src/storage/protocols/vector/adapter/pgvector/metric-cache.ts @@ -2,6 +2,7 @@ import type { DistanceMetric } from '@aws-sdk/client-s3vectors' import { createLruCache, DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS, + DEFAULT_CACHE_TTL_JITTER_RATIO, PGVECTOR_METRIC_CACHE_NAME, } from '@internal/cache' import type { Perf } from 'lru-cache' @@ -11,6 +12,7 @@ const METRIC_CACHE_MAX = 1_000 interface MetricCacheOptions { ttl?: number + ttlJitterRatio?: number ttlResolution?: number perf?: Perf } @@ -18,9 +20,10 @@ interface MetricCacheOptions { export function createMetricCache(options: MetricCacheOptions = {}) { return createLruCache(PGVECTOR_METRIC_CACHE_NAME, { ttl: options.ttl ?? METRIC_CACHE_TTL_MS, + ttlJitterRatio: options.ttlJitterRatio ?? DEFAULT_CACHE_TTL_JITTER_RATIO, ttlResolution: options.ttlResolution, max: METRIC_CACHE_MAX, - updateAgeOnGet: true, + allowStale: false, purgeStaleIntervalMs: DEFAULT_CACHE_PURGE_STALE_INTERVAL_MS, perf: options.perf, }) diff --git a/src/test/tenant.test.ts b/src/test/tenant.test.ts index 036eaaeb5..c91a3b2d6 100644 --- a/src/test/tenant.test.ts +++ b/src/test/tenant.test.ts @@ -154,10 +154,11 @@ type TenantModule = typeof import('../internal/database/tenant') type MultitenantPgModule = typeof import('../internal/database/multitenant-pg') async function loadTenantModule( - maxItems: number + maxItems: number, + cacheOverrides: Record = {} ): Promise<{ tenantModule: TenantModule; multitenantPgModule: MultitenantPgModule }> { vi.resetModules() - mockCreateLruCache({ max: maxItems }) + mockCreateLruCache({ max: maxItems, ...cacheOverrides }) return { tenantModule: await import('../internal/database/tenant'), @@ -1099,6 +1100,39 @@ describe('Tenant configs', () => { } }) + test('Get tenant config hot reads do not renew the absolute ttl', async () => { + const tenantId = 'cache-absolute-ttl' + const encryptedTenant = createEncryptedTenantRow(tenantId) + let now = 1 + + const { tenantModule, multitenantPgModule } = await loadTenantModule(2, { + ttl: 10, + ttlJitterRatio: 0, + ttlResolution: 0, + perf: { now: () => now }, + }) + const querySpy = vi + .spyOn(multitenantPgModule.multitenantPgExecutor, 'query') + .mockResolvedValue(mockTenantQueryResult(encryptedTenant)) + + try { + await tenantModule.getTenantConfig(tenantId) + + now = 5 + await tenantModule.getTenantConfig(tenantId) + expect(querySpy).toHaveBeenCalledTimes(1) + + now = 12 + await tenantModule.getTenantConfig(tenantId) + expect(querySpy).toHaveBeenCalledTimes(2) + } finally { + tenantModule.deleteTenantConfig(tenantId) + vi.doUnmock('@internal/cache') + vi.resetModules() + querySpy.mockRestore() + } + }) + test('An invalidated config load retries through the current generation', async () => { const tenantId = 'tenant-config-invalidation-race' const oldTenant = {