diff --git a/app/src/components/intelligence/MemoryTreeStatusPanel.test.tsx b/app/src/components/intelligence/MemoryTreeStatusPanel.test.tsx index 457b08f40c..6b90db8ff2 100644 --- a/app/src/components/intelligence/MemoryTreeStatusPanel.test.tsx +++ b/app/src/components/intelligence/MemoryTreeStatusPanel.test.tsx @@ -22,14 +22,22 @@ import { const mockPipelineStatus = vi.fn(); const mockSetEnabled = vi.fn(); const mockSyncStatusList = vi.fn(); +const mockRetryFailed = vi.fn(); // #5324: the panel now navigates (budget CTA) and dispatches (escalating the // blocking cause to the shell-mounted UserErrorCenter). Stub both so the // suite keeps rendering the panel bare, without a Router or a Redux store. const mockNavigate = vi.fn(); const mockDispatch = vi.fn(); +// Analytics is a consent-gated side effect that reaches into the core-state +// snapshot; stub it so the panel renders bare and the retry-success path can be +// asserted without a real analytics pipeline. +const mockTrackAnalyticsEvent = vi.fn(); vi.mock('react-router-dom', () => ({ useNavigate: () => mockNavigate })); vi.mock('../../store/hooks', () => ({ useAppDispatch: () => mockDispatch })); +vi.mock('../analytics', () => ({ + trackAnalyticsEvent: (...args: unknown[]) => mockTrackAnalyticsEvent(...args), +})); vi.mock('../../utils/tauriCommands', async importOriginal => { // Inherit everything else (types, sibling wrappers) verbatim so the panel @@ -41,6 +49,7 @@ vi.mock('../../utils/tauriCommands', async importOriginal => { memoryTreePipelineStatus: (...args: unknown[]) => mockPipelineStatus(...args), memoryTreeSetEnabled: (...args: unknown[]) => mockSetEnabled(...args), memorySyncStatusList: (...args: unknown[]) => mockSyncStatusList(...args), + memoryTreeRetryFailed: (...args: unknown[]) => mockRetryFailed(...args), }; }); @@ -73,6 +82,8 @@ describe('', () => { mockPipelineStatus.mockReset(); mockSetEnabled.mockReset(); mockSyncStatusList.mockReset(); + mockRetryFailed.mockReset(); + mockTrackAnalyticsEvent.mockReset(); mockSyncStatusList.mockResolvedValue([]); // default: empty, harmless to existing tests }); @@ -550,6 +561,125 @@ describe('', () => { expect(mockDispatch).toHaveBeenCalled(); }); }); + + // ── Retry-failed affordance ───────────────────────────────────────────── + + it('offers a retry when jobs are parked in failed', async () => { + mockPipelineStatus.mockResolvedValue( + payload({ + status: 'error', + reason: '29 unrecoverable failure(s) need action', + pipeline_jobs: { ready: 0, running: 0, failed: 29 }, + }) + ); + render(); + + await waitFor(() => { + expect(screen.getByTestId('memory-tree-retry-failed')).toBeInTheDocument(); + }); + }); + + /** + * The affordance keys off the failed-job counter, not off the blocking-cause + * banner. A failure the pipeline has already worked past no longer surfaces a + * remediation (the core withholds a superseded cause), but its rows still sit + * in `failed` and still need clearing — so the button must be reachable with + * no banner on screen. Without this the user is left in a permanent `error` + * state with no way out, which is the bug. + */ + it('offers the retry even when no blocking cause is surfaced', async () => { + mockPipelineStatus.mockResolvedValue( + payload({ + status: 'error', + reason: '29 unrecoverable failure(s) need action', + pipeline_jobs: { ready: 0, running: 0, failed: 29 }, + first_blocking_cause: null, + }) + ); + render(); + + await waitFor(() => { + expect(screen.getByTestId('memory-tree-retry-failed')).toBeInTheDocument(); + }); + expect(screen.queryByTestId('memory-tree-blocking-cause')).not.toBeInTheDocument(); + }); + + it('hides the retry when nothing has failed', async () => { + mockPipelineStatus.mockResolvedValue(payload({ status: 'running' })); + render(); + + await waitFor(() => { + expect(screen.getByTestId('memory-tree-status-label')).toHaveTextContent(/running/i); + }); + expect(screen.queryByTestId('memory-tree-retry-failed')).not.toBeInTheDocument(); + }); + + it('requeues the failed jobs, reports the count, and re-fetches', async () => { + mockPipelineStatus.mockResolvedValue( + payload({ + status: 'error', + reason: '29 unrecoverable failure(s) need action', + pipeline_jobs: { ready: 0, running: 0, failed: 29 }, + }) + ); + mockRetryFailed.mockResolvedValue({ requeued: 29 }); + const onToast = vi.fn(); + render(); + + await waitFor(() => { + expect(screen.getByTestId('memory-tree-retry-failed')).toBeInTheDocument(); + }); + const callsBefore = mockPipelineStatus.mock.calls.length; + await act(async () => { + fireEvent.click(screen.getByTestId('memory-tree-retry-failed')); + }); + + expect(mockRetryFailed).toHaveBeenCalledTimes(1); + // Successful domain outcome is tracked with the privacy-safe count only. + expect(mockTrackAnalyticsEvent).toHaveBeenCalledWith('memory_tree_retry_succeeded', { + count: 29, + }); + await waitFor(() => { + expect(onToast).toHaveBeenCalledWith( + expect.objectContaining({ type: 'success', message: expect.stringContaining('29') }) + ); + }); + // Successful domain outcome is tracked with a privacy-safe count only. + expect(mockTrackAnalyticsEvent).toHaveBeenCalledWith('memory_tree_retry_succeeded', { + count: 29, + }); + await waitFor(() => { + expect(mockPipelineStatus.mock.calls.length).toBeGreaterThan(callsBefore); + }); + }); + + it('surfaces an error toast when the requeue fails', async () => { + mockPipelineStatus.mockResolvedValue( + payload({ + status: 'error', + reason: '29 unrecoverable failure(s) need action', + pipeline_jobs: { ready: 0, running: 0, failed: 29 }, + }) + ); + mockRetryFailed.mockRejectedValue(new Error('UNIQUE constraint failed')); + const onToast = vi.fn(); + render(); + + await waitFor(() => { + expect(screen.getByTestId('memory-tree-retry-failed')).toBeInTheDocument(); + }); + await act(async () => { + fireEvent.click(screen.getByTestId('memory-tree-retry-failed')); + }); + + await waitFor(() => { + expect(onToast).toHaveBeenCalledWith( + expect.objectContaining({ type: 'error', message: 'UNIQUE constraint failed' }) + ); + }); + // The button must stay usable so a transient failure is not a dead end. + expect(screen.getByTestId('memory-tree-retry-failed')).not.toBeDisabled(); + }); }); describe('integration health helpers', () => { diff --git a/app/src/components/intelligence/MemoryTreeStatusPanel.tsx b/app/src/components/intelligence/MemoryTreeStatusPanel.tsx index 1cef30c3ca..41287b0201 100644 --- a/app/src/components/intelligence/MemoryTreeStatusPanel.tsx +++ b/app/src/components/intelligence/MemoryTreeStatusPanel.tsx @@ -31,8 +31,10 @@ import { type MemorySyncStatusRow, memoryTreePipelineStatus, type MemoryTreePipelineStatus, + memoryTreeRetryFailed, memoryTreeSetEnabled, } from '../../utils/tauriCommands'; +import { trackAnalyticsEvent } from '../analytics'; import Button from '../ui/Button'; /** Translator function shape exposed by `useT()`. */ @@ -338,6 +340,7 @@ export function MemoryTreeStatusPanel({ onToast }: MemoryTreeStatusPanelProps) { const dispatch = useAppDispatch(); const { status, integrations, loading, error, refresh } = useMemoryTreeStatus(); const [toggleBusy, setToggleBusy] = useState(false); + const [retryBusy, setRetryBusy] = useState(false); // #002 (FR-004): the single first blocking cause. Prefer the explicit // `first_blocking_cause`; fall back to the active degradation cause so older @@ -376,6 +379,46 @@ export function MemoryTreeStatusPanel({ onToast }: MemoryTreeStatusPanelProps) { } }, [status, toggleBusy, refresh, onToast, t]); + /** + * Requeue every terminally-failed job. + * + * An unrecoverable failure (auth, budget, dimension mismatch) is terminal by + * design — the worker never retries it — so a batch that failed under a + * since-fixed config stays parked forever and pins this panel on `error`. + * The `memory_tree_retry_failed` RPC existed for exactly this, but had no + * caller anywhere in the app, leaving the user with a permanent error state + * and no way to clear it. + */ + const handleRetryFailed = useCallback(async () => { + if (retryBusy) { + console.debug('[ui-flow][memory-tree-status] retryFailed: skipped busy=true'); + return; + } + console.debug('[ui-flow][memory-tree-status] retryFailed: entry'); + setRetryBusy(true); + console.debug('[ui-flow][memory-tree-status] retryFailed: busy=true rpc:start'); + try { + const { requeued } = await memoryTreeRetryFailed(); + console.debug('[ui-flow][memory-tree-status] retryFailed: rpc:ok requeued=%d', requeued); + // Record the successful domain outcome (not just the click). Privacy-safe: + // a non-identifying count only, no ids or user text. + trackAnalyticsEvent('memory_tree_retry_succeeded', { count: requeued }); + onToast?.({ + type: 'success', + title: t('memoryTree.status.retryFailedDone'), + message: t('memoryTree.status.retryFailedCount').replace('{count}', String(requeued)), + }); + await refresh(); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.warn('[ui-flow][memory-tree-status] retryFailed: error %s', message); + onToast?.({ type: 'error', title: t('memoryTree.status.retryFailedError'), message }); + } finally { + console.debug('[ui-flow][memory-tree-status] retryFailed: busy=false exit'); + setRetryBusy(false); + } + }, [retryBusy, refresh, onToast, t]); + const statusKind = status?.status ?? 'idle'; // #5324: "Error — 936 unrecoverable failures need action" told the user // nothing they could act on. When the blocking cause is a spent embedding @@ -413,6 +456,12 @@ export function MemoryTreeStatusPanel({ onToast }: MemoryTreeStatusPanelProps) { // with a localized remediation. const degraded = status?.degraded; + // Parked failures are the one panel state the user can act on directly, and + // the affordance is keyed off the counter rather than off the blocking-cause + // banner: a failure the pipeline has already worked past no longer surfaces a + // remediation, but its rows still sit in `failed` and still need clearing. + const failedJobs = status?.pipeline_jobs.failed ?? 0; + const checked = !(status?.is_paused ?? false); const tileClass = @@ -513,6 +562,23 @@ export function MemoryTreeStatusPanel({ onToast }: MemoryTreeStatusPanelProps) { {status.reason ? (
{status.reason}
) : null} + {failedJobs > 0 ? ( +
+ +
+ ) : null} )} diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index 6978e9904e..4611edd4a9 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -1274,6 +1274,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'أبداً', 'memoryTree.status.fetchError': 'لم أستطع الحصول على وضعية شجرة الذاكرة', 'memoryTree.status.retry': 'Retry', + 'memoryTree.status.retryFailed': 'إعادة تشغيل المهام الفاشلة', + 'memoryTree.status.retryFailedBusy': 'جارٍ إعادة المحاولة...', + 'memoryTree.status.retryFailedDone': 'تمت إعادة إدراج المهام الفاشلة', + 'memoryTree.status.retryFailedCount': 'تمت جدولة {count} مهمة للتشغيل من جديد.', + 'memoryTree.status.retryFailedError': 'تعذّرت إعادة إدراج المهام الفاشلة', 'memoryTree.status.toggleFailed': 'لا يمكن أن نهز السيرة الذاتية', 'memoryTree.status.justNow': 'الآن', 'memoryTree.status.secondsAgo': 'اكساكسوكس قبل', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index afa1147bf3..bf2d5a5f47 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -1305,6 +1305,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'কখনো নয়', 'memoryTree.status.fetchError': 'মেমরি প্রাপ্ত করতে ব্যর্থ', 'memoryTree.status.retry': 'পুনরায় চেষ্টা করুন', + 'memoryTree.status.retryFailed': 'ব্যর্থ কাজগুলো আবার চালান', + 'memoryTree.status.retryFailedBusy': 'আবার চেষ্টা করা হচ্ছে...', + 'memoryTree.status.retryFailedDone': 'ব্যর্থ কাজগুলো আবার সারিতে দেওয়া হয়েছে', + 'memoryTree.status.retryFailedCount': '{count}টি কাজ আবার চালানোর জন্য সারিতে রাখা হয়েছে।', + 'memoryTree.status.retryFailedError': 'ব্যর্থ কাজগুলো আবার সারিতে দেওয়া যায়নি', 'memoryTree.status.toggleFailed': 'স্বয়ংক্রিয়ভাবে সনাক্ত করা সম্ভব হয়নি', 'memoryTree.status.justNow': 'এখন', 'memoryTree.status.secondsAgo': 'xqxqx পূর্বে', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index e214421647..bc4b68738b 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -1351,6 +1351,12 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'Nie', 'memoryTree.status.fetchError': 'Speicherbaum-Status konnte nicht abgerufen werden', 'memoryTree.status.retry': 'Wiederholen', + 'memoryTree.status.retryFailed': 'Fehlgeschlagene Jobs erneut ausführen', + 'memoryTree.status.retryFailedBusy': 'Wird wiederholt...', + 'memoryTree.status.retryFailedDone': 'Fehlgeschlagene Jobs neu eingereiht', + 'memoryTree.status.retryFailedCount': 'Erneut eingereihte Jobs: {count}.', + 'memoryTree.status.retryFailedError': + 'Die fehlgeschlagenen Jobs konnten nicht neu eingereiht werden', 'memoryTree.status.toggleFailed': 'Automatische Synchronisierung konnte nicht umgeschaltet werden', 'memoryTree.status.justNow': 'gerade eben', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index 80460c0bb6..db7db4e0e0 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -1246,6 +1246,11 @@ const en: TranslationMap = { 'Memory processing encountered an issue. Check Connections → API keys for configuration.', 'memoryTree.status.fetchError': "Couldn't fetch Memory Tree status", 'memoryTree.status.retry': 'Retry', + 'memoryTree.status.retryFailed': 'Retry failed jobs', + 'memoryTree.status.retryFailedBusy': 'Retrying...', + 'memoryTree.status.retryFailedDone': 'Failed jobs requeued', + 'memoryTree.status.retryFailedCount': 'Jobs queued to run again: {count}.', + 'memoryTree.status.retryFailedError': 'Could not requeue the failed jobs', 'memoryTree.status.toggleFailed': "Couldn't toggle auto-sync", // Relative-time buckets surfaced by the last-sync tile. `{count}` is // replaced client-side at the call site (the runtime `t()` does not diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index 48c6b2015d..4e9004dee2 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -1331,6 +1331,12 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'Nunca', 'memoryTree.status.fetchError': 'No se pudo obtener el estado del Árbol de Memoria', 'memoryTree.status.retry': 'Rever', + 'memoryTree.status.retryFailed': 'Reintentar los trabajos fallidos', + 'memoryTree.status.retryFailedBusy': 'Reintentando...', + 'memoryTree.status.retryFailedDone': 'Trabajos fallidos añadidos de nuevo a la cola', + 'memoryTree.status.retryFailedCount': 'Trabajos en cola para ejecutarse de nuevo: {count}.', + 'memoryTree.status.retryFailedError': + 'No se pudieron volver a poner en cola los trabajos fallidos', 'memoryTree.status.toggleFailed': 'No se pudo activar la sincronización automática', 'memoryTree.status.justNow': 'justo ahora', 'memoryTree.status.secondsAgo': '{count}s hace', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index 4ce6308894..de0f2efb07 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -1344,6 +1344,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'Jamais', 'memoryTree.status.fetchError': "Impossible de récupérer l'état de l'arborescence de mémoire", 'memoryTree.status.retry': 'Réessayer', + 'memoryTree.status.retryFailed': 'Relancer les tâches en échec', + 'memoryTree.status.retryFailedBusy': 'Nouvelle tentative...', + 'memoryTree.status.retryFailedDone': 'Tâches en échec remises en file', + 'memoryTree.status.retryFailedCount': "Tâches remises en file d'attente : {count}.", + 'memoryTree.status.retryFailedError': 'Impossible de remettre en file les tâches en échec', 'memoryTree.status.toggleFailed': "Impossible d'activer/désactiver la synchronisation automatique", 'memoryTree.status.justNow': "à l'instant", diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index 3a7970224a..ef09b3b0b6 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -1302,6 +1302,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'कभी नहीं', 'memoryTree.status.fetchError': 'स्मृति वृक्ष की स्थिति नहीं मिला', 'memoryTree.status.retry': 'रेस्त्री', + 'memoryTree.status.retryFailed': 'विफल कार्य दोबारा चलाएँ', + 'memoryTree.status.retryFailedBusy': 'दोबारा चलाया जा रहा है...', + 'memoryTree.status.retryFailedDone': 'विफल कार्य फिर से कतार में डाले गए', + 'memoryTree.status.retryFailedCount': '{count} कार्य दोबारा चलने के लिए कतार में हैं।', + 'memoryTree.status.retryFailedError': 'विफल कार्यों को फिर से कतार में नहीं डाला जा सका', 'memoryTree.status.toggleFailed': 'ऑटो सिंक को टॉगल नहीं कर सका', 'memoryTree.status.justNow': 'अभी', 'memoryTree.status.secondsAgo': '{count} पहले', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index ac6289e03c..158fa4914e 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -1316,6 +1316,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'Tidak pernah', 'memoryTree.status.fetchError': 'Gagal mengambil status Pohon Memori', 'memoryTree.status.retry': 'Coba lagi', + 'memoryTree.status.retryFailed': 'Jalankan ulang tugas yang gagal', + 'memoryTree.status.retryFailedBusy': 'Mencoba lagi...', + 'memoryTree.status.retryFailedDone': 'Tugas yang gagal masuk antrean lagi', + 'memoryTree.status.retryFailedCount': '{count} tugas diantrekan untuk dijalankan ulang.', + 'memoryTree.status.retryFailedError': 'Tidak dapat mengantrekan ulang tugas yang gagal', 'memoryTree.status.toggleFailed': 'Gagal mengalihkan sinkronisasi otomatis', 'memoryTree.status.justNow': 'baru saja', 'memoryTree.status.secondsAgo': '{count} dtk lalu', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index a409df2f10..5850299948 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -1335,6 +1335,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'Mai', 'memoryTree.status.fetchError': "Impossibile recuperare lo stato dell'Albero della Memoria", 'memoryTree.status.retry': 'Riprova', + 'memoryTree.status.retryFailed': 'Riprova i lavori non riusciti', + 'memoryTree.status.retryFailedBusy': 'Nuovo tentativo...', + 'memoryTree.status.retryFailedDone': 'Lavori non riusciti rimessi in coda', + 'memoryTree.status.retryFailedCount': 'Lavori in coda per una nuova esecuzione: {count}.', + 'memoryTree.status.retryFailedError': 'Impossibile rimettere in coda i lavori non riusciti', 'memoryTree.status.toggleFailed': 'Impossibile attivare/disattivare la sincronizzazione automatica', 'memoryTree.status.justNow': 'proprio adesso', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index 9f38ceebad..7a23395893 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -1292,6 +1292,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': '없음', 'memoryTree.status.fetchError': '메모리 트리 상태를 가져올 수 없습니다.', 'memoryTree.status.retry': '다시 시도', + 'memoryTree.status.retryFailed': '실패한 작업 다시 실행', + 'memoryTree.status.retryFailedBusy': '다시 시도하는 중...', + 'memoryTree.status.retryFailedDone': '실패한 작업을 다시 대기열에 넣었습니다', + 'memoryTree.status.retryFailedCount': '{count}개 작업이 다시 실행되도록 대기열에 있습니다.', + 'memoryTree.status.retryFailedError': '실패한 작업을 다시 대기열에 넣지 못했습니다', 'memoryTree.status.toggleFailed': '자동 동기화를 전환할 수 없습니다.', 'memoryTree.status.justNow': '방금 전', 'memoryTree.status.secondsAgo': '{count}초 전', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index d9c1146d92..6c859248d4 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -1323,6 +1323,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'Nigdy', 'memoryTree.status.fetchError': 'Nie udało się pobrać statusu drzewa pamięci', 'memoryTree.status.retry': 'Ponów', + 'memoryTree.status.retryFailed': 'Ponów nieudane zadania', + 'memoryTree.status.retryFailedBusy': 'Ponawianie...', + 'memoryTree.status.retryFailedDone': 'Nieudane zadania wróciły do kolejki', + 'memoryTree.status.retryFailedCount': 'W kolejce do ponownego uruchomienia: {count}.', + 'memoryTree.status.retryFailedError': 'Nie udało się ponowić nieudanych zadań', 'memoryTree.status.toggleFailed': 'Nie udało się przełączyć automatycznej synchronizacji', 'memoryTree.status.justNow': 'przed chwilą', 'memoryTree.status.secondsAgo': '{count} s temu', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index a24a7f495e..d60ebe0ce8 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -1328,6 +1328,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'Nunca', 'memoryTree.status.fetchError': 'Não foi possível buscar o status da Árvore de Memória', 'memoryTree.status.retry': 'Tentar novamente', + 'memoryTree.status.retryFailed': 'Repetir tarefas com falha', + 'memoryTree.status.retryFailedBusy': 'Tentando novamente...', + 'memoryTree.status.retryFailedDone': 'Tarefas com falha recolocadas na fila', + 'memoryTree.status.retryFailedCount': 'Tarefas na fila para executar de novo: {count}.', + 'memoryTree.status.retryFailedError': 'Não foi possível recolocar as tarefas com falha na fila', 'memoryTree.status.toggleFailed': 'Não foi possível ativar/desativar a sincronização automática', 'memoryTree.status.justNow': 'agora mesmo', 'memoryTree.status.secondsAgo': '{count}s atrás', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index 7e53ff481a..854ecbebe8 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -1316,6 +1316,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': 'Никогда', 'memoryTree.status.fetchError': 'Не удалось получить статус дерева памяти.', 'memoryTree.status.retry': 'Повторить попытку', + 'memoryTree.status.retryFailed': 'Повторить неудавшиеся задачи', + 'memoryTree.status.retryFailedBusy': 'Повторяем...', + 'memoryTree.status.retryFailedDone': 'Неудавшиеся задачи снова в очереди', + 'memoryTree.status.retryFailedCount': 'Задач в очереди на повторный запуск: {count}.', + 'memoryTree.status.retryFailedError': 'Не удалось вернуть неудавшиеся задачи в очередь', 'memoryTree.status.toggleFailed': 'Не удалось включить автосинхронизацию.', 'memoryTree.status.justNow': 'прямо сейчас', 'memoryTree.status.secondsAgo': '{count} сек. назад', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index e3589ad8cb..651efc35a7 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -1239,6 +1239,11 @@ const messages: TranslationMap = { 'memoryTree.status.never': '从未', 'memoryTree.status.fetchError': '无法获取记忆树状态', 'memoryTree.status.retry': '重试', + 'memoryTree.status.retryFailed': '重试失败的任务', + 'memoryTree.status.retryFailedBusy': '正在重试...', + 'memoryTree.status.retryFailedDone': '失败的任务已重新排队', + 'memoryTree.status.retryFailedCount': '已有 {count} 个任务重新排队等待运行。', + 'memoryTree.status.retryFailedError': '无法将失败的任务重新排队', 'memoryTree.status.toggleFailed': '无法切换自动同步', 'memoryTree.status.justNow': '刚刚', 'memoryTree.status.secondsAgo': '{count} 秒前', diff --git a/app/src/services/analytics.ts b/app/src/services/analytics.ts index 3800002d94..19f9f92a6d 100644 --- a/app/src/services/analytics.ts +++ b/app/src/services/analytics.ts @@ -103,6 +103,7 @@ const ALLOWED_EVENT_NAMES = [ 'automation_run_started', 'automation_run_resumed', 'automation_run_cancelled', + 'memory_tree_retry_succeeded', 'skill_install', 'skill_uninstall', 'tab_bar_change', diff --git a/app/src/utils/tauriCommands/memoryTree.test.ts b/app/src/utils/tauriCommands/memoryTree.test.ts index 6bd19c3040..1d30f01543 100644 --- a/app/src/utils/tauriCommands/memoryTree.test.ts +++ b/app/src/utils/tauriCommands/memoryTree.test.ts @@ -21,6 +21,7 @@ import { memoryTreeObsidianVaultStatus, memoryTreeRecall, memoryTreeResetTree, + memoryTreeRetryFailed, memoryTreeSearch, memoryTreeSetLlm, memoryTreeTopEntities, @@ -462,3 +463,27 @@ describe('memorySyncStatusList', () => { expect(rows).toEqual([]); }); }); + +describe('memoryTreeRetryFailed', () => { + test('dispatches memory_tree_retry_failed with empty params and returns the count', async () => { + mockCallCoreRpc.mockResolvedValueOnce({ result: { requeued: 5 }, logs: ['stub'] }); + + const out = await memoryTreeRetryFailed(); + + expect(mockCallCoreRpc).toHaveBeenCalledWith({ + method: 'openhuman.memory_tree_retry_failed', + params: {}, + }); + expect(out).toEqual({ requeued: 5 }); + }); + + test('passes through bare-shape responses (no envelope) unchanged', async () => { + // Defensive path: a handler that stops emitting logs returns the bare + // value, which flows through `unwrapResult` untouched. + mockCallCoreRpc.mockResolvedValueOnce({ requeued: 0 }); + + const out = await memoryTreeRetryFailed(); + + expect(out).toEqual({ requeued: 0 }); + }); +}); diff --git a/app/src/utils/tauriCommands/memoryTree.ts b/app/src/utils/tauriCommands/memoryTree.ts index 95ed58adc5..220e62809b 100644 --- a/app/src/utils/tauriCommands/memoryTree.ts +++ b/app/src/utils/tauriCommands/memoryTree.ts @@ -905,6 +905,34 @@ export async function memoryTreePipelineStatus(): Promise { + console.debug('[memory-tree-rpc] memoryTreeRetryFailed: entry'); + const resp = await callCoreRpc< + MemoryTreeRetryFailedResponse | ResultEnvelope + >({ method: 'openhuman.memory_tree_retry_failed', params: {} }); + const out = unwrapResult(resp); + console.debug('[memory-tree-rpc] memoryTreeRetryFailed: exit requeued=%d', out.requeued); + return out; +} + // ── memory_tree_set_enabled (#1856 Part 1) ─────────────────────────────── /** diff --git a/src/openhuman/config/mod.rs b/src/openhuman/config/mod.rs index fc4639db2f..fea994b297 100644 --- a/src/openhuman/config/mod.rs +++ b/src/openhuman/config/mod.rs @@ -29,6 +29,8 @@ pub use schema::{ default_projects_dir, default_root_openhuman_dir, pre_login_user_dir, read_active_user_id, resolve_action_dir, user_openhuman_dir, write_active_user_id, PRE_LOGIN_USER_ID, }; +// Crate-internal: workspace→config-dir resolver reused by the cloud embedder. +pub(crate) use schema::resolve_config_dir_for_workspace; #[allow(unused_imports)] pub use schema::{ apply_runtime_proxy_to_builder, build_runtime_proxy_client, diff --git a/src/openhuman/config/schema/load/mod.rs b/src/openhuman/config/schema/load/mod.rs index 7f34f6184b..1a932ceb08 100644 --- a/src/openhuman/config/schema/load/mod.rs +++ b/src/openhuman/config/schema/load/mod.rs @@ -32,10 +32,14 @@ pub(crate) use dirs::default_root_dir_name_pub as default_root_dir_name; // re-export; only the load_tests module needs it visible at this level. #[cfg(test)] pub(crate) use dirs::read_active_user_id_checked; +// Non-test: the keyless cloud embedder (`inference::embeddings::cloud_adapter`) +// resolves its `OPENHUMAN_WORKSPACE` credential scope through the same +// workspace→config-dir mapping `config::load` uses, so a legacy `.../workspace` +// override lands on the sibling `.openhuman` root that holds `auth-profiles.json`. +pub(crate) use dirs::resolve_config_dir_for_workspace; #[cfg(test)] pub(crate) use dirs::{ - resolve_config_dir_for_workspace, resolve_runtime_config_dirs, - resolve_runtime_config_dirs_with, ConfigResolutionSource, + resolve_runtime_config_dirs, resolve_runtime_config_dirs_with, ConfigResolutionSource, }; // PathBuf and Config were in scope via `use super::*` in the original load.rs. #[cfg(test)] diff --git a/src/openhuman/config/schema/mod.rs b/src/openhuman/config/schema/mod.rs index 808886d7b3..4642458936 100644 --- a/src/openhuman/config/schema/mod.rs +++ b/src/openhuman/config/schema/mod.rs @@ -28,6 +28,9 @@ pub use load::{ default_projects_dir, default_root_openhuman_dir, pre_login_user_dir, read_active_user_id, resolve_action_dir, user_openhuman_dir, write_active_user_id, PRE_LOGIN_USER_ID, }; +// Crate-internal: the workspace→config-dir resolver, reused by the cloud +// embedder's keyless credential-scope resolution (mirrors `config::load`). +pub(crate) use load::resolve_config_dir_for_workspace; // Contract shared with `core::observability::expected_error_kind`: the loader // appends this marker to a config-read failure when the file's owner differs // from the reading process, and the classifier keys on it to keep that case diff --git a/src/openhuman/inference/embeddings/cloud_adapter.rs b/src/openhuman/inference/embeddings/cloud_adapter.rs index bef1a354c8..280fbd2bbc 100644 --- a/src/openhuman/inference/embeddings/cloud_adapter.rs +++ b/src/openhuman/inference/embeddings/cloud_adapter.rs @@ -52,16 +52,85 @@ impl OpenHumanCloudEmbedding { } } +/// Credential scope used when the caller passes `openhuman_dir = None`. +/// +/// `None` means "wherever this process keeps its credentials", and on a shipped +/// desktop that is **not** the root `~/.openhuman`. Sign-in stores the +/// `app-session` token through `AuthService::from_config`, whose state dir is +/// `config.config_path.parent()` — the user-scoped +/// `~/.openhuman/users//`. This function previously returned the root, +/// so every keyless managed embedder resolved a directory with no +/// `auth-profiles.json` in it and a signed-in user's embeds failed with +/// "No backend session for cloud embeddings" on every call. +/// +/// Resolution mirrors `config::load`'s own directory choice: +/// 1. `OPENHUMAN_WORKSPACE` when set — resolved through the **same** +/// workspace→config-dir mapping `config::load` uses +/// (`resolve_config_dir_for_workspace`), not the raw env value. A legacy +/// `.../workspace` override maps back to its sibling `.openhuman` root, which +/// is where `auth-profiles.json` actually lives; returning the workspace dir +/// itself would reintroduce the "No backend session" failure for that +/// deployment. +/// 2. otherwise `{root}/users/{active_user_id}`, falling back to the pre-login +/// user (`users/local`) when no user has signed in yet — the same directory +/// the pre-login config was written to, so a pre-login process still reads +/// its own store instead of an empty root. +/// +/// Callers holding a `&Config` should still pass the scope explicitly +/// (`create_embedding_provider_with_config`); this is the best available +/// resolution for the call sites that have no `Config` in scope. fn default_state_dir() -> PathBuf { + log::debug!("[embeddings::cloud] default credential scope: resolving"); if let Some(workspace) = std::env::var_os("OPENHUMAN_WORKSPACE") .filter(|value| !value.is_empty()) .map(PathBuf::from) { - return workspace; + // Never log the resolved path: it identifies the user's home layout. + log::debug!( + "[embeddings::cloud] default credential scope = OPENHUMAN_WORKSPACE-derived config dir (env-scoped deployment)" + ); + return env_workspace_state_dir(&workspace); } - directories::UserDirs::new() - .map(|dirs| dirs.home_dir().join(".openhuman")) - .unwrap_or_else(|| PathBuf::from(".openhuman")) + + let root = crate::openhuman::config::default_root_openhuman_dir().unwrap_or_else(|error| { + log::warn!( + "[embeddings::cloud] could not resolve the openhuman root dir ({error}); \ + falling back to a relative .openhuman path" + ); + PathBuf::from(".openhuman") + }); + + // Never log the resolved path or the user id: both identify the user. + let user_id = crate::openhuman::config::read_active_user_id(&root); + log::debug!( + "[embeddings::cloud] default credential scope resolved = user-scoped dir (active_user_present={})", + user_id.is_some() + ); + user_scoped_state_dir(&root, user_id.as_deref()) +} + +/// Pure core of [`default_state_dir`]'s `OPENHUMAN_WORKSPACE` branch, split out +/// so the workspace→config-dir invariant is unit-testable without touching the +/// process environment. +/// +/// Mirrors `config::load`: the credential scope for a workspace override is the +/// config dir [`resolve_config_dir_for_workspace`] derives from it — for a +/// legacy `.../workspace` path that is the sibling `.openhuman` root (which +/// holds `auth-profiles.json`), **not** the workspace dir (which holds none). +fn env_workspace_state_dir(workspace: &std::path::Path) -> PathBuf { + let (config_dir, _workspace_dir) = + crate::openhuman::config::resolve_config_dir_for_workspace(workspace); + config_dir +} + +/// Pure core of [`default_state_dir`]'s non-env branch, split out so the +/// user-scoping invariant is unit-testable without a home directory or a real +/// `active_user.toml`. +fn user_scoped_state_dir(root: &std::path::Path, active_user_id: Option<&str>) -> PathBuf { + crate::openhuman::config::user_openhuman_dir( + root, + active_user_id.unwrap_or(crate::openhuman::config::PRE_LOGIN_USER_ID), + ) } #[async_trait] @@ -125,4 +194,70 @@ mod tests { "unexpected error: {err}" ); } + + /// The keyless credential scope must land in the **user-scoped** directory, + /// never the root. Sign-in writes `auth-profiles.json` to + /// `{root}/users//`; the root itself holds no such file, so the + /// previous root-returning implementation made every keyless managed + /// embedder fail with "No backend session for cloud embeddings" for a user + /// who was signed in. + #[test] + fn default_scope_is_the_active_user_dir_not_the_root() { + let root = std::path::Path::new("/tmp/openhuman-root"); + + let resolved = user_scoped_state_dir(root, Some("user-abc123")); + + assert_eq!( + resolved, + root.join("users").join("user-abc123"), + "managed embedder must read credentials from the active user's dir" + ); + assert_ne!( + resolved, root, + "the root dir holds no auth-profiles.json — resolving to it is the bug" + ); + } + + /// With no user signed in yet, the scope is the pre-login user dir — the + /// same directory the pre-login config and its credential store live in. + /// Falling back to the root here would reintroduce the same empty-store + /// failure one login earlier. + #[test] + fn default_scope_falls_back_to_the_pre_login_user_dir() { + let root = std::path::Path::new("/tmp/openhuman-root"); + + let resolved = user_scoped_state_dir(root, None); + + assert_eq!( + resolved, + root.join("users") + .join(crate::openhuman::config::PRE_LOGIN_USER_ID), + "a pre-login process must read its own store, not the empty root" + ); + } + + /// `OPENHUMAN_WORKSPACE` must resolve through the same workspace→config-dir + /// mapping `config::load` uses, not return the raw workspace path. A legacy + /// `/workspace` override keeps its credentials in the sibling + /// `/.openhuman` dir; returning the workspace dir itself would send the + /// keyless embedder to a directory with no `auth-profiles.json` and + /// reintroduce "No backend session" for that deployment. + #[test] + fn env_workspace_scope_is_the_config_dir_not_the_raw_workspace() { + // A path that does not exist on disk, so the resolver's `config.toml` + // probes both miss and the `"workspace"` basename rule decides. + let workspace = std::path::Path::new("/nonexistent-openhuman-test-root/workspace"); + + let resolved = env_workspace_state_dir(workspace); + + assert_eq!( + resolved, + std::path::Path::new("/nonexistent-openhuman-test-root/.openhuman"), + "a `.../workspace` override must resolve to its sibling .openhuman config dir" + ); + assert_ne!( + resolved, workspace, + "returning the raw workspace dir is the regression this guards against" + ); + } } diff --git a/src/openhuman/memory/tree/tree/rpc.rs b/src/openhuman/memory/tree/tree/rpc.rs index 933bb3105b..9f4becf731 100644 --- a/src/openhuman/memory/tree/tree/rpc.rs +++ b/src/openhuman/memory/tree/tree/rpc.rs @@ -643,33 +643,111 @@ pub async fn retry_failed_rpc(config: &Config) -> Result Result, String> { use crate::openhuman::memory::tree::health::{FailureClass, FailureCode, PipelineFailure}; - let row: Option<(Option, Option)> = - chunk_store::with_connection(config, |conn| { - conn.query_row( - "SELECT failure_reason, failure_class FROM mem_tree_jobs + // Read the newest failed row AND the success watermark on the SAME + // connection. `with_connection` holds the process-global connection mutex + // for the whole closure, so no job can settle between the two reads and + // flip the supersession decision (a race the #5427 review flagged). The + // watermark is only queried when the failed row carries a timestamp to + // compare against. + type FailureWatermark = (Option, Option, Option, Option); + let row: Option = chunk_store::with_connection(config, |conn| { + let failed: Option<(Option, Option, Option)> = conn + .query_row( + "SELECT failure_reason, failure_class, completed_at_ms FROM mem_tree_jobs WHERE status = 'failed' AND failure_reason IS NOT NULL ORDER BY completed_at_ms DESC LIMIT 1", [], - |r| Ok((r.get(0)?, r.get(1)?)), + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .optional()?; + + let Some((reason, class, failed_at_ms)) = failed else { + return Ok(None); + }; + + let last_success_ms: Option = if failed_at_ms.is_some() { + conn.query_row( + "SELECT MAX(completed_at_ms) FROM mem_tree_jobs WHERE status = 'done'", + [], + |r| r.get(0), ) .optional() - .map_err(Into::into) - }) - .map_err(|e| format!("latest_failed_job_failure: {e:#}"))?; + .map(Option::flatten)? + } else { + None + }; + + Ok(Some((reason, class, failed_at_ms, last_success_ms))) + }) + .map_err(|e| format!("latest_failed_job_failure: {e:#}"))?; - let Some((Some(reason), class)) = row else { + let Some((Some(reason), class, failed_at_ms, last_success_ms)) = row else { + log::debug!( + "[memory-tree][rpc] pipeline_status: no typed failed row present — no blocking cause" + ); return Ok(None); }; + + // Log every supersession branch, not only the withheld one, so the decision + // is greppable from the logs alone. + match failed_at_ms { + Some(failed_at_ms) + if last_success_ms.is_some_and(|success_ms| success_ms > failed_at_ms) => + { + log::debug!( + "[memory-tree][rpc] pipeline_status: withholding blocking cause reason={reason} \ + — the queue has completed a job since it failed (superseded)" + ); + return Ok(None); + } + Some(_) => { + log::debug!( + "[memory-tree][rpc] pipeline_status: blocking cause is live reason={reason} \ + — no successful settle since it failed" + ); + } + None => { + log::debug!( + "[memory-tree][rpc] pipeline_status: blocking cause reason={reason} has no \ + completion timestamp — surfacing unconditionally (legacy row)" + ); + } + } + let Some(code) = FailureCode::from_str(&reason) else { return Ok(None); }; @@ -1805,6 +1883,124 @@ mod tests { ); } + /// Plant one terminally-`failed` row carrying a typed reason, and + /// optionally one `done` row, at explicit timestamps. Returns nothing — the + /// tests read the derived cause back through `latest_failed_job_failure`. + fn plant_failed_and_done( + cfg: &Config, + reason: &str, + failed_at_ms: i64, + done_at_ms: Option, + ) { + use crate::openhuman::memory::queue::store as queue_store; + use crate::openhuman::memory::queue::types::{FlushStalePayload, NewJob}; + + let failed_job = + NewJob::flush_stale(&FlushStalePayload::default(), "2026-07-10", 3).unwrap(); + let failed_id = queue_store::enqueue(cfg, &failed_job) + .unwrap() + .expect("enqueue failed-row"); + + let done_id = done_at_ms.map(|_| { + let done_job = + NewJob::flush_stale(&FlushStalePayload::default(), "2026-08-06", 3).unwrap(); + queue_store::enqueue(cfg, &done_job) + .unwrap() + .expect("enqueue done-row") + }); + + chunk_store::with_connection(cfg, |conn| { + conn.execute( + "UPDATE mem_tree_jobs + SET status = 'failed', + failure_reason = ?2, + failure_class = 'unrecoverable', + completed_at_ms = ?3 + WHERE id = ?1", + rusqlite::params![failed_id, reason, failed_at_ms], + )?; + if let (Some(done_id), Some(done_at_ms)) = (done_id.as_ref(), done_at_ms) { + conn.execute( + "UPDATE mem_tree_jobs + SET status = 'done', completed_at_ms = ?2 + WHERE id = ?1", + rusqlite::params![done_id, done_at_ms], + )?; + } + Ok(()) + }) + .unwrap(); + } + + /// The active production defect: a signed-in user was told "No embeddings + /// credentials found. Log in to OpenHuman" because a batch of `auth_missing` + /// jobs had failed 27 days earlier and, being unrecoverable, was never + /// retried. The queue had been completing jobs the whole time since. + /// + /// A failure the pipeline has already worked past is not the current + /// blocking cause, so no remediation is surfaced for it. + #[test] + fn blocking_cause_is_withheld_once_the_queue_has_succeeded_since() { + let (_tmp, cfg) = test_config(); + let failed_at = 1_800_000_000_000_i64; + let succeeded_after = failed_at + 27 * 24 * 60 * 60 * 1000; + + plant_failed_and_done(&cfg, "auth_missing", failed_at, Some(succeeded_after)); + + assert!( + latest_failed_job_failure(&cfg).unwrap().is_none(), + "a month-old auth failure the queue has since worked past must not be \ + presented as the user's current problem" + ); + } + + /// The other half of the same rule: a failure with no successful settle + /// after it IS the current blocking cause and must still surface, otherwise + /// the fix would silence the diagnosis it exists to deliver. + #[test] + fn blocking_cause_surfaces_when_nothing_has_succeeded_since() { + use crate::openhuman::memory::tree::health::{FailureClass, FailureCode}; + + let (_tmp, cfg) = test_config(); + let succeeded_before = 1_800_000_000_000_i64; + let failed_after = succeeded_before + 60_000; + + plant_failed_and_done( + &cfg, + "budget_exhausted", + failed_after, + Some(succeeded_before), + ); + + let failure = latest_failed_job_failure(&cfg) + .unwrap() + .expect("a failure with no success after it is the live cause"); + assert_eq!(failure.code, FailureCode::BudgetExhausted); + assert_eq!(failure.class, FailureClass::Unrecoverable); + assert_eq!( + failure.remediation_key, + "memory.health.remediation.budget_exhausted" + ); + } + + /// A queue that has never completed anything has no watermark to compare + /// against, so the failure stands — this is the "broken from the first + /// sync" shape, where the diagnosis matters most. + #[test] + fn blocking_cause_surfaces_when_the_queue_has_never_succeeded() { + let (_tmp, cfg) = test_config(); + + plant_failed_and_done(&cfg, "auth_invalid", 1_800_000_000_000_i64, None); + + let failure = latest_failed_job_failure(&cfg) + .unwrap() + .expect("no successful settle exists to supersede this failure"); + assert_eq!( + failure.remediation_key, + "memory.health.remediation.auth_invalid" + ); + } + /// On a fresh workspace the panel must report `idle` with zero /// counters — the UI uses this to swap the loading skeleton for a /// "no memory yet" state. diff --git a/vendor/tinycortex b/vendor/tinycortex index e0a8738980..5fabcf18d9 160000 --- a/vendor/tinycortex +++ b/vendor/tinycortex @@ -1 +1 @@ -Subproject commit e0a8738980965411f514f4a62c09f941efdea90c +Subproject commit 5fabcf18d9e3907d6b26b59528ad49cebfc1c271