feat(stovepipe): Show most broken projects first

Author: prathshenoy

doc/rfc/stovepipe/project-status-api.md

@@ -64,15 +64,25 @@ The caller may supply the project IDs for which it wants recorded results. The A
 
 When project IDs are supplied, the response includes only results for those IDs, in the supplied order, and paginates that list. An implementation may record results for every requested project, only requested projects with a non-green result, or another documented subset.
 
-With no project IDs supplied, the response lists all recorded project results in project-ID order, excluding the repository fact. Results are a live view; restart pagination to discover facts recorded behind the cursor.
+With no project IDs supplied, the response lists all recorded project results by descending breakage degree, then ascending project ID, excluding the repository fact. A dedicated ordering mapping makes this a bounded primary-key range read, not a sort over all facts. The continuation token identifies the last project; its immutable fact supplies the degree needed to resume this ordering. A missing cursor fact returns an error. Results are a live view; restart pagination to discover facts recorded before the cursor.
 
 Each project result is an immutable validation fact keyed by queue, commit URI, and project ID. A result must identify the selected request. `project_results_complete` is false until the implementation has durably finished producing results for the supplied project IDs; a completed empty set is valid.
 
 Callers use `project_results_complete`, rather than the presence or absence of an individual project result, to determine whether the implementation has finished producing results for the requested projects. `next_page_token` is empty only on the final page. A terminal lifecycle notification is emitted only after the repository result and any applicable project completion record are durable. If notification delivery fails, consumers can recover the same result through either endpoint.
 
 ## Implementation
 
-Both endpoints return durable lifecycle summaries, the whole-repository fact when present, and recorded facts for requested projects in paginated request order, or all recorded project results in project-ID order when none are requested. `project_results_complete` remains false until a durable completion projection is available. Missing facts are omitted; an explicitly recorded zero degree remains present. `updated_at_ms` is the maximum timestamp of the summary and the facts represented on the page.
+Both endpoints return durable lifecycle summaries, the whole-repository fact when present, and recorded facts for requested projects in paginated request order, or all recorded project results by descending breakage degree then ascending project ID when none are requested. `project_results_complete` remains false until a durable completion projection is available. Missing facts are omitted; an explicitly recorded zero degree remains present. `updated_at_ms` is the maximum timestamp of the summary and the facts represented on the page.
+
+The ordering mapping is keyed by `(queue, URI, degree_order, project)`. `degree_order` reverses nonnegative IEEE-754 degree bit patterns into an ascending integer key without rounding; positive and negative zero share one key. Binary identifiers preserve bytewise tie order. A page reads at most `page_size + 1` mappings and hydrates only the returned facts, plus the cursor fact on continuations. A mapping that disagrees with its authoritative fact is a consistency error.
+
+The record controller persists the winning immutable fact before ensuring its mapping. Redelivery repairs missing mappings even when the fact already exists, and derives their keys from the stored degree rather than a losing candidate. Project-recording logs and downstream terminal notifications follow successful mapping writes. No cross-store transaction is required.
+
+### Schema and rollout
+
+The ordering table is additive; the original fact identity and primary key are unchanged. Install the table before deploying code that writes or reads it. Existing facts require a one-time backfill before index-based listing is enabled. `stovepipe/core/projectindex.Backfill` rebuilds one queue-scoped URI using bounded project-key scans and idempotent mapping writes; it is restartable and does not load all facts into memory.
+
+A write-first rollout or a recording pause during cutover must prevent unindexed concurrent writes while historical backfill is verified. Do not switch readers merely because the new table exists: incomplete mappings would omit recorded facts. Schema deployment and historical backfill are operator-controlled, not performed by an RPC or service startup. Reverting readers does not require dropping the additive table.
 
 ## Errors and Authorization
 

stovepipe/controller/project_status.go

@@ -161,7 +161,11 @@ func (c *projectStatusController) GetProjectStatus(ctx context.Context, req enti
 		}
 	}
 
-	page, err := loadProjectStatusPage(ctx, req, requestSummary, factStore)
+	var indexStore storage.ProjectValidationIndexStore
+	if len(req.Projects) == 0 {
+		indexStore = store.GetProjectValidationIndexStore()
+	}
+	page, err := loadProjectStatusPage(ctx, req, requestSummary, factStore, indexStore)
 	if err != nil {
 		return entity.GetProjectStatusResult{}, err
 	}
@@ -221,6 +225,7 @@ func loadProjectStatusPage(
 	req entity.GetProjectStatusRequest,
 	summary entity.RequestSummary,
 	store storage.ValidationFactStore,
+	indexStore storage.ProjectValidationIndexStore,
 ) (projectStatusPage, error) {
 	token, err := parseProjectStatusPageToken(req, summary)
 	if err != nil {
@@ -233,7 +238,7 @@ func loadProjectStatusPage(
 	}
 
 	if len(req.Projects) == 0 {
-		return listProjectStatusPage(ctx, summary, store, token, pageSize)
+		return listProjectStatusPage(ctx, summary, store, indexStore, token, pageSize)
 	}
 
 	return getRequestedProjectStatusPage(ctx, summary, store, token, pageSize)
@@ -282,32 +287,61 @@ func listProjectStatusPage(
 	ctx context.Context,
 	summary entity.RequestSummary,
 	store storage.ValidationFactStore,
+	indexStore storage.ProjectValidationIndexStore,
 	token projectStatusPageToken,
 	pageSize int,
 ) (projectStatusPage, error) {
-	facts, err := store.ListProjects(ctx, summary.URI, token.AfterProject, pageSize+1)
-	if err != nil {
-		return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to list project facts for request %q: %w", summary.RequestID, err)
-	}
+	var after entity.ProjectValidationIndex
+	if token.AfterProject != "" {
+		fact, err := store.Get(ctx, summary.URI, token.AfterProject)
+		if err != nil {
+			return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to load cursor project %q: %w", token.AfterProject, err)
+		}
 
-	for _, fact := range facts {
-		if err := validateProjectFact(fact, summary, fact.Project); err != nil {
+		if err := validateProjectFact(fact, summary, token.AfterProject); err != nil {
 			return projectStatusPage{}, err
 		}
 
-		if fact.Project == "" {
-			return projectStatusPage{}, &ProjectStatusConsistencyError{message: "project listing contains a repository fact"}
+		after, err = entity.NewProjectValidationIndex(fact)
+		if err != nil {
+			return projectStatusPage{}, err
 		}
 	}
 
-	page := projectStatusPage{facts: facts}
-	if len(facts) > pageSize {
-		page.facts = facts[:pageSize]
-		token.AfterProject = page.facts[pageSize-1].Project
+	entries, err := indexStore.List(ctx, summary.URI, after, pageSize+1)
+	if err != nil {
+		return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to list project index for request %q: %w", summary.RequestID, err)
+	}
+
+	var page projectStatusPage
+	if len(entries) > pageSize {
+		entries = entries[:pageSize]
+		token.AfterProject = entries[pageSize-1].Project
 		page.nextToken, err = encodeProjectStatusPageToken(token)
+		if err != nil {
+			return projectStatusPage{}, err
+		}
 	}
 
-	return page, err
+	for _, entry := range entries {
+		fact, err := store.Get(ctx, summary.URI, entry.Project)
+		if err != nil {
+			return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to load indexed project %q: %w", entry.Project, err)
+		}
+
+		if err := validateProjectFact(fact, summary, entry.Project); err != nil {
+			return projectStatusPage{}, err
+		}
+
+		expected, err := entity.NewProjectValidationIndex(fact)
+		if err != nil || entry != expected {
+			return projectStatusPage{}, &ProjectStatusConsistencyError{message: "project ordering index disagrees with stored fact"}
+		}
+
+		page.facts = append(page.facts, fact)
+	}
+
+	return page, nil
 }
 
 func encodeProjectStatusPageToken(token projectStatusPageToken) (string, error) {

stovepipe/controller/project_status_test.go

@@ -111,6 +111,7 @@ func TestGetProjectStatus(t *testing.T) {
 			uriStore := storagemock.NewMockRequestURIStore(mockCtrl)
 			summaryStore := storagemock.NewMockRequestSummaryStore(mockCtrl)
 			factStore := storagemock.NewMockValidationFactStore(mockCtrl)
+			indexStore := storagemock.NewMockProjectValidationIndexStore(mockCtrl)
 			if !(tt.wantInvalid && tt.request.PageToken == "") {
 				factory.EXPECT().For(storage.Config{QueueName: projectStatusQueue}).Return(store, nil)
 				store.EXPECT().GetRequestURIStore().Return(uriStore)
@@ -123,7 +124,14 @@ func TestGetProjectStatus(t *testing.T) {
 						factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "").Return(tt.fact, tt.factErr)
 						if !tt.noProjectLookup && !(tt.factErr == nil && tt.wantConsistency) {
 							if len(tt.request.Projects) == 0 {
-								factStore.EXPECT().ListProjects(gomock.Any(), projectStatusURI, "", defaultProjectStatusPageSize+1).Return(tt.projectFacts, tt.wantErr)
+								store.EXPECT().GetProjectValidationIndexStore().Return(indexStore)
+								indexStore.EXPECT().List(gomock.Any(), projectStatusURI, entity.ProjectValidationIndex{}, defaultProjectStatusPageSize+1).
+									Return(projectStatusIndexEntries(t, tt.projectFacts), tt.wantErr)
+								if tt.wantErr == nil {
+									for _, fact := range tt.projectFacts {
+										factStore.EXPECT().Get(gomock.Any(), projectStatusURI, fact.Project).Return(fact, nil)
+									}
+								}
 							}
 							for _, project := range tt.request.Projects {
 								factStore.EXPECT().Get(gomock.Any(), projectStatusURI, project).Return(entity.ValidationFact{}, storage.ErrNotFound)
@@ -300,9 +308,9 @@ func TestGetProjectStatusReturnsProjectFactsInRequestedOrder(t *testing.T) {
 	summaryStore.EXPECT().Get(gomock.Any(), projectStatusID).Return(requestSummary, nil)
 	store.EXPECT().GetValidationFactStore().Return(factStore)
 	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "").Return(entity.ValidationFact{}, storage.ErrNotFound)
-	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "project-b").Return(entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Project: "project-b", Degree: entity.DegreeBroken, CreatedAt: 20}, nil)
+	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "project-b").Return(entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Project: "project-b", Degree: entity.DegreeGreen, CreatedAt: 20}, nil)
 	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "missing").Return(entity.ValidationFact{}, storage.ErrNotFound)
-	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "project-a").Return(entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Project: "project-a", Degree: entity.DegreeGreen, CreatedAt: 30}, nil)
+	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "project-a").Return(entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Project: "project-a", Degree: entity.DegreeBroken, CreatedAt: 30}, nil)
 
 	controller := NewProjectStatusController(zap.NewNop().Sugar(), tally.NoopScope, factory)
 	got, err := controller.GetProjectStatus(context.Background(), entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{"project-b", "missing", "project-a"}})
@@ -452,32 +460,38 @@ func TestParseProjectStatusPageTokenRejectsMismatches(t *testing.T) {
 func TestGetProjectStatusPaginatesStoredProjects(t *testing.T) {
 	summary := entity.RequestSummary{RequestID: projectStatusID, Queue: projectStatusQueue, URI: projectStatusURI, StateTimestampMs: 10}
 	stored := []entity.ValidationFact{
+		{URI: projectStatusURI, RequestID: projectStatusID, Project: "c", Degree: 1, CreatedAt: 60},
 		{URI: projectStatusURI, RequestID: projectStatusID, Project: "a", Degree: 0.5, CreatedAt: 20},
 		{URI: projectStatusURI, RequestID: projectStatusID, Project: "b", Degree: 0, CreatedAt: 50},
-		{URI: projectStatusURI, RequestID: projectStatusID, Project: "c", Degree: 1, CreatedAt: 60},
 	}
 
 	ctrl := gomock.NewController(t)
 	factory := storagemock.NewMockFactory(ctrl)
 	store := storagemock.NewMockStorage(ctrl)
 	summaries := storagemock.NewMockRequestSummaryStore(ctrl)
 	facts := storagemock.NewMockValidationFactStore(ctrl)
+	indexStore := storagemock.NewMockProjectValidationIndexStore(ctrl)
+	entries := projectStatusIndexEntries(t, stored)
 
 	factory.EXPECT().For(storage.Config{QueueName: projectStatusQueue}).Return(store, nil).Times(2)
 	store.EXPECT().GetRequestSummaryStore().Return(summaries).Times(2)
 	summaries.EXPECT().Get(gomock.Any(), projectStatusID).Return(summary, nil).Times(2)
 	store.EXPECT().GetValidationFactStore().Return(facts).Times(2)
+	store.EXPECT().GetProjectValidationIndexStore().Return(indexStore).Times(2)
 	facts.EXPECT().Get(gomock.Any(), projectStatusURI, "").Return(entity.ValidationFact{}, storage.ErrNotFound).Times(2)
-	facts.EXPECT().ListProjects(gomock.Any(), projectStatusURI, "", 3).Return(stored, nil)
-	facts.EXPECT().ListProjects(gomock.Any(), projectStatusURI, "b", defaultProjectStatusPageSize+1).Return(stored[2:], nil)
+	indexStore.EXPECT().List(gomock.Any(), projectStatusURI, entity.ProjectValidationIndex{}, 3).Return(entries, nil)
+	indexStore.EXPECT().List(gomock.Any(), projectStatusURI, entries[1], defaultProjectStatusPageSize+1).Return(entries[2:], nil)
+	facts.EXPECT().Get(gomock.Any(), projectStatusURI, "c").Return(stored[0], nil)
+	facts.EXPECT().Get(gomock.Any(), projectStatusURI, "a").Return(stored[1], nil).Times(2)
+	facts.EXPECT().Get(gomock.Any(), projectStatusURI, "b").Return(stored[2], nil)
 
 	controller := NewProjectStatusController(zap.NewNop().Sugar(), tally.NoopScope, factory)
 	req := entity.GetProjectStatusRequest{Queue: projectStatusQueue, RequestID: projectStatusID, PageSize: 2}
 
 	first, err := controller.GetProjectStatus(context.Background(), req)
 	require.NoError(t, err)
 	assert.Equal(t, stored[:2], first.ProjectValidationFacts)
-	assert.Equal(t, int64(50), first.UpdatedAtMs)
+	assert.Equal(t, int64(60), first.UpdatedAtMs)
 	require.NotEmpty(t, first.NextPageToken)
 
 	req.PageToken = first.NextPageToken
@@ -487,10 +501,69 @@ func TestGetProjectStatusPaginatesStoredProjects(t *testing.T) {
 	second, err := controller.GetProjectStatus(context.Background(), req)
 	require.NoError(t, err)
 	assert.Equal(t, stored[2:], second.ProjectValidationFacts)
-	assert.Equal(t, int64(60), second.UpdatedAtMs)
+	assert.Equal(t, int64(50), second.UpdatedAtMs)
 	assert.Empty(t, second.NextPageToken)
 }
 
+func projectStatusIndexEntries(t *testing.T, facts []entity.ValidationFact) []entity.ProjectValidationIndex {
+	t.Helper()
+	var entries []entity.ProjectValidationIndex
+	for _, fact := range facts {
+		entry, err := entity.NewProjectValidationIndex(fact)
+		require.NoError(t, err)
+		entries = append(entries, entry)
+	}
+	return entries
+}
+
+func TestListProjectStatusPageRejectsMissingCursor(t *testing.T) {
+	for _, readErr := range []error{storage.ErrNotFound, errors.New("database unavailable")} {
+		t.Run(readErr.Error(), func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			facts := storagemock.NewMockValidationFactStore(ctrl)
+			index := storagemock.NewMockProjectValidationIndexStore(ctrl)
+			facts.EXPECT().Get(gomock.Any(), projectStatusURI, "cursor").Return(entity.ValidationFact{}, readErr)
+			_, err := listProjectStatusPage(context.Background(), entity.RequestSummary{URI: projectStatusURI}, facts, index, projectStatusPageToken{AfterProject: "cursor"}, 2)
+			require.ErrorIs(t, err, readErr)
+		})
+	}
+}
+
+func TestListProjectStatusPageRejectsInconsistentIndex(t *testing.T) {
+	fact := entity.ValidationFact{URI: projectStatusURI, Project: "p", Degree: 0.5, RequestID: projectStatusID}
+	entry, err := entity.NewProjectValidationIndex(fact)
+	require.NoError(t, err)
+	for _, tt := range []struct {
+		name    string
+		modify  func(*entity.ProjectValidationIndex, *entity.ValidationFact)
+		factErr error
+	}{
+		{name: "missing fact", factErr: storage.ErrNotFound, modify: func(*entity.ProjectValidationIndex, *entity.ValidationFact) {}},
+		{name: "read failure", factErr: errors.New("database unavailable"), modify: func(*entity.ProjectValidationIndex, *entity.ValidationFact) {}},
+		{name: "wrong order key", modify: func(e *entity.ProjectValidationIndex, _ *entity.ValidationFact) { e.DegreeOrder++ }},
+		{name: "wrong URI", modify: func(e *entity.ProjectValidationIndex, _ *entity.ValidationFact) { e.URI = "other" }},
+		{name: "wrong request", modify: func(_ *entity.ProjectValidationIndex, f *entity.ValidationFact) { f.RequestID = "other" }},
+		{name: "repository fact", modify: func(e *entity.ProjectValidationIndex, f *entity.ValidationFact) { e.Project = ""; f.Project = "" }},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			facts := storagemock.NewMockValidationFactStore(ctrl)
+			index := storagemock.NewMockProjectValidationIndexStore(ctrl)
+			e, f := entry, fact
+			tt.modify(&e, &f)
+			index.EXPECT().List(gomock.Any(), projectStatusURI, entity.ProjectValidationIndex{}, 3).Return([]entity.ProjectValidationIndex{e}, nil)
+			facts.EXPECT().Get(gomock.Any(), projectStatusURI, e.Project).Return(f, tt.factErr)
+			_, err := listProjectStatusPage(context.Background(), entity.RequestSummary{URI: projectStatusURI, RequestID: projectStatusID}, facts, index, projectStatusPageToken{}, 2)
+			require.Error(t, err)
+			if tt.factErr != nil {
+				assert.ErrorIs(t, err, tt.factErr)
+			} else {
+				assert.True(t, IsProjectStatusConsistency(err))
+			}
+		})
+	}
+}
+
 func TestValidateProjectFact(t *testing.T) {
 	requestSummary := entity.RequestSummary{RequestID: projectStatusID, URI: projectStatusURI}
 	assert.NoError(t, validateProjectFact(entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Project: "project-a", Degree: entity.DegreeGreen}, requestSummary, "project-a"))

stovepipe/controller/record/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//stovepipe/core/hookevent:go_default_library",
         "//stovepipe/core/loader:go_default_library",
         "//stovepipe/core/messagequeue:go_default_library",
+        "//stovepipe/core/projectindex:go_default_library",
         "//stovepipe/core/requestlog:go_default_library",
         "//stovepipe/entity:go_default_library",
         "//stovepipe/extension/projectstatus:go_default_library",

stovepipe/controller/record/record.go

@@ -46,6 +46,7 @@ import (
 	"github.com/uber/submitqueue/stovepipe/core/hookevent"
 	"github.com/uber/submitqueue/stovepipe/core/loader"
 	stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue"
+	"github.com/uber/submitqueue/stovepipe/core/projectindex"
 	"github.com/uber/submitqueue/stovepipe/core/requestlog"
 	"github.com/uber/submitqueue/stovepipe/entity"
 	"github.com/uber/submitqueue/stovepipe/extension/projectstatus"
@@ -197,7 +198,7 @@ func (c *Controller) recordProjectFacts(ctx context.Context, store storage.Stora
 		return err
 	}
 	for _, result := range results {
-		_, _, err := c.recordValidationFact(ctx, store.GetValidationFactStore(), entity.ValidationFact{
+		fact, _, err := c.recordValidationFact(ctx, store.GetValidationFactStore(), entity.ValidationFact{
 			URI:       request.URI,
 			Project:   result.Project,
 			Degree:    result.Degree,
@@ -207,6 +208,10 @@ func (c *Controller) recordProjectFacts(ctx context.Context, store storage.Stora
 		if err != nil {
 			return err
 		}
+
+		if err := projectindex.Ensure(ctx, store.GetProjectValidationIndexStore(), fact); err != nil {
+			return fmt.Errorf("failed to index project fact for request %q and project %q: %w", request.ID, fact.Project, err)
+		}
 	}
 	if len(results) == 0 {
 		return nil