From 718f75af56338dee7300941b521f96bfe99b1539 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Fri, 29 May 2026 13:22:06 +0200 Subject: [PATCH] Fix known tests pagination --- civisibility/utils/net/known_tests_api.go | 122 +++++++++++---- .../utils/net/known_tests_api_test.go | 146 ++++++++++++++++++ 2 files changed, 240 insertions(+), 28 deletions(-) create mode 100644 civisibility/utils/net/known_tests_api_test.go diff --git a/civisibility/utils/net/known_tests_api.go b/civisibility/utils/net/known_tests_api.go index 3129b35..6f016bf 100644 --- a/civisibility/utils/net/known_tests_api.go +++ b/civisibility/utils/net/known_tests_api.go @@ -6,6 +6,7 @@ package net import ( + "encoding/json" "fmt" ) @@ -26,10 +27,15 @@ type ( } KnownTestsRequestData struct { - Service string `json:"service"` - Env string `json:"env"` - RepositoryURL string `json:"repository_url"` - Configurations testConfigurations `json:"configurations"` + Service string `json:"service"` + Env string `json:"env"` + RepositoryURL string `json:"repository_url"` + Configurations testConfigurations `json:"configurations"` + PageInfo *knownTestsRequestPageInfo `json:"page_info,omitempty"` + } + + knownTestsRequestPageInfo struct { + PageState string `json:"page_state,omitempty"` } knownTestsResponse struct { @@ -41,7 +47,14 @@ type ( } KnownTestsResponseData struct { - Tests KnownTestsResponseDataModules `json:"tests"` + Tests KnownTestsResponseDataModules `json:"tests"` + PageInfo *knownTestsResponsePageInfo `json:"page_info,omitempty"` + } + + knownTestsResponsePageInfo struct { + Cursor string `json:"cursor,omitempty"` + Size int `json:"size,omitempty"` + HasNext bool `json:"has_next"` } KnownTestsResponseDataModules map[string]KnownTestsResponseDataSuites @@ -54,33 +67,86 @@ func (c *client) GetKnownTests() (*KnownTestsResponseData, error) { } c.knownTestsRawResponse = nil - body := knownTestsRequest{ - Data: knownTestsRequestHeader{ - ID: c.id, - Type: knownTestsRequestType, - Attributes: KnownTestsRequestData{ - Service: c.serviceName, - Env: c.environment, - RepositoryURL: c.repositoryURL, - Configurations: c.testConfigurations, + allKnownTests := KnownTestsResponseData{ + Tests: KnownTestsResponseDataModules{}, + } + var allResponse knownTestsResponse + var firstRawResponse json.RawMessage + + cursor := "" + pageCount := 0 + for { + body := knownTestsRequest{ + Data: knownTestsRequestHeader{ + ID: c.id, + Type: knownTestsRequestType, + Attributes: KnownTestsRequestData{ + Service: c.serviceName, + Env: c.environment, + RepositoryURL: c.repositoryURL, + Configurations: c.testConfigurations, + }, }, - }, + } + if cursor != "" { + body.Data.Attributes.PageInfo = &knownTestsRequestPageInfo{PageState: cursor} + } + + request := c.getPostRequestConfig(knownTestsURLPath, body) + response, err := c.handler.SendRequest(*request) + + if err != nil { + return nil, fmt.Errorf("sending known tests request: %s", err) + } + pageCount++ + if pageCount == 1 { + firstRawResponse = cloneRawMessage(response.Body) + } + + var responseObject knownTestsResponse + err = response.Unmarshal(&responseObject) + if err != nil { + return nil, fmt.Errorf("unmarshalling known tests response: %s", err) + } + + if pageCount == 1 { + allResponse.Data.ID = responseObject.Data.ID + allResponse.Data.Type = responseObject.Data.Type + } + + mergeKnownTests(allKnownTests.Tests, responseObject.Data.Attributes.Tests) + allKnownTests.PageInfo = responseObject.Data.Attributes.PageInfo + + if allKnownTests.PageInfo == nil || !allKnownTests.PageInfo.HasNext { + break + } + if allKnownTests.PageInfo.Cursor == "" { + return nil, fmt.Errorf("known tests response page_info is missing cursor") + } + cursor = allKnownTests.PageInfo.Cursor } - request := c.getPostRequestConfig(knownTestsURLPath, body) - - response, err := c.handler.SendRequest(*request) - - if err != nil { - return nil, fmt.Errorf("sending known tests request: %s", err) + if pageCount == 1 { + c.knownTestsRawResponse = firstRawResponse + } else { + allResponse.Data.Attributes = allKnownTests + rawResponse, err := json.Marshal(allResponse) + if err != nil { + return nil, fmt.Errorf("marshalling known tests response: %s", err) + } + c.knownTestsRawResponse = cloneRawMessage(rawResponse) } - c.knownTestsRawResponse = cloneRawMessage(response.Body) - var responseObject knownTestsResponse - err = response.Unmarshal(&responseObject) - if err != nil { - return nil, fmt.Errorf("unmarshalling known tests response: %s", err) - } + return &allKnownTests, nil +} - return &responseObject.Data.Attributes, nil +func mergeKnownTests(dst, src KnownTestsResponseDataModules) { + for module, suites := range src { + if _, ok := dst[module]; !ok { + dst[module] = KnownTestsResponseDataSuites{} + } + for suite, tests := range suites { + dst[module][suite] = append(dst[module][suite], tests...) + } + } } diff --git a/civisibility/utils/net/known_tests_api_test.go b/civisibility/utils/net/known_tests_api_test.go new file mode 100644 index 0000000..eba081d --- /dev/null +++ b/civisibility/utils/net/known_tests_api_test.go @@ -0,0 +1,146 @@ +package net + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestClientGetKnownTestsPaginatesWithAttributesPageInfo(t *testing.T) { + requests := []map[string]any{} + responses := []string{ + `{"data":{"id":"known-tests-id","type":"ci_app_libraries_tests","attributes":{"tests":{"module-a":{"suite-a":["test-a"]}},"page_info":{"cursor":"page-2","size":1,"has_next":true}}}}`, + `{"data":{"id":"known-tests-id","type":"ci_app_libraries_tests","attributes":{"tests":{"module-a":{"suite-a":["test-b"]},"module-b":{"suite-b":["test-c"]}},"page_info":{"size":1,"has_next":false}}}}`, + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Fatalf("expected POST request, got %s", r.Method) + } + if strings.TrimPrefix(r.URL.Path, "/") != knownTestsURLPath { + t.Fatalf("unexpected request path %s", r.URL.Path) + } + if len(requests) >= len(responses) { + t.Fatalf("unexpected extra request") + } + + var request map[string]any + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("failed to decode request: %v", err) + } + requests = append(requests, request) + + w.Header().Set(HeaderContentType, ContentTypeJSON) + _, _ = w.Write([]byte(responses[len(requests)-1])) + })) + defer server.Close() + + client := newRawResponseTestClient(server) + knownTests, err := client.GetKnownTests() + if err != nil { + t.Fatalf("GetKnownTests() returned error: %v", err) + } + + if len(requests) != 2 { + t.Fatalf("expected 2 known-tests requests, got %d", len(requests)) + } + + assertKnownTestsRequestPageInfo(t, requests[0], "", false) + assertKnownTestsRequestPageInfo(t, requests[1], "page-2", true) + + if tests := knownTests.Tests["module-a"]["suite-a"]; len(tests) != 2 || tests[0] != "test-a" || tests[1] != "test-b" { + t.Fatalf("expected module-a/suite-a tests to be merged, got %#v", tests) + } + if tests := knownTests.Tests["module-b"]["suite-b"]; len(tests) != 1 || tests[0] != "test-c" { + t.Fatalf("expected module-b/suite-b tests from page 2, got %#v", tests) + } + + var rawResponse knownTestsResponse + if err := json.Unmarshal(client.GetKnownTestsRawResponse(), &rawResponse); err != nil { + t.Fatalf("failed to decode merged raw response: %v", err) + } + if tests := rawResponse.Data.Attributes.Tests["module-a"]["suite-a"]; len(tests) != 2 { + t.Fatalf("expected raw response to include merged known tests, got %#v", tests) + } +} + +func TestClientGetKnownTestsDoesNotCachePartialRawResponseWhenFollowUpFails(t *testing.T) { + responses := []string{ + `{"data":{"id":"known-tests-id","type":"ci_app_libraries_tests","attributes":{"tests":{"module-a":{"suite-a":["test-a"]}},"page_info":{"cursor":"page-2","size":1,"has_next":true}}}}`, + `{"data":`, + } + requests := 0 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if requests > len(responses) { + t.Fatalf("unexpected extra request") + } + + w.Header().Set(HeaderContentType, ContentTypeJSON) + _, _ = w.Write([]byte(responses[requests-1])) + })) + defer server.Close() + + client := newRawResponseTestClient(server) + if _, err := client.GetKnownTests(); err == nil { + t.Fatalf("GetKnownTests() should fail when a later page is invalid") + } + if rawResponse := client.GetKnownTestsRawResponse(); rawResponse != nil { + t.Fatalf("known tests raw response should stay unset after partial pagination failure, got %s", string(rawResponse)) + } +} + +func TestClientGetKnownTestsDoesNotCachePartialRawResponseWithoutFollowUpCursor(t *testing.T) { + response := `{"data":{"id":"known-tests-id","type":"ci_app_libraries_tests","attributes":{"tests":{"module-a":{"suite-a":["test-a"]}},"page_info":{"size":1,"has_next":true}}}}` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(HeaderContentType, ContentTypeJSON) + _, _ = w.Write([]byte(response)) + })) + defer server.Close() + + client := newRawResponseTestClient(server) + if _, err := client.GetKnownTests(); err == nil { + t.Fatalf("GetKnownTests() should fail when a paginated response omits the follow-up cursor") + } + if rawResponse := client.GetKnownTestsRawResponse(); rawResponse != nil { + t.Fatalf("known tests raw response should stay unset after missing cursor failure, got %s", string(rawResponse)) + } +} + +func assertKnownTestsRequestPageInfo(t *testing.T, request map[string]any, expectedPageState string, expectPageInfo bool) { + t.Helper() + + if _, ok := request["page_info"]; ok { + t.Fatalf("request should not send top-level page_info: %#v", request) + } + + data, ok := request["data"].(map[string]any) + if !ok { + t.Fatalf("request data has unexpected shape: %#v", request["data"]) + } + attributes, ok := data["attributes"].(map[string]any) + if !ok { + t.Fatalf("request attributes have unexpected shape: %#v", data["attributes"]) + } + + pageInfo, ok := attributes["page_info"].(map[string]any) + if !expectPageInfo { + if ok { + t.Fatalf("first request should let the backend choose pagination without page_info, got %#v", pageInfo) + } + return + } + if !ok { + t.Fatalf("follow-up request should include data.attributes.page_info") + } + if pageInfo["page_state"] != expectedPageState { + t.Fatalf("expected page_state %q, got %#v", expectedPageState, pageInfo["page_state"]) + } + if _, ok := pageInfo["page_size"]; ok { + t.Fatalf("request should not send page_size: %#v", pageInfo) + } +}