diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 3adc05d..815bc74 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -9,6 +9,7 @@ import ( "github.com/DataDog/ddtest/internal/buildinfo" "github.com/DataDog/ddtest/internal/constants" "github.com/DataDog/ddtest/internal/git" + "github.com/DataDog/ddtest/internal/planner" "github.com/DataDog/ddtest/internal/runner" "github.com/DataDog/ddtest/internal/settings" "github.com/spf13/cobra" @@ -38,9 +39,8 @@ var planCmd = &cobra.Command{ ), Run: func(cmd *cobra.Command, args []string) { ctx := context.Background() - testRunner := runner.New() - if err := testRunner.Plan(ctx); err != nil { - slog.Error("Runner failed", "error", err) + if err := planner.Plan(ctx); err != nil { + slog.Error("Planner failed", "error", err) os.Exit(1) } }, diff --git a/internal/planner/discovered_tests.go b/internal/planner/discovered_tests.go new file mode 100644 index 0000000..afb8fbe --- /dev/null +++ b/internal/planner/discovered_tests.go @@ -0,0 +1,73 @@ +package planner + +import ( + "log/slog" + + "github.com/DataDog/ddtest/internal/testoptimization" +) + +func (tp *TestPlanner) recordFullDiscoveryResults( + discoveredTests []testoptimization.Test, + skippableTests map[string]bool, + subdirPrefix string, +) { + discoveredTestsCount := len(discoveredTests) + if discoveredTestsCount == 0 { + slog.Info("Full test discovery returned no tests") + return + } + + slog.Info("Using full test discovery results") + skippableTestsCount := 0 + for _, test := range discoveredTests { + normalizedSourceFile := stripCwdSubdirPrefix(test.SuiteSourceFile, subdirPrefix) + if normalizedSourceFile != "" { + tp.testFiles[normalizedSourceFile] = struct{}{} + } + + if !skippableTests[test.FQN()] { + slog.Debug("Test is not skipped", "test", test.FQN(), "sourceFile", test.SuiteSourceFile) + recordRunnableTest(tp.suiteAggregates, test, normalizedSourceFile) + } else { + recordSkippedTest(tp.suiteAggregates, test, normalizedSourceFile) + skippableTestsCount++ + } + } + + slog.Info("Processed the discovered tests", "skippableTestsCount", skippableTestsCount, "discoveredTestsCount", discoveredTestsCount) +} + +func (tp *TestPlanner) recordFastDiscoveryFallbackFiles(discoveredTestFiles []string) { + for _, testFile := range discoveredTestFiles { + if testFile != "" { + tp.testFiles[testFile] = struct{}{} + } + } +} + +func recordRunnableTest(suiteAggregates map[testSuiteKey]testSuiteAggregate, test testoptimization.Test, sourceFile string) { + aggregate := suiteAggregateForTest(suiteAggregates, test, sourceFile) + aggregate.NumTests++ + suiteAggregates[testSuiteKey{Module: test.Module, Suite: test.Suite}] = aggregate +} + +func recordSkippedTest(suiteAggregates map[testSuiteKey]testSuiteAggregate, test testoptimization.Test, sourceFile string) { + aggregate := suiteAggregateForTest(suiteAggregates, test, sourceFile) + aggregate.NumTests++ + aggregate.NumTestsSkipped++ + suiteAggregates[testSuiteKey{Module: test.Module, Suite: test.Suite}] = aggregate +} + +func suiteAggregateForTest(suiteAggregates map[testSuiteKey]testSuiteAggregate, test testoptimization.Test, sourceFile string) testSuiteAggregate { + key := testSuiteKey{ + Module: test.Module, + Suite: test.Suite, + } + aggregate := suiteAggregates[key] + if aggregate.SourceFile == "" { + aggregate.Module = test.Module + aggregate.Suite = test.Suite + aggregate.SourceFile = sourceFile + } + return aggregate +} diff --git a/internal/runner/distribution.go b/internal/planner/distribution.go similarity index 77% rename from internal/runner/distribution.go rename to internal/planner/distribution.go index 0aa9d3a..f249733 100644 --- a/internal/runner/distribution.go +++ b/internal/planner/distribution.go @@ -1,8 +1,10 @@ -package runner +package planner import ( "container/heap" + "errors" "fmt" + "log/slog" "os" "path/filepath" "slices" @@ -12,19 +14,47 @@ import ( "github.com/DataDog/ddtest/internal/constants" ) -// DistributeTestFiles distributes test files across parallel runners using weighted list scheduling. -func DistributeTestFiles(testFiles map[string]int, parallelRunners int) [][]string { +// DistributeTestFiles distributes test files using weights loaded into this planner. +func (tp *TestPlanner) DistributeTestFiles(testFiles []string, parallelRunners int) [][]string { + if !tp.planLoaded { + if err := tp.restoreTestOptimizationPlanCache(); err != nil { + if errors.Is(err, os.ErrNotExist) { + slog.Debug("Test optimization run artifacts not found; distributing test files with default weights") + } else { + slog.Warn("Failed to load test optimization run artifacts; distributing test files with default weights", "error", err) + } + } + } + + testFileWeights := testFileWeightsForFiles(tp.testFileWeights, testFiles) + return tp.DistributeWeightedTestFiles(testFileWeights, parallelRunners) +} + +// DistributeWeightedTestFiles distributes test files across parallel runners using weighted list scheduling. +func (tp *TestPlanner) DistributeWeightedTestFiles(testFiles map[string]int, parallelRunners int) [][]string { builder := newTestSplitBuilder(parallelRunners) return builder.distributeFiles(testFiles) } +func testFileWeightsForFiles(cacheWeights map[string]int, testFiles []string) map[string]int { + testFileWeights := make(map[string]int, len(testFiles)) + for _, testFile := range testFiles { + if cachedWeight, ok := cacheWeights[testFile]; ok && cachedWeight > 0 { + testFileWeights[testFile] = cachedWeight + } else { + testFileWeights[testFile] = DefaultTestFileWeight + } + } + return testFileWeights +} + // CreateTestSplits creates test split files for parallel runners // For multiple runners: distributes files using weighted list scheduling and writes to separate runner files // For single runner: copies test-files.txt content to runner-0 -func CreateTestSplits(testFiles map[string]int, parallelRunners int, testFilesOutputPath string) error { +func (tp *TestPlanner) CreateTestSplits(testFiles map[string]int, parallelRunners int, testFilesOutputPath string) error { if parallelRunners > 1 { // Distribute test files across parallel runners using weighted list scheduling. - distribution := DistributeTestFiles(testFiles, parallelRunners) + distribution := tp.DistributeWeightedTestFiles(testFiles, parallelRunners) if err := writeDistributedTestSplits(distribution, constants.TestsSplitDir); err != nil { return err } diff --git a/internal/runner/distribution_test.go b/internal/planner/distribution_test.go similarity index 71% rename from internal/runner/distribution_test.go rename to internal/planner/distribution_test.go index 2036b1a..491ed74 100644 --- a/internal/runner/distribution_test.go +++ b/internal/planner/distribution_test.go @@ -1,4 +1,4 @@ -package runner +package planner import ( "fmt" @@ -9,11 +9,14 @@ import ( "testing" "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/testoptimization" ) -func TestDistributeTestFiles(t *testing.T) { +func TestTestPlanner_DistributeWeightedTestFiles(t *testing.T) { + planner := newTestPlannerWithDefaults() + t.Run("empty test files", func(t *testing.T) { - result := DistributeTestFiles(map[string]int{}, 3) + result := planner.DistributeWeightedTestFiles(map[string]int{}, 3) if len(result) != 3 { t.Errorf("Expected 3 runners, got %d", len(result)) } @@ -30,7 +33,7 @@ func TestDistributeTestFiles(t *testing.T) { "test2.rb": 3, "test3.rb": 2, } - result := DistributeTestFiles(testFiles, 1) + result := planner.DistributeWeightedTestFiles(testFiles, 1) if len(result) != 1 { t.Errorf("Expected 1 runner, got %d", len(result)) @@ -56,12 +59,12 @@ func TestDistributeTestFiles(t *testing.T) { t.Run("zero or negative runners defaults to 1", func(t *testing.T) { testFiles := map[string]int{"test1.rb": 1} - result := DistributeTestFiles(testFiles, 0) + result := planner.DistributeWeightedTestFiles(testFiles, 0) if len(result) != 1 { t.Errorf("Expected 1 runner for parallelRunners=0, got %d", len(result)) } - result = DistributeTestFiles(testFiles, -1) + result = planner.DistributeWeightedTestFiles(testFiles, -1) if len(result) != 1 { t.Errorf("Expected 1 runner for parallelRunners=-1, got %d", len(result)) } @@ -75,7 +78,7 @@ func TestDistributeTestFiles(t *testing.T) { "test4.rb": 8, "test5.rb": 4, } - result := DistributeTestFiles(testFiles, 3) + result := planner.DistributeWeightedTestFiles(testFiles, 3) if len(result) != 3 { t.Errorf("Expected 3 runners, got %d", len(result)) @@ -140,7 +143,7 @@ func TestDistributeTestFiles(t *testing.T) { "test1.rb": 5, "test2.rb": 3, } - result := DistributeTestFiles(testFiles, 5) + result := planner.DistributeWeightedTestFiles(testFiles, 5) if len(result) != 5 { t.Errorf("Expected 5 runners, got %d", len(result)) @@ -171,7 +174,7 @@ func TestDistributeTestFiles(t *testing.T) { "large.rb": 100, "medium.rb": 50, } - result := DistributeTestFiles(testFiles, 3) + result := planner.DistributeWeightedTestFiles(testFiles, 3) // The largest file should go to the first runner // Find which runner has the large.rb file @@ -205,7 +208,7 @@ func TestDistributeTestFiles(t *testing.T) { "c.rb": 1, } - result := DistributeTestFiles(testFiles, 2) + result := planner.DistributeWeightedTestFiles(testFiles, 2) expected := [][]string{ {"a.rb", "c.rb"}, {"b.rb"}, @@ -230,8 +233,8 @@ func TestDistributeTestFiles(t *testing.T) { } // Run multiple times and check results are consistent - result1 := DistributeTestFiles(testFiles, 2) - result2 := DistributeTestFiles(testFiles, 2) + result1 := planner.DistributeWeightedTestFiles(testFiles, 2) + result2 := planner.DistributeWeightedTestFiles(testFiles, 2) // Results should be identical (same distribution) if len(result1) != len(result2) { @@ -268,6 +271,129 @@ func TestDistributeTestFiles(t *testing.T) { }) } +func TestTestPlanner_DistributeTestFiles_UsesCachedWeights(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + cache := testOptimizationPlanCache{ + TestFileWeights: map[string]int{ + "spec/slow_spec.rb": 10_000, + "spec/fast_a_spec.rb": 1, + "spec/fast_b_spec.rb": 1, + }, + } + if err := testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache); err != nil { + t.Fatalf("StoreTestOptimizationPlanCache() should not return error, got: %v", err) + } + + planner := newTestPlannerWithDefaults() + result := planner.DistributeTestFiles([]string{ + "spec/fast_a_spec.rb", + "spec/fast_b_spec.rb", + "spec/slow_spec.rb", + }, 2) + + expected := [][]string{ + {"spec/slow_spec.rb"}, + {"spec/fast_a_spec.rb", "spec/fast_b_spec.rb"}, + } + assertDistribution(t, result, expected) +} + +func TestTestPlanner_DistributeTestFiles_UsesRestoredWeights(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + cache := testOptimizationPlanCache{ + TestFileWeights: map[string]int{ + "spec/slow_spec.rb": 10_000, + "spec/fast_a_spec.rb": 1, + "spec/fast_b_spec.rb": 1, + }, + } + if err := testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache); err != nil { + t.Fatalf("StoreTestOptimizationPlanCache() should not return error, got: %v", err) + } + + planner := newTestPlannerWithDefaults() + if _, err := planner.LoadPlan(); err != nil { + t.Fatalf("LoadPlan() should not return error, got: %v", err) + } + + cachePath := filepath.Join(constants.RunnerCacheDir, testoptimization.TestOptimizationPlanCacheFile) + if err := os.WriteFile(cachePath, []byte("{"), 0644); err != nil { + t.Fatalf("failed to corrupt cache: %v", err) + } + + result := planner.DistributeTestFiles([]string{ + "spec/fast_a_spec.rb", + "spec/fast_b_spec.rb", + "spec/slow_spec.rb", + }, 2) + + expected := [][]string{ + {"spec/slow_spec.rb"}, + {"spec/fast_a_spec.rb", "spec/fast_b_spec.rb"}, + } + assertDistribution(t, result, expected) +} + +func TestTestPlanner_DistributeTestFiles_FallsBackToDefaultWeights(t *testing.T) { + t.Run("missing cache", func(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + logs := captureLogs(t) + + planner := newTestPlannerWithDefaults() + result := planner.DistributeTestFiles([]string{"a", "b", "c", "d"}, 2) + expected := [][]string{ + {"a", "c"}, + {"b", "d"}, + } + assertDistribution(t, result, expected) + + if !strings.Contains(logs.String(), "level=DEBUG") || + !strings.Contains(logs.String(), "Test optimization run artifacts not found; distributing test files with default weights") { + t.Errorf("Expected DEBUG log for missing cache fallback, got logs: %s", logs.String()) + } + }) + + t.Run("corrupt cache", func(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + logs := captureLogs(t) + + cachePath := filepath.Join(constants.RunnerCacheDir, testoptimization.TestOptimizationPlanCacheFile) + if err := os.MkdirAll(filepath.Dir(cachePath), 0755); err != nil { + t.Fatalf("failed to create cache dir: %v", err) + } + if err := os.WriteFile(cachePath, []byte("{"), 0644); err != nil { + t.Fatalf("failed to write corrupt cache: %v", err) + } + + planner := newTestPlannerWithDefaults() + result := planner.DistributeTestFiles([]string{"a", "b", "c"}, 2) + expected := [][]string{ + {"a", "c"}, + {"b"}, + } + assertDistribution(t, result, expected) + + if !strings.Contains(logs.String(), "level=WARN") || + !strings.Contains(logs.String(), "Failed to load test optimization run artifacts; distributing test files with default weights") { + t.Errorf("Expected WARN log for corrupt cache fallback, got logs: %s", logs.String()) + } + }) +} + func TestSortedWeightedTestFiles(t *testing.T) { testFiles := map[string]int{ "small.rb": 1, @@ -379,7 +505,8 @@ func TestCreateTestSplits(t *testing.T) { "test/file2_test.rb": 1, } - err := CreateTestSplits(testFiles, 1, constants.TestFilesOutputPath) + planner := newTestPlannerWithDefaults() + err := planner.CreateTestSplits(testFiles, 1, constants.TestFilesOutputPath) if err != nil { t.Fatalf("CreateTestSplits() should not return error, got: %v", err) } @@ -418,7 +545,8 @@ func TestCreateTestSplits(t *testing.T) { "test/file3_test.rb": 3, } - err := CreateTestSplits(testFiles, 2, constants.TestFilesOutputPath) + planner := newTestPlannerWithDefaults() + err := planner.CreateTestSplits(testFiles, 2, constants.TestFilesOutputPath) if err != nil { t.Fatalf("CreateTestSplits() should not return error, got: %v", err) } @@ -469,7 +597,8 @@ func TestCreateTestSplits(t *testing.T) { _ = os.MkdirAll(filepath.Dir(constants.TestFilesOutputPath), 0755) _ = os.WriteFile(constants.TestFilesOutputPath, []byte(""), 0644) - err := CreateTestSplits(map[string]int{}, 2, constants.TestFilesOutputPath) + planner := newTestPlannerWithDefaults() + err := planner.CreateTestSplits(map[string]int{}, 2, constants.TestFilesOutputPath) if err != nil { t.Fatalf("CreateTestSplits() should not return error for empty files, got: %v", err) } @@ -492,7 +621,8 @@ func TestCreateTestSplits(t *testing.T) { // Don't create test-files.txt testFiles := map[string]int{"test/file1_test.rb": 1} - err := CreateTestSplits(testFiles, 1, constants.TestFilesOutputPath) + planner := newTestPlannerWithDefaults() + err := planner.CreateTestSplits(testFiles, 1, constants.TestFilesOutputPath) if err == nil { t.Error("CreateTestSplits() should return error when test-files.txt doesn't exist") } diff --git a/internal/runner/high_skippable_integration_test.go b/internal/planner/high_skippable_integration_test.go similarity index 97% rename from internal/runner/high_skippable_integration_test.go rename to internal/planner/high_skippable_integration_test.go index b95e8a7..11dd218 100644 --- a/internal/runner/high_skippable_integration_test.go +++ b/internal/planner/high_skippable_integration_test.go @@ -1,4 +1,4 @@ -package runner +package planner import ( "context" @@ -25,7 +25,7 @@ type highSkippableIntegrationFixture struct { ExpectedParallelRunners int `json:"expectedParallelRunners"` } -func TestTestRunner_Plan_HighSkippableIntegrationSelectsExpectedRunnerCountAndRunnableFiles(t *testing.T) { +func TestTestPlanner_Plan_HighSkippableIntegrationSelectsExpectedRunnerCountAndRunnableFiles(t *testing.T) { fixture := loadHighSkippableIntegrationFixture(t, "spree_26236954724.json") if fixture.OriginalParallelRunners != 3 { t.Fatalf("fixture should capture the original 3-runner plan, got %d", fixture.OriginalParallelRunners) diff --git a/internal/runner/parallelism.go b/internal/planner/parallelism.go similarity index 99% rename from internal/runner/parallelism.go rename to internal/planner/parallelism.go index e4e3a4f..20e695c 100644 --- a/internal/runner/parallelism.go +++ b/internal/planner/parallelism.go @@ -1,4 +1,4 @@ -package runner +package planner import ( "log/slog" diff --git a/internal/runner/parallelism_test.go b/internal/planner/parallelism_test.go similarity index 99% rename from internal/runner/parallelism_test.go rename to internal/planner/parallelism_test.go index 456b79d..d9d10d1 100644 --- a/internal/runner/parallelism_test.go +++ b/internal/planner/parallelism_test.go @@ -1,4 +1,4 @@ -package runner +package planner import ( "encoding/json" diff --git a/internal/runner/path_normalization.go b/internal/planner/path_normalization.go similarity index 99% rename from internal/runner/path_normalization.go rename to internal/planner/path_normalization.go index 5f309e4..d75540e 100644 --- a/internal/runner/path_normalization.go +++ b/internal/planner/path_normalization.go @@ -1,4 +1,4 @@ -package runner +package planner import ( "log/slog" diff --git a/internal/runner/path_normalization_test.go b/internal/planner/path_normalization_test.go similarity index 98% rename from internal/runner/path_normalization_test.go rename to internal/planner/path_normalization_test.go index b9cdd3b..8e3b77b 100644 --- a/internal/runner/path_normalization_test.go +++ b/internal/planner/path_normalization_test.go @@ -1,4 +1,4 @@ -package runner +package planner import ( "os" @@ -155,12 +155,13 @@ func initGitRepoInDir(t *testing.T, dir string) { t.Helper() cmd := exec.Command("git", "init") cmd.Dir = dir + cmd.Env = gitTestEnv() if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("failed to init git repo in %s: %v\n%s", dir, err, string(out)) } cmd = exec.Command("git", "commit", "--allow-empty", "-m", "init") cmd.Dir = dir - cmd.Env = append(os.Environ(), + cmd.Env = append(gitTestEnv(), "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@test.com", "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@test.com", ) diff --git a/internal/planner/plan_files.go b/internal/planner/plan_files.go new file mode 100644 index 0000000..276bd10 --- /dev/null +++ b/internal/planner/plan_files.go @@ -0,0 +1,39 @@ +package planner + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/DataDog/ddtest/internal/constants" +) + +func writeTestFilesArtifact(testFileWeights map[string]int) error { + testFileNames := make([]string, 0, len(testFileWeights)) + for testFile := range testFileWeights { + testFileNames = append(testFileNames, testFile) + } + slices.Sort(testFileNames) + + content := strings.Join(testFileNames, "\n") + if len(testFileNames) > 0 { + content += "\n" + } + + if err := writePlanFile(constants.TestFilesOutputPath, []byte(content)); err != nil { + return fmt.Errorf("failed to write test files: %w", err) + } + return nil +} + +func writePlanFile(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("failed to create output directory for %s: %w", path, err) + } + if err := os.WriteFile(path, data, 0644); err != nil { + return fmt.Errorf("failed to write %s: %w", path, err) + } + return nil +} diff --git a/internal/runner/dd_test_optimization.go b/internal/planner/planner.go similarity index 50% rename from internal/runner/dd_test_optimization.go rename to internal/planner/planner.go index 3bd95e0..88c1c18 100644 --- a/internal/runner/dd_test_optimization.go +++ b/internal/planner/planner.go @@ -1,26 +1,240 @@ -package runner +package planner import ( "context" "encoding/json" "fmt" + "io" "log/slog" "maps" + "os" "slices" "strconv" "time" - "github.com/DataDog/ddtest/civisibility/constants" + ciConstants "github.com/DataDog/ddtest/civisibility/constants" "github.com/DataDog/ddtest/civisibility/utils" + "github.com/DataDog/ddtest/internal/ciprovider" + "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/platform" + "github.com/DataDog/ddtest/internal/runmetadata" "github.com/DataDog/ddtest/internal/settings" "github.com/DataDog/ddtest/internal/testoptimization" "golang.org/x/sync/errgroup" ) -const defaultTestFileWeight = int(time.Second / time.Millisecond) +type Planner interface { + Plan(ctx context.Context) error + LoadPlan() (PlanInfo, error) + DistributeTestFiles(testFiles []string, parallelRunners int) [][]string +} + +type PlanInfo struct { + Platform string `json:"platform"` + Framework string `json:"framework"` + OSTags map[string]string `json:"osTags"` + RuntimeTags map[string]string `json:"runtimeTags"` +} + +func NewPlanInfo(tags map[string]string, platformName, frameworkName string) PlanInfo { + return PlanInfo{ + Platform: platformName, + Framework: frameworkName, + OSTags: selectTags(tags, ciConstants.OSPlatform, ciConstants.OSArchitecture, ciConstants.OSVersion), + RuntimeTags: selectTags(tags, ciConstants.RuntimeName, ciConstants.RuntimeVersion), + } +} + +func (p PlanInfo) IsZero() bool { + return p.Platform == "" && + p.Framework == "" && + len(p.OSTags) == 0 && + len(p.RuntimeTags) == 0 +} + +type TestPlanner struct { + testFiles map[string]struct{} + suiteAggregates map[testSuiteKey]testSuiteAggregate + suitesBySourceFile map[string][]testSuiteKey + testSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo + testFileWeights map[string]int + testFileDurationSources map[string]testFileDurationSource + skippablePercentage float64 + planReport planReport + planLoaded bool + runInfo runmetadata.RunInfo + planInfo PlanInfo + platformDetector platform.PlatformDetector + optimizationClient testoptimization.TestOptimizationClient + durationsClient testoptimization.TestSuiteDurationsClient + ciProviderDetector ciprovider.CIProviderDetector + reportWriter io.Writer +} + +const DefaultTestFileWeight = int(time.Second / time.Millisecond) + +type testSuiteKey struct { + Module string `json:"module"` + Suite string `json:"suite"` +} + +func (key testSuiteKey) MarshalText() ([]byte, error) { + return json.Marshal([2]string{key.Module, key.Suite}) +} + +func (key *testSuiteKey) UnmarshalText(text []byte) error { + var values [2]string + if err := json.Unmarshal(text, &values); err != nil { + return err + } + + key.Module = values[0] + key.Suite = values[1] + return nil +} + +type testFileDurationSource string + +const ( + testFileDurationSourceKnown testFileDurationSource = "known" + testFileDurationSourceDefault testFileDurationSource = "default" +) + +type testSuiteAggregate struct { + Module string `json:"module"` + Suite string `json:"suite"` + SourceFile string `json:"sourceFile"` + TotalDuration float64 `json:"totalDuration"` + EstimatedDuration float64 `json:"estimatedDuration"` + DurationSource testFileDurationSource `json:"durationSource,omitempty"` + NumTests int `json:"numTests"` + NumTestsSkipped int `json:"numTestsSkipped"` +} + +type testFileWeightEstimate struct { + weight int + source testFileDurationSource +} + +func selectTags(tags map[string]string, keys ...string) map[string]string { + selected := make(map[string]string) + for _, key := range keys { + if value := tags[key]; value != "" { + selected[key] = value + } + } + return selected +} + +func Plan(ctx context.Context) error { + return New().Plan(ctx) +} + +func New() *TestPlanner { + planner := newTestPlannerWithDefaults() + planner.platformDetector = platform.NewPlatformDetector() + planner.optimizationClient = testoptimization.NewDatadogClient() + planner.durationsClient = testoptimization.NewDurationsClient() + planner.ciProviderDetector = ciprovider.NewCIProviderDetector() + return planner +} + +func NewWithDependencies( + platformDetector platform.PlatformDetector, + optimizationClient testoptimization.TestOptimizationClient, + durationsClient testoptimization.TestSuiteDurationsClient, + ciProviderDetector ciprovider.CIProviderDetector, +) *TestPlanner { + planner := newTestPlannerWithDefaults() + planner.platformDetector = platformDetector + planner.optimizationClient = optimizationClient + planner.durationsClient = durationsClient + planner.ciProviderDetector = ciProviderDetector + return planner +} + +func newTestPlannerWithDefaults() *TestPlanner { + return &TestPlanner{ + testFiles: make(map[string]struct{}), + suiteAggregates: make(map[testSuiteKey]testSuiteAggregate), + suitesBySourceFile: make(map[string][]testSuiteKey), + testSuiteDurations: make(map[string]map[string]testoptimization.TestSuiteDurationInfo), + testFileWeights: make(map[string]int), + testFileDurationSources: make(map[string]testFileDurationSource), + skippablePercentage: 0.0, + reportWriter: os.Stderr, + } +} + +func (tp *TestPlanner) Plan(ctx context.Context) error { + slog.Info("Planning test execution...") + + if err := tp.PreparePlanningData(ctx); err != nil { + return err + } + + if err := writePlanFile(constants.ManifestPath, []byte(constants.ManifestVersion+"\n")); err != nil { + return fmt.Errorf("failed to write test optimization manifest: %w", err) + } + + if err := tp.storeTestOptimizationPlanCache(); err != nil { + return fmt.Errorf("failed to store test optimization plan cache: %w", err) + } + + if err := writeTestFilesArtifact(tp.testFileWeights); err != nil { + return err + } + + percentageContent := fmt.Sprintf("%.2f", tp.skippablePercentage) + if err := writePlanFile(constants.SkippablePercentageOutputPath, []byte(percentageContent)); err != nil { + return fmt.Errorf("failed to write skippable percentage: %w", err) + } + + parallelRunnerSplit := calculateParallelRunnerSplit( + tp.testFileWeights, + settings.GetMinParallelism(), + settings.GetMaxParallelism(), + settings.GetParallelRunnerOverhead(), + ) + parallelRunners := parallelRunnerSplit.parallelRunners + runnersContent := fmt.Sprintf("%d", parallelRunners) + if err := writePlanFile(constants.ParallelRunnersOutputPath, []byte(runnersContent)); err != nil { + return fmt.Errorf("failed to write parallel runners: %w", err) + } + + if ciProvider, err := tp.ciProviderDetector.DetectCIProvider(); err == nil { + slog.Info("CI provider detected, configuring with parallel runners", + "provider", ciProvider.Name(), "parallelRunners", parallelRunners) + + if err := ciProvider.Configure(parallelRunners); err != nil { + slog.Warn("Failed to configure CI provider", "provider", ciProvider.Name(), "error", err) + } + } else { + slog.Info("No CI provider detected or CI provider is not supported, running tests without CI integration", "error", err) + } + + if err := tp.CreateTestSplits(tp.testFileWeights, parallelRunners, constants.TestFilesOutputPath); err != nil { + return fmt.Errorf("failed to create test splits: %w", err) + } + + tp.planReport.Split = parallelRunnerSplit + slog.Info("Test execution planning completed", + "parallelRunners", parallelRunners, + "expectedWallTime", parallelRunnerSplit.wallTimeDuration(), + "imbalance", parallelRunnerSplit.imbalanceDuration(), + "expectedTotalRuntime", parallelRunnerSplit.totalRuntimeDuration(), + "testFilesCount", len(tp.testFileWeights)) -func (tr *TestRunner) PrepareTestOptimization(ctx context.Context) error { - detectedPlatform, err := tr.platformDetector.DetectPlatform() + if settings.GetReportEnabled() { + printPlanReport(tp.reportWriter, tp.planReport) + } + + tp.planLoaded = true + return nil +} + +func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { + detectedPlatform, err := tp.platformDetector.DetectPlatform() if err != nil { return fmt.Errorf("failed to detect platform: %w", err) } @@ -51,7 +265,8 @@ func (tr *TestRunner) PrepareTestOptimization(ctx context.Context) error { return fmt.Errorf("failed to detect framework: %w", err) } slog.Info("Framework detected", "framework", framework.Name()) - tr.runInfoReport = newRunInfoReport(utils.GetCITags(), tags, detectedPlatform.Name(), framework.Name()) + tp.runInfo = runmetadata.New(utils.GetCITags()) + tp.planInfo = NewPlanInfo(tags, detectedPlatform.Name(), framework.Name()) // Create a cancellable context for test discovery discoveryCtx, cancelDiscovery := context.WithCancel(ctx) @@ -64,41 +279,40 @@ func (tr *TestRunner) PrepareTestOptimization(ctx context.Context) error { var fullDiscoveryErr error var fastDiscoveryErr error - tr.testFiles = make(map[string]struct{}) - tr.suiteAggregates = make(map[testSuiteKey]testSuiteAggregate) - tr.suitesBySourceFile = make(map[string][]testSuiteKey) - tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) + tp.testFiles = make(map[string]struct{}) + tp.suiteAggregates = make(map[testSuiteKey]testSuiteAggregate) + tp.suitesBySourceFile = make(map[string][]testSuiteKey) + tp.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) g, _ := errgroup.WithContext(ctx) // Goroutine 1: Initialize optimization client and check settings g.Go(func() error { - defer tr.optimizationClient.StoreCacheAndExit() + defer tp.optimizationClient.StoreCacheAndExit() - if err := tr.optimizationClient.Initialize(tags); err != nil { + if err := tp.optimizationClient.Initialize(tags); err != nil { return fmt.Errorf("failed to initialize optimization client: %w", err) } - // Get settings to check if ITR is enabled - settings := tr.optimizationClient.GetSettings() - tr.planReport.DatadogSettings = newDatadogSettingsReport(settings) - if settings != nil { - slog.Debug("Repository settings", "itr_enabled", settings.ItrEnabled, "tests_skipping", settings.TestsSkipping) + repositorySettings := tp.optimizationClient.GetSettings() + tp.planReport.DatadogSettings = newDatadogSettingsReport(repositorySettings) + if repositorySettings != nil { + slog.Debug("Repository settings", "itr_enabled", repositorySettings.ItrEnabled, "tests_skipping", repositorySettings.TestsSkipping) - if !settings.ItrEnabled || !settings.TestsSkipping { + if !repositorySettings.ItrEnabled || !repositorySettings.TestsSkipping { slog.Info("ITR or test skipping disabled, cancelling full test discovery") cancelDiscovery() } } - tr.fetchAndStoreTestSuiteDurations() + tp.testSuiteDurations = tp.durationsClient.GetTestSuiteDurations() startTime := time.Now() slog.Info("Fetching skippable tests from Datadog...") - skippableTests = tr.optimizationClient.GetSkippableTests() - tr.planReport.SkippableTestsCount = len(skippableTests) - tr.planReport.KnownTests = newKnownTestsReport(tr.optimizationClient.GetKnownTests()) - tr.planReport.ManagedFlakyTests = newManagedFlakyTestsReport(tr.optimizationClient.GetTestManagementTestsData()) + skippableTests = tp.optimizationClient.GetSkippableTests() + tp.planReport.SkippableTestsCount = len(skippableTests) + tp.planReport.KnownTests = newKnownTestsReport(tp.optimizationClient.GetKnownTests()) + tp.planReport.ManagedFlakyTests = newManagedFlakyTestsReport(tp.optimizationClient.GetTestManagementTestsData()) slog.Info("Fetched skippable tests", "duration", time.Since(startTime)) return nil @@ -114,6 +328,13 @@ func (tr *TestRunner) PrepareTestOptimization(ctx context.Context) error { slog.Warn("Full test discovery failed or was cancelled", "error", discErr) return nil // Don't fail the entire process, we have fast discovery as fallback } + if len(res) == 0 { + fullDiscoveryErr = fmt.Errorf("full test discovery returned no tests") + slog.Warn("Full test discovery returned no tests; using fast test file discovery fallback", + "duration", time.Since(startTime), + "error", fullDiscoveryErr) + return nil + } discoveredTests = res fullDiscoverySucceeded = true slog.Info("Discovered local tests", "duration", time.Since(startTime), "count", len(discoveredTests)) @@ -159,190 +380,41 @@ func (tr *TestRunner) PrepareTestOptimization(ctx context.Context) error { // into a collection of testSuiteAggregate structs. // This collection is used to calculate the skippable percentage and the weighted test files. if fullDiscoverySucceeded { - tr.processDiscoveredTests(discoveredTests, skippableTests, subdirPrefix) - } else { - slog.Info("Full test discovery did not run (failed or test impact analysis is not enabled)") - } - - // Add local test files conforming to test framework pattern (spec/*_spec.rb for example) - tr.processDiscoveredTestFiles(discoveredTestFiles) - - // Enrich test suite aggregates with the duration data we got from the backend - tr.resolveSuiteDurations() - tr.suitesBySourceFile = indexSuitesBySourceFile(tr.suiteAggregates) - // For the test files with no suite info, try to match them to our backend test suites data - tr.addBackendTestSuites(subdirPrefix) - - tr.skippablePercentage = calculateSavedTimePercentage(tr.suiteAggregates) - tr.testFileWeights = tr.weightedTestFiles() - tr.planReport.RunInfo = tr.runInfoReport - tr.planReport.Planning = tr.newPlanningReport() - - slog.Info("Test files prepared", "testFilesCount", len(tr.testFiles)) - - return nil -} - -func initializeDurationsFetchInputs() (string, string, error) { - ciTags := utils.GetCITags() - repositoryURL := ciTags[constants.GitRepositoryURL] - if repositoryURL == "" { - return "", "", fmt.Errorf("repository URL is required") - } + tp.recordFullDiscoveryResults(discoveredTests, skippableTests, subdirPrefix) + tp.estimateDiscoveredSuiteDurations() - service := resolveServiceName(repositoryURL) - - return ciTags[constants.GitRepositoryURL], service, nil -} + slog.Info("Full test discovery succeeded; using full discovery results and ignoring fast-discovered-only files", + "fastDiscoveredTestFilesCount", len(discoveredTestFiles)) + } else { + tp.recordFastDiscoveryFallbackFiles(discoveredTestFiles) + tp.addDurationDataForFastDiscoveryFallback(subdirPrefix) -func (tr *TestRunner) fetchAndStoreTestSuiteDurations() { - startTime := time.Now() - repositoryURL, service, err := initializeDurationsFetchInputs() - if err != nil { - slog.Error("Test durations API errored", "duration", time.Since(startTime), "error", err) - tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) - return + slog.Info("Full test discovery did not run or failed; using fast test file discovery fallback", + "fastDiscoveredTestFilesCount", len(discoveredTestFiles)) } - durations, err := tr.durationsClient.GetTestSuiteDurations(repositoryURL, service) - if err != nil { - slog.Error("Test durations API errored", - "service", service, - "repositoryURL", repositoryURL, - "duration", time.Since(startTime), - "error", err) - tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) - return - } - - totalSuites := countTestSuites(durations) - - if totalSuites == 0 { - slog.Warn("Test durations API returned no test suites", - "service", service, - "repositoryURL", repositoryURL, - "modulesCount", len(durations), - "testSuitesCount", totalSuites, - "duration", time.Since(startTime)) - tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) - return - } - - slog.Info("Fetched test suite durations", - "service", service, - "repositoryURL", repositoryURL, - "modulesCount", len(durations), - "testSuitesCount", totalSuites, - "duration", time.Since(startTime)) - tr.testSuiteDurations = durations -} + tp.suitesBySourceFile = indexSuitesBySourceFile(tp.suiteAggregates) + tp.skippablePercentage = calculateSavedTimePercentage(tp.suiteAggregates) + tp.testFileWeights = tp.calculateFileWeights() -type testSuiteKey struct { - Module string `json:"module"` - Suite string `json:"suite"` -} + tp.planReport.RunInfo = tp.runInfo + tp.planReport.PlanInfo = tp.planInfo + tp.planReport.Planning = tp.newPlanningReport() -func (key testSuiteKey) MarshalText() ([]byte, error) { - return json.Marshal([2]string{key.Module, key.Suite}) -} - -func (key *testSuiteKey) UnmarshalText(text []byte) error { - var values [2]string - if err := json.Unmarshal(text, &values); err != nil { - return err - } + slog.Info("Test files prepared", "testFilesCount", len(tp.testFiles)) - key.Module = values[0] - key.Suite = values[1] return nil } -type testSuiteAggregate struct { - Module string `json:"module"` - Suite string `json:"suite"` - SourceFile string `json:"sourceFile"` - TotalDuration float64 `json:"totalDuration"` - EstimatedDuration float64 `json:"estimatedDuration"` - DurationSource testFileDurationSource `json:"durationSource,omitempty"` - NumTests int `json:"numTests"` - NumTestsSkipped int `json:"numTestsSkipped"` -} - -func (tr *TestRunner) processDiscoveredTests( - discoveredTests []testoptimization.Test, - skippableTests map[string]bool, - subdirPrefix string, -) { - discoveredTestsCount := len(discoveredTests) - if discoveredTestsCount == 0 { - slog.Info("Full test discovery returned no tests, using only fast test discovery results") - return - } - - slog.Info("Using full test discovery results") - skippableTestsCount := 0 - for _, test := range discoveredTests { - normalizedSourceFile := stripCwdSubdirPrefix(test.SuiteSourceFile, subdirPrefix) - if normalizedSourceFile != "" { - tr.testFiles[normalizedSourceFile] = struct{}{} - } - - if !skippableTests[test.FQN()] { - slog.Debug("Test is not skipped", "test", test.FQN(), "sourceFile", test.SuiteSourceFile) - recordRunnableTest(tr.suiteAggregates, test, normalizedSourceFile) - } else { - recordSkippedTest(tr.suiteAggregates, test, normalizedSourceFile) - skippableTestsCount++ - } - } - - slog.Info("Processed the discovered tests", "skippableTestsCount", skippableTestsCount, "discoveredTestsCount", discoveredTestsCount) -} - -func (tr *TestRunner) processDiscoveredTestFiles(discoveredTestFiles []string) { - for _, testFile := range discoveredTestFiles { - if testFile != "" { - tr.testFiles[testFile] = struct{}{} - } - } -} - -func recordRunnableTest(suiteAggregates map[testSuiteKey]testSuiteAggregate, test testoptimization.Test, sourceFile string) { - aggregate := suiteAggregateForTest(suiteAggregates, test, sourceFile) - aggregate.NumTests++ - suiteAggregates[testSuiteKey{Module: test.Module, Suite: test.Suite}] = aggregate -} - -func recordSkippedTest(suiteAggregates map[testSuiteKey]testSuiteAggregate, test testoptimization.Test, sourceFile string) { - aggregate := suiteAggregateForTest(suiteAggregates, test, sourceFile) - aggregate.NumTests++ - aggregate.NumTestsSkipped++ - suiteAggregates[testSuiteKey{Module: test.Module, Suite: test.Suite}] = aggregate -} - -func suiteAggregateForTest(suiteAggregates map[testSuiteKey]testSuiteAggregate, test testoptimization.Test, sourceFile string) testSuiteAggregate { - key := testSuiteKey{ - Module: test.Module, - Suite: test.Suite, - } - aggregate := suiteAggregates[key] - if aggregate.SourceFile == "" { - aggregate.Module = test.Module - aggregate.Suite = test.Suite - aggregate.SourceFile = sourceFile - } - return aggregate -} - -func (tr *TestRunner) resolveSuiteDurations() { - for key, aggregate := range tr.suiteAggregates { +func (tp *TestPlanner) estimateDiscoveredSuiteDurations() { + for key, aggregate := range tp.suiteAggregates { // Without backend timing data, use test counts as the estimate: // TotalDuration is the full suite before ITR skips, while EstimatedDuration // is the runnable remainder after skipped tests are removed. aggregate.TotalDuration = float64(aggregate.NumTests) * float64(time.Second) aggregate.EstimatedDuration = float64(aggregate.NumTests-aggregate.NumTestsSkipped) * float64(time.Second) aggregate.DurationSource = testFileDurationSourceDefault - if suiteInfo, ok := getTestSuiteDuration(tr.testSuiteDurations, key); ok { + if suiteInfo, ok := getTestSuiteDuration(tp.testSuiteDurations, key); ok { if p50, ok := parseDurationP50(suiteInfo); ok { aggregate.TotalDuration = p50 aggregate.EstimatedDuration = p50 @@ -354,20 +426,21 @@ func (tr *TestRunner) resolveSuiteDurations() { } } } - tr.suiteAggregates[key] = aggregate + tp.suiteAggregates[key] = aggregate } } -func (tr *TestRunner) addBackendTestSuites(subdirPrefix string) { - for module, suites := range tr.testSuiteDurations { +func (tp *TestPlanner) addDurationDataForFastDiscoveryFallback(subdirPrefix string) { + seenSourceFiles := make(map[string]struct{}) + for module, suites := range tp.testSuiteDurations { for suite, suiteInfo := range suites { key := testSuiteKey{Module: module, Suite: suite} - if _, ok := tr.suiteAggregates[key]; ok { + if _, ok := tp.suiteAggregates[key]; ok { continue } sourceFile := stripCwdSubdirPrefix(suiteInfo.SourceFile, subdirPrefix) - if _, ok := tr.testFiles[sourceFile]; !ok { + if _, ok := tp.testFiles[sourceFile]; !ok { continue } @@ -376,14 +449,13 @@ func (tr *TestRunner) addBackendTestSuites(subdirPrefix string) { continue } - // Backend durations can contain duplicate suite names for a source file already - // handled by discovery. Treat backend-only suites as a fallback only; otherwise - // stale duplicate rows can make fully skipped files look runnable. - if _, ok := tr.suitesBySourceFile[sourceFile]; ok { + // Backend durations can contain duplicate suite names for the same source file. + // Fast discovery only tells us the file exists, so keep one backend fallback row per file. + if _, ok := seenSourceFiles[sourceFile]; ok { continue } - tr.suiteAggregates[key] = testSuiteAggregate{ + tp.suiteAggregates[key] = testSuiteAggregate{ Module: module, Suite: suite, SourceFile: sourceFile, @@ -393,7 +465,7 @@ func (tr *TestRunner) addBackendTestSuites(subdirPrefix string) { NumTests: 1, NumTestsSkipped: 0, } - tr.suitesBySourceFile[sourceFile] = append(tr.suitesBySourceFile[sourceFile], key) + seenSourceFiles[sourceFile] = struct{}{} } } } @@ -475,45 +547,33 @@ func indexSuitesBySourceFile(suiteAggregates map[testSuiteKey]testSuiteAggregate return sourceFileLookup } -type testFileDurationSource string - -const ( - testFileDurationSourceKnown testFileDurationSource = "known" - testFileDurationSourceDefault testFileDurationSource = "default" -) - -type testFileWeightEstimate struct { - weight int - source testFileDurationSource -} - -func (tr *TestRunner) weightedTestFiles() map[string]int { - return tr.estimateTestFileWeights(tr.testFiles) +func (tp *TestPlanner) calculateFileWeights() map[string]int { + return tp.estimateTestFileWeights(tp.testFiles) } -func (tr *TestRunner) estimateTestFileWeights(testFiles map[string]struct{}) map[string]int { +func (tp *TestPlanner) estimateTestFileWeights(testFiles map[string]struct{}) map[string]int { testFileWeights := make(map[string]int, len(testFiles)) - tr.testFileDurationSources = make(map[string]testFileDurationSource, len(testFiles)) + tp.testFileDurationSources = make(map[string]testFileDurationSource, len(testFiles)) for testFile := range testFiles { - estimate, ok := tr.estimateTestFileWeight(testFile) + estimate, ok := tp.estimateTestFileWeight(testFile) if ok { testFileWeights[testFile] = estimate.weight - tr.testFileDurationSources[testFile] = estimate.source + tp.testFileDurationSources[testFile] = estimate.source } } return testFileWeights } -func (tr *TestRunner) testFileWeight(testFile string) (int, bool) { - estimate, ok := tr.estimateTestFileWeight(testFile) +func (tp *TestPlanner) testFileWeight(testFile string) (int, bool) { + estimate, ok := tp.estimateTestFileWeight(testFile) return estimate.weight, ok } -func (tr *TestRunner) estimateTestFileWeight(testFile string) (testFileWeightEstimate, bool) { - suiteKeys := tr.suitesBySourceFile[testFile] +func (tp *TestPlanner) estimateTestFileWeight(testFile string) (testFileWeightEstimate, bool) { + suiteKeys := tp.suitesBySourceFile[testFile] if len(suiteKeys) == 0 { return testFileWeightEstimate{ - weight: defaultTestFileWeight, + weight: DefaultTestFileWeight, source: testFileDurationSourceDefault, }, true } @@ -522,7 +582,7 @@ func (tr *TestRunner) estimateTestFileWeight(testFile string) (testFileWeightEst var hasRunnableSuite bool var source testFileDurationSource for _, key := range suiteKeys { - aggregate := tr.suiteAggregates[key] + aggregate := tp.suiteAggregates[key] if aggregate.NumTests == aggregate.NumTestsSkipped { continue } @@ -538,7 +598,7 @@ func (tr *TestRunner) estimateTestFileWeight(testFile string) (testFileWeightEst } if duration <= 0 { return testFileWeightEstimate{ - weight: defaultTestFileWeight, + weight: DefaultTestFileWeight, source: source, }, true } diff --git a/internal/runner/dd_test_optimization_test.go b/internal/planner/planner_test.go similarity index 53% rename from internal/runner/dd_test_optimization_test.go rename to internal/planner/planner_test.go index 176ad11..87ff392 100644 --- a/internal/runner/dd_test_optimization_test.go +++ b/internal/planner/planner_test.go @@ -1,24 +1,997 @@ -package runner +package planner import ( "bytes" "context" "errors" + "fmt" "log/slog" + "maps" "os" "os/exec" "path/filepath" + "slices" + "strconv" "strings" + "sync" "testing" "time" ciConstants "github.com/DataDog/ddtest/civisibility/constants" ciUtils "github.com/DataDog/ddtest/civisibility/utils" - ciNet "github.com/DataDog/ddtest/civisibility/utils/net" + "github.com/DataDog/ddtest/civisibility/utils/net" + "github.com/DataDog/ddtest/internal/ciprovider" + "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/platform" "github.com/DataDog/ddtest/internal/settings" "github.com/DataDog/ddtest/internal/testoptimization" ) +// Mock implementations for testing + +// MockPlatformDetector mocks platform detection +type MockPlatformDetector struct { + Platform platform.Platform + Err error +} + +func (m *MockPlatformDetector) DetectPlatform() (platform.Platform, error) { + return m.Platform, m.Err +} + +// MockPlatform mocks a platform +type MockPlatform struct { + PlatformName string + Tags map[string]string + TagsErr error + Framework framework.Framework + FrameworkErr error + SanityErr error +} + +func (m *MockPlatform) Name() string { + return m.PlatformName +} + +func (m *MockPlatform) CreateTagsMap() (map[string]string, error) { + return m.Tags, m.TagsErr +} + +func (m *MockPlatform) DetectFramework() (framework.Framework, error) { + return m.Framework, m.FrameworkErr +} + +func (m *MockPlatform) SanityCheck() error { + return m.SanityErr +} + +// MockFramework mocks a testing framework +type MockFramework struct { + FrameworkName string + Tests []testoptimization.Test + TestFiles []string + Err error // Used by both DiscoverTests and DiscoverTestFiles if specific errors are nil + DiscoverTestsErr error // If set, overrides Err for DiscoverTests + DiscoverTestFilesErr error // If set, overrides Err for DiscoverTestFiles + RunTestsCalls []RunTestsCall + mu sync.Mutex +} + +type RunTestsCall struct { + TestFiles []string + EnvMap map[string]string +} + +func (m *MockFramework) Name() string { + return m.FrameworkName +} + +func (m *MockFramework) DiscoverTests(ctx context.Context) ([]testoptimization.Test, error) { + if m.DiscoverTestsErr != nil { + return m.Tests, m.DiscoverTestsErr + } + return m.Tests, m.Err +} + +func (m *MockFramework) DiscoverTestFiles() ([]string, error) { + if m.DiscoverTestFilesErr != nil { + return m.TestFiles, m.DiscoverTestFilesErr + } + return m.TestFiles, m.Err +} + +func (m *MockFramework) RunTests(ctx context.Context, testFiles []string, envMap map[string]string) error { + // Record the call + m.mu.Lock() + m.RunTestsCalls = append(m.RunTestsCalls, RunTestsCall{ + TestFiles: slices.Clone(testFiles), + EnvMap: maps.Clone(envMap), + }) + m.mu.Unlock() + return m.Err +} + +func (m *MockFramework) SetPlatformEnv(platformEnv map[string]string) { + // No-op for mock +} + +func (m *MockFramework) GetPlatformEnv() map[string]string { + return nil +} + +func (m *MockFramework) GetRunTestsCallsCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.RunTestsCalls) +} + +func (m *MockFramework) GetRunTestsCalls() []RunTestsCall { + m.mu.Lock() + defer m.mu.Unlock() + return slices.Clone(m.RunTestsCalls) +} + +// MockTestOptimizationClient mocks the test optimization client +type MockTestOptimizationClient struct { + InitializeCalled bool + InitializeErr error + Settings *net.SettingsResponseData + SkippableTests map[string]bool + KnownTests *net.KnownTestsResponseData + TestManagementTests *net.TestManagementTestsResponseDataModules + ShutdownCalled bool + Tags map[string]string +} + +func (m *MockTestOptimizationClient) Initialize(tags map[string]string) error { + m.InitializeCalled = true + if m.Tags == nil { + m.Tags = make(map[string]string) + } + maps.Copy(m.Tags, tags) + return m.InitializeErr +} + +func (m *MockTestOptimizationClient) GetSettings() *net.SettingsResponseData { + return m.Settings +} + +func (m *MockTestOptimizationClient) GetSkippableTests() map[string]bool { + return m.SkippableTests +} + +func (m *MockTestOptimizationClient) GetKnownTests() *net.KnownTestsResponseData { + return m.KnownTests +} + +func (m *MockTestOptimizationClient) GetTestManagementTestsData() *net.TestManagementTestsResponseDataModules { + return m.TestManagementTests +} + +func (m *MockTestOptimizationClient) StoreCacheAndExit() { + m.ShutdownCalled = true +} + +type MockTestSuiteDurationsClient struct { + Durations map[string]map[string]testoptimization.TestSuiteDurationInfo + Called bool +} + +func (m *MockTestSuiteDurationsClient) GetTestSuiteDurations() map[string]map[string]testoptimization.TestSuiteDurationInfo { + m.Called = true + if m.Durations == nil { + return map[string]map[string]testoptimization.TestSuiteDurationInfo{} + } + return m.Durations +} + +// MockCIProvider mocks a CI provider +type MockCIProvider struct { + ProviderName string + ConfigureCalled bool + ConfigureErr error + ParallelRunners int +} + +func (m *MockCIProvider) Name() string { + return m.ProviderName +} + +func (m *MockCIProvider) Configure(parallelRunners int) error { + m.ConfigureCalled = true + m.ParallelRunners = parallelRunners + return m.ConfigureErr +} + +// MockCIProviderDetector mocks CI provider detection +type MockCIProviderDetector struct { + CIProvider ciprovider.CIProvider + Err error +} + +func (m *MockCIProviderDetector) DetectCIProvider() (ciprovider.CIProvider, error) { + return m.CIProvider, m.Err +} + +// Helper function to create a default mock CI provider detector that returns no provider +func newDefaultMockCIProviderDetector() *MockCIProviderDetector { + return &MockCIProviderDetector{ + Err: errors.New("no CI provider detected"), + } +} + +func assertFileContent(t *testing.T, path string, expected string) { + t.Helper() + + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read %s: %v", path, err) + } + if string(content) != expected { + t.Fatalf("expected %s content %q, got %q", path, expected, string(content)) + } +} + +func gitTestEnv() []string { + return append(os.Environ(), + "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_CONFIG_SYSTEM=/dev/null", + "GIT_CONFIG_NOSYSTEM=1", + ) +} + +func TestNew(t *testing.T) { + runner := New() + + if runner == nil { + t.Error("New() should return non-nil TestPlanner") + return + } + + if len(runner.testFiles) != 0 { + t.Error("New() should initialize testFiles to empty map") + } + + if len(runner.suiteAggregates) != 0 { + t.Error("New() should initialize suiteAggregates to empty map") + } + + if len(runner.suitesBySourceFile) != 0 { + t.Error("New() should initialize suitesBySourceFile to empty map") + } + + if runner.skippablePercentage != 0.0 { + t.Errorf("New() should initialize skippablePercentage to 0.0, got %f", runner.skippablePercentage) + } + + if runner.platformDetector == nil { + t.Error("New() should initialize platformDetector") + } + + if runner.optimizationClient == nil { + t.Error("New() should initialize optimizationClient") + } + + if runner.durationsClient == nil { + t.Error("New() should initialize durationsClient") + } +} + +func TestNewWithDependencies(t *testing.T) { + mockPlatformDetector := &MockPlatformDetector{} + mockOptimizationClient := &MockTestOptimizationClient{} + mockDurationsClient := &MockTestSuiteDurationsClient{} + mockCIProviderDetector := newDefaultMockCIProviderDetector() + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, mockCIProviderDetector) + + if runner == nil { + t.Error("NewWithDependencies() should return non-nil TestPlanner") + return + } + + if runner.platformDetector != mockPlatformDetector { + t.Error("NewWithDependencies() should use injected platformDetector") + } + + if runner.optimizationClient != mockOptimizationClient { + t.Error("NewWithDependencies() should use injected optimizationClient") + } + + if runner.durationsClient != mockDurationsClient { + t.Error("NewWithDependencies() should use injected durationsClient") + } + + if len(runner.testFiles) != 0 { + t.Error("NewWithDependencies() should initialize testFiles to empty map") + } + + if len(runner.suiteAggregates) != 0 { + t.Error("NewWithDependencies() should initialize suiteAggregates to empty map") + } + + if len(runner.suitesBySourceFile) != 0 { + t.Error("NewWithDependencies() should initialize suitesBySourceFile to empty map") + } +} + +func TestTestPlanner_Setup_WithParallelRunners(t *testing.T) { + // Create a temporary directory for test output + tempDir := t.TempDir() + + // Save current working directory and change to temp dir + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + // Create .testoptimization directory + _ = os.MkdirAll(constants.PlanDirectory, 0755) + + // Set parallelism to 1 to test single runner behavior + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") + defer func() { + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") + }() + settings.Init() + logs := captureLogs(t) + + // Setup mocks for a test with 40% skippable percentage + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, + {Suite: "TestSuite1", Name: "test2", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, + {Suite: "TestSuite2", Name: "test3", Parameters: "", SuiteSourceFile: "test/file2_test.rb"}, + {Suite: "TestSuite3", Name: "test4", Parameters: "", SuiteSourceFile: "test/file3_test.rb"}, + {Suite: "TestSuite4", Name: "test5", Parameters: "", SuiteSourceFile: "test/file4_test.rb"}, + }, + } + + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} + mockOptimizationClient := &MockTestOptimizationClient{ + SkippableTests: map[string]bool{ + "TestSuite1.test2.": true, // Skip test2 + "TestSuite4.test5.": true, // Skip test5 + }, + } + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) + + // Run Setup + err := runner.Plan(context.Background()) + if err != nil { + t.Fatalf("Setup() should not return error, got: %v", err) + } + + // Expected: 1 (since max=1) + content, err := os.ReadFile(constants.ParallelRunnersOutputPath) + if err != nil { + t.Fatalf("Failed to read parallel runners file: %v", err) + } + + expected := "1" + if string(content) != expected { + t.Errorf("Expected parallel runners file content '%s', got '%s'", expected, string(content)) + } + + logOutput := logs.String() + if !strings.Contains(logOutput, "Test execution planning completed") || + !strings.Contains(logOutput, "parallelRunners=1") || + !strings.Contains(logOutput, "expectedWallTime=") || + !strings.Contains(logOutput, "imbalance=") || + !strings.Contains(logOutput, "expectedTotalRuntime=") { + t.Errorf("Expected planning log with selected split information, got logs: %s", logOutput) + } +} + +func TestTestPlanner_Plan_WritesManifestAndRunnerLayout(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") + defer func() { + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") + }() + settings.Init() + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, + {Suite: "TestSuite2", Name: "test2", Parameters: "", SuiteSourceFile: "test/file2_test.rb"}, + }, + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + runner := NewWithDependencies( + &MockPlatformDetector{Platform: mockPlatform}, + &MockTestOptimizationClient{SkippableTests: map[string]bool{}}, + &MockTestSuiteDurationsClient{}, + newDefaultMockCIProviderDetector(), + ) + + if err := runner.Plan(context.Background()); err != nil { + t.Fatalf("Plan() should not return error, got: %v", err) + } + + assertFileContent(t, constants.ManifestPath, constants.ManifestVersion+"\n") + + expectedTestFiles := "test/file1_test.rb\ntest/file2_test.rb\n" + assertFileContent(t, constants.TestFilesOutputPath, expectedTestFiles) + + assertFileContent(t, constants.ParallelRunnersOutputPath, "1") + assertFileContent(t, constants.SkippablePercentageOutputPath, "0.00") + + assertFileContent(t, filepath.Join(constants.TestsSplitDir, "runner-0"), expectedTestFiles) +} + +func TestTestPlanner_Plan_DoesNotPrintReportWhenDisabled(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED", "false") + defer func() { + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED") + settings.Init() + }() + settings.Init() + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, + }, + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + runner := NewWithDependencies( + &MockPlatformDetector{Platform: mockPlatform}, + &MockTestOptimizationClient{SkippableTests: map[string]bool{}}, + &MockTestSuiteDurationsClient{}, + newDefaultMockCIProviderDetector(), + ) + var output strings.Builder + runner.reportWriter = &output + + if err := runner.Plan(context.Background()); err != nil { + t.Fatalf("Plan() should not return error, got: %v", err) + } + if output.Len() != 0 { + t.Errorf("Expected no report output when report is disabled, got: %s", output.String()) + } +} + +func TestTestPlanner_Plan_ChoosesParallelismFromFanoutAdjustedSplit(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "2") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "4") + defer func() { + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") + }() + settings.Init() + + var tests []testoptimization.Test + skippableTests := map[string]bool{} + for suiteIndex := range 4 { + suite := fmt.Sprintf("TestSuite%d", suiteIndex) + sourceFile := fmt.Sprintf("test/file%d_test.rb", suiteIndex) + for testIndex := range 10 { + name := fmt.Sprintf("test%d", testIndex) + tests = append(tests, testoptimization.Test{ + Suite: suite, + Name: name, + Parameters: "", + SuiteSourceFile: sourceFile, + }) + if testIndex > 0 { + skippableTests[fmt.Sprintf("%s.%s.", suite, name)] = true + } + } + } + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: tests, + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + runner := NewWithDependencies( + &MockPlatformDetector{Platform: mockPlatform}, + &MockTestOptimizationClient{SkippableTests: skippableTests}, + &MockTestSuiteDurationsClient{}, + newDefaultMockCIProviderDetector(), + ) + + if err := runner.Plan(context.Background()); err != nil { + t.Fatalf("Plan() should not return error, got: %v", err) + } + + assertFileContent(t, constants.SkippablePercentageOutputPath, "90.00") + assertFileContent(t, constants.ParallelRunnersOutputPath, "2") +} + +func TestTestPlanner_Setup_WithCIProvider(t *testing.T) { + tempDir := t.TempDir() + + // Save current working directory and change to temp dir + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + // Create .testoptimization directory + _ = os.MkdirAll(constants.PlanDirectory, 0755) + + // Set parallelism to 1 to test single runner behavior + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") + defer func() { + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") + }() + settings.Init() + + // Setup mocks for test with CI provider + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, + {Suite: "TestSuite2", Name: "test2", Parameters: "", SuiteSourceFile: "test/file2_test.rb"}, + }, + } + + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} + mockOptimizationClient := &MockTestOptimizationClient{ + SkippableTests: map[string]bool{ + "TestSuite1.test1": true, // Skip test1 = 50% skippable + }, + } + + // Mock CI provider that should be called + mockCIProvider := &MockCIProvider{ + ProviderName: "github", + } + mockCIProviderDetector := &MockCIProviderDetector{ + CIProvider: mockCIProvider, + } + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, mockCIProviderDetector) + + // Run Setup + err := runner.Plan(context.Background()) + if err != nil { + t.Fatalf("Setup() should not return error, got: %v", err) + } + + // Verify CI provider Configure was called + if !mockCIProvider.ConfigureCalled { + t.Error("Expected CI provider Configure to be called") + } + + // Verify Configure was called with the correct parallel runners count (1, since max=1) + expectedRunners := 1 + if mockCIProvider.ParallelRunners != expectedRunners { + t.Errorf("Expected CI provider Configure called with %d parallel runners, got %d", + expectedRunners, mockCIProvider.ParallelRunners) + } +} + +func TestTestPlanner_Setup_CIProviderDetectionFailure(t *testing.T) { + tempDir := t.TempDir() + + // Save current working directory and change to temp dir + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + // Create .testoptimization directory + _ = os.MkdirAll(constants.PlanDirectory, 0755) + + // Setup mocks for test without CI provider + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, + }, + } + + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} + mockOptimizationClient := &MockTestOptimizationClient{SkippableTests: map[string]bool{}} + + // Mock CI provider detector that fails + mockCIProviderDetector := &MockCIProviderDetector{ + Err: errors.New("no CI provider detected"), + } + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, mockCIProviderDetector) + + // Run Setup - should succeed even if CI provider detection fails + err := runner.Plan(context.Background()) + if err != nil { + t.Fatalf("Setup() should not fail when CI provider detection fails, got: %v", err) + } +} + +func TestTestPlanner_Setup_CIProviderConfigureFailure(t *testing.T) { + tempDir := t.TempDir() + + // Save current working directory and change to temp dir + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + _ = os.MkdirAll(constants.PlanDirectory, 0755) + + // Setup mocks for test with failing CI provider + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, + }, + } + + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} + mockOptimizationClient := &MockTestOptimizationClient{SkippableTests: map[string]bool{}} + + // Mock CI provider that fails during configuration + mockCIProvider := &MockCIProvider{ + ProviderName: "github", + ConfigureErr: errors.New("configuration failed"), + } + mockCIProviderDetector := &MockCIProviderDetector{ + CIProvider: mockCIProvider, + } + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, mockCIProviderDetector) + + // Run Setup - should succeed even if CI provider configuration fails + err := runner.Plan(context.Background()) + if err != nil { + t.Fatalf("Setup() should not fail when CI provider configuration fails, got: %v", err) + } + + // Verify CI provider Configure was attempted + if !mockCIProvider.ConfigureCalled { + t.Error("Expected CI provider Configure to be called even if it fails") + } +} + +func TestTestPlanner_Setup_WithTestSplit(t *testing.T) { + t.Run("single runner - copy test-files.txt to runner-0", func(t *testing.T) { + // Create a temporary directory for test output + tempDir := t.TempDir() + + // Save current working directory and change to temp dir + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + // Create .testoptimization directory + _ = os.MkdirAll(constants.PlanDirectory, 0755) + + // Set parallelism to 1 to test single runner behavior + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") + defer func() { + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") + }() + settings.Init() + + // Setup mocks for single runner scenario + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, + {Suite: "TestSuite2", Name: "test2", Parameters: "", SuiteSourceFile: "test/file2_test.rb"}, + }, + } + + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} + mockOptimizationClient := &MockTestOptimizationClient{ + SkippableTests: map[string]bool{}, // No tests skipped + } + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) + + // Run Setup + err := runner.Plan(context.Background()) + if err != nil { + t.Fatalf("Setup() should not return error, got: %v", err) + } + + // Verify that tests-split directory was created + if _, err := os.Stat(constants.TestsSplitDir); os.IsNotExist(err) { + t.Error("Expected tests-split directory to be created when parallelRunners = 1") + } + + // Verify that runner-0 file was created + runnerFilePath := filepath.Join(constants.TestsSplitDir, "runner-0") + if _, err := os.Stat(runnerFilePath); os.IsNotExist(err) { + t.Error("Expected runner-0 file to be created when parallelRunners = 1") + } + + // Verify that runner-0 contains the same content as test-files.txt + testFilesContent, err := os.ReadFile(constants.TestFilesOutputPath) + if err != nil { + t.Fatalf("Failed to read test-files.txt: %v", err) + } + + runnerContent, err := os.ReadFile(runnerFilePath) + if err != nil { + t.Fatalf("Failed to read runner-0 file: %v", err) + } + + if string(testFilesContent) != string(runnerContent) { + t.Errorf("Expected runner-0 content to match test-files.txt content.\ntest-files.txt: %q\nrunner-0: %q", + string(testFilesContent), string(runnerContent)) + } + + // Verify the content contains the expected test files + expectedContent := "test/file1_test.rb\ntest/file2_test.rb\n" + if string(runnerContent) != expectedContent { + t.Errorf("Expected runner-0 content %q, got %q", expectedContent, string(runnerContent)) + } + }) + + t.Run("multiple runners - split files created", func(t *testing.T) { + // Create a temporary directory for test output + tempDir := t.TempDir() + + // Save current working directory and change to temp dir + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + // Create .testoptimization directory + _ = os.MkdirAll(constants.PlanDirectory, 0755) + + // Setup mocks with test files that will create a predictable distribution + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, + {Suite: "TestSuite1", Name: "test2", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, // 2 tests in file1 + {Suite: "TestSuite2", Name: "test3", Parameters: "", SuiteSourceFile: "test/file2_test.rb"}, // 1 test in file2 + {Suite: "TestSuite3", Name: "test4", Parameters: "", SuiteSourceFile: "test/file3_test.rb"}, // 1 test in file3 + }, + } + + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} + mockOptimizationClient := &MockTestOptimizationClient{ + SkippableTests: map[string]bool{}, // No tests skipped + } + + expectedParallelRunnersCount := 2 + maxParallelism := 4 + // Set environment variables to force multiple parallel runners + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "2") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", strconv.Itoa(maxParallelism)) + defer func() { + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") + }() + + // Reinitialize settings to pick up environment variables + settings.Init() + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) + + // Run Setup + err := runner.Plan(context.Background()) + if err != nil { + t.Fatalf("Setup() should not return error, got: %v", err) + } + + // Verify that tests-split directory was created + if _, err := os.Stat(constants.TestsSplitDir); os.IsNotExist(err) { + t.Error("Expected tests-split directory to be created") + } + + // With this split, 2 runners are as fast as 3 and more balanced. + // Verify runner files exist + for i := range expectedParallelRunnersCount { + runnerPath := filepath.Join(constants.TestsSplitDir, fmt.Sprintf("runner-%d", i)) + if _, err := os.Stat(runnerPath); os.IsNotExist(err) { + t.Errorf("Expected runner-%d file to exist", i) + } + } + + // Verify content of runner files + // With the test distribution (file1: 2 tests, file2: 1 test, file3: 1 test), + // expected: runner 0 gets file1 (2 tests), runner 1 gets file2+file3 (2 tests). + runner0Content, err := os.ReadFile(filepath.Join(constants.TestsSplitDir, "runner-0")) + if err != nil { + t.Fatalf("Failed to read runner-0 file: %v", err) + } + + // Verify runner-0 has the largest file (file1 with 2 tests) + runner0Files := strings.Fields(strings.TrimSpace(string(runner0Content))) + if !slices.Contains(runner0Files, "test/file1_test.rb") { + t.Error("Expected runner-0 to contain test/file1_test.rb (largest file)") + } + + // Count total files across all runners + totalFiles := 0 + for i := range expectedParallelRunnersCount { + runnerPath := filepath.Join(constants.TestsSplitDir, fmt.Sprintf("runner-%d", i)) + content, err := os.ReadFile(runnerPath) + if err != nil { + continue + } + files := strings.Fields(strings.TrimSpace(string(content))) + totalFiles += len(files) + } + + // Should have all 3 test files distributed + if totalFiles != 3 { + t.Errorf("Expected 3 total files distributed across runners, got %d", totalFiles) + } + }) +} + +// TestTestPlanner_Plan_SubdirRootRelativeDiscovery_WritesNormalizedPaths +// reproduces the end-to-end bug from issue #33: Plan writes repo-root-relative paths +// that become invalid for workers running from a monorepo subdirectory. +func TestTestPlanner_Plan_SubdirRootRelativeDiscovery_WritesNormalizedPaths(t *testing.T) { + // Create a temp monorepo: repoRoot/core/spec/... + repoRoot := t.TempDir() + + // Initialize git repo at the root + cmd := exec.Command("git", "init") + cmd.Dir = repoRoot + cmd.Env = gitTestEnv() + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("failed to init git repo: %v\n%s", err, string(out)) + } + cmd = exec.Command("git", "commit", "--allow-empty", "-m", "init") + cmd.Dir = repoRoot + cmd.Env = append(gitTestEnv(), + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@test.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@test.com", + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("failed to create initial commit: %v\n%s", err, string(out)) + } + + coreDir := filepath.Join(repoRoot, "core") + _ = os.MkdirAll(filepath.Join(coreDir, "spec", "models"), 0755) + _ = os.WriteFile(filepath.Join(coreDir, "spec", "models", "order_spec.rb"), []byte("# spec"), 0644) + _ = os.WriteFile(filepath.Join(coreDir, "spec", "models", "payment_spec.rb"), []byte("# spec"), 0644) + + // chdir into subdirectory + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(coreDir) + + // Set parallelism to 1 + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") + defer func() { + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") + }() + settings.Init() + + // Full discovery returns repo-root-relative paths (the bug) + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "Order", Name: "should be valid", Parameters: "", SuiteSourceFile: "core/spec/models/order_spec.rb"}, + {Suite: "Payment", Name: "should process", Parameters: "", SuiteSourceFile: "core/spec/models/payment_spec.rb"}, + }, + } + + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} + mockOptimizationClient := &MockTestOptimizationClient{ + SkippableTests: map[string]bool{}, + } + + runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) + + err := runner.Plan(context.Background()) + if err != nil { + t.Fatalf("Plan() should not return error, got: %v", err) + } + + // Verify test-files.txt contains CWD-relative paths + testFilesContent, err := os.ReadFile(constants.TestFilesOutputPath) + if err != nil { + t.Fatalf("Failed to read test-files.txt: %v", err) + } + + testFilesStr := string(testFilesContent) + if strings.Contains(testFilesStr, "core/") { + t.Errorf("test-files.txt should not contain repo-root prefix 'core/', got:\n%s", testFilesStr) + } + + expectedContent := "spec/models/order_spec.rb\nspec/models/payment_spec.rb\n" + if testFilesStr != expectedContent { + t.Errorf("Expected test-files.txt content:\n%s\nGot:\n%s", expectedContent, testFilesStr) + } + + // Verify runner-0 split file also contains CWD-relative paths + runnerContent, err := os.ReadFile(filepath.Join(constants.TestsSplitDir, "runner-0")) + if err != nil { + t.Fatalf("Failed to read runner-0: %v", err) + } + + runnerStr := string(runnerContent) + if strings.Contains(runnerStr, "core/") { + t.Errorf("runner-0 should not contain repo-root prefix 'core/', got:\n%s", runnerStr) + } +} func captureLogs(t *testing.T) *bytes.Buffer { t.Helper() var buf bytes.Buffer @@ -31,7 +1004,7 @@ func captureLogs(t *testing.T) *bytes.Buffer { return &buf } -func TestTestRunner_PrepareTestOptimization_Success(t *testing.T) { +func TestTestPlanner_PreparePlanningData_Success(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) @@ -84,45 +1057,43 @@ func TestTestRunner_PrepareTestOptimization_Success(t *testing.T) { runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Errorf("PrepareTestOptimization() should not return error, got: %v", err) + t.Errorf("PreparePlanningData() should not return error, got: %v", err) } // Verify optimization client was initialized if !mockOptimizationClient.InitializeCalled { - t.Error("PrepareTestOptimization() should initialize optimization client") + t.Error("PreparePlanningData() should initialize optimization client") } // Verify tags were passed to optimization client if mockOptimizationClient.Tags["platform"] != "ruby" { - t.Error("PrepareTestOptimization() should pass platform tags to optimization client") + t.Error("PreparePlanningData() should pass platform tags to optimization client") } // Verify optimization client was shut down if !mockOptimizationClient.ShutdownCalled { - t.Error("PrepareTestOptimization() should shutdown optimization client") + t.Error("PreparePlanningData() should shutdown optimization client") } expectedFiles := map[string]bool{ - "test/file1_test.rb": true, // test1 is not skipped - "test/file2_test.rb": true, // test3 is not skipped - "test/file3_test.rb": true, // test4 is skipped but the source file is discovered - "test/fast_only_test.rb": true, // from fast discovery only + "test/file1_test.rb": true, // test1 is not skipped + "test/file2_test.rb": true, // test3 is not skipped + "test/file3_test.rb": true, // test4 is skipped but the source file is discovered } if len(runner.testFiles) != len(expectedFiles) { - t.Errorf("PrepareTestOptimization() should result in %d test files, got %d", len(expectedFiles), len(runner.testFiles)) + t.Errorf("PreparePlanningData() should result in %d test files, got %d", len(expectedFiles), len(runner.testFiles)) } - if weightedFiles := runner.weightedTestFiles(); len(weightedFiles) != 3 { - t.Errorf("Expected weighted files to omit fully skipped file and keep 3 files, got %v", weightedFiles) + if weightedFiles := runner.calculateFileWeights(); len(weightedFiles) != 2 { + t.Errorf("Expected weighted files to omit fully skipped and fast-only files, got %v", weightedFiles) } expectedTestFileWeights := map[string]int{ - "test/file1_test.rb": 3, - "test/file2_test.rb": defaultTestFileWeight, - "test/fast_only_test.rb": defaultTestFileWeight, + "test/file1_test.rb": 3, + "test/file2_test.rb": DefaultTestFileWeight, } if len(runner.testFileWeights) != len(expectedTestFileWeights) { t.Errorf("Expected precomputed test file weights to have %d entries, got %v", len(expectedTestFileWeights), runner.testFileWeights) @@ -156,27 +1127,22 @@ func TestTestRunner_PrepareTestOptimization_Success(t *testing.T) { t.Errorf("Expected file2 weight to use count fallback %d, got weight=%d ok=%t", expectedFile2Weight, weight, ok) } - if weight, ok := runner.testFileWeight("test/fast_only_test.rb"); !ok || weight != expectedFile2Weight { - t.Errorf("Expected fast-only file weight to use default %d, got weight=%d ok=%t", expectedFile2Weight, weight, ok) - } - // Verify skippable percentage was calculated correctly (2 out of 4 tests skipped = 50%) expectedPercentage := 50.0 if runner.skippablePercentage != expectedPercentage { - t.Errorf("PrepareTestOptimization() should calculate skippable percentage as %.2f, got %.2f", + t.Errorf("PreparePlanningData() should calculate skippable percentage as %.2f, got %.2f", expectedPercentage, runner.skippablePercentage) } if !mockDurationsClient.Called { - t.Error("PrepareTestOptimization() should fetch test suite durations") + t.Error("PreparePlanningData() should fetch test suite durations") } } -func TestTestRunner_PrepareTestOptimization_DurationsErrorContinues(t *testing.T) { +func TestTestPlanner_PreparePlanningData_EmptyDurationsContinues(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) - logs := captureLogs(t) mockFramework := &MockFramework{ FrameworkName: "rspec", @@ -193,73 +1159,28 @@ func TestTestRunner_PrepareTestOptimization_DurationsErrorContinues(t *testing.T } mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} mockOptimizationClient := &MockTestOptimizationClient{} - mockDurationsClient := &MockTestSuiteDurationsClient{Err: errors.New("durations backend failed")} + mockDurationsClient := &MockTestSuiteDurationsClient{} runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail when durations API errors, got: %v", err) - } - - if len(runner.testSuiteDurations) != 0 { - t.Errorf("Expected empty in-memory test suite durations on error, got %v", runner.testSuiteDurations) - } - - if !strings.Contains(logs.String(), "level=ERROR") || !strings.Contains(logs.String(), "Test durations API errored") || !strings.Contains(logs.String(), "duration=") { - t.Errorf("Expected ERROR log for durations API failure, got logs: %s", logs.String()) - } -} - -func TestTestRunner_PrepareTestOptimization_EmptyDurationsWarns(t *testing.T) { - ctx := context.Background() - ciUtils.ResetCITags() - t.Cleanup(ciUtils.ResetCITags) - logs := captureLogs(t) - - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: []testoptimization.Test{ - {Suite: "Suite", Name: "test1", Parameters: "", SuiteSourceFile: "spec/file1_test.rb"}, - }, - } - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{ - ciConstants.GitRepositoryURL: "github.com/DataDog/ddtest", - }, - Framework: mockFramework, - } - mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} - mockOptimizationClient := &MockTestOptimizationClient{} - mockDurationsClient := &MockTestSuiteDurationsClient{ - Durations: map[string]map[string]testoptimization.TestSuiteDurationInfo{}, + t.Fatalf("PreparePlanningData() should not fail with empty durations, got: %v", err) } - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) - - err := runner.PrepareTestOptimization(ctx) - if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail with empty durations, got: %v", err) + if !mockDurationsClient.Called { + t.Error("PreparePlanningData() should fetch test suite durations") } if len(runner.testSuiteDurations) != 0 { t.Errorf("Expected empty in-memory test suite durations on empty response, got %v", runner.testSuiteDurations) } - - if !strings.Contains(logs.String(), "level=WARN") || - !strings.Contains(logs.String(), "Test durations API returned no test suites") || - !strings.Contains(logs.String(), "testSuitesCount=0") || - !strings.Contains(logs.String(), "duration=") { - t.Errorf("Expected WARN log for empty durations response, got logs: %s", logs.String()) - } } -func TestTestRunner_PrepareTestOptimization_NonEmptyDurationsUsesP50ForMatchingSuites(t *testing.T) { +func TestTestPlanner_PreparePlanningData_NonEmptyDurationsUsesP50ForMatchingSuites(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) - logs := captureLogs(t) mockFramework := &MockFramework{ FrameworkName: "rspec", @@ -296,9 +1217,9 @@ func TestTestRunner_PrepareTestOptimization_NonEmptyDurationsUsesP50ForMatchingS runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail with durations data, got: %v", err) + t.Fatalf("PreparePlanningData() should not fail with durations data, got: %v", err) } if len(runner.testSuiteDurations) != 1 { @@ -318,17 +1239,9 @@ func TestTestRunner_PrepareTestOptimization_NonEmptyDurationsUsesP50ForMatchingS if weight, ok := runner.testFileWeight("spec/file2_test.rb"); !ok || weight != 30 { t.Errorf("Expected file2 weight to use backend p50 converted to 30ms, got weight=%d ok=%t", weight, ok) } - - if !strings.Contains(logs.String(), "level=INFO") || - !strings.Contains(logs.String(), "Fetched test suite durations") || - !strings.Contains(logs.String(), "modulesCount=1") || - !strings.Contains(logs.String(), "testSuitesCount=2") || - !strings.Contains(logs.String(), "duration=") { - t.Errorf("Expected INFO log for non-empty durations response, got logs: %s", logs.String()) - } } -func TestTestRunner_PrepareTestOptimization_SkippablePercentageUsesDurations(t *testing.T) { +func TestTestPlanner_PreparePlanningData_SkippablePercentageUsesDurations(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) @@ -371,9 +1284,9 @@ func TestTestRunner_PrepareTestOptimization_SkippablePercentageUsesDurations(t * runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail, got: %v", err) + t.Fatalf("PreparePlanningData() should not fail, got: %v", err) } expectedPercentage := 40.0 @@ -382,7 +1295,7 @@ func TestTestRunner_PrepareTestOptimization_SkippablePercentageUsesDurations(t * } } -func TestTestRunner_TestFileWeight_CountFallbackForMissingSuiteDuration(t *testing.T) { +func TestTestPlanner_TestFileWeight_CountFallbackForMissingSuiteDuration(t *testing.T) { runner := NewWithDependencies(&MockPlatformDetector{}, &MockTestOptimizationClient{}, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) runner.testFiles = map[string]struct{}{ "spec/file1_test.rb": {}, @@ -413,7 +1326,7 @@ func TestTestRunner_TestFileWeight_CountFallbackForMissingSuiteDuration(t *testi }, } - runner.resolveSuiteDurations() + runner.estimateDiscoveredSuiteDurations() runner.suitesBySourceFile = indexSuitesBySourceFile(runner.suiteAggregates) if weight, ok := runner.testFileWeight("spec/file1_test.rb"); !ok || weight != 11 { @@ -429,7 +1342,7 @@ func TestTestRunner_TestFileWeight_CountFallbackForMissingSuiteDuration(t *testi t.Errorf("Expected unknown file weight to use default 1 second, got weight=%d ok=%t", weight, ok) } - runner.weightedTestFiles() + runner.calculateFileWeights() if source := runner.testFileDurationSources["spec/file1_test.rb"]; source != testFileDurationSourceKnown { t.Errorf("Expected Suite1 file duration source to be known, got %q", source) } @@ -441,7 +1354,7 @@ func TestTestRunner_TestFileWeight_CountFallbackForMissingSuiteDuration(t *testi } } -func TestTestRunner_TestFileWeight_InvalidP50FallsBackForFullDiscoveryAggregate(t *testing.T) { +func TestTestPlanner_TestFileWeight_InvalidP50FallsBackForFullDiscoveryAggregate(t *testing.T) { runner := NewWithDependencies(&MockPlatformDetector{}, &MockTestOptimizationClient{}, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) runner.testFiles = map[string]struct{}{ "spec/file1_test.rb": {}, @@ -465,7 +1378,7 @@ func TestTestRunner_TestFileWeight_InvalidP50FallsBackForFullDiscoveryAggregate( }, } - runner.resolveSuiteDurations() + runner.estimateDiscoveredSuiteDurations() runner.suitesBySourceFile = indexSuitesBySourceFile(runner.suiteAggregates) aggregate := runner.suiteAggregates[testSuiteKey{Module: "rspec", Suite: "Suite1"}] @@ -480,7 +1393,7 @@ func TestTestRunner_TestFileWeight_InvalidP50FallsBackForFullDiscoveryAggregate( } } -func TestTestRunner_TestFileWeight_ZeroP50FallsBackForFullDiscoveryAggregate(t *testing.T) { +func TestTestPlanner_TestFileWeight_ZeroP50FallsBackForFullDiscoveryAggregate(t *testing.T) { runner := NewWithDependencies(&MockPlatformDetector{}, &MockTestOptimizationClient{}, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) runner.testFiles = map[string]struct{}{ "spec/file1_test.rb": {}, @@ -504,7 +1417,7 @@ func TestTestRunner_TestFileWeight_ZeroP50FallsBackForFullDiscoveryAggregate(t * }, } - runner.resolveSuiteDurations() + runner.estimateDiscoveredSuiteDurations() runner.suitesBySourceFile = indexSuitesBySourceFile(runner.suiteAggregates) aggregate := runner.suiteAggregates[testSuiteKey{Module: "rspec", Suite: "Suite1"}] @@ -519,7 +1432,7 @@ func TestTestRunner_TestFileWeight_ZeroP50FallsBackForFullDiscoveryAggregate(t * } } -func TestTestRunner_TestFileWeight_SubMillisecondP50MinimumWeight(t *testing.T) { +func TestTestPlanner_TestFileWeight_SubMillisecondP50MinimumWeight(t *testing.T) { runner := NewWithDependencies(&MockPlatformDetector{}, &MockTestOptimizationClient{}, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) runner.testFiles = map[string]struct{}{ "spec/fast_test.rb": {}, @@ -542,7 +1455,7 @@ func TestTestRunner_TestFileWeight_SubMillisecondP50MinimumWeight(t *testing.T) }, } - runner.resolveSuiteDurations() + runner.estimateDiscoveredSuiteDurations() runner.suitesBySourceFile = indexSuitesBySourceFile(runner.suiteAggregates) if weight, ok := runner.testFileWeight("spec/fast_test.rb"); !ok || weight != 1 { @@ -550,7 +1463,7 @@ func TestTestRunner_TestFileWeight_SubMillisecondP50MinimumWeight(t *testing.T) } } -func TestTestRunner_TestFileWeight_SkipsFullySkippedSuites(t *testing.T) { +func TestTestPlanner_TestFileWeight_SkipsFullySkippedSuites(t *testing.T) { runner := NewWithDependencies(&MockPlatformDetector{}, &MockTestOptimizationClient{}, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) runner.testFiles = map[string]struct{}{ "spec/skipped_test.rb": {}, @@ -575,7 +1488,7 @@ func TestTestRunner_TestFileWeight_SkipsFullySkippedSuites(t *testing.T) { t.Errorf("Expected fully skipped suite file to have no weight, got weight=%d ok=%t", weight, ok) } - if weightedFiles := runner.weightedTestFiles(); len(weightedFiles) != 0 { + if weightedFiles := runner.calculateFileWeights(); len(weightedFiles) != 0 { t.Errorf("Expected fully skipped suite file to be omitted from weighted files, got %v", weightedFiles) } } @@ -627,7 +1540,7 @@ func TestIndexSuitesBySourceFile_IgnoresEmptySourceFile(t *testing.T) { } } -func TestTestRunner_PrepareTestOptimization_FastDiscoveryUsesBackendDurations(t *testing.T) { +func TestTestPlanner_PreparePlanningData_FastDiscoveryUsesBackendDurations(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) @@ -657,9 +1570,9 @@ func TestTestRunner_PrepareTestOptimization_FastDiscoveryUsesBackendDurations(t runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, &MockTestOptimizationClient{}, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail when full discovery fails but fast discovery succeeds, got: %v", err) + t.Fatalf("PreparePlanningData() should not fail when full discovery fails but fast discovery succeeds, got: %v", err) } if weight, ok := runner.testFileWeight("spec/backend_only_spec.rb"); !ok || weight != 42 { @@ -667,7 +1580,54 @@ func TestTestRunner_PrepareTestOptimization_FastDiscoveryUsesBackendDurations(t } } -func TestTestRunner_PrepareTestOptimization_IgnoresZeroBackendDurationForFastDiscovery(t *testing.T) { +func TestTestPlanner_PreparePlanningData_FastDiscoveryUsesOneBackendDurationPerSourceFile(t *testing.T) { + ctx := context.Background() + ciUtils.ResetCITags() + t.Cleanup(ciUtils.ResetCITags) + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + TestFiles: []string{"spec/backend_only_spec.rb"}, + DiscoverTestsErr: errors.New("full discovery failed"), + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{ + ciConstants.GitRepositoryURL: "github.com/DataDog/ddtest", + }, + Framework: mockFramework, + } + mockDurationsClient := &MockTestSuiteDurationsClient{ + Durations: map[string]map[string]testoptimization.TestSuiteDurationInfo{ + "rspec": { + "BackendOnlySuite": { + SourceFile: "spec/backend_only_spec.rb", + Duration: testoptimization.DurationPercentiles{P50: "42000000"}, + }, + "DuplicateBackendOnlySuite": { + SourceFile: "spec/backend_only_spec.rb", + Duration: testoptimization.DurationPercentiles{P50: "84000000"}, + }, + }, + }, + } + + runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, &MockTestOptimizationClient{}, mockDurationsClient, newDefaultMockCIProviderDetector()) + + err := runner.PreparePlanningData(ctx) + if err != nil { + t.Fatalf("PreparePlanningData() should not fail when full discovery fails but fast discovery succeeds, got: %v", err) + } + + if len(runner.suiteAggregates) != 1 { + t.Fatalf("Expected one backend fallback suite aggregate per source file, got %v", runner.suiteAggregates) + } + if suiteKeys := runner.suitesBySourceFile["spec/backend_only_spec.rb"]; len(suiteKeys) != 1 { + t.Fatalf("Expected rebuilt source-file index to contain one suite key, got %v", runner.suitesBySourceFile) + } +} + +func TestTestPlanner_PreparePlanningData_IgnoresZeroBackendDurationForFastDiscovery(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) @@ -697,21 +1657,21 @@ func TestTestRunner_PrepareTestOptimization_IgnoresZeroBackendDurationForFastDis runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, &MockTestOptimizationClient{}, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail when full discovery fails but fast discovery succeeds, got: %v", err) + t.Fatalf("PreparePlanningData() should not fail when full discovery fails but fast discovery succeeds, got: %v", err) } if len(runner.suiteAggregates) != 0 { t.Errorf("Expected zero-duration backend suite to be ignored, got aggregates: %v", runner.suiteAggregates) } - if weight, ok := runner.testFileWeight("spec/backend_only_spec.rb"); !ok || weight != defaultTestFileWeight { + if weight, ok := runner.testFileWeight("spec/backend_only_spec.rb"); !ok || weight != DefaultTestFileWeight { t.Errorf("Expected fast-discovery file with broken backend duration to use default weight, got weight=%d ok=%t", weight, ok) } } -func TestTestRunner_PrepareTestOptimization_BackendDurationSubdirMatchesFastDiscovery(t *testing.T) { +func TestTestPlanner_PreparePlanningData_BackendDurationSubdirMatchesFastDiscovery(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) @@ -751,9 +1711,9 @@ func TestTestRunner_PrepareTestOptimization_BackendDurationSubdirMatchesFastDisc runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, &MockTestOptimizationClient{}, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail, got: %v", err) + t.Fatalf("PreparePlanningData() should not fail, got: %v", err) } if !mockDurationsClient.Called { t.Fatal("Expected durations client to be called") @@ -768,7 +1728,7 @@ func TestTestRunner_PrepareTestOptimization_BackendDurationSubdirMatchesFastDisc } } -func TestTestRunner_PrepareTestOptimization_IgnoresBackendDurationsForUndiscoveredFiles(t *testing.T) { +func TestTestPlanner_PreparePlanningData_IgnoresBackendDurationsForUndiscoveredFiles(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) @@ -798,9 +1758,9 @@ func TestTestRunner_PrepareTestOptimization_IgnoresBackendDurationsForUndiscover runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, &MockTestOptimizationClient{}, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail, got: %v", err) + t.Fatalf("PreparePlanningData() should not fail, got: %v", err) } if len(runner.suiteAggregates) != 0 { @@ -812,7 +1772,110 @@ func TestTestRunner_PrepareTestOptimization_IgnoresBackendDurationsForUndiscover } } -func TestTestRunner_PrepareTestOptimization_FastDiscoveryDoesNotRunStaleBackendFilesWhenSkippingDisabled(t *testing.T) { +func TestTestPlanner_PreparePlanningData_FullDiscoveryIgnoresFastOnlyFiles(t *testing.T) { + ctx := context.Background() + ciUtils.ResetCITags() + t.Cleanup(ciUtils.ResetCITags) + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + TestFiles: []string{"spec/discovered_spec.rb", "spec/fast_only_spec.rb"}, + Tests: []testoptimization.Test{ + { + Module: "rspec", + Suite: "DiscoveredSuite", + Name: "test1", + SuiteSourceFile: "spec/discovered_spec.rb", + }, + }, + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{ + ciConstants.GitRepositoryURL: "github.com/DataDog/ddtest", + }, + Framework: mockFramework, + } + + runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, &MockTestOptimizationClient{}, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) + + err := runner.PreparePlanningData(ctx) + if err != nil { + t.Fatalf("PreparePlanningData() should not fail, got: %v", err) + } + + if _, ok := runner.testFiles["spec/discovered_spec.rb"]; !ok { + t.Fatalf("Expected full-discovered file to be planned, got %v", runner.testFiles) + } + if _, ok := runner.testFiles["spec/fast_only_spec.rb"]; ok { + t.Errorf("Expected fast-only file to be ignored after successful full discovery, got %v", runner.testFiles) + } + if _, ok := runner.testFileWeights["spec/fast_only_spec.rb"]; ok { + t.Errorf("Expected fast-only file not to have a weight, got %v", runner.testFileWeights) + } + if _, ok := runner.testFileDurationSources["spec/fast_only_spec.rb"]; ok { + t.Errorf("Expected fast-only file not to have a duration source, got %v", runner.testFileDurationSources) + } +} + +func TestTestPlanner_PreparePlanningData_FullDiscoveryDoesNotReintroduceFastOnlyBackendSuite(t *testing.T) { + ctx := context.Background() + ciUtils.ResetCITags() + t.Cleanup(ciUtils.ResetCITags) + ciUtils.AddCITagsMap(map[string]string{ciConstants.GitRepositoryURL: "github.com/DataDog/ddtest"}) + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + TestFiles: []string{"spec/discovered_spec.rb", "spec/fast_only_spec.rb"}, + Tests: []testoptimization.Test{ + { + Module: "rspec", + Suite: "DiscoveredSuite", + Name: "test1", + SuiteSourceFile: "spec/discovered_spec.rb", + }, + }, + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{ + ciConstants.GitRepositoryURL: "github.com/DataDog/ddtest", + }, + Framework: mockFramework, + } + mockDurationsClient := &MockTestSuiteDurationsClient{ + Durations: map[string]map[string]testoptimization.TestSuiteDurationInfo{ + "rspec": { + "FastOnlySuite": { + SourceFile: "spec/fast_only_spec.rb", + Duration: testoptimization.DurationPercentiles{P50: "42000000", P90: "84000000"}, + }, + }, + }, + } + + runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, &MockTestOptimizationClient{}, mockDurationsClient, newDefaultMockCIProviderDetector()) + + err := runner.PreparePlanningData(ctx) + if err != nil { + t.Fatalf("PreparePlanningData() should not fail, got: %v", err) + } + if !mockDurationsClient.Called { + t.Fatal("Expected durations client to be called") + } + + if _, ok := runner.suiteAggregates[testSuiteKey{Module: "rspec", Suite: "FastOnlySuite"}]; ok { + t.Errorf("Expected backend suite for fast-only file not to be added, got aggregates %v", runner.suiteAggregates) + } + if _, ok := runner.testFiles["spec/fast_only_spec.rb"]; ok { + t.Errorf("Expected fast-only file not to be planned despite backend duration, got %v", runner.testFiles) + } + if _, ok := runner.testFileWeights["spec/fast_only_spec.rb"]; ok { + t.Errorf("Expected fast-only file not to be runnable despite backend duration, got %v", runner.testFileWeights) + } +} + +func TestTestPlanner_PreparePlanningData_FastDiscoveryDoesNotRunStaleBackendFilesWhenSkippingDisabled(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) @@ -830,7 +1893,7 @@ func TestTestRunner_PrepareTestOptimization_FastDiscoveryDoesNotRunStaleBackendF Framework: mockFramework, } mockOptimizationClient := &MockTestOptimizationClient{ - Settings: &ciNet.SettingsResponseData{ + Settings: &net.SettingsResponseData{ ItrEnabled: true, TestsSkipping: false, }, @@ -852,12 +1915,12 @@ func TestTestRunner_PrepareTestOptimization_FastDiscoveryDoesNotRunStaleBackendF runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail, got: %v", err) + t.Fatalf("PreparePlanningData() should not fail, got: %v", err) } - weightedFiles := runner.weightedTestFiles() + weightedFiles := runner.calculateFileWeights() if len(weightedFiles) != 1 { t.Fatalf("Expected only local fast-discovery file to be runnable, got %v", weightedFiles) } @@ -872,7 +1935,7 @@ func TestTestRunner_PrepareTestOptimization_FastDiscoveryDoesNotRunStaleBackendF } } -func TestTestRunner_PrepareTestOptimization_BackendDoesNotReintroduceFullySkippedSuite(t *testing.T) { +func TestTestPlanner_PreparePlanningData_BackendDoesNotReintroduceFullySkippedSuite(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) @@ -911,9 +1974,9 @@ func TestTestRunner_PrepareTestOptimization_BackendDoesNotReintroduceFullySkippe runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail, got: %v", err) + t.Fatalf("PreparePlanningData() should not fail, got: %v", err) } aggregate := runner.suiteAggregates[testSuiteKey{Module: "rspec", Suite: "SkippedSuite"}] @@ -926,7 +1989,7 @@ func TestTestRunner_PrepareTestOptimization_BackendDoesNotReintroduceFullySkippe } } -func TestTestRunner_PrepareTestOptimization_BackendDoesNotDuplicateDiscoveredSourceFile(t *testing.T) { +func TestTestPlanner_PreparePlanningData_BackendDoesNotDuplicateDiscoveredSourceFile(t *testing.T) { ctx := context.Background() ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) @@ -965,9 +2028,9 @@ func TestTestRunner_PrepareTestOptimization_BackendDoesNotDuplicateDiscoveredSou runner := NewWithDependencies(&MockPlatformDetector{Platform: mockPlatform}, mockOptimizationClient, mockDurationsClient, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not fail, got: %v", err) + t.Fatalf("PreparePlanningData() should not fail, got: %v", err) } duplicateKey := testSuiteKey{Module: "rspec", Suite: "BackendDuplicateSuite"} @@ -1022,7 +2085,7 @@ func TestRecordRunnableAndSkippedTest_CountsTestsPerSuite(t *testing.T) { } } -func TestTestRunner_PrepareTestOptimization_PlatformDetectionError(t *testing.T) { +func TestTestPlanner_PreparePlanningData_PlatformDetectionError(t *testing.T) { ctx := context.Background() mockPlatformDetector := &MockPlatformDetector{ @@ -1033,19 +2096,19 @@ func TestTestRunner_PrepareTestOptimization_PlatformDetectionError(t *testing.T) runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err == nil { - t.Error("PrepareTestOptimization() should return error when platform detection fails") + t.Error("PreparePlanningData() should return error when platform detection fails") } expectedMsg := "failed to detect platform" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("PrepareTestOptimization() error should contain '%s', got: %v", expectedMsg, err) + t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } } -func TestTestRunner_PrepareTestOptimization_TagsCreationError(t *testing.T) { +func TestTestPlanner_PreparePlanningData_TagsCreationError(t *testing.T) { ctx := context.Background() mockPlatform := &MockPlatform{ @@ -1060,19 +2123,19 @@ func TestTestRunner_PrepareTestOptimization_TagsCreationError(t *testing.T) { runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err == nil { - t.Error("PrepareTestOptimization() should return error when tags creation fails") + t.Error("PreparePlanningData() should return error when tags creation fails") } expectedMsg := "failed to create platform tags" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("PrepareTestOptimization() error should contain '%s', got: %v", expectedMsg, err) + t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } } -func TestTestRunner_PrepareTestOptimization_OptimizationClientInitError(t *testing.T) { +func TestTestPlanner_PreparePlanningData_OptimizationClientInitError(t *testing.T) { ctx := context.Background() mockFramework := &MockFramework{ @@ -1096,19 +2159,19 @@ func TestTestRunner_PrepareTestOptimization_OptimizationClientInitError(t *testi runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err == nil { - t.Error("PrepareTestOptimization() should return error when optimization client initialization fails") + t.Error("PreparePlanningData() should return error when optimization client initialization fails") } expectedMsg := "failed to initialize optimization client" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("PrepareTestOptimization() error should contain '%s', got: %v", expectedMsg, err) + t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } } -func TestTestRunner_PrepareTestOptimization_FrameworkDetectionError(t *testing.T) { +func TestTestPlanner_PreparePlanningData_FrameworkDetectionError(t *testing.T) { ctx := context.Background() mockPlatform := &MockPlatform{ @@ -1124,19 +2187,19 @@ func TestTestRunner_PrepareTestOptimization_FrameworkDetectionError(t *testing.T runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err == nil { - t.Error("PrepareTestOptimization() should return error when framework detection fails") + t.Error("PreparePlanningData() should return error when framework detection fails") } expectedMsg := "failed to detect framework" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("PrepareTestOptimization() error should contain '%s', got: %v", expectedMsg, err) + t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } } -func TestTestRunner_PrepareTestOptimization_TestDiscoveryError(t *testing.T) { +func TestTestPlanner_PreparePlanningData_TestDiscoveryError(t *testing.T) { ctx := context.Background() mockFramework := &MockFramework{ @@ -1156,23 +2219,25 @@ func TestTestRunner_PrepareTestOptimization_TestDiscoveryError(t *testing.T) { runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err == nil { - t.Error("PrepareTestOptimization() should return error when test discovery fails") + t.Error("PreparePlanningData() should return error when test discovery fails") } expectedMsg := "test discovery failed" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("PrepareTestOptimization() error should contain '%s', got: %v", expectedMsg, err) + t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } } -func TestTestRunner_PrepareTestOptimization_EmptyTests(t *testing.T) { +func TestTestPlanner_PreparePlanningData_EmptyTests(t *testing.T) { ctx := context.Background() + logs := captureLogs(t) mockFramework := &MockFramework{ - Tests: []testoptimization.Test{}, // Empty test list + TestFiles: []string{"file1.rb"}, // Fast discovery should be used when full discovery returns no tests. + Tests: []testoptimization.Test{}, // Empty test list } mockPlatform := &MockPlatform{ @@ -1185,14 +2250,24 @@ func TestTestRunner_PrepareTestOptimization_EmptyTests(t *testing.T) { runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Errorf("PrepareTestOptimization() should handle empty tests, got: %v", err) + t.Errorf("PreparePlanningData() should handle empty tests, got: %v", err) } - if len(runner.testFiles) != 0 { - t.Errorf("PrepareTestOptimization() should result in 0 test files for empty tests, got %d", len(runner.testFiles)) + if len(runner.testFiles) != 1 { + t.Errorf("PreparePlanningData() should use fast discovery fallback for empty full discovery, got %d files", len(runner.testFiles)) + } + if _, ok := runner.testFiles["file1.rb"]; !ok { + t.Errorf("PreparePlanningData() should include fast-discovered file after empty full discovery, got %v", runner.testFiles) + } + if _, ok := runner.testFileWeights["file1.rb"]; !ok { + t.Errorf("PreparePlanningData() should schedule fast-discovered file after empty full discovery, got %v", runner.testFileWeights) + } + if !strings.Contains(logs.String(), "level=WARN") || + !strings.Contains(logs.String(), "Full test discovery returned no tests") { + t.Errorf("Expected WARN log for empty full discovery, got logs: %s", logs.String()) } // Division by zero should be handled gracefully @@ -1201,7 +2276,7 @@ func TestTestRunner_PrepareTestOptimization_EmptyTests(t *testing.T) { } } -func TestTestRunner_PrepareTestOptimization_AllTestsSkipped(t *testing.T) { +func TestTestPlanner_PreparePlanningData_AllTestsSkipped(t *testing.T) { ctx := context.Background() mockFramework := &MockFramework{ @@ -1226,26 +2301,26 @@ func TestTestRunner_PrepareTestOptimization_AllTestsSkipped(t *testing.T) { runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Errorf("PrepareTestOptimization() should handle all tests skipped, got: %v", err) + t.Errorf("PreparePlanningData() should handle all tests skipped, got: %v", err) } if len(runner.testFiles) != 2 { - t.Errorf("PrepareTestOptimization() should keep all discovered files even when all tests are skipped, got %d", len(runner.testFiles)) + t.Errorf("PreparePlanningData() should keep all discovered files even when all tests are skipped, got %d", len(runner.testFiles)) } - if weightedFiles := runner.weightedTestFiles(); len(weightedFiles) != 0 { - t.Errorf("PrepareTestOptimization() should result in 0 weighted files when all tests are skipped, got %v", weightedFiles) + if weightedFiles := runner.calculateFileWeights(); len(weightedFiles) != 0 { + t.Errorf("PreparePlanningData() should result in 0 weighted files when all tests are skipped, got %v", weightedFiles) } if runner.skippablePercentage != 100.0 { - t.Errorf("PrepareTestOptimization() should calculate 100%% skippable when all tests are skipped, got %.2f", runner.skippablePercentage) + t.Errorf("PreparePlanningData() should calculate 100%% skippable when all tests are skipped, got %.2f", runner.skippablePercentage) } } -func TestTestRunner_PrepareTestOptimization_RuntimeTagsOverride(t *testing.T) { +func TestTestPlanner_PreparePlanningData_RuntimeTagsOverride(t *testing.T) { ctx := context.Background() // Set runtime tags override via environment variable - only override some tags @@ -1288,15 +2363,15 @@ func TestTestRunner_PrepareTestOptimization_RuntimeTagsOverride(t *testing.T) { runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Errorf("PrepareTestOptimization() should not return error, got: %v", err) + t.Errorf("PreparePlanningData() should not return error, got: %v", err) } // Verify optimization client was initialized if !mockOptimizationClient.InitializeCalled { - t.Error("PrepareTestOptimization() should initialize optimization client") + t.Error("PreparePlanningData() should initialize optimization client") } // Check that override tags replaced the detected values @@ -1322,7 +2397,7 @@ func TestTestRunner_PrepareTestOptimization_RuntimeTagsOverride(t *testing.T) { } } -func TestTestRunner_PrepareTestOptimization_RuntimeTagsOverrideInvalidJSON(t *testing.T) { +func TestTestPlanner_PreparePlanningData_RuntimeTagsOverrideInvalidJSON(t *testing.T) { ctx := context.Background() // Set invalid JSON as runtime tags override @@ -1353,24 +2428,24 @@ func TestTestRunner_PrepareTestOptimization_RuntimeTagsOverrideInvalidJSON(t *te runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err == nil { - t.Error("PrepareTestOptimization() should return error when runtime tags JSON is invalid") + t.Error("PreparePlanningData() should return error when runtime tags JSON is invalid") } expectedMsg := "failed to parse runtime tags override" if !strings.Contains(err.Error(), expectedMsg) { - t.Errorf("PrepareTestOptimization() error should contain '%s', got: %v", expectedMsg, err) + t.Errorf("PreparePlanningData() error should contain '%s', got: %v", expectedMsg, err) } // Optimization client should not be initialized when there's a parse error if mockOptimizationClient.InitializeCalled { - t.Error("PrepareTestOptimization() should not initialize optimization client when runtime tags JSON is invalid") + t.Error("PreparePlanningData() should not initialize optimization client when runtime tags JSON is invalid") } } -func TestTestRunner_PrepareTestOptimization_NoRuntimeTagsOverride(t *testing.T) { +func TestTestPlanner_PreparePlanningData_NoRuntimeTagsOverride(t *testing.T) { ctx := context.Background() // Ensure no runtime tags override is set @@ -1407,15 +2482,15 @@ func TestTestRunner_PrepareTestOptimization_NoRuntimeTagsOverride(t *testing.T) runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Errorf("PrepareTestOptimization() should not return error, got: %v", err) + t.Errorf("PreparePlanningData() should not return error, got: %v", err) } // Verify optimization client was initialized with platform tags if !mockOptimizationClient.InitializeCalled { - t.Error("PrepareTestOptimization() should initialize optimization client") + t.Error("PreparePlanningData() should initialize optimization client") } // Check that platform tags were used (not override) @@ -1434,13 +2509,14 @@ func initGitRepo(t *testing.T, dir string) { t.Helper() cmd := exec.Command("git", "init") cmd.Dir = dir + cmd.Env = gitTestEnv() if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("failed to init git repo in %s: %v\n%s", dir, err, string(out)) } // Need at least one commit for some git operations to work cmd = exec.Command("git", "commit", "--allow-empty", "-m", "init") cmd.Dir = dir - cmd.Env = append(os.Environ(), + cmd.Env = append(gitTestEnv(), "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@test.com", "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@test.com", ) @@ -1449,10 +2525,10 @@ func initGitRepo(t *testing.T, dir string) { } } -// TestPrepareTestOptimization_ITRFullDiscovery_SubdirRootRelativePath_NormalizesToCwdRelative +// TestPreparePlanningData_ITRFullDiscovery_SubdirRootRelativePath_NormalizesToCwdRelative // reproduces issue #33: full discovery returns repo-root-relative SuiteSourceFile paths // (e.g. "core/spec/...") but workers run from subdirectory "core/", causing double-prefix. -func TestPrepareTestOptimization_ITRFullDiscovery_SubdirRootRelativePath_NormalizesToCwdRelative(t *testing.T) { +func TestPreparePlanningData_ITRFullDiscovery_SubdirRootRelativePath_NormalizesToCwdRelative(t *testing.T) { ctx := context.Background() // Create a temp monorepo: repoRoot/core/spec/models/order_spec.rb @@ -1492,9 +2568,9 @@ func TestPrepareTestOptimization_ITRFullDiscovery_SubdirRootRelativePath_Normali runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not return error, got: %v", err) + t.Fatalf("PreparePlanningData() should not return error, got: %v", err) } // The key assertion: testFiles should contain CWD-relative paths, not repo-root-relative paths @@ -1519,9 +2595,9 @@ func TestPrepareTestOptimization_ITRFullDiscovery_SubdirRootRelativePath_Normali } } -// TestPrepareTestOptimization_RepoRootRun_LeavesRepoRelativePathsUnchanged +// TestPreparePlanningData_RepoRootRun_LeavesRepoRelativePathsUnchanged // ensures that when running from the repo root (not a subdirectory), paths are not modified. -func TestPrepareTestOptimization_RepoRootRun_LeavesRepoRelativePathsUnchanged(t *testing.T) { +func TestPreparePlanningData_RepoRootRun_LeavesRepoRelativePathsUnchanged(t *testing.T) { ctx := context.Background() // Create a temp repo root with spec files @@ -1556,9 +2632,9 @@ func TestPrepareTestOptimization_RepoRootRun_LeavesRepoRelativePathsUnchanged(t runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not return error, got: %v", err) + t.Fatalf("PreparePlanningData() should not return error, got: %v", err) } // Paths should remain unchanged when running from repo root @@ -1567,9 +2643,9 @@ func TestPrepareTestOptimization_RepoRootRun_LeavesRepoRelativePathsUnchanged(t } } -// TestPrepareTestOptimization_FastDiscovery_PathsRemainUnchanged +// TestPreparePlanningData_FastDiscovery_PathsRemainUnchanged // ensures the fast discovery path (ITR disabled) does not modify paths. -func TestPrepareTestOptimization_FastDiscovery_PathsRemainUnchanged(t *testing.T) { +func TestPreparePlanningData_FastDiscovery_PathsRemainUnchanged(t *testing.T) { ctx := context.Background() // Fast discovery returns CWD-relative paths directly from glob @@ -1593,9 +2669,9 @@ func TestPrepareTestOptimization_FastDiscovery_PathsRemainUnchanged(t *testing.T runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not return error, got: %v", err) + t.Fatalf("PreparePlanningData() should not return error, got: %v", err) } // Fast discovery paths should be used as-is @@ -1615,10 +2691,10 @@ func TestPrepareTestOptimization_FastDiscovery_PathsRemainUnchanged(t *testing.T } } -// TestPrepareTestOptimization_ITRPathNormalization_PrefixMismatchUnchanged +// TestPreparePlanningData_ITRPathNormalization_PrefixMismatchUnchanged // ensures that when SuiteSourceFile does not match the current subdir prefix, // the path is not modified (conservative behavior). -func TestPrepareTestOptimization_ITRPathNormalization_PrefixMismatchUnchanged(t *testing.T) { +func TestPreparePlanningData_ITRPathNormalization_PrefixMismatchUnchanged(t *testing.T) { ctx := context.Background() // Create monorepo with "api" and "core" subdirs @@ -1658,9 +2734,9 @@ func TestPrepareTestOptimization_ITRPathNormalization_PrefixMismatchUnchanged(t runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not return error, got: %v", err) + t.Fatalf("PreparePlanningData() should not return error, got: %v", err) } // "core/spec/..." doesn't match "api" subdir prefix, should be unchanged @@ -1681,13 +2757,13 @@ func TestPrepareTestOptimization_ITRPathNormalization_PrefixMismatchUnchanged(t } } -// TestPrepareTestOptimization_ITRSubdir_SkipMatching_WithSuitePathsMatchingCwd +// TestPreparePlanningData_ITRSubdir_SkipMatching_WithSuitePathsMatchingCwd // verifies that when running from a monorepo subdirectory, skip matching works // correctly: both the API (skippable tests) and framework discovery use the same // CWD-relative Suite names (e.g. "Spree::Role at ./spec/models/role_spec.rb"), // while SuiteSourceFile is repo-root-relative (e.g. "core/spec/models/role_spec.rb") // and needs normalization for worker splitting. -func TestPrepareTestOptimization_ITRSubdir_SkipMatching_WithSuitePathsMatchingCwd(t *testing.T) { +func TestPreparePlanningData_ITRSubdir_SkipMatching_WithSuitePathsMatchingCwd(t *testing.T) { ctx := context.Background() // Create a temp monorepo: repoRoot/core/spec/models/ @@ -1739,9 +2815,9 @@ func TestPrepareTestOptimization_ITRSubdir_SkipMatching_WithSuitePathsMatchingCw runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - err := runner.PrepareTestOptimization(ctx) + err := runner.PreparePlanningData(ctx) if err != nil { - t.Fatalf("PrepareTestOptimization() should not return error, got: %v", err) + t.Fatalf("PreparePlanningData() should not return error, got: %v", err) } // 2 of 3 tests should be skipped (the role_spec.rb tests) @@ -1751,7 +2827,7 @@ func TestPrepareTestOptimization_ITRSubdir_SkipMatching_WithSuitePathsMatchingCw expectedSkippablePercentage, runner.skippablePercentage) } - // All discovered source files should remain in testFiles, while weightedTestFiles omits the fully skipped role_spec.rb. + // All discovered source files should remain in testFiles, while calculateFileWeights omits the fully skipped role_spec.rb. // The SuiteSourceFile paths should be normalized from "core/spec/..." to "spec/..." (CWD-relative). expectedFiles := map[string]bool{ "spec/models/role_spec.rb": true, @@ -1768,7 +2844,7 @@ func TestPrepareTestOptimization_ITRSubdir_SkipMatching_WithSuitePathsMatchingCw } } - weightedFiles := runner.weightedTestFiles() + weightedFiles := runner.calculateFileWeights() if len(weightedFiles) != 1 { t.Fatalf("Expected 1 weighted test file (only order_spec.rb), got %d: %v", len(weightedFiles), weightedFiles) } diff --git a/internal/planner/report.go b/internal/planner/report.go new file mode 100644 index 0000000..67b9eb5 --- /dev/null +++ b/internal/planner/report.go @@ -0,0 +1,336 @@ +package planner + +import ( + "fmt" + "io" + "strconv" + "strings" + "time" + + ciConstants "github.com/DataDog/ddtest/civisibility/constants" + "github.com/DataDog/ddtest/civisibility/utils/net" + "github.com/DataDog/ddtest/internal/runmetadata" +) + +type datadogSettingsReport struct { + Available bool + TestImpactAnalysis bool + TestSkipping bool + TestImpactCollection bool + KnownTests bool + ImpactedTests bool + EarlyFlakeDetection bool + AutoTestRetries bool + FlakyTestManagement bool +} + +func newDatadogSettingsReport(settings *net.SettingsResponseData) datadogSettingsReport { + if settings == nil { + return datadogSettingsReport{} + } + return datadogSettingsReport{ + Available: true, + TestImpactAnalysis: settings.ItrEnabled, + TestSkipping: settings.TestsSkipping, + TestImpactCollection: settings.CodeCoverage, + KnownTests: settings.KnownTestsEnabled, + ImpactedTests: settings.ImpactedTestsEnabled, + EarlyFlakeDetection: settings.EarlyFlakeDetection.Enabled, + AutoTestRetries: settings.FlakyTestRetriesEnabled, + FlakyTestManagement: settings.TestManagement.Enabled, + } +} + +type knownTestsReport struct { + Available bool + Modules int + Suites int + Tests int +} + +func newKnownTestsReport(knownTests *net.KnownTestsResponseData) knownTestsReport { + if knownTests == nil { + return knownTestsReport{} + } + + report := knownTestsReport{ + Available: true, + Modules: len(knownTests.Tests), + } + for _, suites := range knownTests.Tests { + report.Suites += len(suites) + for _, tests := range suites { + report.Tests += len(tests) + } + } + return report +} + +type managedFlakyTestsReport struct { + Available bool + Total int + Quarantined int + Disabled int + AttemptToFix int +} + +func newManagedFlakyTestsReport(testManagementTests *net.TestManagementTestsResponseDataModules) managedFlakyTestsReport { + if testManagementTests == nil { + return managedFlakyTestsReport{} + } + + report := managedFlakyTestsReport{Available: true} + for _, suites := range testManagementTests.Modules { + for _, tests := range suites.Suites { + for _, test := range tests.Tests { + report.Total++ + if test.Properties.Quarantined { + report.Quarantined++ + } + if test.Properties.Disabled { + report.Disabled++ + } + if test.Properties.AttemptToFix { + report.AttemptToFix++ + } + } + } + } + return report +} + +type durationSourceReport struct { + Known int + Default int +} + +type planningReport struct { + TestFilesDiscovered int + FullySkippedFiles int + TestFilesToRun int + DurationSources durationSourceReport + EstimatedTimeSaved float64 +} + +type planReport struct { + RunInfo runmetadata.RunInfo + PlanInfo PlanInfo + DatadogSettings datadogSettingsReport + KnownTests knownTestsReport + SkippableTestsCount int + ManagedFlakyTests managedFlakyTestsReport + Planning planningReport + Split splitScore +} + +func (tp *TestPlanner) newPlanningReport() planningReport { + fullySkippedFiles := len(tp.testFiles) - len(tp.testFileWeights) + if fullySkippedFiles < 0 { + fullySkippedFiles = 0 + } + + return planningReport{ + TestFilesDiscovered: len(tp.testFiles), + FullySkippedFiles: fullySkippedFiles, + TestFilesToRun: len(tp.testFileWeights), + DurationSources: tp.durationSourceReport(), + EstimatedTimeSaved: tp.skippablePercentage, + } +} + +func (tp *TestPlanner) durationSourceReport() durationSourceReport { + var report durationSourceReport + for _, source := range tp.testFileDurationSources { + switch source { + case testFileDurationSourceKnown: + report.Known++ + default: + report.Default++ + } + } + return report +} + +func printPlanReport(w io.Writer, report planReport) { + reportFprintln(w, "+++ DDTest: plan report") + reportFprintln(w) + printRunInfoReport(w, report.RunInfo, report.PlanInfo) + reportFprintln(w) + printDatadogSettingsReport(w, report.DatadogSettings) + reportFprintln(w) + printBackendDataReport(w, report) + reportFprintln(w) + printPlanningReport(w, report.Planning) + reportFprintln(w) + printSplitReport(w, report.Split) +} + +func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planInfo PlanInfo) { + reportFprintln(w, "Run") + reportFprintf(w, " Service: %s\n", valueOrNotAvailable(runInfo.Service)) + reportFprintf(w, " Repository: %s\n", valueOrNotAvailable(runInfo.Repository)) + reportFprintf(w, " Commit: %s\n", valueOrNotAvailable(runInfo.Commit)) + reportFprintf(w, " Branch: %s\n", valueOrNotAvailable(runInfo.Branch)) + reportFprintf(w, " Platform: %s\n", formatPlatform(planInfo.Platform, planInfo.Framework)) + reportFprintf(w, " OS tags: %s\n", formatTagList(planInfo.OSTags, ciConstants.OSPlatform, ciConstants.OSArchitecture, ciConstants.OSVersion)) + reportFprintf(w, " Runtime tags: %s\n", formatTagList(planInfo.RuntimeTags, ciConstants.RuntimeName, ciConstants.RuntimeVersion)) +} + +func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { + reportFprintln(w, "Datadog") + if !report.Available { + reportFprintln(w, " Settings: not available") + return + } + + reportFprintf(w, " Test Impact Analysis: %s\n", enabledWord(report.TestImpactAnalysis)) + reportFprintf(w, " Test skipping: %s\n", enabledWord(report.TestSkipping)) + reportFprintf(w, " Test impact collection: %s\n", enabledWord(report.TestImpactCollection)) + reportFprintf(w, " Known tests: %s\n", enabledWord(report.KnownTests)) + reportFprintf(w, " Impacted tests: %s\n", enabledWord(report.ImpactedTests)) + reportFprintf(w, " Early flake detection: %s\n", enabledWord(report.EarlyFlakeDetection)) + reportFprintf(w, " Auto test retries: %s\n", enabledWord(report.AutoTestRetries)) + reportFprintf(w, " Flaky test management: %s\n", enabledWord(report.FlakyTestManagement)) +} + +func printBackendDataReport(w io.Writer, report planReport) { + reportFprintln(w, "Backend data") + reportFprintf(w, " Known tests: %s\n", formatKnownTests(report.DatadogSettings, report.KnownTests)) + reportFprintf(w, " Skippable tests for this run: %s\n", formatSkippableTests(report.DatadogSettings, report.SkippableTestsCount)) + reportFprintf(w, " Managed flaky tests: %s\n", formatManagedFlakyTests(report.DatadogSettings, report.ManagedFlakyTests)) +} + +func printPlanningReport(w io.Writer, report planningReport) { + reportFprintln(w, "Planning") + reportFprintf(w, " Test files discovered: %s\n", formatCount(report.TestFilesDiscovered)) + reportFprintf(w, " Fully skipped files: %s\n", formatCount(report.FullySkippedFiles)) + reportFprintf(w, " Test files to run: %s\n", formatCount(report.TestFilesToRun)) + reportFprintf(w, " Duration source: %s known, %s default\n", + formatCount(report.DurationSources.Known), + formatCount(report.DurationSources.Default)) + reportFprintf(w, " Estimated time saved: %.2f%%\n", report.EstimatedTimeSaved) +} + +func printSplitReport(w io.Writer, report splitScore) { + reportFprintln(w, "Split") + reportFprintf(w, " Runners: %s\n", formatCount(report.parallelRunners)) + reportFprintf(w, " Expected wall time: %s\n", formatDuration(report.wallTimeDuration())) + reportFprintf(w, " Imbalance: %s\n", formatDuration(report.imbalanceDuration())) + reportFprintf(w, " Total estimated runtime: %s\n", formatDuration(report.totalRuntimeDuration())) +} + +func reportFprintln(w io.Writer, args ...any) { + _, _ = fmt.Fprintln(w, args...) +} + +func reportFprintf(w io.Writer, format string, args ...any) { + _, _ = fmt.Fprintf(w, format, args...) +} + +func formatKnownTests(settings datadogSettingsReport, known knownTestsReport) string { + if settings.Available && !settings.KnownTests { + return "disabled" + } + if !known.Available { + return "not available" + } + return fmt.Sprintf("%s modules, %s suites, %s tests", + formatCount(known.Modules), + formatCount(known.Suites), + formatCount(known.Tests)) +} + +func formatSkippableTests(settings datadogSettingsReport, count int) string { + if settings.Available && !settings.TestSkipping { + return "disabled" + } + return formatCount(count) +} + +func formatManagedFlakyTests(settings datadogSettingsReport, managed managedFlakyTestsReport) string { + if settings.Available && !settings.FlakyTestManagement { + return "disabled" + } + if !managed.Available { + return "not available" + } + return fmt.Sprintf("%s total, %s quarantined, %s disabled, %s attempt-to-fix", + formatCount(managed.Total), + formatCount(managed.Quarantined), + formatCount(managed.Disabled), + formatCount(managed.AttemptToFix)) +} + +func formatTagList(tags map[string]string, keys ...string) string { + parts := make([]string, 0, len(keys)) + for _, key := range keys { + if value := tags[key]; value != "" { + parts = append(parts, key+"="+value) + } + } + if len(parts) == 0 { + return "not available" + } + return strings.Join(parts, ", ") +} + +func formatPlatform(platformName, frameworkName string) string { + switch { + case platformName == "" && frameworkName == "": + return "not available" + case platformName == "": + return frameworkName + case frameworkName == "": + return platformName + default: + return platformName + " / " + frameworkName + } +} + +func valueOrNotAvailable(value string) string { + if value == "" { + return "not available" + } + return value +} + +func enabledWord(enabled bool) string { + if enabled { + return "enabled" + } + return "disabled" +} + +func formatCount(count int) string { + sign := "" + if count < 0 { + sign = "-" + count = -count + } + + value := strconv.Itoa(count) + if len(value) <= 3 { + return sign + value + } + + prefixLength := len(value) % 3 + if prefixLength == 0 { + prefixLength = 3 + } + + var builder strings.Builder + builder.WriteString(sign) + builder.WriteString(value[:prefixLength]) + for i := prefixLength; i < len(value); i += 3 { + builder.WriteByte(',') + builder.WriteString(value[i : i+3]) + } + return builder.String() +} + +func formatDuration(duration time.Duration) string { + if duration < time.Millisecond { + return duration.String() + } + return duration.Round(time.Millisecond).String() +} diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go new file mode 100644 index 0000000..716868a --- /dev/null +++ b/internal/planner/report_test.go @@ -0,0 +1,193 @@ +package planner + +import ( + "strings" + "testing" + + "github.com/DataDog/ddtest/civisibility/utils/net" + "github.com/DataDog/ddtest/internal/runmetadata" +) + +func TestPrintPlanReport_AllData(t *testing.T) { + var output strings.Builder + + printPlanReport(&output, planReport{ + RunInfo: runmetadata.RunInfo{ + Service: "checkout-api", + Repository: "https://github.com/acme/checkout.git", + Commit: "9f3a1c7d2b4e", + Branch: "feature/split-report", + }, + PlanInfo: PlanInfo{ + Platform: "ruby", + Framework: "rspec", + OSTags: map[string]string{ + "os.platform": "linux", + "os.architecture": "amd64", + "os.version": "6.8.0", + }, + RuntimeTags: map[string]string{ + "runtime.name": "ruby", + "runtime.version": "3.3.4", + }, + }, + DatadogSettings: datadogSettingsReport{ + Available: true, + TestImpactAnalysis: true, + TestSkipping: true, + TestImpactCollection: false, + KnownTests: true, + ImpactedTests: false, + EarlyFlakeDetection: true, + AutoTestRetries: true, + FlakyTestManagement: true, + }, + KnownTests: knownTestsReport{ + Available: true, + Modules: 4, + Suites: 1284, + Tests: 18921, + }, + SkippableTestsCount: 312, + ManagedFlakyTests: managedFlakyTestsReport{ + Available: true, + Total: 26, + Quarantined: 8, + Disabled: 3, + AttemptToFix: 5, + }, + Planning: planningReport{ + TestFilesDiscovered: 642, + FullySkippedFiles: 118, + TestFilesToRun: 524, + DurationSources: durationSourceReport{ + Known: 431, + Default: 90, + }, + EstimatedTimeSaved: 38.4, + }, + Split: splitScore{ + parallelRunners: 6, + wallTime: 252000, + imbalance: 11000, + totalRuntime: 1426000, + }, + }) + + expected := `+++ DDTest: plan report + +Run + Service: checkout-api + Repository: https://github.com/acme/checkout.git + Commit: 9f3a1c7d2b4e + Branch: feature/split-report + Platform: ruby / rspec + OS tags: os.platform=linux, os.architecture=amd64, os.version=6.8.0 + Runtime tags: runtime.name=ruby, runtime.version=3.3.4 + +Datadog + Test Impact Analysis: enabled + Test skipping: enabled + Test impact collection: disabled + Known tests: enabled + Impacted tests: disabled + Early flake detection: enabled + Auto test retries: enabled + Flaky test management: enabled + +Backend data + Known tests: 4 modules, 1,284 suites, 18,921 tests + Skippable tests for this run: 312 + Managed flaky tests: 26 total, 8 quarantined, 3 disabled, 5 attempt-to-fix + +Planning + Test files discovered: 642 + Fully skipped files: 118 + Test files to run: 524 + Duration source: 431 known, 90 default + Estimated time saved: 38.40% + +Split + Runners: 6 + Expected wall time: 4m12s + Imbalance: 11s + Total estimated runtime: 23m46s +` + if output.String() != expected { + t.Errorf("unexpected plan report:\n%s", output.String()) + } +} + +func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { + var output strings.Builder + + printPlanReport(&output, planReport{}) + + report := output.String() + if !strings.Contains(report, " Settings: not available") { + t.Errorf("expected missing settings message, got:\n%s", report) + } + if !strings.Contains(report, " Known tests: not available") { + t.Errorf("expected missing known tests message, got:\n%s", report) + } + if !strings.Contains(report, " Managed flaky tests: not available") { + t.Errorf("expected missing managed flaky tests message, got:\n%s", report) + } +} + +func TestPrintPlanReport_DisabledFeatures(t *testing.T) { + var output strings.Builder + + printPlanReport(&output, planReport{ + DatadogSettings: datadogSettingsReport{ + Available: true, + }, + }) + + report := output.String() + if !strings.Contains(report, " Known tests: disabled") { + t.Errorf("expected disabled known tests, got:\n%s", report) + } + if !strings.Contains(report, " Skippable tests for this run: disabled") { + t.Errorf("expected disabled skippable tests, got:\n%s", report) + } + if !strings.Contains(report, " Managed flaky tests: disabled") { + t.Errorf("expected disabled managed flaky tests, got:\n%s", report) + } +} + +func TestReportSummaries(t *testing.T) { + known := newKnownTestsReport(&net.KnownTestsResponseData{ + Tests: net.KnownTestsResponseDataModules{ + "module-a": net.KnownTestsResponseDataSuites{ + "suite-a": []string{"test-a", "test-b"}, + }, + "module-b": net.KnownTestsResponseDataSuites{ + "suite-b": []string{"test-c"}, + "suite-c": []string{"test-d", "test-e"}, + }, + }, + }) + if known.Modules != 2 || known.Suites != 3 || known.Tests != 5 { + t.Errorf("unexpected known test summary: %+v", known) + } + + managed := newManagedFlakyTestsReport(&net.TestManagementTestsResponseDataModules{ + Modules: map[string]net.TestManagementTestsResponseDataSuites{ + "module-a": { + Suites: map[string]net.TestManagementTestsResponseDataTests{ + "suite-a": { + Tests: map[string]net.TestManagementTestsResponseDataTestProperties{ + "test-a": {Properties: net.TestManagementTestsResponseDataTestPropertiesAttributes{Quarantined: true}}, + "test-b": {Properties: net.TestManagementTestsResponseDataTestPropertiesAttributes{Disabled: true}}, + "test-c": {Properties: net.TestManagementTestsResponseDataTestPropertiesAttributes{AttemptToFix: true}}, + }, + }, + }, + }, + }, + }) + if managed.Total != 3 || managed.Quarantined != 1 || managed.Disabled != 1 || managed.AttemptToFix != 1 { + t.Errorf("unexpected managed flaky test summary: %+v", managed) + } +} diff --git a/internal/planner/test_optimization_plan_cache.go b/internal/planner/test_optimization_plan_cache.go new file mode 100644 index 0000000..4118b6a --- /dev/null +++ b/internal/planner/test_optimization_plan_cache.go @@ -0,0 +1,179 @@ +package planner + +import ( + "encoding/json" + "log/slog" + + "github.com/DataDog/ddtest/internal/runmetadata" + "github.com/DataDog/ddtest/internal/testoptimization" +) + +type testOptimizationPlanCache struct { + TestSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo `json:"testSuiteDurations"` + SuiteAggregates map[testSuiteKey]testSuiteAggregate `json:"suiteAggregates"` + SuitesBySourceFile map[string][]testSuiteKey `json:"suitesBySourceFile"` + TestFileWeights map[string]int `json:"testFileWeights"` + TestFileDurationSources map[string]testFileDurationSource `json:"testFileDurationSources"` + RunInfo runmetadata.RunInfo `json:"runInfo"` + PlanInfo PlanInfo `json:"planInfo"` +} + +func (tp *TestPlanner) storeTestOptimizationPlanCache() error { + cache := testOptimizationPlanCache{ + TestSuiteDurations: tp.testSuiteDurations, + SuiteAggregates: tp.suiteAggregates, + SuitesBySourceFile: tp.suitesBySourceFile, + TestFileWeights: tp.testFileWeights, + TestFileDurationSources: tp.testFileDurationSources, + RunInfo: tp.runInfo, + PlanInfo: tp.planInfo, + } + + return testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache) +} + +func LoadPlan() (PlanInfo, error) { + return newTestPlannerWithDefaults().LoadPlan() +} + +func (tp *TestPlanner) LoadPlan() (PlanInfo, error) { + if !tp.planLoaded { + if err := tp.restoreTestOptimizationPlanCache(); err != nil { + return PlanInfo{}, err + } + } + + return tp.planInfo, nil +} + +func (tp *TestPlanner) restoreTestOptimizationPlanCache() error { + var cache testOptimizationPlanCache + if err := readAndNormalizeTestOptimizationPlanCache(&cache); err != nil { + return err + } + + tp.testSuiteDurations = cache.TestSuiteDurations + tp.suiteAggregates = cache.SuiteAggregates + tp.suitesBySourceFile = cache.SuitesBySourceFile + tp.testFileWeights = cache.TestFileWeights + tp.testFileDurationSources = cache.TestFileDurationSources + tp.runInfo = cache.RunInfo + tp.planInfo = cache.PlanInfo + tp.planLoaded = true + + testSuitesCount := countTestSuites(tp.testSuiteDurations) + suiteAggregatesCount := len(tp.suiteAggregates) + suitesBySourceFileCount := len(tp.suitesBySourceFile) + testFileWeightsCount := len(tp.testFileWeights) + slog.Info("Restored test optimization plan cache", + "objectsCount", testSuitesCount+suiteAggregatesCount+suitesBySourceFileCount+testFileWeightsCount, + "modulesCount", len(tp.testSuiteDurations), + "testSuitesCount", testSuitesCount, + "suiteAggregatesCount", suiteAggregatesCount, + "suitesBySourceFileCount", suitesBySourceFileCount, + "testFileWeightsCount", testFileWeightsCount) + + return nil +} + +type legacyRunInfo struct { + Service string `json:"service"` + Repository string `json:"repository"` + Commit string `json:"commit"` + Branch string `json:"branch"` + Platform string `json:"platform"` + Framework string `json:"framework"` + OSTags map[string]string `json:"osTags"` + RuntimeTags map[string]string `json:"runtimeTags"` +} + +func (c *testOptimizationPlanCache) UnmarshalJSON(data []byte) error { + var decoded struct { + TestSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo `json:"testSuiteDurations"` + SuiteAggregates map[testSuiteKey]testSuiteAggregate `json:"suiteAggregates"` + SuitesBySourceFile map[string][]testSuiteKey `json:"suitesBySourceFile"` + TestFileWeights map[string]int `json:"testFileWeights"` + TestFileDurationSources map[string]testFileDurationSource `json:"testFileDurationSources"` + RunInfo legacyRunInfo `json:"runInfo"` + PlanInfo PlanInfo `json:"planInfo"` + } + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + + c.TestSuiteDurations = decoded.TestSuiteDurations + c.SuiteAggregates = decoded.SuiteAggregates + c.SuitesBySourceFile = decoded.SuitesBySourceFile + c.TestFileWeights = decoded.TestFileWeights + c.TestFileDurationSources = decoded.TestFileDurationSources + c.RunInfo = decoded.RunInfo.runInfo() + c.PlanInfo = decoded.PlanInfo + if c.PlanInfo.IsZero() { + c.PlanInfo = decoded.RunInfo.planInfo() + } + + return nil +} + +func (r legacyRunInfo) runInfo() runmetadata.RunInfo { + return runmetadata.RunInfo{ + Service: r.Service, + Repository: r.Repository, + Commit: r.Commit, + Branch: r.Branch, + } +} + +func (r legacyRunInfo) planInfo() PlanInfo { + return PlanInfo{ + Platform: r.Platform, + Framework: r.Framework, + OSTags: r.OSTags, + RuntimeTags: r.RuntimeTags, + } +} + +func readAndNormalizeTestOptimizationPlanCache(cache *testOptimizationPlanCache) error { + if err := testoptimization.NewCacheManager().ReadTestOptimizationPlanCache(cache); err != nil { + return err + } + + if cache.TestSuiteDurations == nil { + cache.TestSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) + } + if cache.SuiteAggregates == nil { + cache.SuiteAggregates = make(map[testSuiteKey]testSuiteAggregate) + } + if cache.SuitesBySourceFile == nil { + cache.SuitesBySourceFile = indexSuitesBySourceFile(cache.SuiteAggregates) + } + if cache.TestFileWeights == nil { + cache.TestFileWeights = testFileWeightsFromSuites(cache.SuiteAggregates, cache.SuitesBySourceFile) + } + if cache.TestFileDurationSources == nil { + cache.TestFileDurationSources = make(map[string]testFileDurationSource) + } + return nil +} + +func countTestSuites(testSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo) int { + totalSuites := 0 + for _, suites := range testSuiteDurations { + totalSuites += len(suites) + } + return totalSuites +} + +func testFileWeightsFromSuites( + suiteAggregates map[testSuiteKey]testSuiteAggregate, + suitesBySourceFile map[string][]testSuiteKey, +) map[string]int { + planner := newTestPlannerWithDefaults() + planner.suiteAggregates = suiteAggregates + planner.suitesBySourceFile = suitesBySourceFile + testFiles := make(map[string]struct{}, len(suitesBySourceFile)) + for testFile := range suitesBySourceFile { + testFiles[testFile] = struct{}{} + } + return planner.estimateTestFileWeights(testFiles) +} diff --git a/internal/runner/test_optimization_plan_cache_test.go b/internal/planner/test_optimization_plan_cache_test.go similarity index 75% rename from internal/runner/test_optimization_plan_cache_test.go rename to internal/planner/test_optimization_plan_cache_test.go index cbc7c7d..29123ff 100644 --- a/internal/runner/test_optimization_plan_cache_test.go +++ b/internal/planner/test_optimization_plan_cache_test.go @@ -1,4 +1,4 @@ -package runner +package planner import ( "context" @@ -13,7 +13,7 @@ import ( "github.com/DataDog/ddtest/internal/testoptimization" ) -func TestTestRunner_Plan_StoresTestOptimizationPlanCache(t *testing.T) { +func TestTestPlanner_Plan_StoresTestOptimizationPlanCache(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -75,9 +75,12 @@ func TestTestRunner_Plan_StoresTestOptimizationPlanCache(t *testing.T) { if !reflect.DeepEqual(restored.testFileWeights, runner.testFileWeights) { t.Errorf("Expected restored test file weights to match stored weights.\nexpected: %v\nactual: %v", runner.testFileWeights, restored.testFileWeights) } + if !reflect.DeepEqual(restored.planInfo, runner.planInfo) { + t.Errorf("Expected restored plan info to match stored plan info.\nexpected: %v\nactual: %v", runner.planInfo, restored.planInfo) + } } -func TestTestRunner_StoreAndRestoreTestOptimizationPlanCache_RoundTripDurations(t *testing.T) { +func TestTestPlanner_StoreAndRestoreTestOptimizationPlanCache_RoundTripDurations(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -156,7 +159,7 @@ func TestTestRunner_StoreAndRestoreTestOptimizationPlanCache_RoundTripDurations( } } -func TestTestRunner_RestoreTestOptimizationPlanCache_ComputesMissingWeights(t *testing.T) { +func TestTestPlanner_RestoreTestOptimizationPlanCache_ComputesMissingWeights(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -215,6 +218,94 @@ func TestTestRunner_RestoreTestOptimizationPlanCache_ComputesMissingWeights(t *t } } +func TestLoadPlan_MigratesLegacyRunInfoPlanFields(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + type legacyTestOptimizationPlanCache struct { + RunInfo legacyRunInfo `json:"runInfo"` + } + cache := legacyTestOptimizationPlanCache{ + RunInfo: legacyRunInfo{ + Service: "checkout-api", + Repository: "https://github.com/acme/checkout.git", + Commit: "9f3a1c7d2b4e", + Branch: "feature/split-report", + Platform: "ruby", + Framework: "rspec", + OSTags: map[string]string{ + "os.platform": "linux", + "os.architecture": "amd64", + "os.version": "6.8.0", + }, + RuntimeTags: map[string]string{ + "runtime.name": "ruby", + "runtime.version": "3.3.4", + }, + }, + } + if err := testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache); err != nil { + t.Fatalf("StoreTestOptimizationPlanCache() should not return error, got: %v", err) + } + + plan, err := LoadPlan() + if err != nil { + t.Fatalf("LoadPlan() should not return error, got: %v", err) + } + + expected := PlanInfo{ + Platform: "ruby", + Framework: "rspec", + OSTags: map[string]string{ + "os.platform": "linux", + "os.architecture": "amd64", + "os.version": "6.8.0", + }, + RuntimeTags: map[string]string{ + "runtime.name": "ruby", + "runtime.version": "3.3.4", + }, + } + if !reflect.DeepEqual(plan, expected) { + t.Errorf("Expected LoadPlan() to migrate legacy plan fields.\nexpected: %v\nactual: %v", expected, plan) + } +} + +func TestTestPlanner_LoadPlan_UsesExistingPlannerState(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + cachePath := filepath.Join(constants.RunnerCacheDir, testoptimization.TestOptimizationPlanCacheFile) + if err := os.MkdirAll(filepath.Dir(cachePath), 0755); err != nil { + t.Fatalf("failed to create cache dir: %v", err) + } + if err := os.WriteFile(cachePath, []byte("{"), 0644); err != nil { + t.Fatalf("failed to write corrupt cache: %v", err) + } + + planner := newTestPlannerWithDefaults() + planner.planInfo = PlanInfo{ + Platform: "ruby", + Framework: "rspec", + } + planner.testFileWeights = map[string]int{ + "spec/example_spec.rb": 123, + } + planner.planLoaded = true + + plan, err := planner.LoadPlan() + if err != nil { + t.Fatalf("LoadPlan() should use existing planner state, got error: %v", err) + } + if plan.Platform != "ruby" || plan.Framework != "rspec" { + t.Fatalf("LoadPlan() returned wrong plan info: %v", plan) + } +} + func TestTestSuiteKey_JSONMapKeyRoundTrip(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() diff --git a/internal/runner/testdata/high_skippable_integration/spree_26236954724.json b/internal/planner/testdata/high_skippable_integration/spree_26236954724.json similarity index 100% rename from internal/runner/testdata/high_skippable_integration/spree_26236954724.json rename to internal/planner/testdata/high_skippable_integration/spree_26236954724.json diff --git a/internal/runner/testdata/split_selection/forem-26214779547.json b/internal/planner/testdata/split_selection/forem-26214779547.json similarity index 100% rename from internal/runner/testdata/split_selection/forem-26214779547.json rename to internal/planner/testdata/split_selection/forem-26214779547.json diff --git a/internal/runner/testdata/split_selection/spree-26223858840.json b/internal/planner/testdata/split_selection/spree-26223858840.json similarity index 100% rename from internal/runner/testdata/split_selection/spree-26223858840.json rename to internal/planner/testdata/split_selection/spree-26223858840.json diff --git a/internal/runner/testdata/split_selection/spree-26224156491.json b/internal/planner/testdata/split_selection/spree-26224156491.json similarity index 100% rename from internal/runner/testdata/split_selection/spree-26224156491.json rename to internal/planner/testdata/split_selection/spree-26224156491.json diff --git a/internal/runner/testdata/split_selection/spree-26224387824.json b/internal/planner/testdata/split_selection/spree-26224387824.json similarity index 100% rename from internal/runner/testdata/split_selection/spree-26224387824.json rename to internal/planner/testdata/split_selection/spree-26224387824.json diff --git a/internal/runmetadata/runmetadata.go b/internal/runmetadata/runmetadata.go new file mode 100644 index 0000000..1c3670e --- /dev/null +++ b/internal/runmetadata/runmetadata.go @@ -0,0 +1,51 @@ +package runmetadata + +import ( + "os" + "regexp" + "strings" + + ciConstants "github.com/DataDog/ddtest/civisibility/constants" +) + +var repositoryNameRegex = regexp.MustCompile(`(?m)/([a-zA-Z0-9\-_.]+)$`) + +type RunInfo struct { + Service string `json:"service"` + Repository string `json:"repository"` + Commit string `json:"commit"` + Branch string `json:"branch"` +} + +func New(ciTags map[string]string) RunInfo { + repository := ciTags[ciConstants.GitRepositoryURL] + return RunInfo{ + Service: ResolveServiceName(repository), + Repository: repository, + Commit: ciTags[ciConstants.GitCommitSHA], + Branch: ciTags[ciConstants.GitBranch], + } +} + +func (r RunInfo) IsZero() bool { + return r.Service == "" && + r.Repository == "" && + r.Commit == "" && + r.Branch == "" +} + +func ResolveServiceName(repositoryURL string) string { + if service := os.Getenv("DD_SERVICE"); service != "" { + return service + } + return ServiceNameFromRepositoryURL(repositoryURL) +} + +func ServiceNameFromRepositoryURL(repositoryURL string) string { + normalizedRepositoryURL := strings.TrimRight(repositoryURL, "/") + matches := repositoryNameRegex.FindStringSubmatch(normalizedRepositoryURL) + if len(matches) > 1 { + return strings.TrimSuffix(matches[1], ".git") + } + return normalizedRepositoryURL +} diff --git a/internal/runner/service_test.go b/internal/runmetadata/runmetadata_test.go similarity index 81% rename from internal/runner/service_test.go rename to internal/runmetadata/runmetadata_test.go index 03bab77..ce2f39f 100644 --- a/internal/runner/service_test.go +++ b/internal/runmetadata/runmetadata_test.go @@ -1,4 +1,4 @@ -package runner +package runmetadata import "testing" @@ -37,9 +37,9 @@ func TestServiceNameFromRepositoryURL(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := serviceNameFromRepositoryURL(tt.repositoryURL) + got := ServiceNameFromRepositoryURL(tt.repositoryURL) if got != tt.want { - t.Errorf("serviceNameFromRepositoryURL(%q) = %q, want %q", tt.repositoryURL, got, tt.want) + t.Errorf("ServiceNameFromRepositoryURL(%q) = %q, want %q", tt.repositoryURL, got, tt.want) } }) } @@ -48,8 +48,8 @@ func TestServiceNameFromRepositoryURL(t *testing.T) { func TestResolveServiceNamePrefersDDService(t *testing.T) { t.Setenv("DD_SERVICE", "custom-service") - got := resolveServiceName("https://github.com/DataDog/ddtest.git") + got := ResolveServiceName("https://github.com/DataDog/ddtest.git") if got != "custom-service" { - t.Errorf("resolveServiceName() = %q, want %q", got, "custom-service") + t.Errorf("ResolveServiceName() = %q, want %q", got, "custom-service") } } diff --git a/internal/runner/ci_node_executor.go b/internal/runner/ci_node_executor.go index 56af333..b049fe5 100644 --- a/internal/runner/ci_node_executor.go +++ b/internal/runner/ci_node_executor.go @@ -4,13 +4,15 @@ import ( "fmt" "log/slog" "os" + "path/filepath" + "github.com/DataDog/ddtest/internal/constants" "golang.org/x/sync/errgroup" ) // runCINode executes tests for a specific CI node (one split, not the whole tests set). // It further splits the node's tests among local workers based on ci_node_workers setting. -func (e testExecutor) runCINode(ciNode int, ciNodeWorkers int, testFileWeights map[string]int) runExecutionResult { +func (e testExecutor) runCINode(ciNode int, ciNodeWorkers int) runExecutionResult { report := newCINodeExecutionReport(ciNode, ciNodeWorkers) testFiles, err := loadCINodeTestFiles(ciNode) if err != nil { @@ -21,7 +23,7 @@ func (e testExecutor) runCINode(ciNode int, ciNodeWorkers int, testFileWeights m if report.LocalWorkers <= 1 { err = e.runCINodeSingleWorker(ciNode, testFiles) } else { - err = e.runCINodeWorkers(ciNode, report.LocalWorkers, testFiles, testFileWeights) + err = e.runCINodeWorkers(ciNode, report.LocalWorkers, testFiles) } if err != nil { return report.failure(err) @@ -42,7 +44,7 @@ func newCINodeExecutionReport(ciNode int, ciNodeWorkers int) runExecutionReport } func loadCINodeTestFiles(ciNode int) ([]string, error) { - runnerFilePath := runnerSplitPath(ciNode) + runnerFilePath := filepath.Join(constants.TestsSplitDir, fmt.Sprintf("runner-%d", ciNode)) testFiles, err := loadTestBatch(runnerFilePath) if os.IsNotExist(err) { return nil, fmt.Errorf("runner file for ci-node %d does not exist: %s", ciNode, runnerFilePath) @@ -62,7 +64,7 @@ func (e testExecutor) runCINodeSingleWorker(ciNode int, testFiles []string) erro return e.runBatch(testFiles, ciNode, 0) } -func (e testExecutor) runCINodeWorkers(ciNode int, ciNodeWorkers int, testFiles []string, testFileWeights map[string]int) error { +func (e testExecutor) runCINodeWorkers(ciNode int, ciNodeWorkers int, testFiles []string) error { if len(testFiles) == 0 { slog.Info("No tests to run for CI node", "ciNode", ciNode) return nil @@ -71,10 +73,7 @@ func (e testExecutor) runCINodeWorkers(ciNode int, ciNodeWorkers int, testFiles slog.Info("Running tests for CI node in parallel mode", "ciNode", ciNode, "ciNodeWorkers", ciNodeWorkers, "testFilesCount", len(testFiles)) - if testFileWeights == nil { - testFileWeights = map[string]int{} - } - groups := subsplitTestsBetweenWorkers(testFiles, ciNodeWorkers, testFileWeights) + groups := e.subsplitTestsBetweenWorkers(testFiles, ciNodeWorkers) return e.runCINodeWorkerGroups(ciNode, groups) } @@ -101,21 +100,6 @@ func (e testExecutor) runCINodeWorkerGroups(ciNode int, groups [][]string) error return nil } -// subsplitTestsBetweenWorkers splits a CI node's test files among local workers -// using the same weighted distribution algorithm used for CI node splits. -func subsplitTestsBetweenWorkers(testFiles []string, n int, testFileWeights map[string]int) [][]string { - if n <= 0 { - n = 1 - } - - nodeTestFiles := make(map[string]int, len(testFiles)) - for _, testFile := range testFiles { - if cachedWeight, ok := testFileWeights[testFile]; ok && cachedWeight > 0 { - nodeTestFiles[testFile] = cachedWeight - } else { - nodeTestFiles[testFile] = defaultTestFileWeight - } - } - - return DistributeTestFiles(nodeTestFiles, n) +func (e testExecutor) subsplitTestsBetweenWorkers(testFiles []string, n int) [][]string { + return e.planner.DistributeTestFiles(testFiles, n) } diff --git a/internal/runner/ci_node_executor_test.go b/internal/runner/ci_node_executor_test.go index c40d741..e77193c 100644 --- a/internal/runner/ci_node_executor_test.go +++ b/internal/runner/ci_node_executor_test.go @@ -27,7 +27,7 @@ func TestRunCINode_SingleWorker(t *testing.T) { } // Test with single worker (ciNodeWorkers=1) - result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runCINode(1, 1, nil) + result := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runCINode(1, 1) report, err := result.report, result.err if err != nil { t.Fatalf("runCINode() should not return error, got: %v", err) @@ -67,7 +67,8 @@ func TestRunCINode_MultipleWorkers(t *testing.T) { } // Test with 2 workers on ci-node 1 - result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runCINode(1, 2, nil) + executor := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}) + result := executor.runCINode(1, 2) report, err := result.report, result.err if err != nil { t.Fatalf("runCINode() should not return error, got: %v", err) @@ -135,7 +136,8 @@ func TestRunCINode_NodeIndexMatchesCINode(t *testing.T) { } // Test with 2 workers on ci-node 1 - result := newTestExecutor(context.Background(), mockFramework, workerEnvMap).runCINode(1, 2, nil) + executor := newTestExecutor(context.Background(), mockFramework, workerEnvMap, roundRobinTestPlanner{}) + result := executor.runCINode(1, 2) err := result.err if err != nil { t.Fatalf("runCINode() should not return error, got: %v", err) @@ -189,7 +191,7 @@ func TestRunCINode_SingleWorkerNodeIndex(t *testing.T) { "WORKER_INDEX": "{{workerIndex}}", } - result := newTestExecutor(context.Background(), mockFramework, workerEnvMap).runCINode(2, 1, nil) + result := newTestExecutor(context.Background(), mockFramework, workerEnvMap, roundRobinTestPlanner{}).runCINode(2, 1) err := result.err if err != nil { t.Fatalf("runCINode() should not return error, got: %v", err) @@ -219,7 +221,7 @@ func TestRunCINode_FileNotFound(t *testing.T) { mockFramework := &MockFramework{FrameworkName: "rspec"} - result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runCINode(2, 1, nil) + result := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runCINode(2, 1) err := result.err if err == nil { t.Error("runCINode() should return error when runner file doesn't exist") @@ -244,7 +246,7 @@ func TestRunCINode_EmptyFile(t *testing.T) { mockFramework := &MockFramework{FrameworkName: "rspec"} // Should not error for empty file, just not run any tests - result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runCINode(0, 2, nil) + result := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runCINode(0, 2) report, err := result.report, result.err if err != nil { t.Fatalf("runCINode() should not return error for empty file, got: %v", err) @@ -260,9 +262,11 @@ func TestRunCINode_EmptyFile(t *testing.T) { } func TestSubsplitTestsBetweenWorkers(t *testing.T) { + executor := testExecutor{planner: roundRobinTestPlanner{}} + t.Run("even split", func(t *testing.T) { files := []string{"a", "b", "c", "d"} - result := subsplitTestsBetweenWorkers(files, 2, map[string]int{}) + result := executor.subsplitTestsBetweenWorkers(files, 2) if len(result) != 2 { t.Fatalf("Expected 2 groups, got %d", len(result)) @@ -282,7 +286,7 @@ func TestSubsplitTestsBetweenWorkers(t *testing.T) { t.Run("uneven split", func(t *testing.T) { files := []string{"a", "b", "c", "d", "e"} - result := subsplitTestsBetweenWorkers(files, 2, map[string]int{}) + result := executor.subsplitTestsBetweenWorkers(files, 2) if len(result) != 2 { t.Fatalf("Expected 2 groups, got %d", len(result)) @@ -302,7 +306,7 @@ func TestSubsplitTestsBetweenWorkers(t *testing.T) { t.Run("more groups than files", func(t *testing.T) { files := []string{"a", "b"} - result := subsplitTestsBetweenWorkers(files, 4, map[string]int{}) + result := executor.subsplitTestsBetweenWorkers(files, 4) if len(result) != 4 { t.Fatalf("Expected 4 groups, got %d", len(result)) @@ -325,7 +329,7 @@ func TestSubsplitTestsBetweenWorkers(t *testing.T) { t.Run("single group", func(t *testing.T) { files := []string{"a", "b", "c"} - result := subsplitTestsBetweenWorkers(files, 1, map[string]int{}) + result := executor.subsplitTestsBetweenWorkers(files, 1) if len(result) != 1 { t.Fatalf("Expected 1 group, got %d", len(result)) @@ -337,7 +341,7 @@ func TestSubsplitTestsBetweenWorkers(t *testing.T) { }) t.Run("empty input", func(t *testing.T) { - result := subsplitTestsBetweenWorkers([]string{}, 3, map[string]int{}) + result := executor.subsplitTestsBetweenWorkers([]string{}, 3) if len(result) != 3 { t.Fatalf("Expected 3 groups, got %d", len(result)) @@ -352,7 +356,7 @@ func TestSubsplitTestsBetweenWorkers(t *testing.T) { t.Run("zero groups defaults to 1", func(t *testing.T) { files := []string{"a", "b"} - result := subsplitTestsBetweenWorkers(files, 0, map[string]int{}) + result := executor.subsplitTestsBetweenWorkers(files, 0) if len(result) != 1 { t.Fatalf("Expected 1 group for n=0, got %d", len(result)) @@ -363,26 +367,4 @@ func TestSubsplitTestsBetweenWorkers(t *testing.T) { } }) - t.Run("weighted split keeps heavy file isolated", func(t *testing.T) { - files := []string{"a", "b", "c", "d"} - weights := map[string]int{ - "a": 100, - "b": 1, - "c": 1, - "d": 1, - } - - result := subsplitTestsBetweenWorkers(files, 2, weights) - - if len(result) != 2 { - t.Fatalf("Expected 2 groups, got %d", len(result)) - } - - if !slices.Equal(result[0], []string{"a"}) { - t.Errorf("Expected heavy file to be alone in group 0, got %v", result[0]) - } - if !slices.Equal(result[1], []string{"b", "c", "d"}) { - t.Errorf("Expected light files in group 1, got %v", result[1]) - } - }) } diff --git a/internal/runner/parallel_executor_test.go b/internal/runner/parallel_executor_test.go index 90efd64..caacdf5 100644 --- a/internal/runner/parallel_executor_test.go +++ b/internal/runner/parallel_executor_test.go @@ -26,7 +26,7 @@ func TestRunParallel_Success(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runParallel() + result := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runParallel() report, err := result.report, result.err if err != nil { t.Fatalf("runParallel() should not return error, got: %v", err) @@ -53,7 +53,7 @@ func TestRunParallel_MissingSplitDirectory(t *testing.T) { // Don't create tests-split directory mockFramework := &MockFramework{FrameworkName: "rspec"} - result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runParallel() + result := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runParallel() err := result.err if err == nil { t.Error("runParallel() should return error when tests-split directory is missing") diff --git a/internal/runner/plan_files.go b/internal/runner/plan_files.go deleted file mode 100644 index 119197b..0000000 --- a/internal/runner/plan_files.go +++ /dev/null @@ -1,23 +0,0 @@ -package runner - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/DataDog/ddtest/internal/constants" -) - -func runnerSplitPath(ciNode int) string { - return filepath.Join(constants.TestsSplitDir, fmt.Sprintf("runner-%d", ciNode)) -} - -func writePlanFile(path string, data []byte) error { - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return fmt.Errorf("failed to create output directory for %s: %w", path, err) - } - if err := os.WriteFile(path, data, 0644); err != nil { - return fmt.Errorf("failed to write %s: %w", path, err) - } - return nil -} diff --git a/internal/runner/report.go b/internal/runner/report.go index 3687663..29f0cc6 100644 --- a/internal/runner/report.go +++ b/internal/runner/report.go @@ -8,7 +8,8 @@ import ( "time" ciConstants "github.com/DataDog/ddtest/civisibility/constants" - "github.com/DataDog/ddtest/civisibility/utils/net" + "github.com/DataDog/ddtest/internal/planner" + "github.com/DataDog/ddtest/internal/runmetadata" ) const ( @@ -17,152 +18,6 @@ const ( runModeCINode = "CI node" ) -type runInfoReport struct { - Service string `json:"service"` - Repository string `json:"repository"` - Commit string `json:"commit"` - Branch string `json:"branch"` - Platform string `json:"platform"` - Framework string `json:"framework"` - OSTags map[string]string `json:"osTags"` - RuntimeTags map[string]string `json:"runtimeTags"` -} - -func newRunInfoReport(ciTags map[string]string, runtimeTags map[string]string, platformName, frameworkName string) runInfoReport { - repository := ciTags[ciConstants.GitRepositoryURL] - return runInfoReport{ - Service: resolveServiceName(repository), - Repository: repository, - Commit: ciTags[ciConstants.GitCommitSHA], - Branch: ciTags[ciConstants.GitBranch], - Platform: platformName, - Framework: frameworkName, - OSTags: selectTags(runtimeTags, ciConstants.OSPlatform, ciConstants.OSArchitecture, ciConstants.OSVersion), - RuntimeTags: selectTags(runtimeTags, ciConstants.RuntimeName, ciConstants.RuntimeVersion), - } -} - -func (r runInfoReport) isZero() bool { - return r.Service == "" && - r.Repository == "" && - r.Commit == "" && - r.Branch == "" && - r.Platform == "" && - r.Framework == "" && - len(r.OSTags) == 0 && - len(r.RuntimeTags) == 0 -} - -type datadogSettingsReport struct { - Available bool - TestImpactAnalysis bool - TestSkipping bool - TestImpactCollection bool - KnownTests bool - ImpactedTests bool - EarlyFlakeDetection bool - AutoTestRetries bool - FlakyTestManagement bool -} - -func newDatadogSettingsReport(settings *net.SettingsResponseData) datadogSettingsReport { - if settings == nil { - return datadogSettingsReport{} - } - return datadogSettingsReport{ - Available: true, - TestImpactAnalysis: settings.ItrEnabled, - TestSkipping: settings.TestsSkipping, - TestImpactCollection: settings.CodeCoverage, - KnownTests: settings.KnownTestsEnabled, - ImpactedTests: settings.ImpactedTestsEnabled, - EarlyFlakeDetection: settings.EarlyFlakeDetection.Enabled, - AutoTestRetries: settings.FlakyTestRetriesEnabled, - FlakyTestManagement: settings.TestManagement.Enabled, - } -} - -type knownTestsReport struct { - Available bool - Modules int - Suites int - Tests int -} - -func newKnownTestsReport(knownTests *net.KnownTestsResponseData) knownTestsReport { - if knownTests == nil { - return knownTestsReport{} - } - - report := knownTestsReport{ - Available: true, - Modules: len(knownTests.Tests), - } - for _, suites := range knownTests.Tests { - report.Suites += len(suites) - for _, tests := range suites { - report.Tests += len(tests) - } - } - return report -} - -type managedFlakyTestsReport struct { - Available bool - Total int - Quarantined int - Disabled int - AttemptToFix int -} - -func newManagedFlakyTestsReport(testManagementTests *net.TestManagementTestsResponseDataModules) managedFlakyTestsReport { - if testManagementTests == nil { - return managedFlakyTestsReport{} - } - - report := managedFlakyTestsReport{Available: true} - for _, suites := range testManagementTests.Modules { - for _, tests := range suites.Suites { - for _, test := range tests.Tests { - report.Total++ - if test.Properties.Quarantined { - report.Quarantined++ - } - if test.Properties.Disabled { - report.Disabled++ - } - if test.Properties.AttemptToFix { - report.AttemptToFix++ - } - } - } - } - return report -} - -type durationSourceReport struct { - Known int - Default int -} - -type planningReport struct { - TestFilesDiscovered int - FullySkippedFiles int - TestFilesToRun int - DurationSources durationSourceReport - EstimatedTimeSaved float64 -} - -type planReport struct { - RunInfo runInfoReport - DatadogSettings datadogSettingsReport - KnownTests knownTestsReport - SkippableTestsCount int - ManagedFlakyTests managedFlakyTestsReport - Planning planningReport - Split splitScore -} - type runExecutionReport struct { Mode string CINode int @@ -171,114 +26,30 @@ type runExecutionReport struct { } type runReport struct { - RunInfo runInfoReport + RunInfo runmetadata.RunInfo + PlanInfo planner.PlanInfo Execution runExecutionReport Duration time.Duration Err error } -func (tr *TestRunner) newPlanningReport() planningReport { - fullySkippedFiles := len(tr.testFiles) - len(tr.testFileWeights) - if fullySkippedFiles < 0 { - fullySkippedFiles = 0 - } - - return planningReport{ - TestFilesDiscovered: len(tr.testFiles), - FullySkippedFiles: fullySkippedFiles, - TestFilesToRun: len(tr.testFileWeights), - DurationSources: tr.durationSourceReport(), - EstimatedTimeSaved: tr.skippablePercentage, - } -} - -func (tr *TestRunner) durationSourceReport() durationSourceReport { - var report durationSourceReport - for _, source := range tr.testFileDurationSources { - switch source { - case testFileDurationSourceKnown: - report.Known++ - default: - report.Default++ - } - } - return report -} - -func printPlanReport(w io.Writer, report planReport) { - reportFprintln(w, "+++ DDTest: plan report") - reportFprintln(w) - printRunInfoReport(w, report.RunInfo) - reportFprintln(w) - printDatadogSettingsReport(w, report.DatadogSettings) - reportFprintln(w) - printBackendDataReport(w, report) - reportFprintln(w) - printPlanningReport(w, report.Planning) - reportFprintln(w) - printSplitReport(w, report.Split) -} - func printRunReport(w io.Writer, report runReport) { reportFprintln(w, "+++ DDTest: run report") reportFprintln(w) - printRunInfoReport(w, report.RunInfo) + printRunInfoReport(w, report.RunInfo, report.PlanInfo) reportFprintln(w) printExecutionReport(w, report) } -func printRunInfoReport(w io.Writer, report runInfoReport) { +func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planInfo planner.PlanInfo) { reportFprintln(w, "Run") - reportFprintf(w, " Service: %s\n", valueOrNotAvailable(report.Service)) - reportFprintf(w, " Repository: %s\n", valueOrNotAvailable(report.Repository)) - reportFprintf(w, " Commit: %s\n", valueOrNotAvailable(report.Commit)) - reportFprintf(w, " Branch: %s\n", valueOrNotAvailable(report.Branch)) - reportFprintf(w, " Platform: %s\n", formatPlatform(report.Platform, report.Framework)) - reportFprintf(w, " OS tags: %s\n", formatTagList(report.OSTags, ciConstants.OSPlatform, ciConstants.OSArchitecture, ciConstants.OSVersion)) - reportFprintf(w, " Runtime tags: %s\n", formatTagList(report.RuntimeTags, ciConstants.RuntimeName, ciConstants.RuntimeVersion)) -} - -func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { - reportFprintln(w, "Datadog") - if !report.Available { - reportFprintln(w, " Settings: not available") - return - } - - reportFprintf(w, " Test Impact Analysis: %s\n", enabledWord(report.TestImpactAnalysis)) - reportFprintf(w, " Test skipping: %s\n", enabledWord(report.TestSkipping)) - reportFprintf(w, " Test impact collection: %s\n", enabledWord(report.TestImpactCollection)) - reportFprintf(w, " Known tests: %s\n", enabledWord(report.KnownTests)) - reportFprintf(w, " Impacted tests: %s\n", enabledWord(report.ImpactedTests)) - reportFprintf(w, " Early flake detection: %s\n", enabledWord(report.EarlyFlakeDetection)) - reportFprintf(w, " Auto test retries: %s\n", enabledWord(report.AutoTestRetries)) - reportFprintf(w, " Flaky test management: %s\n", enabledWord(report.FlakyTestManagement)) -} - -func printBackendDataReport(w io.Writer, report planReport) { - reportFprintln(w, "Backend data") - reportFprintf(w, " Known tests: %s\n", formatKnownTests(report.DatadogSettings, report.KnownTests)) - reportFprintf(w, " Skippable tests for this run: %s\n", formatSkippableTests(report.DatadogSettings, report.SkippableTestsCount)) - reportFprintf(w, " Managed flaky tests: %s\n", formatManagedFlakyTests(report.DatadogSettings, report.ManagedFlakyTests)) -} - -func printPlanningReport(w io.Writer, report planningReport) { - reportFprintln(w, "Planning") - reportFprintf(w, " Test files discovered: %s\n", formatCount(report.TestFilesDiscovered)) - reportFprintf(w, " Fully skipped files: %s\n", formatCount(report.FullySkippedFiles)) - reportFprintf(w, " Test files to run: %s\n", formatCount(report.TestFilesToRun)) - reportFprintf(w, " Duration source: %s known, %s default\n", - formatCount(report.DurationSources.Known), - formatCount(report.DurationSources.Default)) - reportFprintf(w, " Estimated time saved: %.2f%%\n", report.EstimatedTimeSaved) -} - -func printSplitReport(w io.Writer, report splitScore) { - reportFprintln(w, "Split") - reportFprintf(w, " Runners: %s\n", formatCount(report.parallelRunners)) - reportFprintf(w, " Expected wall time: %s\n", formatDuration(report.wallTimeDuration())) - reportFprintf(w, " Imbalance: %s\n", formatDuration(report.imbalanceDuration())) - reportFprintf(w, " Total estimated runtime: %s\n", formatDuration(report.totalRuntimeDuration())) + reportFprintf(w, " Service: %s\n", valueOrNotAvailable(runInfo.Service)) + reportFprintf(w, " Repository: %s\n", valueOrNotAvailable(runInfo.Repository)) + reportFprintf(w, " Commit: %s\n", valueOrNotAvailable(runInfo.Commit)) + reportFprintf(w, " Branch: %s\n", valueOrNotAvailable(runInfo.Branch)) + reportFprintf(w, " Platform: %s\n", formatPlatform(planInfo.Platform, planInfo.Framework)) + reportFprintf(w, " OS tags: %s\n", formatTagList(planInfo.OSTags, ciConstants.OSPlatform, ciConstants.OSArchitecture, ciConstants.OSVersion)) + reportFprintf(w, " Runtime tags: %s\n", formatTagList(planInfo.RuntimeTags, ciConstants.RuntimeName, ciConstants.RuntimeVersion)) } func printExecutionReport(w io.Writer, report runReport) { @@ -306,50 +77,6 @@ func reportFprintf(w io.Writer, format string, args ...any) { _, _ = fmt.Fprintf(w, format, args...) } -func formatKnownTests(settings datadogSettingsReport, known knownTestsReport) string { - if settings.Available && !settings.KnownTests { - return "disabled" - } - if !known.Available { - return "not available" - } - return fmt.Sprintf("%s modules, %s suites, %s tests", - formatCount(known.Modules), - formatCount(known.Suites), - formatCount(known.Tests)) -} - -func formatSkippableTests(settings datadogSettingsReport, count int) string { - if settings.Available && !settings.TestSkipping { - return "disabled" - } - return formatCount(count) -} - -func formatManagedFlakyTests(settings datadogSettingsReport, managed managedFlakyTestsReport) string { - if settings.Available && !settings.FlakyTestManagement { - return "disabled" - } - if !managed.Available { - return "not available" - } - return fmt.Sprintf("%s total, %s quarantined, %s disabled, %s attempt-to-fix", - formatCount(managed.Total), - formatCount(managed.Quarantined), - formatCount(managed.Disabled), - formatCount(managed.AttemptToFix)) -} - -func selectTags(tags map[string]string, keys ...string) map[string]string { - selected := make(map[string]string) - for _, key := range keys { - if value := tags[key]; value != "" { - selected[key] = value - } - } - return selected -} - func formatTagList(tags map[string]string, keys ...string) string { parts := make([]string, 0, len(keys)) for _, key := range keys { @@ -383,13 +110,6 @@ func valueOrNotAvailable(value string) string { return value } -func enabledWord(enabled bool) string { - if enabled { - return "enabled" - } - return "disabled" -} - func formatCount(count int) string { sign := "" if count < 0 { diff --git a/internal/runner/report_test.go b/internal/runner/report_test.go index 12008e4..f0f1c2f 100644 --- a/internal/runner/report_test.go +++ b/internal/runner/report_test.go @@ -6,166 +6,23 @@ import ( "testing" "time" - "github.com/DataDog/ddtest/civisibility/utils/net" + "github.com/DataDog/ddtest/internal/planner" + "github.com/DataDog/ddtest/internal/runmetadata" ) -func TestPrintPlanReport_AllData(t *testing.T) { - var output strings.Builder - - printPlanReport(&output, planReport{ - RunInfo: runInfoReport{ - Service: "checkout-api", - Repository: "https://github.com/acme/checkout.git", - Commit: "9f3a1c7d2b4e", - Branch: "feature/split-report", - Platform: "ruby", - Framework: "rspec", - OSTags: map[string]string{ - "os.platform": "linux", - "os.architecture": "amd64", - "os.version": "6.8.0", - }, - RuntimeTags: map[string]string{ - "runtime.name": "ruby", - "runtime.version": "3.3.4", - }, - }, - DatadogSettings: datadogSettingsReport{ - Available: true, - TestImpactAnalysis: true, - TestSkipping: true, - TestImpactCollection: false, - KnownTests: true, - ImpactedTests: false, - EarlyFlakeDetection: true, - AutoTestRetries: true, - FlakyTestManagement: true, - }, - KnownTests: knownTestsReport{ - Available: true, - Modules: 4, - Suites: 1284, - Tests: 18921, - }, - SkippableTestsCount: 312, - ManagedFlakyTests: managedFlakyTestsReport{ - Available: true, - Total: 26, - Quarantined: 8, - Disabled: 3, - AttemptToFix: 5, - }, - Planning: planningReport{ - TestFilesDiscovered: 642, - FullySkippedFiles: 118, - TestFilesToRun: 524, - DurationSources: durationSourceReport{ - Known: 431, - Default: 90, - }, - EstimatedTimeSaved: 38.4, - }, - Split: splitScore{ - parallelRunners: 6, - wallTime: 252000, - imbalance: 11000, - totalRuntime: 1426000, - }, - }) - - expected := `+++ DDTest: plan report - -Run - Service: checkout-api - Repository: https://github.com/acme/checkout.git - Commit: 9f3a1c7d2b4e - Branch: feature/split-report - Platform: ruby / rspec - OS tags: os.platform=linux, os.architecture=amd64, os.version=6.8.0 - Runtime tags: runtime.name=ruby, runtime.version=3.3.4 - -Datadog - Test Impact Analysis: enabled - Test skipping: enabled - Test impact collection: disabled - Known tests: enabled - Impacted tests: disabled - Early flake detection: enabled - Auto test retries: enabled - Flaky test management: enabled - -Backend data - Known tests: 4 modules, 1,284 suites, 18,921 tests - Skippable tests for this run: 312 - Managed flaky tests: 26 total, 8 quarantined, 3 disabled, 5 attempt-to-fix - -Planning - Test files discovered: 642 - Fully skipped files: 118 - Test files to run: 524 - Duration source: 431 known, 90 default - Estimated time saved: 38.40% - -Split - Runners: 6 - Expected wall time: 4m12s - Imbalance: 11s - Total estimated runtime: 23m46s -` - if output.String() != expected { - t.Errorf("unexpected plan report:\n%s", output.String()) - } -} - -func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { - var output strings.Builder - - printPlanReport(&output, planReport{}) - - report := output.String() - if !strings.Contains(report, " Settings: not available") { - t.Errorf("expected missing settings message, got:\n%s", report) - } - if !strings.Contains(report, " Known tests: not available") { - t.Errorf("expected missing known tests message, got:\n%s", report) - } - if !strings.Contains(report, " Managed flaky tests: not available") { - t.Errorf("expected missing managed flaky tests message, got:\n%s", report) - } -} - -func TestPrintPlanReport_DisabledFeatures(t *testing.T) { - var output strings.Builder - - printPlanReport(&output, planReport{ - DatadogSettings: datadogSettingsReport{ - Available: true, - }, - }) - - report := output.String() - if !strings.Contains(report, " Known tests: disabled") { - t.Errorf("expected disabled known tests, got:\n%s", report) - } - if !strings.Contains(report, " Skippable tests for this run: disabled") { - t.Errorf("expected disabled skippable tests, got:\n%s", report) - } - if !strings.Contains(report, " Managed flaky tests: disabled") { - t.Errorf("expected disabled managed flaky tests, got:\n%s", report) - } -} - func TestPrintRunReport_Passed(t *testing.T) { var output strings.Builder printRunReport(&output, runReport{ - RunInfo: runInfoReport{ + RunInfo: runmetadata.RunInfo{ Service: "checkout-api", Repository: "https://github.com/acme/checkout.git", Commit: "9f3a1c7d2b4e", Branch: "feature/split-report", - Platform: "ruby", - Framework: "rspec", + }, + PlanInfo: planner.PlanInfo{ + Platform: "ruby", + Framework: "rspec", OSTags: map[string]string{ "os.platform": "linux", "os.architecture": "amd64", @@ -227,39 +84,3 @@ func TestPrintRunReport_Failed(t *testing.T) { t.Errorf("expected failed run report, got:\n%s", report) } } - -func TestReportSummaries(t *testing.T) { - known := newKnownTestsReport(&net.KnownTestsResponseData{ - Tests: net.KnownTestsResponseDataModules{ - "module-a": net.KnownTestsResponseDataSuites{ - "suite-a": []string{"test-a", "test-b"}, - }, - "module-b": net.KnownTestsResponseDataSuites{ - "suite-b": []string{"test-c"}, - "suite-c": []string{"test-d", "test-e"}, - }, - }, - }) - if known.Modules != 2 || known.Suites != 3 || known.Tests != 5 { - t.Errorf("unexpected known test summary: %+v", known) - } - - managed := newManagedFlakyTestsReport(&net.TestManagementTestsResponseDataModules{ - Modules: map[string]net.TestManagementTestsResponseDataSuites{ - "module-a": { - Suites: map[string]net.TestManagementTestsResponseDataTests{ - "suite-a": { - Tests: map[string]net.TestManagementTestsResponseDataTestProperties{ - "test-a": {Properties: net.TestManagementTestsResponseDataTestPropertiesAttributes{Quarantined: true}}, - "test-b": {Properties: net.TestManagementTestsResponseDataTestPropertiesAttributes{Disabled: true}}, - "test-c": {Properties: net.TestManagementTestsResponseDataTestPropertiesAttributes{AttemptToFix: true}}, - }, - }, - }, - }, - }, - }) - if managed.Total != 3 || managed.Quarantined != 1 || managed.Disabled != 1 || managed.AttemptToFix != 1 { - t.Errorf("unexpected managed flaky test summary: %+v", managed) - } -} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 36bbe0a..fc23c32 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -2,159 +2,56 @@ package runner import ( "context" - "errors" "fmt" "io" "log/slog" "os" - "slices" "strings" "time" ciUtils "github.com/DataDog/ddtest/civisibility/utils" - "github.com/DataDog/ddtest/internal/ciprovider" "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/planner" "github.com/DataDog/ddtest/internal/platform" + "github.com/DataDog/ddtest/internal/runmetadata" "github.com/DataDog/ddtest/internal/settings" - "github.com/DataDog/ddtest/internal/testoptimization" ) type Runner interface { - Plan(ctx context.Context) error Run(ctx context.Context) error } +type Planner interface { + Plan(ctx context.Context) error + LoadPlan() (planner.PlanInfo, error) + DistributeTestFiles(testFiles []string, parallelRunners int) [][]string +} + type TestRunner struct { - testFiles map[string]struct{} - suiteAggregates map[testSuiteKey]testSuiteAggregate - suitesBySourceFile map[string][]testSuiteKey - testSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo - testFileWeights map[string]int - testFileDurationSources map[string]testFileDurationSource - skippablePercentage float64 - planReport planReport - runInfoReport runInfoReport - platformDetector platform.PlatformDetector - optimizationClient testoptimization.TestOptimizationClient - durationsClient testoptimization.TestSuiteDurationsClient - ciProviderDetector ciprovider.CIProviderDetector - reportWriter io.Writer + platformDetector platform.PlatformDetector + planner Planner + reportWriter io.Writer } func New() *TestRunner { - runner := newTestRunnerWithDefaults() - runner.platformDetector = platform.NewPlatformDetector() - runner.optimizationClient = testoptimization.NewDatadogClient() - runner.durationsClient = testoptimization.NewDurationsClient() - runner.ciProviderDetector = ciprovider.NewCIProviderDetector() - return runner + return NewWithDependencies(platform.NewPlatformDetector(), planner.New()) } func NewWithDependencies( platformDetector platform.PlatformDetector, - optimizationClient testoptimization.TestOptimizationClient, - durationsClient testoptimization.TestSuiteDurationsClient, - ciProviderDetector ciprovider.CIProviderDetector, + testPlanner Planner, ) *TestRunner { runner := newTestRunnerWithDefaults() runner.platformDetector = platformDetector - runner.optimizationClient = optimizationClient - runner.durationsClient = durationsClient - runner.ciProviderDetector = ciProviderDetector + runner.planner = testPlanner return runner } func newTestRunnerWithDefaults() *TestRunner { return &TestRunner{ - testFiles: make(map[string]struct{}), - suiteAggregates: make(map[testSuiteKey]testSuiteAggregate), - suitesBySourceFile: make(map[string][]testSuiteKey), - testSuiteDurations: make(map[string]map[string]testoptimization.TestSuiteDurationInfo), - testFileWeights: make(map[string]int), - testFileDurationSources: make(map[string]testFileDurationSource), - skippablePercentage: 0.0, - reportWriter: os.Stderr, - } -} - -func (tr *TestRunner) Plan(ctx context.Context) error { - slog.Info("Planning test execution...") - - if err := tr.PrepareTestOptimization(ctx); err != nil { - return err - } - - if err := writePlanFile(constants.ManifestPath, []byte(constants.ManifestVersion+"\n")); err != nil { - return fmt.Errorf("failed to write test optimization manifest: %w", err) - } - - if err := tr.storeTestOptimizationPlanCache(); err != nil { - return fmt.Errorf("failed to store test optimization plan cache: %w", err) - } - - testFileNames := make([]string, 0, len(tr.testFileWeights)) - for testFile := range tr.testFileWeights { - testFileNames = append(testFileNames, testFile) - } - slices.Sort(testFileNames) - - content := strings.Join(testFileNames, "\n") - if len(testFileNames) > 0 { - content += "\n" - } - - if err := writePlanFile(constants.TestFilesOutputPath, []byte(content)); err != nil { - return fmt.Errorf("failed to write test files: %w", err) - } - - percentageContent := fmt.Sprintf("%.2f", tr.skippablePercentage) - if err := writePlanFile(constants.SkippablePercentageOutputPath, []byte(percentageContent)); err != nil { - return fmt.Errorf("failed to write skippable percentage: %w", err) - } - - // Calculate and write parallel runners count - parallelRunnerSplit := calculateParallelRunnerSplit( - tr.testFileWeights, - settings.GetMinParallelism(), - settings.GetMaxParallelism(), - settings.GetParallelRunnerOverhead(), - ) - parallelRunners := parallelRunnerSplit.parallelRunners - runnersContent := fmt.Sprintf("%d", parallelRunners) - if err := writePlanFile(constants.ParallelRunnersOutputPath, []byte(runnersContent)); err != nil { - return fmt.Errorf("failed to write parallel runners: %w", err) - } - - // Detect and configure CI provider if available - if ciProvider, err := tr.ciProviderDetector.DetectCIProvider(); err == nil { - slog.Info("CI provider detected, configuring with parallel runners", - "provider", ciProvider.Name(), "parallelRunners", parallelRunners) - - if err := ciProvider.Configure(parallelRunners); err != nil { - slog.Warn("Failed to configure CI provider", "provider", ciProvider.Name(), "error", err) - } - } else { - slog.Info("No CI provider detected or CI provider is not supported, running tests without CI integration", "error", err) - } - - // Split test files for runners - if err := CreateTestSplits(tr.testFileWeights, parallelRunners, constants.TestFilesOutputPath); err != nil { - return fmt.Errorf("failed to create test splits: %w", err) - } - - tr.planReport.Split = parallelRunnerSplit - slog.Info("Test execution planning completed", - "parallelRunners", parallelRunners, - "expectedWallTime", parallelRunnerSplit.wallTimeDuration(), - "imbalance", parallelRunnerSplit.imbalanceDuration(), - "expectedTotalRuntime", parallelRunnerSplit.totalRuntimeDuration(), - "testFilesCount", len(tr.testFileWeights)) - - if settings.GetReportEnabled() { - printPlanReport(tr.reportWriter, tr.planReport) + planner: planner.New(), + reportWriter: os.Stderr, } - - return nil } func (tr *TestRunner) Run(ctx context.Context) error { @@ -163,34 +60,23 @@ func (tr *TestRunner) Run(ctx context.Context) error { slog.Info("Test optimization planning data not found, running planning phase...") // Run Setup if the file doesn't exist - if err := tr.Plan(ctx); err != nil { + if err := tr.planner.Plan(ctx); err != nil { return fmt.Errorf("failed to run planning phase: %w", err) } } else if err != nil { return fmt.Errorf("failed to check parallel runners count at %s: %w", constants.ParallelRunnersOutputPath, err) } - if err := tr.restoreTestOptimizationPlanCache(); err != nil { - if errors.Is(err, os.ErrNotExist) { - slog.Debug("Test optimization plan cache not found; CI-node subsplits will use default weights", - "file", testoptimization.TestOptimizationPlanCacheFile) - } else { - slog.Warn("Failed to restore test optimization plan cache; CI-node subsplits will use default weights", - "file", testoptimization.TestOptimizationPlanCacheFile, "error", err) - } - } - - runnersData, err := os.ReadFile(constants.ParallelRunnersOutputPath) + planInfo, err := tr.planner.LoadPlan() if err != nil { - return fmt.Errorf("failed to read parallel runners count from %s: %w", constants.ParallelRunnersOutputPath, err) + slog.Error("Test optimization plan is not available", "error", err) + return fmt.Errorf("test optimization plan is not available: %w", err) } - runnersString := strings.TrimSpace(string(runnersData)) - parallelRunners := 0 - if _, err := fmt.Sscanf(runnersString, "%d", ¶llelRunners); err != nil { - return fmt.Errorf("failed to parse parallel runners count from %s: %w", runnersString, err) + parallelRunners, err := readParallelRunnersCount() + if err != nil { + return err } - slog.Info("Got parallel runners count", "parallelRunners", parallelRunners) // Parse worker environment variables if provided in settings @@ -209,16 +95,17 @@ func (tr *TestRunner) Run(ctx context.Context) error { return fmt.Errorf("failed to detect framework: %w", err) } slog.Info("Framework detected", "framework", framework.Name()) - if tr.runInfoReport.isZero() { - tr.runInfoReport = newRunInfoReport(ciUtils.GetCITags(), nil, detectedPlatform.Name(), framework.Name()) + runInfo := runmetadata.New(ciUtils.GetCITags()) + if planInfo.IsZero() { + planInfo = planner.NewPlanInfo(nil, detectedPlatform.Name(), framework.Name()) } ciNode := settings.GetCiNode() startTime := time.Now() - executor := newTestExecutor(ctx, framework, workerEnvMap) + executor := newTestExecutor(ctx, framework, workerEnvMap, tr.planner) var executionResult runExecutionResult if ciNode >= 0 { - executionResult = executor.runCINode(ciNode, settings.GetCiNodeWorkers(), tr.testFileWeights) + executionResult = executor.runCINode(ciNode, settings.GetCiNodeWorkers()) } else if parallelRunners > 1 { executionResult = executor.runParallel() } else { @@ -227,7 +114,8 @@ func (tr *TestRunner) Run(ctx context.Context) error { if settings.GetReportEnabled() { printRunReport(tr.reportWriter, runReport{ - RunInfo: tr.runInfoReport, + RunInfo: runInfo, + PlanInfo: planInfo, Execution: executionResult.report, Duration: time.Since(startTime), Err: executionResult.err, @@ -235,3 +123,18 @@ func (tr *TestRunner) Run(ctx context.Context) error { } return executionResult.err } + +func readParallelRunnersCount() (int, error) { + runnersData, err := os.ReadFile(constants.ParallelRunnersOutputPath) + if err != nil { + return 0, fmt.Errorf("failed to read parallel runners count from %s: %w", constants.ParallelRunnersOutputPath, err) + } + runnersString := strings.TrimSpace(string(runnersData)) + + parallelRunners := 0 + if _, err := fmt.Sscanf(runnersString, "%d", ¶llelRunners); err != nil { + return 0, fmt.Errorf("failed to parse parallel runners count from %s: %w", runnersString, err) + } + + return parallelRunners, nil +} diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 09a37a1..4bb7f14 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -3,986 +3,253 @@ package runner import ( "context" "errors" - "fmt" - "maps" "os" - "os/exec" "path/filepath" "slices" - "strconv" "strings" - "sync" "testing" - "github.com/DataDog/ddtest/civisibility/utils/net" - "github.com/DataDog/ddtest/internal/ciprovider" "github.com/DataDog/ddtest/internal/constants" - "github.com/DataDog/ddtest/internal/framework" - "github.com/DataDog/ddtest/internal/platform" + "github.com/DataDog/ddtest/internal/planner" "github.com/DataDog/ddtest/internal/settings" - "github.com/DataDog/ddtest/internal/testoptimization" + "github.com/spf13/viper" ) -// Mock implementations for testing - -// MockPlatformDetector mocks platform detection -type MockPlatformDetector struct { - Platform platform.Platform - Err error -} - -func (m *MockPlatformDetector) DetectPlatform() (platform.Platform, error) { - return m.Platform, m.Err -} - -// MockPlatform mocks a platform -type MockPlatform struct { - PlatformName string - Tags map[string]string - TagsErr error - Framework framework.Framework - FrameworkErr error - SanityErr error -} - -func (m *MockPlatform) Name() string { - return m.PlatformName -} - -func (m *MockPlatform) CreateTagsMap() (map[string]string, error) { - return m.Tags, m.TagsErr -} - -func (m *MockPlatform) DetectFramework() (framework.Framework, error) { - return m.Framework, m.FrameworkErr -} - -func (m *MockPlatform) SanityCheck() error { - return m.SanityErr -} - -// MockFramework mocks a testing framework -type MockFramework struct { - FrameworkName string - Tests []testoptimization.Test - TestFiles []string - Err error // Used by both DiscoverTests and DiscoverTestFiles if specific errors are nil - DiscoverTestsErr error // If set, overrides Err for DiscoverTests - DiscoverTestFilesErr error // If set, overrides Err for DiscoverTestFiles - RunTestsCalls []RunTestsCall - mu sync.Mutex -} - -type RunTestsCall struct { - TestFiles []string - EnvMap map[string]string -} - -func (m *MockFramework) Name() string { - return m.FrameworkName -} - -func (m *MockFramework) DiscoverTests(ctx context.Context) ([]testoptimization.Test, error) { - if m.DiscoverTestsErr != nil { - return m.Tests, m.DiscoverTestsErr - } - return m.Tests, m.Err +type fakePlanner struct { + planCalls int + loadCalls int + distributeCalls int + planFunc func(context.Context) error + distributeFunc func([]string, int) [][]string + plan planner.PlanInfo + loadErr error + distributedTestFiles [][]string + distributedWorkerNums []int } -func (m *MockFramework) DiscoverTestFiles() ([]string, error) { - if m.DiscoverTestFilesErr != nil { - return m.TestFiles, m.DiscoverTestFilesErr +func (f *fakePlanner) Plan(ctx context.Context) error { + f.planCalls++ + if f.planFunc != nil { + return f.planFunc(ctx) } - return m.TestFiles, m.Err -} - -func (m *MockFramework) RunTests(ctx context.Context, testFiles []string, envMap map[string]string) error { - // Record the call - m.mu.Lock() - m.RunTestsCalls = append(m.RunTestsCalls, RunTestsCall{ - TestFiles: slices.Clone(testFiles), - EnvMap: maps.Clone(envMap), - }) - m.mu.Unlock() - return m.Err -} - -func (m *MockFramework) SetPlatformEnv(platformEnv map[string]string) { - // No-op for mock -} - -func (m *MockFramework) GetPlatformEnv() map[string]string { return nil } -func (m *MockFramework) GetRunTestsCallsCount() int { - m.mu.Lock() - defer m.mu.Unlock() - return len(m.RunTestsCalls) -} - -func (m *MockFramework) GetRunTestsCalls() []RunTestsCall { - m.mu.Lock() - defer m.mu.Unlock() - return slices.Clone(m.RunTestsCalls) -} - -// MockTestOptimizationClient mocks the test optimization client -type MockTestOptimizationClient struct { - InitializeCalled bool - InitializeErr error - Settings *net.SettingsResponseData - SkippableTests map[string]bool - KnownTests *net.KnownTestsResponseData - TestManagementTests *net.TestManagementTestsResponseDataModules - ShutdownCalled bool - Tags map[string]string -} - -func (m *MockTestOptimizationClient) Initialize(tags map[string]string) error { - m.InitializeCalled = true - if m.Tags == nil { - m.Tags = make(map[string]string) - } - maps.Copy(m.Tags, tags) - return m.InitializeErr -} - -func (m *MockTestOptimizationClient) GetSettings() *net.SettingsResponseData { - return m.Settings -} - -func (m *MockTestOptimizationClient) GetSkippableTests() map[string]bool { - return m.SkippableTests -} - -func (m *MockTestOptimizationClient) GetKnownTests() *net.KnownTestsResponseData { - return m.KnownTests -} - -func (m *MockTestOptimizationClient) GetTestManagementTestsData() *net.TestManagementTestsResponseDataModules { - return m.TestManagementTests -} - -func (m *MockTestOptimizationClient) StoreCacheAndExit() { - m.ShutdownCalled = true -} - -type MockTestSuiteDurationsClient struct { - Durations map[string]map[string]testoptimization.TestSuiteDurationInfo - Err error - Called bool - RepositoryURL string - Service string +func (f *fakePlanner) LoadPlan() (planner.PlanInfo, error) { + f.loadCalls++ + return f.plan, f.loadErr } -func (m *MockTestSuiteDurationsClient) GetTestSuiteDurations(repositoryURL, service string) (map[string]map[string]testoptimization.TestSuiteDurationInfo, error) { - m.Called = true - m.RepositoryURL = repositoryURL - m.Service = service - if m.Err != nil { - return nil, m.Err - } - if m.Durations == nil { - return map[string]map[string]testoptimization.TestSuiteDurationInfo{}, nil - } - return m.Durations, nil -} - -// MockCIProvider mocks a CI provider -type MockCIProvider struct { - ProviderName string - ConfigureCalled bool - ConfigureErr error - ParallelRunners int -} - -func (m *MockCIProvider) Name() string { - return m.ProviderName -} - -func (m *MockCIProvider) Configure(parallelRunners int) error { - m.ConfigureCalled = true - m.ParallelRunners = parallelRunners - return m.ConfigureErr -} - -// MockCIProviderDetector mocks CI provider detection -type MockCIProviderDetector struct { - CIProvider ciprovider.CIProvider - Err error -} - -func (m *MockCIProviderDetector) DetectCIProvider() (ciprovider.CIProvider, error) { - return m.CIProvider, m.Err -} - -// Helper function to create a default mock CI provider detector that returns no provider -func newDefaultMockCIProviderDetector() *MockCIProviderDetector { - return &MockCIProviderDetector{ - Err: errors.New("no CI provider detected"), +func (f *fakePlanner) DistributeTestFiles(testFiles []string, parallelRunners int) [][]string { + f.distributeCalls++ + f.distributedTestFiles = append(f.distributedTestFiles, slices.Clone(testFiles)) + f.distributedWorkerNums = append(f.distributedWorkerNums, parallelRunners) + if f.distributeFunc != nil { + return f.distributeFunc(testFiles, parallelRunners) } + return distributeRoundRobin(testFiles, parallelRunners) } -func assertFileContent(t *testing.T, path string, expected string) { - t.Helper() +func TestTestRunner_Run_PlansThroughPublicClientWhenArtifactsMissing(t *testing.T) { + withRunnerTestSettings(t) + chdirTemp(t) - content, err := os.ReadFile(path) - if err != nil { - t.Fatalf("failed to read %s: %v", path, err) - } - if string(content) != expected { - t.Fatalf("expected %s content %q, got %q", path, expected, string(content)) - } -} - -func TestNew(t *testing.T) { - runner := New() - - if runner == nil { - t.Error("New() should return non-nil TestRunner") - return - } - - if len(runner.testFiles) != 0 { - t.Error("New() should initialize testFiles to empty map") - } - - if len(runner.suiteAggregates) != 0 { - t.Error("New() should initialize suiteAggregates to empty map") - } - - if len(runner.suitesBySourceFile) != 0 { - t.Error("New() should initialize suitesBySourceFile to empty map") - } - - if runner.skippablePercentage != 0.0 { - t.Errorf("New() should initialize skippablePercentage to 0.0, got %f", runner.skippablePercentage) - } - - if runner.platformDetector == nil { - t.Error("New() should initialize platformDetector") - } - - if runner.optimizationClient == nil { - t.Error("New() should initialize optimizationClient") - } - - if runner.durationsClient == nil { - t.Error("New() should initialize durationsClient") - } -} - -func TestNewWithDependencies(t *testing.T) { - mockPlatformDetector := &MockPlatformDetector{} - mockOptimizationClient := &MockTestOptimizationClient{} - mockDurationsClient := &MockTestSuiteDurationsClient{} - mockCIProviderDetector := newDefaultMockCIProviderDetector() - - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, mockDurationsClient, mockCIProviderDetector) - - if runner == nil { - t.Error("NewWithDependencies() should return non-nil TestRunner") - return - } - - if runner.platformDetector != mockPlatformDetector { - t.Error("NewWithDependencies() should use injected platformDetector") - } - - if runner.optimizationClient != mockOptimizationClient { - t.Error("NewWithDependencies() should use injected optimizationClient") - } - - if runner.durationsClient != mockDurationsClient { - t.Error("NewWithDependencies() should use injected durationsClient") - } - - if len(runner.testFiles) != 0 { - t.Error("NewWithDependencies() should initialize testFiles to empty map") - } - - if len(runner.suiteAggregates) != 0 { - t.Error("NewWithDependencies() should initialize suiteAggregates to empty map") - } - - if len(runner.suitesBySourceFile) != 0 { - t.Error("NewWithDependencies() should initialize suitesBySourceFile to empty map") - } -} - -func TestTestRunner_Setup_WithParallelRunners(t *testing.T) { - // Create a temporary directory for test output - tempDir := t.TempDir() - - // Save current working directory and change to temp dir - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) - - // Create .testoptimization directory - _ = os.MkdirAll(constants.PlanDirectory, 0755) - - // Set parallelism to 1 to test single runner behavior - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") - defer func() { - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") - }() - settings.Init() - logs := captureLogs(t) - - // Setup mocks for a test with 40% skippable percentage - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: []testoptimization.Test{ - {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, - {Suite: "TestSuite1", Name: "test2", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, - {Suite: "TestSuite2", Name: "test3", Parameters: "", SuiteSourceFile: "test/file2_test.rb"}, - {Suite: "TestSuite3", Name: "test4", Parameters: "", SuiteSourceFile: "test/file3_test.rb"}, - {Suite: "TestSuite4", Name: "test5", Parameters: "", SuiteSourceFile: "test/file4_test.rb"}, + framework := &MockFramework{FrameworkName: "rspec"} + platform := &MockPlatform{PlatformName: "ruby", Framework: framework} + testPlanner := &fakePlanner{ + planFunc: func(ctx context.Context) error { + writeRunnerTestFile(t, constants.ParallelRunnersOutputPath, "1") + writeRunnerTestFile(t, constants.TestFilesOutputPath, "spec/a_spec.rb\n") + return nil }, - } - - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, - } - - mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} - mockOptimizationClient := &MockTestOptimizationClient{ - SkippableTests: map[string]bool{ - "TestSuite1.test2.": true, // Skip test2 - "TestSuite4.test5.": true, // Skip test5 + plan: planner.PlanInfo{ + Platform: "ruby", + Framework: "rspec", }, } + runner := NewWithDependencies(&MockPlatformDetector{Platform: platform}, testPlanner) - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - - // Run Setup - err := runner.Plan(context.Background()) - if err != nil { - t.Fatalf("Setup() should not return error, got: %v", err) + if err := runner.Run(context.Background()); err != nil { + t.Fatalf("Run() returned error: %v", err) } - // Expected: 1 (since max=1) - content, err := os.ReadFile(constants.ParallelRunnersOutputPath) - if err != nil { - t.Fatalf("Failed to read parallel runners file: %v", err) + if testPlanner.planCalls != 1 { + t.Fatalf("expected planner Plan() to be called once, got %d", testPlanner.planCalls) } - - expected := "1" - if string(content) != expected { - t.Errorf("Expected parallel runners file content '%s', got '%s'", expected, string(content)) + if testPlanner.loadCalls != 1 { + t.Fatalf("expected LoadPlan() to be called once, got %d", testPlanner.loadCalls) } - - logOutput := logs.String() - if !strings.Contains(logOutput, "Test execution planning completed") || - !strings.Contains(logOutput, "parallelRunners=1") || - !strings.Contains(logOutput, "expectedWallTime=") || - !strings.Contains(logOutput, "imbalance=") || - !strings.Contains(logOutput, "expectedTotalRuntime=") { - t.Errorf("Expected planning log with selected split information, got logs: %s", logOutput) + if testPlanner.distributeCalls != 0 { + t.Fatalf("expected DistributeTestFiles() not to be called outside CI-node worker mode, got %d", testPlanner.distributeCalls) + } + calls := framework.GetRunTestsCalls() + if len(calls) != 1 || !slices.Equal(calls[0].TestFiles, []string{"spec/a_spec.rb"}) { + t.Fatalf("expected runner to execute planned test file, got %+v", calls) } } -func TestTestRunner_Plan_WritesManifestAndRunnerLayout(t *testing.T) { - tempDir := t.TempDir() - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) +func TestTestRunner_Run_UsesExistingArtifactsWithoutPlanning(t *testing.T) { + withRunnerTestSettings(t) + chdirTemp(t) + writeRunnerTestFile(t, constants.ParallelRunnersOutputPath, "1") + writeRunnerTestFile(t, constants.TestFilesOutputPath, "spec/existing_spec.rb\n") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") - defer func() { - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") - }() - settings.Init() + framework := &MockFramework{FrameworkName: "rspec"} + platform := &MockPlatform{PlatformName: "ruby", Framework: framework} + testPlanner := &fakePlanner{} + runner := NewWithDependencies(&MockPlatformDetector{Platform: platform}, testPlanner) - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: []testoptimization.Test{ - {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, - {Suite: "TestSuite2", Name: "test2", Parameters: "", SuiteSourceFile: "test/file2_test.rb"}, - }, - } - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, + if err := runner.Run(context.Background()); err != nil { + t.Fatalf("Run() returned error: %v", err) } - runner := NewWithDependencies( - &MockPlatformDetector{Platform: mockPlatform}, - &MockTestOptimizationClient{SkippableTests: map[string]bool{}}, - &MockTestSuiteDurationsClient{}, - newDefaultMockCIProviderDetector(), - ) - - if err := runner.Plan(context.Background()); err != nil { - t.Fatalf("Plan() should not return error, got: %v", err) + if testPlanner.planCalls != 0 { + t.Fatalf("expected planner Plan() not to be called, got %d calls", testPlanner.planCalls) } - - assertFileContent(t, constants.ManifestPath, constants.ManifestVersion+"\n") - - expectedTestFiles := "test/file1_test.rb\ntest/file2_test.rb\n" - assertFileContent(t, constants.TestFilesOutputPath, expectedTestFiles) - - assertFileContent(t, constants.ParallelRunnersOutputPath, "1") - assertFileContent(t, constants.SkippablePercentageOutputPath, "0.00") - - assertFileContent(t, filepath.Join(constants.TestsSplitDir, "runner-0"), expectedTestFiles) -} - -func TestTestRunner_Plan_DoesNotPrintReportWhenDisabled(t *testing.T) { - tempDir := t.TempDir() - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) - - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED", "false") - defer func() { - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED") - settings.Init() - }() - settings.Init() - - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: []testoptimization.Test{ - {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, - }, + if testPlanner.loadCalls != 1 { + t.Fatalf("expected LoadPlan() to be called once, got %d", testPlanner.loadCalls) } - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, - } - - runner := NewWithDependencies( - &MockPlatformDetector{Platform: mockPlatform}, - &MockTestOptimizationClient{SkippableTests: map[string]bool{}}, - &MockTestSuiteDurationsClient{}, - newDefaultMockCIProviderDetector(), - ) - var output strings.Builder - runner.reportWriter = &output - - if err := runner.Plan(context.Background()); err != nil { - t.Fatalf("Plan() should not return error, got: %v", err) + if testPlanner.distributeCalls != 0 { + t.Fatalf("expected DistributeTestFiles() not to be called outside CI-node worker mode, got %d", testPlanner.distributeCalls) } - if output.Len() != 0 { - t.Errorf("Expected no report output when report is disabled, got: %s", output.String()) + calls := framework.GetRunTestsCalls() + if len(calls) != 1 || !slices.Equal(calls[0].TestFiles, []string{"spec/existing_spec.rb"}) { + t.Fatalf("expected runner to execute existing artifact test file, got %+v", calls) } } -func TestTestRunner_Plan_ChoosesParallelismFromFanoutAdjustedSplit(t *testing.T) { - tempDir := t.TempDir() - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) +func TestTestRunner_Run_ReturnsErrorWhenPlanUnavailable(t *testing.T) { + withRunnerTestSettings(t) + chdirTemp(t) + writeRunnerTestFile(t, constants.ParallelRunnersOutputPath, "1") + writeRunnerTestFile(t, constants.TestFilesOutputPath, "spec/existing_spec.rb\n") + logs := captureLogs(t) - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "2") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "4") - defer func() { - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") - }() - settings.Init() + framework := &MockFramework{FrameworkName: "rspec"} + platform := &MockPlatform{PlatformName: "ruby", Framework: framework} + loadErr := errors.New("plan cache missing") + testPlanner := &fakePlanner{loadErr: loadErr} + runner := NewWithDependencies(&MockPlatformDetector{Platform: platform}, testPlanner) - var tests []testoptimization.Test - skippableTests := map[string]bool{} - for suiteIndex := range 4 { - suite := fmt.Sprintf("TestSuite%d", suiteIndex) - sourceFile := fmt.Sprintf("test/file%d_test.rb", suiteIndex) - for testIndex := range 10 { - name := fmt.Sprintf("test%d", testIndex) - tests = append(tests, testoptimization.Test{ - Suite: suite, - Name: name, - Parameters: "", - SuiteSourceFile: sourceFile, - }) - if testIndex > 0 { - skippableTests[fmt.Sprintf("%s.%s.", suite, name)] = true - } - } + err := runner.Run(context.Background()) + if !errors.Is(err, loadErr) { + t.Fatalf("expected Run() to return LoadPlan() error, got %v", err) } - - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: tests, + if testPlanner.loadCalls != 1 { + t.Fatalf("expected LoadPlan() to be called once, got %d", testPlanner.loadCalls) } - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, + if framework.GetRunTestsCallsCount() != 0 { + t.Fatalf("expected runner not to execute tests when plan is unavailable, got %d calls", framework.GetRunTestsCallsCount()) } - - runner := NewWithDependencies( - &MockPlatformDetector{Platform: mockPlatform}, - &MockTestOptimizationClient{SkippableTests: skippableTests}, - &MockTestSuiteDurationsClient{}, - newDefaultMockCIProviderDetector(), - ) - - if err := runner.Plan(context.Background()); err != nil { - t.Fatalf("Plan() should not return error, got: %v", err) + if !strings.Contains(logs.String(), "level=ERROR") || + !strings.Contains(logs.String(), "Test optimization plan is not available") { + t.Fatalf("expected error log for unavailable plan, got logs: %s", logs.String()) } - - assertFileContent(t, constants.SkippablePercentageOutputPath, "90.00") - assertFileContent(t, constants.ParallelRunnersOutputPath, "2") } -func TestTestRunner_Setup_WithCIProvider(t *testing.T) { - tempDir := t.TempDir() - - // Save current working directory and change to temp dir - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) - - // Create .testoptimization directory - _ = os.MkdirAll(constants.PlanDirectory, 0755) - - // Set parallelism to 1 to test single runner behavior - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") - defer func() { - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") - }() +func TestTestRunner_Run_CINodeWorkersRunWithoutLoadedWeights(t *testing.T) { + withRunnerTestSettings(t) + t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_CI_NODE", "0") + t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_CI_NODE_WORKERS", "2") + viper.Reset() settings.Init() + chdirTemp(t) + writeRunnerTestFile(t, constants.ParallelRunnersOutputPath, "1") + writeRunnerTestFile(t, filepath.Join(constants.TestsSplitDir, "runner-0"), strings.Join([]string{ + "spec/fast_a_spec.rb", + "spec/fast_b_spec.rb", + "spec/slow_spec.rb", + }, "\n")+"\n") - // Setup mocks for test with CI provider - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: []testoptimization.Test{ - {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, - {Suite: "TestSuite2", Name: "test2", Parameters: "", SuiteSourceFile: "test/file2_test.rb"}, - }, - } + framework := &MockFramework{FrameworkName: "rspec"} + platform := &MockPlatform{PlatformName: "ruby", Framework: framework} + testPlanner := &fakePlanner{} + runner := NewWithDependencies(&MockPlatformDetector{Platform: platform}, testPlanner) - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, + if err := runner.Run(context.Background()); err != nil { + t.Fatalf("Run() returned error: %v", err) } - mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} - mockOptimizationClient := &MockTestOptimizationClient{ - SkippableTests: map[string]bool{ - "TestSuite1.test1": true, // Skip test1 = 50% skippable - }, + calls := framework.GetRunTestsCalls() + if len(calls) != 2 { + t.Fatalf("expected two CI-node worker calls, got %+v", calls) } - - // Mock CI provider that should be called - mockCIProvider := &MockCIProvider{ - ProviderName: "github", + if testPlanner.loadCalls != 1 { + t.Fatalf("expected LoadPlan() to be called once, got %d", testPlanner.loadCalls) } - mockCIProviderDetector := &MockCIProviderDetector{ - CIProvider: mockCIProvider, + if testPlanner.distributeCalls != 1 { + t.Fatalf("expected DistributeTestFiles() to be called once, got %d", testPlanner.distributeCalls) } - - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, mockCIProviderDetector) - - // Run Setup - err := runner.Plan(context.Background()) - if err != nil { - t.Fatalf("Setup() should not return error, got: %v", err) + if len(testPlanner.distributedTestFiles) != 1 || + !slices.Equal(testPlanner.distributedTestFiles[0], []string{ + "spec/fast_a_spec.rb", + "spec/fast_b_spec.rb", + "spec/slow_spec.rb", + }) { + t.Fatalf("expected runner to pass only CI-node file list to planner distribution, got %v", testPlanner.distributedTestFiles) } - - // Verify CI provider Configure was called - if !mockCIProvider.ConfigureCalled { - t.Error("Expected CI provider Configure to be called") + if len(testPlanner.distributedWorkerNums) != 1 || testPlanner.distributedWorkerNums[0] != 2 { + t.Fatalf("expected runner to request 2 worker groups, got %v", testPlanner.distributedWorkerNums) } - - // Verify Configure was called with the correct parallel runners count (1, since max=1) - expectedRunners := 1 - if mockCIProvider.ParallelRunners != expectedRunners { - t.Errorf("Expected CI provider Configure called with %d parallel runners, got %d", - expectedRunners, mockCIProvider.ParallelRunners) - } -} - -func TestTestRunner_Setup_CIProviderDetectionFailure(t *testing.T) { - tempDir := t.TempDir() - - // Save current working directory and change to temp dir - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) - - // Create .testoptimization directory - _ = os.MkdirAll(constants.PlanDirectory, 0755) - - // Setup mocks for test without CI provider - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: []testoptimization.Test{ - {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, - }, - } - - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, + allFiles := make([]string, 0) + for _, call := range calls { + allFiles = append(allFiles, call.TestFiles...) } - - mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} - mockOptimizationClient := &MockTestOptimizationClient{SkippableTests: map[string]bool{}} - - // Mock CI provider detector that fails - mockCIProviderDetector := &MockCIProviderDetector{ - Err: errors.New("no CI provider detected"), - } - - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, mockCIProviderDetector) - - // Run Setup - should succeed even if CI provider detection fails - err := runner.Plan(context.Background()) - if err != nil { - t.Fatalf("Setup() should not fail when CI provider detection fails, got: %v", err) + slices.Sort(allFiles) + expectedFiles := []string{"spec/fast_a_spec.rb", "spec/fast_b_spec.rb", "spec/slow_spec.rb"} + if !slices.Equal(allFiles, expectedFiles) { + t.Fatalf("expected CI-node workers to run all node files without loaded weights, got %+v", calls) } } -func TestTestRunner_Setup_CIProviderConfigureFailure(t *testing.T) { - tempDir := t.TempDir() - - // Save current working directory and change to temp dir - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) - - _ = os.MkdirAll(constants.PlanDirectory, 0755) - - // Setup mocks for test with failing CI provider - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: []testoptimization.Test{ - {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, - }, - } - - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, +func distributeRoundRobin(testFiles []string, parallelRunners int) [][]string { + if parallelRunners <= 0 { + parallelRunners = 1 } - - mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} - mockOptimizationClient := &MockTestOptimizationClient{SkippableTests: map[string]bool{}} - - // Mock CI provider that fails during configuration - mockCIProvider := &MockCIProvider{ - ProviderName: "github", - ConfigureErr: errors.New("configuration failed"), - } - mockCIProviderDetector := &MockCIProviderDetector{ - CIProvider: mockCIProvider, - } - - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, mockCIProviderDetector) - - // Run Setup - should succeed even if CI provider configuration fails - err := runner.Plan(context.Background()) - if err != nil { - t.Fatalf("Setup() should not fail when CI provider configuration fails, got: %v", err) + groups := make([][]string, parallelRunners) + for i := range groups { + groups[i] = []string{} } - - // Verify CI provider Configure was attempted - if !mockCIProvider.ConfigureCalled { - t.Error("Expected CI provider Configure to be called even if it fails") + for index, testFile := range testFiles { + groups[index%parallelRunners] = append(groups[index%parallelRunners], testFile) } + return groups } -func TestTestRunner_Setup_WithTestSplit(t *testing.T) { - t.Run("single runner - copy test-files.txt to runner-0", func(t *testing.T) { - // Create a temporary directory for test output - tempDir := t.TempDir() - - // Save current working directory and change to temp dir - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) - - // Create .testoptimization directory - _ = os.MkdirAll(constants.PlanDirectory, 0755) - - // Set parallelism to 1 to test single runner behavior - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") - defer func() { - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") - }() +func withRunnerTestSettings(t *testing.T) { + t.Helper() + t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED", "false") + t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_CI_NODE", "-1") + t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_CI_NODE_WORKERS", "1") + viper.Reset() + settings.Init() + t.Cleanup(func() { + viper.Reset() settings.Init() - - // Setup mocks for single runner scenario - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: []testoptimization.Test{ - {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, - {Suite: "TestSuite2", Name: "test2", Parameters: "", SuiteSourceFile: "test/file2_test.rb"}, - }, - } - - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, - } - - mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} - mockOptimizationClient := &MockTestOptimizationClient{ - SkippableTests: map[string]bool{}, // No tests skipped - } - - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - - // Run Setup - err := runner.Plan(context.Background()) - if err != nil { - t.Fatalf("Setup() should not return error, got: %v", err) - } - - // Verify that tests-split directory was created - if _, err := os.Stat(constants.TestsSplitDir); os.IsNotExist(err) { - t.Error("Expected tests-split directory to be created when parallelRunners = 1") - } - - // Verify that runner-0 file was created - runnerFilePath := filepath.Join(constants.TestsSplitDir, "runner-0") - if _, err := os.Stat(runnerFilePath); os.IsNotExist(err) { - t.Error("Expected runner-0 file to be created when parallelRunners = 1") - } - - // Verify that runner-0 contains the same content as test-files.txt - testFilesContent, err := os.ReadFile(constants.TestFilesOutputPath) - if err != nil { - t.Fatalf("Failed to read test-files.txt: %v", err) - } - - runnerContent, err := os.ReadFile(runnerFilePath) - if err != nil { - t.Fatalf("Failed to read runner-0 file: %v", err) - } - - if string(testFilesContent) != string(runnerContent) { - t.Errorf("Expected runner-0 content to match test-files.txt content.\ntest-files.txt: %q\nrunner-0: %q", - string(testFilesContent), string(runnerContent)) - } - - // Verify the content contains the expected test files - expectedContent := "test/file1_test.rb\ntest/file2_test.rb\n" - if string(runnerContent) != expectedContent { - t.Errorf("Expected runner-0 content %q, got %q", expectedContent, string(runnerContent)) - } - }) - - t.Run("multiple runners - split files created", func(t *testing.T) { - // Create a temporary directory for test output - tempDir := t.TempDir() - - // Save current working directory and change to temp dir - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) - - // Create .testoptimization directory - _ = os.MkdirAll(constants.PlanDirectory, 0755) - - // Setup mocks with test files that will create a predictable distribution - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: []testoptimization.Test{ - {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, - {Suite: "TestSuite1", Name: "test2", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, // 2 tests in file1 - {Suite: "TestSuite2", Name: "test3", Parameters: "", SuiteSourceFile: "test/file2_test.rb"}, // 1 test in file2 - {Suite: "TestSuite3", Name: "test4", Parameters: "", SuiteSourceFile: "test/file3_test.rb"}, // 1 test in file3 - }, - } - - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, - } - - mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} - mockOptimizationClient := &MockTestOptimizationClient{ - SkippableTests: map[string]bool{}, // No tests skipped - } - - expectedParallelRunnersCount := 2 - maxParallelism := 4 - // Set environment variables to force multiple parallel runners - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "2") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", strconv.Itoa(maxParallelism)) - defer func() { - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") - }() - - // Reinitialize settings to pick up environment variables - settings.Init() - - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - - // Run Setup - err := runner.Plan(context.Background()) - if err != nil { - t.Fatalf("Setup() should not return error, got: %v", err) - } - - // Verify that tests-split directory was created - if _, err := os.Stat(constants.TestsSplitDir); os.IsNotExist(err) { - t.Error("Expected tests-split directory to be created") - } - - // With this split, 2 runners are as fast as 3 and more balanced. - // Verify runner files exist - for i := range expectedParallelRunnersCount { - runnerPath := filepath.Join(constants.TestsSplitDir, fmt.Sprintf("runner-%d", i)) - if _, err := os.Stat(runnerPath); os.IsNotExist(err) { - t.Errorf("Expected runner-%d file to exist", i) - } - } - - // Verify content of runner files - // With the test distribution (file1: 2 tests, file2: 1 test, file3: 1 test), - // expected: runner 0 gets file1 (2 tests), runner 1 gets file2+file3 (2 tests). - runner0Content, err := os.ReadFile(filepath.Join(constants.TestsSplitDir, "runner-0")) - if err != nil { - t.Fatalf("Failed to read runner-0 file: %v", err) - } - - // Verify runner-0 has the largest file (file1 with 2 tests) - runner0Files := strings.Fields(strings.TrimSpace(string(runner0Content))) - if !slices.Contains(runner0Files, "test/file1_test.rb") { - t.Error("Expected runner-0 to contain test/file1_test.rb (largest file)") - } - - // Count total files across all runners - totalFiles := 0 - for i := range expectedParallelRunnersCount { - runnerPath := filepath.Join(constants.TestsSplitDir, fmt.Sprintf("runner-%d", i)) - content, err := os.ReadFile(runnerPath) - if err != nil { - continue - } - files := strings.Fields(strings.TrimSpace(string(content))) - totalFiles += len(files) - } - - // Should have all 3 test files distributed - if totalFiles != 3 { - t.Errorf("Expected 3 total files distributed across runners, got %d", totalFiles) - } }) } -// TestTestRunner_Plan_SubdirRootRelativeDiscovery_WritesNormalizedPaths -// reproduces the end-to-end bug from issue #33: Plan writes repo-root-relative paths -// that become invalid for workers running from a monorepo subdirectory. -func TestTestRunner_Plan_SubdirRootRelativeDiscovery_WritesNormalizedPaths(t *testing.T) { - // Create a temp monorepo: repoRoot/core/spec/... - repoRoot := t.TempDir() - - // Initialize git repo at the root - cmd := exec.Command("git", "init") - cmd.Dir = repoRoot - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("failed to init git repo: %v\n%s", err, string(out)) - } - cmd = exec.Command("git", "commit", "--allow-empty", "-m", "init") - cmd.Dir = repoRoot - cmd.Env = append(os.Environ(), - "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@test.com", - "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@test.com", - ) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("failed to create initial commit: %v\n%s", err, string(out)) - } - - coreDir := filepath.Join(repoRoot, "core") - _ = os.MkdirAll(filepath.Join(coreDir, "spec", "models"), 0755) - _ = os.WriteFile(filepath.Join(coreDir, "spec", "models", "order_spec.rb"), []byte("# spec"), 0644) - _ = os.WriteFile(filepath.Join(coreDir, "spec", "models", "payment_spec.rb"), []byte("# spec"), 0644) - - // chdir into subdirectory - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(coreDir) - - // Set parallelism to 1 - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") - defer func() { - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") - _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") - }() - settings.Init() - - // Full discovery returns repo-root-relative paths (the bug) - mockFramework := &MockFramework{ - FrameworkName: "rspec", - Tests: []testoptimization.Test{ - {Suite: "Order", Name: "should be valid", Parameters: "", SuiteSourceFile: "core/spec/models/order_spec.rb"}, - {Suite: "Payment", Name: "should process", Parameters: "", SuiteSourceFile: "core/spec/models/payment_spec.rb"}, - }, - } - - mockPlatform := &MockPlatform{ - PlatformName: "ruby", - Tags: map[string]string{"platform": "ruby"}, - Framework: mockFramework, - } - - mockPlatformDetector := &MockPlatformDetector{Platform: mockPlatform} - mockOptimizationClient := &MockTestOptimizationClient{ - SkippableTests: map[string]bool{}, - } - - runner := NewWithDependencies(mockPlatformDetector, mockOptimizationClient, &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector()) - - err := runner.Plan(context.Background()) - if err != nil { - t.Fatalf("Plan() should not return error, got: %v", err) - } - - // Verify test-files.txt contains CWD-relative paths - testFilesContent, err := os.ReadFile(constants.TestFilesOutputPath) +func chdirTemp(t *testing.T) { + t.Helper() + oldWd, err := os.Getwd() if err != nil { - t.Fatalf("Failed to read test-files.txt: %v", err) - } - - testFilesStr := string(testFilesContent) - if strings.Contains(testFilesStr, "core/") { - t.Errorf("test-files.txt should not contain repo-root prefix 'core/', got:\n%s", testFilesStr) + t.Fatalf("failed to get cwd: %v", err) } - - expectedContent := "spec/models/order_spec.rb\nspec/models/payment_spec.rb\n" - if testFilesStr != expectedContent { - t.Errorf("Expected test-files.txt content:\n%s\nGot:\n%s", expectedContent, testFilesStr) + tempDir := t.TempDir() + if err := os.Chdir(tempDir); err != nil { + t.Fatalf("failed to chdir to temp dir: %v", err) } + t.Cleanup(func() { + _ = os.Chdir(oldWd) + }) +} - // Verify runner-0 split file also contains CWD-relative paths - runnerContent, err := os.ReadFile(filepath.Join(constants.TestsSplitDir, "runner-0")) - if err != nil { - t.Fatalf("Failed to read runner-0: %v", err) +func writeRunnerTestFile(t *testing.T, path string, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatalf("failed to create directory for %s: %v", path, err) } - - runnerStr := string(runnerContent) - if strings.Contains(runnerStr, "core/") { - t.Errorf("runner-0 should not contain repo-root prefix 'core/', got:\n%s", runnerStr) + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("failed to write %s: %v", path, err) } } diff --git a/internal/runner/sequential_executor_test.go b/internal/runner/sequential_executor_test.go index c739170..0d7e1c2 100644 --- a/internal/runner/sequential_executor_test.go +++ b/internal/runner/sequential_executor_test.go @@ -26,7 +26,7 @@ func TestRunSequential_Success(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runSequential() + result := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runSequential() report, err := result.report, result.err if err != nil { t.Fatalf("runSequential() should not return error, got: %v", err) diff --git a/internal/runner/service.go b/internal/runner/service.go deleted file mode 100644 index 9ac3321..0000000 --- a/internal/runner/service.go +++ /dev/null @@ -1,25 +0,0 @@ -package runner - -import ( - "os" - "regexp" - "strings" -) - -var repositoryNameRegex = regexp.MustCompile(`(?m)/([a-zA-Z0-9\-_.]+)$`) - -func resolveServiceName(repositoryURL string) string { - if service := os.Getenv("DD_SERVICE"); service != "" { - return service - } - return serviceNameFromRepositoryURL(repositoryURL) -} - -func serviceNameFromRepositoryURL(repositoryURL string) string { - normalizedRepositoryURL := strings.TrimRight(repositoryURL, "/") - matches := repositoryNameRegex.FindStringSubmatch(normalizedRepositoryURL) - if len(matches) > 1 { - return strings.TrimSuffix(matches[1], ".git") - } - return normalizedRepositoryURL -} diff --git a/internal/runner/test_batch_executor.go b/internal/runner/test_batch_executor.go index 6e61b3c..a29ae27 100644 --- a/internal/runner/test_batch_executor.go +++ b/internal/runner/test_batch_executor.go @@ -9,17 +9,23 @@ import ( "github.com/DataDog/ddtest/internal/framework" ) +type testFilePlanner interface { + DistributeTestFiles(testFiles []string, parallelRunners int) [][]string +} + type testExecutor struct { ctx context.Context framework framework.Framework workerEnvMap map[string]string + planner testFilePlanner } -func newTestExecutor(ctx context.Context, framework framework.Framework, workerEnvMap map[string]string) testExecutor { +func newTestExecutor(ctx context.Context, framework framework.Framework, workerEnvMap map[string]string, planner testFilePlanner) testExecutor { return testExecutor{ ctx: ctx, framework: framework, workerEnvMap: workerEnvMap, + planner: planner, } } diff --git a/internal/runner/test_batch_test.go b/internal/runner/test_batch_executor_test.go similarity index 97% rename from internal/runner/test_batch_test.go rename to internal/runner/test_batch_executor_test.go index 94f1700..eb8a68c 100644 --- a/internal/runner/test_batch_test.go +++ b/internal/runner/test_batch_executor_test.go @@ -59,7 +59,7 @@ func TestRunBatch(t *testing.T) { "STATIC": "value", } - err := newTestExecutor(context.Background(), mockFramework, workerEnvMap).runBatch(testFiles, 5, 3) + err := newTestExecutor(context.Background(), mockFramework, workerEnvMap, roundRobinTestPlanner{}).runBatch(testFiles, 5, 3) if err != nil { t.Fatalf("runBatch() should not return error, got: %v", err) } diff --git a/internal/runner/test_helpers_test.go b/internal/runner/test_helpers_test.go new file mode 100644 index 0000000..f8a9092 --- /dev/null +++ b/internal/runner/test_helpers_test.go @@ -0,0 +1,141 @@ +package runner + +import ( + "bytes" + "context" + "log/slog" + "maps" + "slices" + "sync" + "testing" + + "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/platform" + "github.com/DataDog/ddtest/internal/testoptimization" +) + +func captureLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + originalLogger := slog.Default() + logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + slog.SetDefault(logger) + t.Cleanup(func() { + slog.SetDefault(originalLogger) + }) + return &buf +} + +type MockPlatformDetector struct { + Platform platform.Platform + Err error +} + +func (m *MockPlatformDetector) DetectPlatform() (platform.Platform, error) { + return m.Platform, m.Err +} + +type MockPlatform struct { + PlatformName string + Tags map[string]string + TagsErr error + Framework framework.Framework + FrameworkErr error + SanityErr error +} + +func (m *MockPlatform) Name() string { + return m.PlatformName +} + +func (m *MockPlatform) CreateTagsMap() (map[string]string, error) { + return m.Tags, m.TagsErr +} + +func (m *MockPlatform) DetectFramework() (framework.Framework, error) { + return m.Framework, m.FrameworkErr +} + +func (m *MockPlatform) SanityCheck() error { + return m.SanityErr +} + +type MockFramework struct { + FrameworkName string + Tests []testoptimization.Test + TestFiles []string + Err error + DiscoverTestsErr error + DiscoverTestFilesErr error + RunTestsCalls []RunTestsCall + mu sync.Mutex +} + +type RunTestsCall struct { + TestFiles []string + EnvMap map[string]string +} + +func (m *MockFramework) Name() string { + return m.FrameworkName +} + +func (m *MockFramework) DiscoverTests(ctx context.Context) ([]testoptimization.Test, error) { + if m.DiscoverTestsErr != nil { + return m.Tests, m.DiscoverTestsErr + } + return m.Tests, m.Err +} + +func (m *MockFramework) DiscoverTestFiles() ([]string, error) { + if m.DiscoverTestFilesErr != nil { + return m.TestFiles, m.DiscoverTestFilesErr + } + return m.TestFiles, m.Err +} + +func (m *MockFramework) RunTests(ctx context.Context, testFiles []string, envMap map[string]string) error { + m.mu.Lock() + defer m.mu.Unlock() + m.RunTestsCalls = append(m.RunTestsCalls, RunTestsCall{ + TestFiles: slices.Clone(testFiles), + EnvMap: maps.Clone(envMap), + }) + return m.Err +} + +func (m *MockFramework) SetPlatformEnv(platformEnv map[string]string) { +} + +func (m *MockFramework) GetPlatformEnv() map[string]string { + return nil +} + +func (m *MockFramework) GetRunTestsCallsCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.RunTestsCalls) +} + +func (m *MockFramework) GetRunTestsCalls() []RunTestsCall { + m.mu.Lock() + defer m.mu.Unlock() + return slices.Clone(m.RunTestsCalls) +} + +type roundRobinTestPlanner struct{} + +func (roundRobinTestPlanner) DistributeTestFiles(testFiles []string, parallelRunners int) [][]string { + if parallelRunners <= 0 { + parallelRunners = 1 + } + + groups := make([][]string, parallelRunners) + for i := range groups { + groups[i] = []string{} + } + for index, testFile := range testFiles { + groups[index%parallelRunners] = append(groups[index%parallelRunners], testFile) + } + return groups +} diff --git a/internal/runner/test_optimization_plan_cache.go b/internal/runner/test_optimization_plan_cache.go deleted file mode 100644 index 322bdc8..0000000 --- a/internal/runner/test_optimization_plan_cache.go +++ /dev/null @@ -1,93 +0,0 @@ -package runner - -import ( - "log/slog" - - "github.com/DataDog/ddtest/internal/testoptimization" -) - -type testOptimizationPlanCache struct { - TestSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo `json:"testSuiteDurations"` - SuiteAggregates map[testSuiteKey]testSuiteAggregate `json:"suiteAggregates"` - SuitesBySourceFile map[string][]testSuiteKey `json:"suitesBySourceFile"` - TestFileWeights map[string]int `json:"testFileWeights"` - TestFileDurationSources map[string]testFileDurationSource `json:"testFileDurationSources"` - RunInfo runInfoReport `json:"runInfo"` -} - -func (tr *TestRunner) storeTestOptimizationPlanCache() error { - cache := testOptimizationPlanCache{ - TestSuiteDurations: tr.testSuiteDurations, - SuiteAggregates: tr.suiteAggregates, - SuitesBySourceFile: tr.suitesBySourceFile, - TestFileWeights: tr.testFileWeights, - TestFileDurationSources: tr.testFileDurationSources, - RunInfo: tr.runInfoReport, - } - - return testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache) -} - -func (tr *TestRunner) restoreTestOptimizationPlanCache() error { - var cache testOptimizationPlanCache - if err := testoptimization.NewCacheManager().ReadTestOptimizationPlanCache(&cache); err != nil { - return err - } - - tr.testSuiteDurations = cache.TestSuiteDurations - if tr.testSuiteDurations == nil { - tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) - } - - tr.suiteAggregates = cache.SuiteAggregates - if tr.suiteAggregates == nil { - tr.suiteAggregates = make(map[testSuiteKey]testSuiteAggregate) - } - - tr.suitesBySourceFile = cache.SuitesBySourceFile - if tr.suitesBySourceFile == nil { - tr.suitesBySourceFile = indexSuitesBySourceFile(tr.suiteAggregates) - } - - tr.testFileWeights = cache.TestFileWeights - if tr.testFileWeights == nil { - tr.testFileWeights = tr.testFileWeightsFromSuites() - } - - tr.testFileDurationSources = cache.TestFileDurationSources - if tr.testFileDurationSources == nil { - tr.testFileDurationSources = make(map[string]testFileDurationSource) - } - - tr.runInfoReport = cache.RunInfo - - testSuitesCount := countTestSuites(tr.testSuiteDurations) - suiteAggregatesCount := len(tr.suiteAggregates) - suitesBySourceFileCount := len(tr.suitesBySourceFile) - testFileWeightsCount := len(tr.testFileWeights) - slog.Info("Restored test optimization plan cache", - "objectsCount", testSuitesCount+suiteAggregatesCount+suitesBySourceFileCount+testFileWeightsCount, - "modulesCount", len(tr.testSuiteDurations), - "testSuitesCount", testSuitesCount, - "suiteAggregatesCount", suiteAggregatesCount, - "suitesBySourceFileCount", suitesBySourceFileCount, - "testFileWeightsCount", testFileWeightsCount) - - return nil -} - -func countTestSuites(testSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo) int { - totalSuites := 0 - for _, suites := range testSuiteDurations { - totalSuites += len(suites) - } - return totalSuites -} - -func (tr *TestRunner) testFileWeightsFromSuites() map[string]int { - testFiles := make(map[string]struct{}, len(tr.suitesBySourceFile)) - for testFile := range tr.suitesBySourceFile { - testFiles[testFile] = struct{}{} - } - return tr.estimateTestFileWeights(testFiles) -} diff --git a/internal/runner/worker_env.go b/internal/runner/worker_env.go index d7f9518..57be687 100644 --- a/internal/runner/worker_env.go +++ b/internal/runner/worker_env.go @@ -10,6 +10,7 @@ import ( ciConstants "github.com/DataDog/ddtest/civisibility/constants" ciUtils "github.com/DataDog/ddtest/civisibility/utils" "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/runmetadata" ) func createWorkerEnv(workerEnvMap map[string]string, nodeIndex int, workerIndex int) map[string]string { @@ -53,7 +54,7 @@ func ensureTestSessionName(workerEnv map[string]string, nodeIndex int, workerInd return } - service := resolveServiceName(ciUtils.GetCITags()[ciConstants.GitRepositoryURL]) + service := runmetadata.ResolveServiceName(ciUtils.GetCITags()[ciConstants.GitRepositoryURL]) workerEnv[ciConstants.CIVisibilityTestSessionNameEnvironmentVariable] = fmt.Sprintf("%s-node-%d-worker-%d", service, nodeIndex, workerIndex) } diff --git a/internal/runner/worker_env_test.go b/internal/runner/worker_env_test.go index 23b66be..f0bbe15 100644 --- a/internal/runner/worker_env_test.go +++ b/internal/runner/worker_env_test.go @@ -57,7 +57,7 @@ func TestRunBatch_DefaultTestSessionName(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runBatch([]string{"test/file1_test.rb"}, 2, 4) + err := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runBatch([]string{"test/file1_test.rb"}, 2, 4) if err != nil { t.Fatalf("runBatch() should not return error, got: %v", err) } @@ -81,7 +81,7 @@ func TestRunBatch_DefaultTestSessionNameUsesDDService(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runBatch([]string{"test/file1_test.rb"}, 3, 7) + err := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runBatch([]string{"test/file1_test.rb"}, 3, 7) if err != nil { t.Fatalf("runBatch() should not return error, got: %v", err) } @@ -103,7 +103,7 @@ func TestRunBatch_UserTestSessionNameSupportsPlaceholders(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runBatch([]string{"test/file1_test.rb"}, 5, 8) + err := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runBatch([]string{"test/file1_test.rb"}, 5, 8) if err != nil { t.Fatalf("runBatch() should not return error, got: %v", err) } @@ -128,7 +128,7 @@ func TestRunBatch_WorkerEnvTestSessionNameTakesPrecedence(t *testing.T) { ciConstants.CIVisibilityTestSessionNameEnvironmentVariable: "worker-node-{{nodeIndex}}-worker-{{workerIndex}}", } - err := newTestExecutor(context.Background(), mockFramework, workerEnvMap).runBatch([]string{"test/file1_test.rb"}, 9, 1) + err := newTestExecutor(context.Background(), mockFramework, workerEnvMap, roundRobinTestPlanner{}).runBatch([]string{"test/file1_test.rb"}, 9, 1) if err != nil { t.Fatalf("runBatch() should not return error, got: %v", err) } @@ -150,7 +150,7 @@ func TestRunBatch_DefaultManifestFile(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runBatch([]string{"test/file1_test.rb"}, 2, 4) + err := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runBatch([]string{"test/file1_test.rb"}, 2, 4) if err != nil { t.Fatalf("runBatch() should not return error, got: %v", err) } @@ -181,7 +181,7 @@ func TestRunBatch_ManifestFileUsesProcessEnv(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runBatch([]string{"test/file1_test.rb"}, 2, 4) + err := newTestExecutor(context.Background(), mockFramework, map[string]string{}, roundRobinTestPlanner{}).runBatch([]string{"test/file1_test.rb"}, 2, 4) if err != nil { t.Fatalf("runBatch() should not return error, got: %v", err) } @@ -203,7 +203,7 @@ func TestRunBatch_WorkerEnvManifestFileTakesPrecedence(t *testing.T) { constants.TestOptimizationManifestFileEnvVar: "/tmp/worker-manifest.txt", } - err := newTestExecutor(context.Background(), mockFramework, workerEnvMap).runBatch([]string{"test/file1_test.rb"}, 2, 4) + err := newTestExecutor(context.Background(), mockFramework, workerEnvMap, roundRobinTestPlanner{}).runBatch([]string{"test/file1_test.rb"}, 2, 4) if err != nil { t.Fatalf("runBatch() should not return error, got: %v", err) } diff --git a/internal/testoptimization/durations_client.go b/internal/testoptimization/durations_client.go index aeec37b..8f54c3f 100644 --- a/internal/testoptimization/durations_client.go +++ b/internal/testoptimization/durations_client.go @@ -14,7 +14,9 @@ import ( "github.com/DataDog/ddtest/civisibility" "github.com/DataDog/ddtest/civisibility/constants" + "github.com/DataDog/ddtest/civisibility/utils" "github.com/DataDog/ddtest/internal/httptransport" + "github.com/DataDog/ddtest/internal/runmetadata" ) const ( @@ -84,7 +86,7 @@ type ( // TestSuiteDurationsClient defines the interface for fetching test suite durations type TestSuiteDurationsClient interface { - GetTestSuiteDurations(repositoryURL, service string) (map[string]map[string]TestSuiteDurationInfo, error) + GetTestSuiteDurations() map[string]map[string]TestSuiteDurationInfo } // DurationsAPI abstracts the HTTP endpoint for testability (equivalent of CIVisibilityIntegrations) @@ -109,7 +111,56 @@ func NewDurationsClientWithDependencies(api DurationsAPI) *DatadogDurationsClien } } -func (c *DatadogDurationsClient) GetTestSuiteDurations(repositoryURL, service string) (map[string]map[string]TestSuiteDurationInfo, error) { +func (c *DatadogDurationsClient) GetTestSuiteDurations() map[string]map[string]TestSuiteDurationInfo { + startTime := time.Now() + repositoryURL, service, err := testSuiteDurationsFetchInputs() + if err != nil { + slog.Error("Test durations API errored", "duration", time.Since(startTime), "error", err) + return map[string]map[string]TestSuiteDurationInfo{} + } + + durations, err := c.fetchTestSuiteDurations(repositoryURL, service) + if err != nil { + slog.Error("Test durations API errored", + "service", service, + "repositoryURL", repositoryURL, + "duration", time.Since(startTime), + "error", err) + return map[string]map[string]TestSuiteDurationInfo{} + } + + totalSuites := countTestSuiteDurations(durations) + if totalSuites == 0 { + slog.Warn("Test durations API returned no test suites", + "service", service, + "repositoryURL", repositoryURL, + "modulesCount", len(durations), + "testSuitesCount", totalSuites, + "duration", time.Since(startTime)) + return map[string]map[string]TestSuiteDurationInfo{} + } + + slog.Info("Fetched test suite durations", + "service", service, + "repositoryURL", repositoryURL, + "modulesCount", len(durations), + "testSuitesCount", totalSuites, + "duration", time.Since(startTime)) + return durations +} + +func testSuiteDurationsFetchInputs() (string, string, error) { + ciTags := utils.GetCITags() + repositoryURL := ciTags[constants.GitRepositoryURL] + if repositoryURL == "" { + return "", "", fmt.Errorf("repository URL is required") + } + + service := runmetadata.ResolveServiceName(repositoryURL) + return repositoryURL, service, nil +} + +func (c *DatadogDurationsClient) fetchTestSuiteDurations(repositoryURL, service string) (map[string]map[string]TestSuiteDurationInfo, error) { startTime := time.Now() allSuites := make(map[string]map[string]TestSuiteDurationInfo) @@ -147,6 +198,14 @@ func (c *DatadogDurationsClient) GetTestSuiteDurations(repositoryURL, service st return allSuites, nil } +func countTestSuiteDurations(testSuiteDurations map[string]map[string]TestSuiteDurationInfo) int { + totalSuites := 0 + for _, suites := range testSuiteDurations { + totalSuites += len(suites) + } + return totalSuites +} + // DatadogDurationsAPI implements DurationsAPI using real HTTP calls (equivalent of DatadogCIVisibilityIntegrations) type DatadogDurationsAPI struct { baseURL string diff --git a/internal/testoptimization/durations_client_test.go b/internal/testoptimization/durations_client_test.go index c887185..8cf01e7 100644 --- a/internal/testoptimization/durations_client_test.go +++ b/internal/testoptimization/durations_client_test.go @@ -1,13 +1,16 @@ package testoptimization import ( + "bytes" "fmt" + "log/slog" "net/http" "os" "strings" "testing" "github.com/DataDog/ddtest/civisibility/constants" + ciUtils "github.com/DataDog/ddtest/civisibility/utils" ) // MockDurationsAPI implements DurationsAPI for testing (equivalent of MockCIVisibilityIntegrations) @@ -44,6 +47,24 @@ func (m *MockDurationsAPI) FetchTestSuiteDurations(repositoryURL, service, curso }, nil } +func captureLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + originalLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { + slog.SetDefault(originalLogger) + }) + return &buf +} + +func addRepositoryTag(t *testing.T, repositoryURL string) { + t.Helper() + ciUtils.ResetCITags() + t.Cleanup(ciUtils.ResetCITags) + ciUtils.AddCITagsMap(map[string]string{constants.GitRepositoryURL: repositoryURL}) +} + func TestNewDurationsClientWithDependencies(t *testing.T) { mockAPI := &MockDurationsAPI{} client := NewDurationsClientWithDependencies(mockAPI) @@ -53,7 +74,125 @@ func TestNewDurationsClientWithDependencies(t *testing.T) { } } -func TestDurationsClient_GetTestSuiteDurations_SinglePage(t *testing.T) { +func TestDurationsClient_GetTestSuiteDurations_DerivesInputsAndLogsSuccess(t *testing.T) { + addRepositoryTag(t, "github.com/DataDog/foo") + t.Setenv("DD_SERVICE", "my-service") + logs := captureLogs(t) + mockAPI := &MockDurationsAPI{ + Responses: []*durationsResponseAttributes{ + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{ + "module1": { + "suite1": { + SourceFile: "spec/user_spec.rb", + Duration: DurationPercentiles{P50: "280000000", P90: "350000000"}, + }, + }, + }, + }, + }, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result := client.GetTestSuiteDurations() + + if !mockAPI.FetchCalled { + t.Error("GetTestSuiteDurations() should call FetchTestSuiteDurations") + } + if mockAPI.RepositoryURL != "github.com/DataDog/foo" { + t.Errorf("Expected repository URL 'github.com/DataDog/foo', got '%s'", mockAPI.RepositoryURL) + } + if mockAPI.Service != "my-service" { + t.Errorf("Expected service 'my-service', got '%s'", mockAPI.Service) + } + if len(result) != 1 { + t.Errorf("Expected 1 module, got %d", len(result)) + } + if !strings.Contains(logs.String(), "level=INFO") || + !strings.Contains(logs.String(), "Fetched test suite durations") || + !strings.Contains(logs.String(), "modulesCount=1") || + !strings.Contains(logs.String(), "testSuitesCount=1") || + !strings.Contains(logs.String(), "duration=") { + t.Errorf("Expected INFO log for non-empty durations response, got logs: %s", logs.String()) + } +} + +func TestDurationsClient_GetTestSuiteDurations_MissingRepositoryReturnsEmptyAndLogsError(t *testing.T) { + ciUtils.ResetCITags() + t.Cleanup(ciUtils.ResetCITags) + ciUtils.AddCITagsMap(map[string]string{constants.GitRepositoryURL: ""}) + logs := captureLogs(t) + mockAPI := &MockDurationsAPI{} + + client := NewDurationsClientWithDependencies(mockAPI) + result := client.GetTestSuiteDurations() + + if mockAPI.FetchCalled { + t.Error("GetTestSuiteDurations() should not fetch without a repository URL") + } + if len(result) != 0 { + t.Errorf("Expected empty durations on missing repository URL, got %v", result) + } + if !strings.Contains(logs.String(), "level=ERROR") || + !strings.Contains(logs.String(), "Test durations API errored") || + !strings.Contains(logs.String(), "repository URL is required") || + !strings.Contains(logs.String(), "duration=") { + t.Errorf("Expected ERROR log for missing repository URL, got logs: %s", logs.String()) + } +} + +func TestDurationsClient_GetTestSuiteDurations_APIErrorReturnsEmptyAndLogsError(t *testing.T) { + addRepositoryTag(t, "github.com/DataDog/foo") + t.Setenv("DD_SERVICE", "") + logs := captureLogs(t) + mockAPI := &MockDurationsAPI{ + ResponseErrors: []error{fmt.Errorf("connection refused")}, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result := client.GetTestSuiteDurations() + + if len(result) != 0 { + t.Errorf("Expected empty durations on API error, got %v", result) + } + if !strings.Contains(logs.String(), "level=ERROR") || + !strings.Contains(logs.String(), "Test durations API errored") || + !strings.Contains(logs.String(), "repositoryURL=github.com/DataDog/foo") || + !strings.Contains(logs.String(), "service=foo") || + !strings.Contains(logs.String(), "connection refused") || + !strings.Contains(logs.String(), "duration=") { + t.Errorf("Expected ERROR log for durations API failure, got logs: %s", logs.String()) + } +} + +func TestDurationsClient_GetTestSuiteDurations_EmptyResponseReturnsEmptyAndLogsWarn(t *testing.T) { + addRepositoryTag(t, "github.com/DataDog/foo") + t.Setenv("DD_SERVICE", "") + logs := captureLogs(t) + mockAPI := &MockDurationsAPI{ + Responses: []*durationsResponseAttributes{ + { + TestSuites: map[string]map[string]TestSuiteDurationInfo{}, + }, + }, + } + + client := NewDurationsClientWithDependencies(mockAPI) + result := client.GetTestSuiteDurations() + + if len(result) != 0 { + t.Errorf("Expected empty durations on empty response, got %v", result) + } + if !strings.Contains(logs.String(), "level=WARN") || + !strings.Contains(logs.String(), "Test durations API returned no test suites") || + !strings.Contains(logs.String(), "modulesCount=0") || + !strings.Contains(logs.String(), "testSuitesCount=0") || + !strings.Contains(logs.String(), "duration=") { + t.Errorf("Expected WARN log for empty durations response, got logs: %s", logs.String()) + } +} + +func TestDurationsClient_FetchTestSuiteDurations_SinglePage(t *testing.T) { mockAPI := &MockDurationsAPI{ Responses: []*durationsResponseAttributes{ { @@ -80,14 +219,14 @@ func TestDurationsClient_GetTestSuiteDurations_SinglePage(t *testing.T) { } client := NewDurationsClientWithDependencies(mockAPI) - result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + result, err := client.fetchTestSuiteDurations("github.com/DataDog/foo", "my-service") if err != nil { - t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + t.Errorf("fetchTestSuiteDurations() should not return error, got: %v", err) } if !mockAPI.FetchCalled { - t.Error("GetTestSuiteDurations() should call FetchTestSuiteDurations") + t.Error("fetchTestSuiteDurations() should call FetchTestSuiteDurations") } if mockAPI.RepositoryURL != "github.com/DataDog/foo" { @@ -139,7 +278,7 @@ func TestDurationsClient_GetTestSuiteDurations_SinglePage(t *testing.T) { } } -func TestDurationsClient_GetTestSuiteDurations_Pagination(t *testing.T) { +func TestDurationsClient_FetchTestSuiteDurations_Pagination(t *testing.T) { mockAPI := &MockDurationsAPI{ Responses: []*durationsResponseAttributes{ { @@ -182,10 +321,10 @@ func TestDurationsClient_GetTestSuiteDurations_Pagination(t *testing.T) { } client := NewDurationsClientWithDependencies(mockAPI) - result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + result, err := client.fetchTestSuiteDurations("github.com/DataDog/foo", "my-service") if err != nil { - t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + t.Errorf("fetchTestSuiteDurations() should not return error, got: %v", err) } // Verify pagination cursors were passed correctly @@ -234,7 +373,7 @@ func TestDurationsClient_GetTestSuiteDurations_Pagination(t *testing.T) { } } -func TestDurationsClient_GetTestSuiteDurations_EmptyResponse(t *testing.T) { +func TestDurationsClient_FetchTestSuiteDurations_EmptyResponse(t *testing.T) { mockAPI := &MockDurationsAPI{ Responses: []*durationsResponseAttributes{ { @@ -244,22 +383,22 @@ func TestDurationsClient_GetTestSuiteDurations_EmptyResponse(t *testing.T) { } client := NewDurationsClientWithDependencies(mockAPI) - result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + result, err := client.fetchTestSuiteDurations("github.com/DataDog/foo", "my-service") if err != nil { - t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + t.Errorf("fetchTestSuiteDurations() should not return error, got: %v", err) } if result == nil { - t.Error("GetTestSuiteDurations() should return non-nil map even with empty data") + t.Error("fetchTestSuiteDurations() should return non-nil map even with empty data") } if len(result) != 0 { - t.Errorf("GetTestSuiteDurations() should return empty map, got %d modules", len(result)) + t.Errorf("fetchTestSuiteDurations() should return empty map, got %d modules", len(result)) } } -func TestDurationsClient_GetTestSuiteDurations_NilTestSuites(t *testing.T) { +func TestDurationsClient_FetchTestSuiteDurations_NilTestSuites(t *testing.T) { mockAPI := &MockDurationsAPI{ Responses: []*durationsResponseAttributes{ { @@ -269,39 +408,39 @@ func TestDurationsClient_GetTestSuiteDurations_NilTestSuites(t *testing.T) { } client := NewDurationsClientWithDependencies(mockAPI) - result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + result, err := client.fetchTestSuiteDurations("github.com/DataDog/foo", "my-service") if err != nil { - t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + t.Errorf("fetchTestSuiteDurations() should not return error, got: %v", err) } if result == nil { - t.Error("GetTestSuiteDurations() should return non-nil map even with nil test suites") + t.Error("fetchTestSuiteDurations() should return non-nil map even with nil test suites") } if len(result) != 0 { - t.Errorf("GetTestSuiteDurations() should return empty map, got %d modules", len(result)) + t.Errorf("fetchTestSuiteDurations() should return empty map, got %d modules", len(result)) } } -func TestDurationsClient_GetTestSuiteDurations_APIError(t *testing.T) { +func TestDurationsClient_FetchTestSuiteDurations_APIError(t *testing.T) { mockAPI := &MockDurationsAPI{ ResponseErrors: []error{fmt.Errorf("connection refused")}, } client := NewDurationsClientWithDependencies(mockAPI) - result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + result, err := client.fetchTestSuiteDurations("github.com/DataDog/foo", "my-service") if err == nil { - t.Error("GetTestSuiteDurations() should return error when API fails") + t.Error("fetchTestSuiteDurations() should return error when API fails") } if result != nil { - t.Error("GetTestSuiteDurations() should return nil result when API fails") + t.Error("fetchTestSuiteDurations() should return nil result when API fails") } } -func TestDurationsClient_GetTestSuiteDurations_PaginationError(t *testing.T) { +func TestDurationsClient_FetchTestSuiteDurations_PaginationError(t *testing.T) { mockAPI := &MockDurationsAPI{ Responses: []*durationsResponseAttributes{ { @@ -324,18 +463,18 @@ func TestDurationsClient_GetTestSuiteDurations_PaginationError(t *testing.T) { } client := NewDurationsClientWithDependencies(mockAPI) - result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + result, err := client.fetchTestSuiteDurations("github.com/DataDog/foo", "my-service") if err == nil { - t.Error("GetTestSuiteDurations() should return error when pagination fails") + t.Error("fetchTestSuiteDurations() should return error when pagination fails") } if result != nil { - t.Error("GetTestSuiteDurations() should return nil result when pagination fails") + t.Error("fetchTestSuiteDurations() should return nil result when pagination fails") } } -func TestDurationsClient_GetTestSuiteDurations_NilPageInfo(t *testing.T) { +func TestDurationsClient_FetchTestSuiteDurations_NilPageInfo(t *testing.T) { mockAPI := &MockDurationsAPI{ Responses: []*durationsResponseAttributes{ { @@ -353,10 +492,10 @@ func TestDurationsClient_GetTestSuiteDurations_NilPageInfo(t *testing.T) { } client := NewDurationsClientWithDependencies(mockAPI) - result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + result, err := client.fetchTestSuiteDurations("github.com/DataDog/foo", "my-service") if err != nil { - t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + t.Errorf("fetchTestSuiteDurations() should not return error, got: %v", err) } if len(result) != 1 { @@ -369,7 +508,7 @@ func TestDurationsClient_GetTestSuiteDurations_NilPageInfo(t *testing.T) { } } -func TestDurationsClient_GetTestSuiteDurations_ThreePages(t *testing.T) { +func TestDurationsClient_FetchTestSuiteDurations_ThreePages(t *testing.T) { mockAPI := &MockDurationsAPI{ Responses: []*durationsResponseAttributes{ { @@ -409,10 +548,10 @@ func TestDurationsClient_GetTestSuiteDurations_ThreePages(t *testing.T) { } client := NewDurationsClientWithDependencies(mockAPI) - result, err := client.GetTestSuiteDurations("github.com/DataDog/foo", "my-service") + result, err := client.fetchTestSuiteDurations("github.com/DataDog/foo", "my-service") if err != nil { - t.Errorf("GetTestSuiteDurations() should not return error, got: %v", err) + t.Errorf("fetchTestSuiteDurations() should not return error, got: %v", err) } if len(mockAPI.Cursors) != 3 {