fix: save datasets selected by run mode

src/inference_endpoint/commands/benchmark/execute.py

@@ -509,7 +509,14 @@ def setup_benchmark(
     # Report directory
     report_dir = resolve_report_dir(config)
     report_dir.mkdir(parents=True, exist_ok=True)
-    config.to_yaml_file(report_dir / "config.yaml", redact_secrets=True)
+    datasets = config.datasets
+    if test_mode == TestMode.PERF:
+        datasets = [ds for ds in datasets if ds.type == DatasetType.PERFORMANCE]
+    elif test_mode == TestMode.ACC:
+        datasets = [ds for ds in datasets if ds.type == DatasetType.ACCURACY]
+    config.with_updates(datasets=datasets).to_yaml_file(
+        report_dir / "config.yaml", redact_secrets=True
+    )
 
     # Tokenizer check (light API call, no download)
     model_name = config.model_params.name

tests/integration/commands/test_sigint.py

@@ -276,6 +276,7 @@ def test_sigint_grace_expiry_abandons_wedged_drain(mock_http_echo_server, tmp_pa
     """
     report_dir = tmp_path / "report"
     config_path = tmp_path / "bench.yaml"
+    log_file = tmp_path / "run.log"
     _write_config(
         report_dir,
         mock_http_echo_server.url,
@@ -286,10 +287,18 @@ def test_sigint_grace_expiry_abandons_wedged_drain(mock_http_echo_server, tmp_pa
     agg_pid: int | None = None
     try:
         with _benchmark_proc(
-            [_cli(), "benchmark", "from-config", "-c", str(config_path)]
+            [_cli(), "benchmark", "from-config", "-c", str(config_path)],
+            log_file,
         ) as proc:
             _wait_services_ready(proc, report_dir)
-            time.sleep(3.0)  # comfortably inside the ~120 s performance phase
+            # Service readiness precedes client startup. Wait for the session
+            # before allowing live metrics ticks to reach the parent.
+            deadline = time.monotonic() + 60.0
+            while "Starting phase:" not in log_file.read_text(errors="replace"):
+                assert proc.poll() is None, log_file.read_text(errors="replace")
+                assert time.monotonic() < deadline, "performance phase never started"
+                time.sleep(0.05)
+            time.sleep(3.0)
 
             agg_pid = _pid_of_child("metrics_aggregator", str(report_dir))
             assert agg_pid is not None, "aggregator child not found"

tests/unit/commands/test_benchmark.py

@@ -3303,9 +3303,16 @@ def test_agentic_trajectory_repeats_are_included_in_total_samples(
 
 class TestReportConfigSecretRedaction:
     @pytest.mark.unit
-    @pytest.mark.parametrize("test_mode", [TestMode.PERF, TestMode.ACC])
-    def test_report_config_redacts_secrets_without_mutating_runtime(
-        self, tmp_path, test_mode
+    @pytest.mark.parametrize(
+        ("test_mode", "expected_types"),
+        [
+            (TestMode.PERF, ["performance"]),
+            (TestMode.ACC, ["accuracy"]),
+            (TestMode.BOTH, ["accuracy", "performance"]),
+        ],
+    )
+    def test_report_config_matches_mode_and_redacts_secrets(
+        self, tmp_path, test_mode, expected_types
     ):
         endpoint_key = "sentinel-endpoint-key"
         service_token = "sentinel-service-token"
@@ -3327,7 +3334,8 @@ def test_report_config_redacts_secrets_without_mutating_runtime(
                             "swebench_service_auth_token": service_token,
                         },
                     },
-                }
+                },
+                {"name": "random", "type": "performance", "path": "unused.jsonl"},
             ],
             report_dir=str(tmp_path),
         )
@@ -3351,7 +3359,10 @@ def test_report_config_redacts_secrets_without_mutating_runtime(
         persisted = (tmp_path / "config.yaml").read_text()
         assert endpoint_key not in persisted
         assert service_token not in persisted
-        assert persisted.count("<redacted>") >= 2
+        assert persisted.count("<redacted>") >= (1 if test_mode == TestMode.PERF else 2)
+        saved = BenchmarkConfig.from_yaml_file(tmp_path / "config.yaml")
+        assert [ds.type.value for ds in saved.datasets] == expected_types
+        assert len(config.datasets) == 2
         assert ctx.config.endpoint_config.api_key == endpoint_key
         assert (
             ctx.config.datasets[0].accuracy_config.extras["swebench_service_auth_token"]