diff --git a/configs/config_hepatitis.yaml b/configs/config_hepatitis.yaml index e491ea6..aff824e 100644 --- a/configs/config_hepatitis.yaml +++ b/configs/config_hepatitis.yaml @@ -86,7 +86,7 @@ generation: stats: [prdc, alpha_precision, wasserstein_dist, inv_kl_divergence] sanity: [nearest_syn_neighbor_distance] performance: [xgb] - privacy: [identifiability_score, DomiasMIA_prior] + privacy: [identifiability_score] storage: null best_params_path: null final_n_iter_override: null diff --git a/configs/config_loris.yaml b/configs/config_loris.yaml index e5c1b51..08a49d5 100644 --- a/configs/config_loris.yaml +++ b/configs/config_loris.yaml @@ -117,7 +117,7 @@ generation: stats: [prdc, alpha_precision, wasserstein_dist, inv_kl_divergence] sanity: [nearest_syn_neighbor_distance] performance: [xgb] - privacy: [identifiability_score, DomiasMIA_prior] + privacy: [identifiability_score] storage: null best_params_path: null final_n_iter_override: null diff --git a/pyproject.toml b/pyproject.toml index 6593111..dfb7b44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "synthdata" -version = "0.7.3" +version = "0.7.4" description = "Sandbox for synthetic data generation and evaluation" authors = [ {name = "Alp Erkent", email = "alp.erkent@childmind.org"}, diff --git a/synthdata/config.py b/synthdata/config.py index 31e8562..dcde083 100644 --- a/synthdata/config.py +++ b/synthdata/config.py @@ -293,7 +293,9 @@ class HPOConfig: ], "sanity": ["nearest_syn_neighbor_distance"], "performance": ["xgb"], - "privacy": ["identifiability_score", "DomiasMIA_prior"], + # DOMIAS is deferred from HPO until its high-dimensional KDE + # failure modes have a separately validated treatment. + "privacy": ["identifiability_score"], } ) #: Optuna storage URL, e.g. "sqlite:///output/dataset/optuna_studies.db". diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index cfee4c5..cd2b9b1 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -32,6 +32,11 @@ def test_empty_dict_returns_defaults(self): cfg = _from_dict(Config, {}) assert cfg == Config() + def test_default_hpo_privacy_objective_excludes_domias(self): + cfg = _from_dict(Config, {}) + + assert cfg.generation.hpo.metric_config["privacy"] == ["identifiability_score"] + def test_flat_fields_applied(self): cfg = _from_dict(Config, {"name": "mydata", "seed": 7}) assert cfg.name == "mydata" @@ -91,6 +96,13 @@ def test_refidiff_benchmark_profiles_load(self, config_name): assert cfg.imputation.refidiff.catboost_warmup_iterations == 1000 assert cfg.imputation.benchmark.enabled + @pytest.mark.parametrize("config_name", ["config_hepatitis.yaml", "config_loris.yaml"]) + def test_shipped_hpo_profiles_exclude_domias(self, config_name): + root = Path(__file__).parents[2] + cfg = load_config(root / "configs" / config_name) + + assert cfg.generation.hpo.metric_config["privacy"] == ["identifiability_score"] + def test_evaluation_binary_target_nested_dict_builds_nested_dataclass(self): cfg = _from_dict( Config, diff --git a/tests/unit/test_synthcity_eval.py b/tests/unit/test_synthcity_eval.py index 6acaa21..dbafed3 100644 --- a/tests/unit/test_synthcity_eval.py +++ b/tests/unit/test_synthcity_eval.py @@ -16,6 +16,9 @@ def test_default_selection_uses_native_attack_category(self): assert result["attack"] == SYNTHCITY_METRIC_CONFIG["attack"] assert "attacks" not in result + def test_final_evaluation_catalog_retains_domias(self): + assert "DomiasMIA_prior" in SYNTHCITY_METRIC_CONFIG["privacy"] + def test_privacy_category_includes_attack_metrics(self): result = resolve_metric_config(FrameworkSelectionConfig(categories=["privacy"]))