config.yaml

# Versioned directories (used with config.get_dir_path())
versions:
  osm_data: "20261022"
  model_output: "20260727_by_shared_label"
  snapshot_osm: "20261022"
  # 2026-10-21.0 release. The prior month's snapshot (20260923, release
  # 2026-09-23.1) stays on openpois-01 as the baseline for
  # scripts/overture/compare_confidence.py. The July 20260722* dirs stay on
  # the laptop as the method-paper basis.
  snapshot_overture: "20261021"
  # 20260730 (kept, July release) added the two private-property / non-POI
  # exclusions: the residential-landuse spatial rule (250,905 unnamed POIs)
  # and the wildcard -> inclusion-set taxonomy change (61,764 unlabeled
  # rows). 20260902 is the first run with min_shadow_match_score 0.70 and
  # the August Overture taxonomy fixes (beauty_service rename,
  # food_delivery_service move).
  conflation: "20261022"
  # Validation round whose condensed handoff drives the confidence
  # calibration (see .claude/docs/confidence-calibration.md). Produced by
  # openpois-validator's scripts/08_export_handoff.py; the curves are
  # release-specific and do not transport across Overture releases.
  calibration: "20261001"
  source_coop: "2026-10-22-v1"   # Source Cooperative upload folder (YYYY-MM-DD-v<IDX>); bump v<IDX> only for same-day re-uploads
  # Ghost POI dataset reconstructed from OSM history (one row per
  # detected previous-state event). Pinned to the same value as
  # ``osm_data`` since it is derived from the same history parquets,
  # and regenerated together with the monthly snapshot refresh.
  ghost_osm: "20261022"
  # Hand-curated exclude/include pins applied after calibration (see
  # conflation.manual_overrides). Rows are appended by the Close triage
  # lane; bump the version when the file is re-baselined, not per row.
  manual_overrides: "20260924"
  # The PRIOR release's id registry (source key -> stable unified_id), read by
  # scripts/conflation/assign_stable_ids.py. Bump with ``conflation`` each
  # month, to the version being replaced. 20261022 is the stable-id backfill
  # registry (2026-10-22-v1).
  id_registry: "20261022"

# Settings for downloading data
download:
  general:
    timeout: 1_000
    # Census 1:5M cartographic state boundary file; includes 50 states + DC
    # + 5 inhabited US territories (PR, USVI, GU, MP, AS). The 1:20M variant
    # used previously only covered states + DC + PR. Used by all three
    # snapshot downloads to restrict POIs to the US footprint. The coastline
    # buffer expands the dissolved polygon outward by N metres so near-shore
    # POIs are retained; internal state borders disappear on dissolve so the
    # buffer only affects the coast.
    boundary:
      source_url: "https://www2.census.gov/geo/tiger/GENZ2023/shp/cb_2023_us_state_5m.zip"
      coastline_buffer_m: 100
  osm:
    start_date: 2016-01-01
    end_date: 2026-10-21
    # Snapshot extracts. Geofabrik publishes a dedicated PBF for PR and USVI
    # under north-america/us/, but the western-Pacific inhabited territories
    # (Guam, Northern Mariana Islands, American Samoa) have no per-territory
    # files. They are bundled into a single `american-oceania` extract that
    # also includes the uninhabited US Pacific possessions (Wake, Midway,
    # Howland, Baker, Jarvis, Palmyra, Kingman) — those contribute near-zero
    # POIs and are accepted as bonus coverage.
    pbf_url: "https://download.geofabrik.de/north-america/us-latest.osm.pbf"
    pr_pbf_url: "https://download.geofabrik.de/north-america/us/puerto-rico-latest.osm.pbf"
    usvi_pbf_url: "https://download.geofabrik.de/north-america/us/us-virgin-islands-latest.osm.pbf"
    american_oceania_pbf_url: "https://download.geofabrik.de/australia-oceania/american-oceania-latest.osm.pbf"
    # Full-history PBFs live on Geofabrik's OAuth-protected internal server.
    # Any OSM account grants access; generate a Netscape-format cookie jar by
    # logging in at https://osm-internal.download.geofabrik.de/ and exporting
    # cookies, or by running Geofabrik's oauth_cookie_client.py. The
    # `usvi_*` and `american_oceania_*` history URLs follow the same path
    # convention as the snapshot URLs; if any are missing on the server, the
    # history loader logs a warning and continues without that territory's
    # history (the rater then falls back to the global-mean delta).
    history_pbf_url: "https://osm-internal.download.geofabrik.de/north-america/us-internal.osh.pbf"
    pr_history_pbf_url: "https://osm-internal.download.geofabrik.de/north-america/us/puerto-rico-internal.osh.pbf"
    usvi_history_pbf_url: "https://osm-internal.download.geofabrik.de/north-america/us/us-virgin-islands-internal.osh.pbf"
    american_oceania_history_pbf_url: "https://osm-internal.download.geofabrik.de/australia-oceania/american-oceania-internal.osh.pbf"
    history_cookie_file: "~/data/openpois/.creds/geofabrik_cookies.txt"
    # How scripts/osm_data/download_history.py builds versions.osm_data:
    #   full        - download the full-history PBFs above (~23 GB) and parse.
    #                 Required for any turnover-model refit.
    #   incremental - roll incremental_history.base_version forward with
    #                 Geofabrik's public daily diffs (~0.4 GB/month). Output is
    #                 ghost-grade only: same-day edits are collapsed and there is
    #                 no user/changeset, so format_tabular / osm_turnover refuse
    #                 it. Diffs are kept ~131 days, so a base older than that
    #                 needs a full run. See .claude/plans/incremental-ghost-history.md.
    history_mode: "incremental"
    incremental_history:
      # osm_data version to roll forward (the previous month's run).
      base_version: "20261001"
      # Warn once this many incremental runs are chained without a full build.
      max_chain_months: 12
      # Keep the downloaded .osc.gz files under osm_data/<v>/diffs/.
      keep_diffs: false
      # Public replication feeds, keyed like the history extracts. The US
      # mainland feed must come first.
      replication_urls:
        us: "https://download.geofabrik.de/north-america/us-updates/"
        pr: "https://download.geofabrik.de/north-america/us/puerto-rico-updates/"
        usvi: "https://download.geofabrik.de/north-america/us/us-virgin-islands-updates/"
        american_oceania: "https://download.geofabrik.de/australia-oceania/american-oceania-updates/"
    overwrite_download: true
    overwrite_filter: true
    overwrite_parse: true
    source_label: "osm"
    keep_all_keys: false
    chunk_size: 100_000
    max_area_nodes: 1_000
    verbose: true
    # Priority order for shared-label assignment (first non-null wins). The
    # osmium ingest filter is derived separately from the taxonomy crosswalk
    # (taxonomy.build_osm_tag_filter_expressions), which value-scopes keys
    # lacking a wildcard row (landuse -> cemetery,religious; craft -> listed
    # values) so we don't drag in every value of those keys.
    filter_keys: ['shop', 'healthcare', 'leisure', 'amenity', 'tourism', 'office', 'craft', 'historic', 'landuse']
    # POIs that are unnamed AND carry one of these access values are dropped
    # at parse time — anonymous non-public features (overwhelmingly
    # residential/HOA swimming pools) that add noise without value. Both
    # conditions must hold: a named feature is always kept, and an untagged
    # access never drops a row. `restricted` is deliberately absent — those
    # rows are mostly named facilities. ~477k of 5.49M in the July 2026
    # snapshot (8.7%). Set to [] to disable.
    #
    # NOTE: applied only at snapshot build. The 2026-07 run predates this and
    # applied it after rating via scripts/osm_snapshot/apply_access_exclusion.py;
    # from the next OSM pull onward the snapshot arrives already filtered and
    # that script becomes a no-op. The history/observations path deliberately
    # keeps these rows — a nameless POI carries no name-change signal, so it
    # cannot affect the turnover fit.
    excluded_access: ['private', 'no']
    # Second, mapper-independent private-property exclusion. `access` only
    # catches features somebody bothered to tag; the 2026-07 validation round
    # found the OSM-only segment's unverifiable rate is still dominated by
    # unnamed, imagery-traced features on private land (backyard pools, HOA
    # gardens, unnamed pitches). landuse=residential polygons are extracted
    # from the raw PBFs by a second osmium pass, and an unnamed POI whose
    # representative point falls inside one is dropped.
    #
    # Deliberately tag-scoped: in the US landuse=residential is routinely drawn
    # around a whole subdivision, so an unscoped rule would also remove unnamed
    # convenience stores, places of worship, fuel stations and schools that
    # legitimately sit inside those blocks. Named features are never dropped.
    # Scoping applies to the POI's *primary* tag (first non-null key in
    # filter_keys order), matching how shared_label is assigned.
    #
    # Set landuse_values to [] to disable the whole step.
    residential_exclusion:
      landuse_values: ['residential']
      scoped_tags:
        leisure: ['pitch', 'swimming_pool', 'playground', 'garden', 'track',
                  'common', 'batting_cage', 'practice_pitch', 'paddling_pool']
        amenity: ['fountain', 'shower', 'boat_storage', 'rv_storage']
        tourism: ['rv_pitch']
    extract_keys: [
      'addr:city','addr:country','addr:housename','addr:housenumber','addr:postcode',
      'addr:state','addr:street','addr:unit','access','amenity','atm','bar','bicycle',
      'bicycle_parking','brand','brand:wikidata','building','check_date',
      'check_date:opening_hours','craft','cuisine','description','education','email',
      'emergency','fountain','geometry','healthcare','healthcare:speciality','historic',
      'image','landuse','leisure','name','nursery','office','official_name','old_name',
      'opening_date','opening_hours','osm_id','osm_type','phone','playground','preschool',
      'recycling_type','religion','self_service','service','shelter','shop','short_name',
      'social_facility','social_facility:for','source','sport','tourism','type','url',
      'website','wikidata','wikipedia'
    ]
  overture:
    release_date: "2026-10-21.0"  # pin for determinism; null = auto-detect latest
    s3_bucket: "overturemaps-us-west-2"
    s3_region: "us-west-2"
    # DuckDB resource caps for the per-part S3 scans and the final polygon
    # filter. Peak host RAM ~= workers * memory_limit, peak CPU ~= workers *
    # threads. Scale per-worker values down if raising workers above 1.
    duckdb:
      memory_limit: "8GB"  # raised from 4GB to cut disk spill in the finalize polygon-filter step
      threads: 2
      workers: 2
    # (L0, L1) allowlist. L1 = null means "all of this L0".
    # Entries intentionally exclude office/B2B-style L1s (corporate offices,
    # media services, etc.), transit/parking/airports (covered elsewhere), and
    # private lodging (Airbnb-style — duplicates residential addresses).
    taxonomy_allowlist:
      - [food_and_drink, null]
      - [shopping, null]
      - [arts_and_entertainment, null]
      - [sports_and_recreation, null]
      - [health_care, null]
      - [lodging, null]
      - [cultural_and_historic, null]
      - [education, null]
      - [lifestyle_services, personal_or_beauty_service]
      - [lifestyle_services, wellness_service]
      - [lifestyle_services, animal_or_pet_service]
      - [lifestyle_services, beauty_service]
      - [lifestyle_services, food_service]
      - [services_and_business, financial_service]
      - [services_and_business, legal_service]
      - [services_and_business, professional_service]
      - [services_and_business, real_estate_service]
      - [services_and_business, home_service]
      - [services_and_business, family_service]
      - [community_and_government, social_or_community_service]
      - [community_and_government, government_office]
      - [community_and_government, civic_organization]
      - [community_and_government, public_facility]
      - [community_and_government, public_safety_service]
      - [travel_and_transportation, fueling_station]
      - [travel_and_transportation, vehicle_service]
      # Added 2026-07 so the labels below stop being OSM-only. Each of
      # these L1s is the sole home of a shared_label the OSM side already
      # maps; without them the label can never match.
      - [services_and_business, laundry_service]        # Laundromat, Dry Cleaning
      - [services_and_business, shipping_or_delivery_service]  # Post Office
      - [services_and_business, storage_facility]       # Self-Storage
      - [services_and_business, printing_service]       # Print and Copy Shop
      # Marina lives here (as do garden/pier); the rest of this L0 is
      # lakes/rivers/mountains and stays out.
      - [geographic_entities, built_feature]            # Marina
  # Census reference areas for the msa_code + urban_rural indicators. The
  # cartographic-boundary (cb_*_500k) line is used because Census publishes
  # single national "us" files there for both CBSA and Place. Place population
  # comes from the 2020 Decennial (covers CDPs, which PEP omits) via the Census
  # API; it requires a free key (CENSUS_API_KEY in the env or ~/.Renviron) and
  # is cached to a static CSV on first run.
  census_areas:
    cbsa_url: "https://www2.census.gov/geo/tiger/GENZ2023/shp/cb_2023_us_cbsa_500k.zip"
    place_url: "https://www2.census.gov/geo/tiger/GENZ2023/shp/cb_2023_us_place_500k.zip"
    cbsa_shp_name: "cb_2023_us_cbsa_500k.shp"
    place_shp_name: "cb_2023_us_place_500k.shp"
    population_api_url: "https://api.census.gov/data/2020/dec/pl?get=NAME,P1_001N&for=place:*&in=state:*"

# Settings for OSM exploratory data analysis
osm_data:
  tag_key: name
  # OSM lifecycle namespaces whose appearance on a POI's primary tag (one of
  # download.osm.filter_keys) marks a closure — e.g. shop=supermarket becoming
  # disused:shop=supermarket while `name` is kept plain. Treated as a turnover
  # (soft-delete) event in observation formatting; removal is a reopening.
  # Set to [] to disable (recovers the name-only change signal for ablation).
  lifecycle_closure_prefixes:
    - 'disused:'
    - 'abandoned:'
    - 'demolished:'
    - 'was:'
    - 'removed:'
    - 'razed:'
  top_n_types: 10
  timestamp_cols:
    - obs_timestamp
    - last_obs_timestamp
    - last_tag_timestamp
  apply_model:
    model_stub: '20260727'

# Settings for scripts/models/osm_turnover.py (JAX turnover model)
osm_turnover_model:
  # Overridable at the CLI via --model-type
  # {constant,constant_breakpoint,random_by_type,random_effects}.
  # random_effects is the production model (location-aware: λ on POI type +
  # MSA + urbanicity + interaction). Rate the snapshot with
  # apply_model_random_effects.py, NOT apply_model.py (which is per-group only).
  default_model_type: random_effects
  var_prior: [-1.0, 5.0]
  # Time-varying λ — only used by --model-type constant_breakpoint. Log-normal
  # prior on the breakpoint age t_B (years since the tag was established):
  # (loc, scale) on log t_B; loc = 0 → median 1 year.
  t_breakpoint_prior: [0.0, 1.0]
  # Tight hyperprior on log_tau (random-effect scale for per-group logit_delta
  # in RandomByTypeModel). Tau median ≈ exp(-2) ≈ 0.135 on the logit scale —
  # shrinks per-group δ toward the global intercept.
  logit_delta_var_prior: [-2.0, 0.5]
  # Column in osm_observations.parquet for grouping random effects.
  # "shared_label" = shared taxonomy category
  group_key: shared_label
  group_values: null
  min_value_count: 5
  # POI subsampling for the fit. A simple random sample of this fraction of
  # POIs (unique osm_type+id, all their observation rows kept). The draw is
  # deterministic given poi_sample_seed, so reruns train on identical POIs.
  #
  # DEFAULT IS 1.0 (full data) AND SHOULD STAY THAT WAY. Every fit used to
  # rate a snapshot must be a full-data fit: at 1% the random_effects
  # amenity_msa interaction thins out from ~4,000 active cells to ~18 (it
  # needs interaction_min_count POIs per cell), and the per-label and per-MSA
  # levels thin out with it, so the resulting confidence estimates are much
  # weaker and are not comparable to a previous full-data run. A full fit runs
  # ~5.5 h; a 1% fit runs in minutes.
  #
  # Sampling is for *exploratory* sweeps only — pass --sample-fraction 0.01 on
  # the command line rather than editing this value. apply_model_random_effects
  # refuses to rate from a sampled fit unless --allow-sampled-fit is passed.
  poi_sample_fraction: 1.0
  poi_sample_seed: 0
  # NUTS warmup (window adaptation) and retained-sample counts. Warmup should
  # generally be >= n_samples for hierarchical models.
  n_warmup: 500
  n_samples: 500
  # Number of independent chains (vmapped in parallel). n_chains > 1 enables
  # R-hat and bulk ESS diagnostics at roughly linear wall-time cost on CPU.
  n_chains: 4
  save_full_model: true
  # ---- random_effects model (multi-term additive random intercepts) --------
  # Used only when --model-type random_effects. Every hyperparameter the model
  # uses is wired from this block — no magic numbers in the script or model
  # body. Each term is included iff enabled: true, so any combination composes.
  # var_prior / prior are (loc, scale) on the relevant log_sigma or coefficient.
  random_effects:
    # Minimum DISTINCT POIs in an (amenity, MSA) cell to warrant its own
    # interaction level; smaller cells fall back on the main effects.
    interaction_min_count: 100
    terms:
      amenity:
        enabled: true
        column: shared_label
        var_prior: [-1.0, 5.0]      # matches the legacy random_by_type prior
      msa:
        enabled: true
        column: msa_code
        var_prior: [-1.0, 1.0]
      amenity_msa:
        enabled: true
        columns: [shared_label, msa_code]
        var_prior: [-2.0, 0.5]      # tight: shrink interactions toward zero
      urbanicity:
        enabled: true
        column: urban_rural          # urban = reference; suburban/rural coeffs
        prior: [0.0, 1.0]
    # δ (zero-inflation) random intercepts — composable and toggled
    # independently of the λ terms above. Each enabled term adds a random
    # intercept on logit δ (amenity grouped by shared_label, msa by msa_code),
    # with its own log_tau scale (var_prior). No terms enabled → single global
    # δ. Unseen levels back off to the global intercept, same as the λ terms.
    delta_terms:
      amenity:
        enabled: true
        column: shared_label
        var_prior: [-2.0, 0.5]       # tight, matches the legacy per-group δ
      msa:
        enabled: false
        column: msa_code
        var_prior: [-2.0, 0.5]

# Directory definitions (used with config.get_dir_path())
directories:
  osm_data:
    versioned: true
    path: ~/data/openpois/osm_data
    files:
      # US + territories full-history pipeline (PBF-based)
      osm_changes: osm_changes.parquet
      osm_versions: osm_versions.parquet
      raw_history_pbf: us-internal.osh.pbf
      filtered_history_pbf: us-pois.osh.pbf
      time_filtered_history_pbf: us-pois-timefilt.osh.pbf
      raw_pr_history_pbf: puerto-rico-internal.osh.pbf
      filtered_pr_history_pbf: puerto-rico-pois.osh.pbf
      time_filtered_pr_history_pbf: puerto-rico-pois-timefilt.osh.pbf
      raw_usvi_history_pbf: us-virgin-islands-internal.osh.pbf
      filtered_usvi_history_pbf: us-virgin-islands-pois.osh.pbf
      time_filtered_usvi_history_pbf: us-virgin-islands-pois-timefilt.osh.pbf
      raw_american_oceania_history_pbf: american-oceania-internal.osh.pbf
      filtered_american_oceania_history_pbf: american-oceania-pois.osh.pbf
      time_filtered_american_oceania_history_pbf: american-oceania-pois-timefilt.osh.pbf
      us_versions: us_osm_versions.parquet
      us_changes: us_osm_changes.parquet
      pr_versions: pr_osm_versions.parquet
      pr_changes: pr_osm_changes.parquet
      usvi_versions: usvi_osm_versions.parquet
      usvi_changes: usvi_osm_changes.parquet
      american_oceania_versions: american_oceania_osm_versions.parquet
      american_oceania_changes: american_oceania_osm_changes.parquet
      # Modelling-ready observations (one row per POI version × shared_label)
      osm_observations: osm_observations.parquet
      # Shared k-fold holdout assignment (id → fold), built once by
      # scripts/models/make_holdout_folds.py and reused by every OOS run so
      # all model specifications are scored on identical folds.
      holdout_folds: holdout_folds.parquet
      # Provenance of this history run (full vs incremental, replication
      # sequences reached); see openpois.io.osm_history_incremental.
      coverage: history_coverage.json
  model_output:
    versioned: true
    path: ~/data/openpois/osm_turnover_model
    files:
      fitted_params: fitted_params.csv
      param_draws: param_draws.parquet
      predictions: predictions.csv
      diagnostics: diagnostics.csv
      inference_data: inference_data.nc
      metrics_summary: metrics_summary.csv
      metrics_subgroup: metrics_subgroup.csv
      # Overdispersion diagnostics (scripts/models/osm_turnover.py via
      # openpois.models.dispersion): φ̂ + posterior-predictive p-values for the
      # covariate-cell and per-POI frailty statistics, subgroup/age breakdown,
      # and the fitted-probability calibration table.
      dispersion_summary: dispersion_summary.csv
      dispersion_subgroup: dispersion_subgroup.csv
      dispersion_calibration: dispersion_calibration.csv
      # Long-form factor → level-name map (random_effects), used by the
      # apply step to reconstruct per-cell curves from the fitted params.
      factor_lookups: factor_lookups.csv
      # Out-of-sample cross-validation outputs (scripts/models/osm_turnover_cv.py)
      oos_metrics_per_fold: oos_metrics_per_fold.csv
      oos_metrics_aggregate: oos_metrics_aggregate.csv
      oos_metrics_subgroup: oos_metrics_subgroup.csv
  snapshot_osm:
    versioned: true
    path: ~/data/openpois/snapshots/osm
    files:
      raw_pbf: us-latest.osm.pbf
      filtered_pbf: us-pois.osm.pbf
      raw_pr_pbf: puerto-rico-latest.osm.pbf
      filtered_pr_pbf: puerto-rico-pois.osm.pbf
      raw_usvi_pbf: us-virgin-islands-latest.osm.pbf
      filtered_usvi_pbf: us-virgin-islands-pois.osm.pbf
      raw_american_oceania_pbf: american-oceania-latest.osm.pbf
      filtered_american_oceania_pbf: american-oceania-pois.osm.pbf
      # landuse=residential polygons, extracted from the raw PBFs by a second
      # osmium pass (the POI ingest filter is value-scoped to
      # landuse=cemetery,religious so residential never reaches it). Persisted
      # because download.py unlinks the raw PBFs, and rebuilding the layer
      # otherwise means another ~11 GB Geofabrik pull.
      residential_filtered_pbf: us-residential.osm.pbf
      residential_pr_filtered_pbf: puerto-rico-residential.osm.pbf
      residential_usvi_filtered_pbf: us-virgin-islands-residential.osm.pbf
      residential_american_oceania_filtered_pbf: american-oceania-residential.osm.pbf
      residential_areas: landuse_residential.parquet
      snapshot: osm_snapshot.parquet
      rated_snapshot: osm_snapshot_rated.parquet
      partitioned: osm_snapshot_partitioned
      pmtiles: osm_snapshot.pmtiles
  boundary:
    versioned: false
    path: ~/data/openpois/boundary
  census_areas:
    versioned: false
    path: ~/data/openpois/census_areas
    files:
      cbsa_shapefile: cb_2023_us_cbsa_500k.shp
      place_shapefile: cb_2023_us_place_500k.shp
      place_population: place_population_2020.csv
  snapshot_overture:
    versioned: true
    path: ~/data/openpois/snapshots/overture
    files:
      snapshot: overture_snapshot.parquet
      # Self-hosted web tiles (tiles.overturemaps.org blocks openpois.org
      # referers since 2026-10) and the filter-panel category counts the site
      # reads alongside them. Built by scripts/overture/prepare_pmtiles.py.
      pmtiles: overture_snapshot.pmtiles
      categories: overture_categories.json
  conflation:
    versioned: true
    path: ~/data/openpois/conflation
    files:
      conflated: conflated.parquet
      match_diagnostics: match_diagnostics.parquet
      partitioned: conflated_partitioned
      pmtiles: conflated.pmtiles
      summary_by_label: summary_by_label.csv
      match_status: match_status_by_label.csv
      id_registry: id_registry.parquet
      id_changes: id_changes.parquet
  # Prior release's id registry; same tree as ``conflation``, versioned on
  # ``versions.id_registry``.
  id_registry:
    versioned: true
    path: ~/data/openpois/conflation
    files:
      registry: id_registry.parquet
  ghost_osm:
    versioned: true
    path: ~/data/openpois/ghost_osm
    files:
      ghosts: ghosts.parquet
  # Manual confidence overrides (columns: unified_id, overture_id, osm_type,
  # osm_id, action, reason, date, report_id). Read by scripts/conflation/
  # apply_manual_overrides.py, the last stage of `make conflate`. A
  # missing file is a no-op.
  manual_overrides:
    versioned: true
    path: ~/data/openpois/manual_overrides
    files:
      overrides: manual_overrides.csv
  # Fixed validation input for the confidence calibration, exported from
  # openpois-validator. Lives in the repo tree but is gitignored: the labels
  # are the validation moat. Versioned on ``versions.calibration``.
  calibration:
    versioned: true
    path: ~/repos/openpois/data/calibration
    files:
      validation_rows: validation_rows.parquet
      metadata: metadata.json
      reference_curves: reference_curves
  testing:
    versioned: false
    path: ~/data/openpois/testing
    files:
      osm_snippet: osm_snippet.csv
      overture_snippet: overture_snippet.csv
      # Small national-scale modelling fixture (~10k obs, 5 MSAs x 10 amenity
      # types) built by scripts/exploratory/build_test_dataset.py.
      test_observations: test_observations.parquet

# Settings for POI conflation
conflation:
  overture_confidence_weight: 0.7
  # Raised 0.50 -> 0.70 on 2026-07-26 after a manual review of 30 sampled
  # matches per score band: precision was ~33% in 0.5-0.6 and ~60% in
  # 0.6-0.7, against ~87% at 0.7-0.8 and ~97%+ above that. Drops ~427k
  # matches, the large majority of them false. Unnamed-OSM matches are
  # unaffected: they cap at 0.80 (neutral name score) and sampled as
  # ~83% correct, so they sit above the new floor.
  min_match_score: 0.70
  # Shrinkage constant for the type-affinity table: the number of
  # identifier-confirmed pairs at which the empirical confusion term and the
  # Overture-hierarchy prior carry equal weight. See
  # docs/type-affinity-metric.md. Rebuild the table after changing this:
  #   python scripts/conflation/build_type_affinity.py
  type_affinity_k: 100
  # Drop conflated points that never received a shared_label (uncategorized
  # POIs) from the canonical conflated.parquet. Applied in the change-detection
  # step; conflated_baseline.parquet keeps the full unfiltered set.
  drop_unlabeled: true
  max_radius_m: 200
  default_radius_m: 100
  # Composite-score component weights, in two sets selected per candidate
  # pair. Identifier evidence (website / phone agreement) is decisive when
  # present but absent on ~5 of 6 pairs — OSM carries a website on 17% of
  # rows and a phone on 14%, against 83% / 92% on the Overture side — so a
  # single fixed identifier weight would spend score mass on a blank for most
  # candidates. These are the defaults, used when neither side has a
  # comparable identifier; its share is redistributed to the others.
  distance_weight: 0.20
  name_weight: 0.40
  type_weight: 0.40
  identifier_weight: 0.0
  # Used instead when both sides carry a website or a phone. Ratio 30:30:15:15
  # (name : type : distance : identifier) normalised to sum to 1.
  weights_with_identifier:
    distance: 0.1667
    name: 0.3333
    type: 0.3333
    identifier: 0.1667
  chunk_size: 500_000
  chunk_target_pois: 200_000
  # Overture-internal deduplication runs before OSM × Overture
  # matching. Self-matches Overture POIs, groups them into clusters,
  # and drops non-winners so they never reach the cross-source stage.
  # See ``openpois.conflation.dedup_overture.mark_no_conflate``.
  overture_internal_dedup:
    enabled: true
    min_match_score: 0.75
    max_radius_m: 100
    chunk_target_pois: 200_000
    duckdb_memory_limit: "4GB"
  test_bbox:
    xmin: -122.45
    ymin: 47.50
    xmax: -122.25
    ymax: 47.70
  # Change-detection feature: use OSM history to penalize Overture POIs
  # that co-locate with a "ghost" — a previous state of an OSM element
  # (primary-tag deletion, lifecycle-prefix addition, or substantial
  # rename). Disabled by default for clean A/B testing.
  change_detection:
    enabled: false
    # Minimum composite score for an Overture × ghost shadow match.
    # Same scale as the main matcher's min_match_score, and raised with it
    # from 0.50 to 0.70 on 2026-07-27. The shadow matcher shares
    # ``compute_match_scores`` and reads the same component weights, so the
    # 0.50 floor had been calibrated against the pre-2026-07 weights
    # (0.0/0.50/0.30/0.20 with a constant identifier stub). Note its scores
    # are not on quite the same scale as the main matcher's: it passes
    # all-zero L0 bits and no affinity table, so its type score is binary on
    # exact shared_label equality, and it supplies no identifier arrays so it
    # always uses the no-identifier weight set.
    # NOT yet exercised: set after the 20260729 run, takes effect next run.
    min_shadow_match_score: 0.70
    # rapidfuzz.fuzz.token_set_ratio threshold below which an OSM name
    # change is considered a "substantial rename" rather than a typo
    # fix. Range 0-100. Lower = stricter (fewer events emitted).
    name_change_similarity_threshold: 50
    # Fallback delta for ghosts whose shared_label isn't in the fitted
    # model's per-group params. Equals sigmoid(logit_delta_0) for the
    # current 20260727_by_shared_label fit (logit_delta_0 = -1.807;
    # recomputed 2026-09-01, unchanged at this precision).
    default_delta: 0.141
    # Same-entity name gate: the token_set_ratio (0-100) a ghost's prior
    # name / brand must reach against the Overture name / brand — both
    # sides normalised (accents, case, punctuation, legal suffixes, a
    # trailing category token) — before a shadow match can fire; token
    # subset / superset pairs pass too. Both names are always required,
    # so unnamed ghosts never demote anything.
    #
    # 70 deliberately REVERSES the May-2026 "decision rule A" choice,
    # which kept this at 0 so that a differently named current business
    # at a churned address (Sleep Train -> Roosevelt Square) would still
    # be demoted. The September-2026 web-verified sample of 57 shadow_cd
    # rows measured 31% precision for that loose matcher (different-name
    # demotions 76% spurious: the listing was still open), so the rule
    # is now precision over recall: demote only when OSM closed the
    # *same* business Overture still shows. Do not lower without re-
    # running the >= 70% precision release gate in docs/change-detection.md.
    min_prior_name_match_score: 70
    # Ghosts whose event_timestamp is older than this are dropped when
    # apply_change_detection.py loads ghosts.parquet (the build itself
    # stays a pure history product). Pre-2021 deletions were 90% spurious
    # as closure evidence in the September-2026 sample. 0 keeps all.
    max_ghost_age_years: 3
    suppress_if_current_survivor:
      # Post-filter: drop the penalty if a *current* OSM element within
      # radius_m has name token_set_ratio >= threshold against the
      # Overture name (both normalised). Catches the POI that survives in
      # OSM under different geometry — a node merged into a building
      # way, a duplicate cleanup, a respelling — which the primary
      # matcher missed. Widened 50 -> 150 m on 2026-09-24: the sample's
      # misses (Blue Harbor Bank, Clark University Campus Store, Girl
      # Scouts of Nassau County, Mendocino Masonic Hall) were named ways
      # 50-150 m from the Overture point.
      enabled: true
      radius_m: 150
      name_similarity_threshold: 70
      # Candidate set is the FULL filtered osm_snapshot.parquet (nodes,
      # ways and relations by centroid; directories.snapshot_osm.snapshot)
      # rather than only the rated POIs, so building ways count as
      # survivors. false falls back to the rated snapshot.
      use_full_snapshot: true
  # Hand-curated confidence pins, applied by apply_manual_overrides.py as
  # the LAST stage of `make conflate` (after calibration, so a forced
  # conf_mean is never re-scaled). CSV columns: unified_id, overture_id,
  # osm_type, osm_id, action (exclude | include), reason, date, report_id;
  # any one id hitting a row applies the pin. `exclude` forces
  # conf_mean/lower/upper = 0 with calibration_flag 'manual_exclude';
  # `include` forces them to 1 with 'manual_include'. This is the fix path
  # for Overture-only closures / wrong POIs at any threshold; rows are
  # appended by the Close triage lane with the report id.
  manual_overrides:
    enabled: true
    # Explicit CSV path. null resolves the versioned default,
    # directories.manual_overrides/<versions.manual_overrides>/
    # manual_overrides.csv. A missing file is a no-op (logged).
    path: null
  # Release-stable unified_id (scripts/conflation/assign_stable_ids.py, run
  # inside `make conflate_baseline`; see openpois.conflation.stable_ids). Ids
  # are carried forward through the prior release's registry
  # (versions.id_registry) by source key; new places get id_prefix + 16
  # base32 characters of a hash of their founding key. guard_threshold is the
  # names_match threshold below which a retagged OSM element counts as a
  # different business and gets a new id (the change-detection name gate).
  stable_ids:
    guard_threshold: 70
    id_prefix: "opoi_"
  # Existence-confidence calibration (see
  # .claude/docs/confidence-calibration.md). From October 2026 production is
  # three Bayesian monotone-spline fits with the fixed-rate mixture label layer
  # (``bayes`` below), exported as grid curves and applied to every conflated
  # POI after change detection. The v4 estimator (fit_calibration.py) is
  # retired; its keys stay because the HT review, the CV comparator and the
  # evaluation scripts still read them.
  calibration:
    # The production fit: ``make fit_calibration`` (run_bayes_phase1.sh,
    # MODE=mixture) fits each segment separately with arm C and
    # label_noise = fixed_mixture, then ``make export_calibration``
    # (export_bayes_curves.py) writes the grid curves to
    # conflation/<version>/calibration/ if every segment passes acceptance.
    bayes:
      segments: [overture, osm, matched]
      warmup: 1000                  # NUTS iterations per chain
      samples: 1000
      chains: 4
      # NUTS target acceptance per segment (null = BlackJAX's 0.80). Fit alone,
      # each segment adapts a larger step size than the matched surface allowed
      # in the joint fit: on the July round at 0.80 the osm fit had 19
      # divergences and matched 12, failing the acceptance rule; at 0.99 osm had
      # none (2026-10-01 tuning on openpois-01). On the pooled October round
      # matched at 0.99 had 1 divergence (chain 3) and failed; refit at 0.995
      # with a new seed (Nat, 2026-10-01). With the forward rates over all
      # verdicts, 0.995 again had 1 divergence (chain 3); 0.998 and a new seed.
      target_accept:
        overture: null
        osm: 0.99
        matched: 0.998
      # NUTS seed per segment (null = fit_bayes_calibration.py's 20260927).
      seed:
        overture: null
        osm: null
        matched: 20261002
      grid_1d: 2001                 # 1-D curve nodes on [0, 1] (step 0.0005)
      grid_2d: 201                  # matched nodes per axis (step 0.005)
      # Fit outputs land in conflation/<version>/<eval_dir_name>/.
      eval_dir_name: calibration_bayes
    # Minimum gold labels for a refined (verdict x LLM-confidence) cell to
    # keep its own rate; thinner cells merge into the parent verdict class.
    # Still read by the HT review's correction rates.
    min_cell_gold: 25
    # Cross the LLM verdict with the LLM's own verdict confidence. Worth it
    # on round 20260730: LLM-exists splits 449/450 correct at high confidence
    # against 141/148 at medium.
    refine_by_confidence: true
    # RETIRED (v4) from here to band_alpha: matched_index_mode,
    # band_aggregation, pool_min_coef, grid_points, curve_output_bins,
    # bootstrap_reps and band_alpha set the v4 fit. Do not delete them: the
    # Bayesian scripts' shared FitConfig (bayes_calibration_common.py), the CV
    # comparator, simulate_band_coverage.py and the matched-mode evaluation
    # scripts still read them.
    # How the matched segment combines its two source scores (see
    # .claude/docs/confidence-calibration.md, "The matched segment"):
    #   interaction - monotone bilinear logistic index in rescaled logits,
    #                 a0 + a1 x + a2 y + a3 x y with Gupta constraints (4 params)
    #   pool        - constrained log-odds pool (3 params; production to 2026-09)
    #   additive | surface | average - compared and not adopted
    # Chosen 2026-09-26 from the five-mode comparison on round 20260730
    # (~/data/library/writeups/2026-09-26-openpois-matched-segment-modes.md):
    # it captures the substitutive "either source suffices" structure the pool
    # cannot represent, nests the pool (a3 = 0), and was never measurably worse
    # than the pool on either proper score across 24 cross-fit runs.
    matched_index_mode: interaction
    # How the 1-D pipeline's bootstrap band reaches the published bins:
    #   bin             - each replicate's map averaged over the production POIs
    #                     in each published bin (adopted 2026-09-26)
    #   anchored_kernel - pre-2026-09 method; re-smoothed each replicate, which
    #                     shrank and off-centred the band (simulated coverage of
    #                     a nominal 95% band: 0.76 matched, 0.87 osm, 0.46
    #                     overture; 0.89 / 0.91 / 0.66 with "bin").
    band_aggregation: bin
    # Lower bound on every slope/edge-derivative of a fitted matched index.
    pool_min_coef: 0.001
    grid_points: 200
    curve_output_bins: 40           # equal-mass value bins in the lookup
    bootstrap_reps: 500
    band_alpha: 0.05                # 95% pointwise band
    rng_seed: 20260730              # also read by the Bayesian scripts
    # Production (October 2026 on): earlier validation rounds pooled with the
    # current one (versions.calibration) in the Bayesian fit; see
    # .claude/plans/bayesian-monotone-calibration.md, execution log decision
    # 23. Their phase-1 rows join the curve fit and their gold joins the fixed
    # silver-label rates P(exists | segment, LLM verdict), each round under
    # its own design weights. CV holds out and scores the current round only.
    # When versions.calibration moves to the October 2026 round, set
    # ["20260730"].
    pooled_rounds: ["20260730"]

# Settings for publishing snapshots to Source Cooperative
# (https://source.coop/henryspatialanalysis/openpois). Source Coop is
# S3-compatible: uploads go to the literal bucket name below with keys
# prefixed by ``{repo_prefix}/``; public reads are served at
# ``{public_base_url}/``.
publish:
  # Source Coop serves S3 through a data proxy where the BUCKET is the account
  # and the repository is the first key segment. The former direct-S3 layout
  # (bucket us-west-2.opendata.source.coop, keys henryspatialanalysis/openpois/)
  # no longer authenticates with proxy-issued credentials.
  endpoint_url: "https://data.source.coop"
  bucket: "henryspatialanalysis"
  repo_prefix: "openpois"
  public_base_url: "https://data.source.coop/henryspatialanalysis/openpois"
  credentials_file: "~/repos/openpois/.env.json"
  geohash_precision_sort: 6       # ~0.6 km x 1.2 km; within-partition sort key for spatial row-group pruning
  # Values surfaced in the per-version README. Set each round until the
  # pipeline captures them automatically — see .claude/TODO.md.
  version_metadata:
    osm_snapshot_date: "2026-10-21"   # YYYY-MM-DD Geofabrik extract date
    overture_release: "2026-10-21.0"  # Overture Maps release ID, https://docs.overturemaps.org/release-calendar/
    model_commit: null                # null → use current git HEAD; set a short SHA to pin
  # PMTiles generation — multi-zoom archives for OSM, conflated and Overture.
  # Site's View.minZoom is 10 (full-metro view). Zooms up to the archive's
  # top zoom are served directly; deeper views are lossless OL over-zooms of
  # the top-zoom tile. drop-densest-as-needed drops features to keep each
  # tile under ~500 KB, and it sets ONE spacing per zoom level, so the densest
  # tile in the country (Midtown Manhattan) thins every tile at that zoom.
  # With max_zoom 14 alone, the 2026-10-01 archives kept only 19-75% of POIs
  # at z14, and over-zooming never brings the dropped ones back.
  # extend_zooms_if_still_dropping keeps adding zoom levels past max_zoom
  # until the top zoom drops nothing, so every POI renders somewhere; the
  # site reads each archive's top zoom from its header.
  pmtiles:
    min_zoom: 10
    max_zoom: 14
    extend_zooms_if_still_dropping: true
    drop_strategy: "drop-densest-as-needed"
    osm_layer_name: "osm_pois"
    conflated_layer_name: "conflated_pois"
    overture_layer_name: "overture_pois"
    osm_properties:
      - osm_id
      - osm_type       # node|way|relation — drives popup OSM link path
      - source
      - name
      - conf_mean      # drives confidence-based point coloring
      - amenity
      - shop
      - leisure
      - healthcare
      - craft
      - historic
      - landuse
      - office
      - tourism
    conflated_properties:
      - unified_id
      - source
      - overture_id    # drives the popup's Overture link
      - osm_id
      - osm_type       # node|way|relation — drives popup OSM link path
      - shared_label
      - conf_mean      # calibrated P(exists and open) since 2026-07-30
      - name
      - brand
      - match_score
      - match_distance_m
      # Calibration provenance, so a tile consumer can tell a curve-derived
      # confidence from a change-detection-demoted or extrapolated one without
      # fetching the parquet. Null for the ordinary on-curve case.
      - calibration_flag
    # Overture snapshot columns read into the Overture tiles. The prepare
    # script renames them to short tile properties (id, name, brand, website,
    # phone, addr_*) and flattens the LIST columns: websites/phones to their
    # first entry, taxonomy_hierarchy to a " > "-joined path for the popup.
    overture_properties:
      - overture_id
      - overture_name
      - brand_name
      - confidence       # Overture's provider confidence; drives point coloring
      - taxonomy_l0      # filter-panel group
      - basic_category   # filter key (Overture's recommended map category)
      - taxonomy_primary
      - taxonomy_hierarchy
      - overture_addr_street
      - overture_addr_city
      - overture_addr_state
      - overture_websites
      - overture_phones