From 40ed8572e22736a81620627a4804c8e983578c05 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 11:12:01 -0700 Subject: [PATCH 01/37] feat: add services/ scaffold + unwpp-service (Phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a services/ directory for per-datasource REST microservices. Each service: FastAPI app, Dockerfile, requirements.txt, test_client.py, managed by a top-level services/Makefile. unwpp-service (port 8100): - Pre-warms on startup: downloads all 4 WPP 2024 CSV.gz files to CACHE_DIR (mounted volume) if not already cached, then loads into memory - GET /health — liveness + loaded dataset names - GET /demographics/{ISO}?start_year=&end_year= — CBR/CDR, age distribution, cumulative life-table deaths; 404 on unknown ISO, 400 on inverted range - test_client.py: 27 checks covering happy path, lowercase ISO, single year, future years (2024-file branch), unknown ISO, and inverted range Co-Authored-By: Claude Sonnet 4.6 --- services/Makefile | 36 ++++++++ services/unwpp/Dockerfile | 12 +++ services/unwpp/main.py | 141 ++++++++++++++++++++++++++++++++ services/unwpp/requirements.txt | 5 ++ services/unwpp/test_client.py | 124 ++++++++++++++++++++++++++++ 5 files changed, 318 insertions(+) create mode 100644 services/Makefile create mode 100644 services/unwpp/Dockerfile create mode 100644 services/unwpp/main.py create mode 100644 services/unwpp/requirements.txt create mode 100644 services/unwpp/test_client.py diff --git a/services/Makefile b/services/Makefile new file mode 100644 index 0000000..50977a2 --- /dev/null +++ b/services/Makefile @@ -0,0 +1,36 @@ +CACHE_ROOT ?= $(HOME)/.laser/cache +PYTHON ?= python3.11 + +# ── unwpp-service (port 8100) ────────────────────────────────────────────────── + +build-unwpp: + docker build -t laser-unwpp-service unwpp/ + +run-unwpp: build-unwpp + @mkdir -p $(CACHE_ROOT)/unwpp + docker run -d \ + --name laser-unwpp-service \ + -p 8100:8000 \ + -v $(CACHE_ROOT)/unwpp:/cache \ + laser-unwpp-service + @echo "unwpp-service running on http://localhost:8100" + +stop-unwpp: + docker stop laser-unwpp-service 2>/dev/null || true + docker rm laser-unwpp-service 2>/dev/null || true + +restart-unwpp: stop-unwpp run-unwpp + +logs-unwpp: + docker logs -f laser-unwpp-service + +test-unwpp: + $(PYTHON) unwpp/test_client.py --url http://localhost:8100 + +# ── all ──────────────────────────────────────────────────────────────────────── + +run-all: run-unwpp +stop-all: stop-unwpp + +.PHONY: build-unwpp run-unwpp stop-unwpp restart-unwpp logs-unwpp test-unwpp \ + run-all stop-all diff --git a/services/unwpp/Dockerfile b/services/unwpp/Dockerfile new file mode 100644 index 0000000..377fddd --- /dev/null +++ b/services/unwpp/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY main.py . + +ENV CACHE_DIR=/cache + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/services/unwpp/main.py b/services/unwpp/main.py new file mode 100644 index 0000000..bfd30e8 --- /dev/null +++ b/services/unwpp/main.py @@ -0,0 +1,141 @@ +""" +unwpp-service — serves filtered UN World Population Prospects demographic data. + +GET /health +GET /demographics/{ISO}?start_year=YYYY&end_year=YYYY + +On startup, downloads all four WPP dataset files to CACHE_DIR if not already +present (pre-warm), then loads them into memory. Requests are served entirely +from in-memory DataFrames. +""" + +import logging +import os +from contextlib import asynccontextmanager +from pathlib import Path + +import httpx +import numpy as np +import pandas as pd +from fastapi import FastAPI, HTTPException, Query + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + +CACHE_DIR = Path(os.environ.get("CACHE_DIR", Path.home() / ".laser" / "cache" / "unwpp")) + +_WPP_BASE = ( + "https://population.un.org/wpp/assets/Excel%20Files/" + "1_Indicator%20(Standard)/CSV_FILES" +) + +_FILES = { + "age_dist": "WPP2024_Population1JanuaryByAge5GroupSex_Medium.csv.gz", + "indicators": "WPP2024_Demographic_Indicators_Medium.csv.gz", + "life_1950": "WPP2024_Life_Table_Complete_Medium_Both_1950-2023.csv.gz", + "life_2024": "WPP2024_Life_Table_Complete_Medium_Both_2024-2100.csv.gz", +} + +_CSV_DTYPES = {"Notes": str, "ISO3_code": str, "ISO2_code": str, "LocTypeName": str} + +# In-memory DataFrames populated at startup +_data: dict[str, pd.DataFrame] = {} + + +def _download(key: str) -> Path: + filename = _FILES[key] + dest = CACHE_DIR / filename + if dest.exists(): + logger.info("Cache hit: %s", dest) + return dest + CACHE_DIR.mkdir(parents=True, exist_ok=True) + url = f"{_WPP_BASE}/{filename}" + logger.info("Downloading %s ...", url) + with httpx.stream("GET", url, follow_redirects=True, timeout=600) as r: + r.raise_for_status() + with dest.open("wb") as f: + for chunk in r.iter_bytes(chunk_size=65_536): + f.write(chunk) + logger.info("Saved %s (%.1f MB)", dest, dest.stat().st_size / 1e6) + return dest + + +def _load(key: str) -> pd.DataFrame: + path = _download(key) + logger.info("Loading %s ...", path.name) + df = pd.read_csv(path, compression="gzip", dtype=_CSV_DTYPES) + logger.info("Loaded %s — %d rows", path.name, len(df)) + return df + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logger.info("Pre-warming: downloading and loading all WPP datasets ...") + for key in _FILES: + _data[key] = _load(key) + logger.info("Pre-warm complete — ready to serve.") + yield + + +app = FastAPI(title="unwpp-service", lifespan=lifespan) + + +@app.get("/health") +def health(): + return {"status": "ok", "loaded": list(_data.keys())} + + +@app.get("/demographics/{iso}") +def demographics( + iso: str, + start_year: int = Query(..., ge=1950, le=2100), + end_year: int = Query(..., ge=1950, le=2100), +): + if start_year > end_year: + raise HTTPException(400, "start_year must be <= end_year") + iso = iso.upper() + + # ── CBR / CDR ────────────────────────────────────────────────────────────── + country_demo = _data["indicators"][_data["indicators"]["ISO3_code"] == iso] + if country_demo.empty: + raise HTTPException(404, f"ISO code not found: {iso!r}") + + cxr = ( + country_demo[country_demo["Time"].between(start_year, end_year)] + .sort_values("Time")[["Time", "CBR", "CDR"]] + .rename(columns={"Time": "year"}) + .to_dict(orient="records") + ) + + # ── Age distribution at start_year ───────────────────────────────────────── + country_pop = _data["age_dist"][ + (_data["age_dist"]["ISO3_code"] == iso) & (_data["age_dist"]["Time"] == start_year) + ] + age_dist = ( + country_pop.sort_values("AgeGrpStart")[["AgeGrpStart", "PopTotal"]] + .rename(columns={"AgeGrpStart": "age_start", "PopTotal": "pop_total"}) + .to_dict(orient="records") + ) + + # ── Life expectancy (cumulative deaths) at start_year ────────────────────── + life_key = "life_1950" if start_year <= 2023 else "life_2024" + country_life = _data[life_key][ + (_data[life_key]["ISO3_code"] == iso) & (_data[life_key]["Time"] == start_year) + ].sort_values("AgeGrpStart").reset_index(drop=True) + + survival = country_life["lx"].to_numpy() + cumulative = 100_000 - np.round(survival) + cumulative = np.append(cumulative[1:], 100_000) + life_exp = [ + {"age": int(age), "cumulative_deaths": float(cd)} + for age, cd in zip(country_life["AgeGrpStart"].tolist(), cumulative) + ] + + return { + "iso": iso, + "start_year": start_year, + "end_year": end_year, + "cxr": cxr, + "age_dist": age_dist, + "life_exp": life_exp, + } diff --git a/services/unwpp/requirements.txt b/services/unwpp/requirements.txt new file mode 100644 index 0000000..1365e5f --- /dev/null +++ b/services/unwpp/requirements.txt @@ -0,0 +1,5 @@ +fastapi>=0.110 +uvicorn[standard]>=0.27 +httpx>=0.27 +pandas>=2.0 +numpy>=1.26 diff --git a/services/unwpp/test_client.py b/services/unwpp/test_client.py new file mode 100644 index 0000000..b9870ca --- /dev/null +++ b/services/unwpp/test_client.py @@ -0,0 +1,124 @@ +""" +Test client for unwpp-service. + +Usage: + python test_client.py [--url http://localhost:8100] + +Runs a series of checks against a live unwpp-service instance: + - /health responds OK + - /demographics returns expected shape and value ranges + - Edge cases: unknown ISO, inverted year range, boundary years +""" + +import argparse +import sys + +import httpx + +BASE = "http://localhost:8100" + + +def get(path: str, **params) -> httpx.Response: + return httpx.get(f"{BASE}{path}", params=params, timeout=30) + + +def check(label: str, cond: bool, detail: str = "") -> None: + status = "PASS" if cond else "FAIL" + print(f" [{status}] {label}" + (f" — {detail}" if detail else "")) + if not cond: + sys.exit(1) + + +def test_health(): + print("── /health ─────────────────────────────────────────────") + r = get("/health") + check("HTTP 200", r.status_code == 200) + body = r.json() + check("status == ok", body.get("status") == "ok") + loaded = body.get("loaded", []) + for key in ("age_dist", "indicators", "life_1950", "life_2024"): + check(f"dataset loaded: {key}", key in loaded) + + +def test_demographics_basic(): + print("── GET /demographics/NGA (2000–2025) ───────────────────") + r = get("/demographics/NGA", start_year=2000, end_year=2025) + check("HTTP 200", r.status_code == 200) + body = r.json() + check("iso == NGA", body["iso"] == "NGA") + + cxr = body["cxr"] + check("cxr has 26 rows", len(cxr) == 26, f"got {len(cxr)}") + check("cxr first year == 2000", cxr[0]["year"] == 2000) + check("cxr last year == 2025", cxr[-1]["year"] == 2025) + check("CBR > 0", all(row["CBR"] > 0 for row in cxr)) + check("CDR > 0", all(row["CDR"] > 0 for row in cxr)) + + age_dist = body["age_dist"] + check("age_dist non-empty", len(age_dist) > 0, f"got {len(age_dist)}") + check("age_dist has age_start=0", any(r["age_start"] == 0 for r in age_dist)) + check("age_dist pop_total > 0", all(r["pop_total"] > 0 for r in age_dist)) + + life_exp = body["life_exp"] + check("life_exp non-empty", len(life_exp) > 0, f"got {len(life_exp)}") + check("life_exp has age=0", any(r["age"] == 0 for r in life_exp)) + check("life_exp cumulative_deaths >= 0", all(r["cumulative_deaths"] >= 0 for r in life_exp)) + + +def test_demographics_lowercase_iso(): + print("── GET /demographics/nga (lowercase ISO) ───────────────") + r = get("/demographics/nga", start_year=2010, end_year=2010) + check("HTTP 200 (normalised to uppercase)", r.status_code == 200) + check("iso == NGA", r.json()["iso"] == "NGA") + + +def test_single_year(): + print("── GET /demographics/ETH (start_year == end_year) ──────") + r = get("/demographics/ETH", start_year=2015, end_year=2015) + check("HTTP 200", r.status_code == 200) + cxr = r.json()["cxr"] + check("cxr has exactly 1 row", len(cxr) == 1, f"got {len(cxr)}") + + +def test_future_years(): + print("── GET /demographics/IND (2030–2050) ───────────────────") + r = get("/demographics/IND", start_year=2030, end_year=2050) + check("HTTP 200", r.status_code == 200) + cxr = r.json()["cxr"] + check("cxr has 21 rows", len(cxr) == 21, f"got {len(cxr)}") + life_exp = r.json()["life_exp"] + check("life_exp uses 2024 file", len(life_exp) > 0) + + +def test_unknown_iso(): + print("── GET /demographics/ZZZ (unknown ISO) ─────────────────") + r = get("/demographics/ZZZ", start_year=2000, end_year=2010) + check("HTTP 404", r.status_code == 404, f"got {r.status_code}") + + +def test_inverted_years(): + print("── GET /demographics/NGA (start > end) ─────────────────") + r = get("/demographics/NGA", start_year=2025, end_year=2000) + check("HTTP 400", r.status_code == 400, f"got {r.status_code}") + + +def main(): + global BASE + parser = argparse.ArgumentParser() + parser.add_argument("--url", default=BASE) + args = parser.parse_args() + BASE = args.url.rstrip("/") + + print(f"\nRunning unwpp-service tests against {BASE}\n") + test_health() + test_demographics_basic() + test_demographics_lowercase_iso() + test_single_year() + test_future_years() + test_unknown_iso() + test_inverted_years() + print("\nAll tests passed.") + + +if __name__ == "__main__": + main() From 210d871961c1865cd240b5cbb108009d6ed08341 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 11:18:14 -0700 Subject: [PATCH 02/37] feat: add gadm-service (Phase 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /health GET /boundaries/{ISO}/{level} → GeoJSON FeatureCollection Downloads gadm41_{ISO}_shp.zip on first request for a country (temp-file write to avoid partial-download cache poisoning), caches under CACHE_DIR. All admin levels for a country are served from the single cached zip via pyogrio (bundled GDAL — no system GDAL needed in the Docker image). Properties per feature: nodeid (sequential int), name, gid. Returns 404 for unknown ISO or unavailable level, 400 for level outside 0–5. test_client.py: 16 checks — structure, level 0/1/2, cache speed, error cases. Co-Authored-By: Claude Sonnet 4.6 --- services/Makefile | 31 +++++++- services/gadm/Dockerfile | 13 ++++ services/gadm/main.py | 100 ++++++++++++++++++++++++++ services/gadm/requirements.txt | 5 ++ services/gadm/test_client.py | 125 +++++++++++++++++++++++++++++++++ 5 files changed, 272 insertions(+), 2 deletions(-) create mode 100644 services/gadm/Dockerfile create mode 100644 services/gadm/main.py create mode 100644 services/gadm/requirements.txt create mode 100644 services/gadm/test_client.py diff --git a/services/Makefile b/services/Makefile index 50977a2..59d0e4a 100644 --- a/services/Makefile +++ b/services/Makefile @@ -27,10 +27,37 @@ logs-unwpp: test-unwpp: $(PYTHON) unwpp/test_client.py --url http://localhost:8100 +# ── gadm-service (port 8101) ─────────────────────────────────────────────────── + +build-gadm: + docker build -t laser-gadm-service gadm/ + +run-gadm: build-gadm + @mkdir -p $(CACHE_ROOT)/gadm + docker run -d \ + --name laser-gadm-service \ + -p 8101:8000 \ + -v $(CACHE_ROOT)/gadm:/cache \ + laser-gadm-service + @echo "gadm-service running on http://localhost:8101" + +stop-gadm: + docker stop laser-gadm-service 2>/dev/null || true + docker rm laser-gadm-service 2>/dev/null || true + +restart-gadm: stop-gadm run-gadm + +logs-gadm: + docker logs -f laser-gadm-service + +test-gadm: + $(PYTHON) gadm/test_client.py --url http://localhost:8101 + # ── all ──────────────────────────────────────────────────────────────────────── -run-all: run-unwpp -stop-all: stop-unwpp +run-all: run-unwpp run-gadm +stop-all: stop-unwpp stop-gadm .PHONY: build-unwpp run-unwpp stop-unwpp restart-unwpp logs-unwpp test-unwpp \ + build-gadm run-gadm stop-gadm restart-gadm logs-gadm test-gadm \ run-all stop-all diff --git a/services/gadm/Dockerfile b/services/gadm/Dockerfile new file mode 100644 index 0000000..49f8db1 --- /dev/null +++ b/services/gadm/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.11-slim + +WORKDIR /app + +# pyogrio bundles its own GDAL — no system GDAL install needed +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY main.py . + +ENV CACHE_DIR=/cache + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/services/gadm/main.py b/services/gadm/main.py new file mode 100644 index 0000000..b480cfd --- /dev/null +++ b/services/gadm/main.py @@ -0,0 +1,100 @@ +""" +gadm-service — serves GADM 4.1 admin boundary GeoJSON. + +GET /health +GET /boundaries/{ISO}/{admin_level} → GeoJSON FeatureCollection + +Downloads gadm41_{ISO}_shp.zip on first request for a country, caches it. +All admin levels for that country are then served from the single cached zip. +""" + +import logging +import os +import shutil +import tempfile +from pathlib import Path + +import geopandas as gpd +import httpx +from fastapi import FastAPI, HTTPException +from fastapi.responses import JSONResponse + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + +CACHE_DIR = Path(os.environ.get("CACHE_DIR", Path.home() / ".laser" / "cache" / "gadm")) + +_GADM_BASE = "https://geodata.ucdavis.edu/gadm/gadm4.1/shp" + + +def _zip_path(iso: str) -> Path: + return CACHE_DIR / iso / f"gadm41_{iso}_shp.zip" + + +def _download(iso: str) -> Path: + dest = _zip_path(iso) + if dest.exists(): + logger.info("Cache hit: %s", dest) + return dest + + dest.parent.mkdir(parents=True, exist_ok=True) + url = f"{_GADM_BASE}/gadm41_{iso}_shp.zip" + logger.info("Downloading %s ...", url) + + # Write to a temp file first — avoids a partial file being treated as cached + tmp = Path(tempfile.mktemp(dir=dest.parent, suffix=".tmp")) + try: + with httpx.stream("GET", url, follow_redirects=True, timeout=600) as r: + if r.status_code == 404: + raise HTTPException(404, f"GADM has no data for ISO {iso!r}") + r.raise_for_status() + with tmp.open("wb") as f: + for chunk in r.iter_bytes(chunk_size=65_536): + f.write(chunk) + shutil.move(str(tmp), str(dest)) + except Exception: + tmp.unlink(missing_ok=True) + raise + + logger.info("Saved %s (%.1f MB)", dest, dest.stat().st_size / 1e6) + return dest + + +def _read(iso: str, level: int) -> gpd.GeoDataFrame: + zip_path = _download(iso) + layer = f"gadm41_{iso}_{level}" + logger.info("Reading %s from %s ...", layer, zip_path.name) + + try: + gdf = gpd.read_file(f"zip://{zip_path}!{layer}.shp", engine="pyogrio") + except Exception as exc: + raise HTTPException(404, f"Admin level {level} not available for {iso}: {exc}") from exc + + gdf["nodeid"] = range(len(gdf)) + + if level == 0: + gdf["name"] = gdf["GID_0"] + elif level <= 3: + gdf["name"] = gdf[f"NAME_{level}"] + else: + gdf["name"] = gdf["NAME_3"].astype(str) + ":" + gdf["NAME_4"].astype(str) + + gid_col = f"GID_{level}" + return gdf[["nodeid", "name", gid_col, "geometry"]].rename(columns={gid_col: "gid"}) + + +app = FastAPI(title="gadm-service") + + +@app.get("/health") +def health(): + return {"status": "ok"} + + +@app.get("/boundaries/{iso}/{level}") +def boundaries(iso: str, level: int): + if not 0 <= level <= 5: + raise HTTPException(400, f"admin_level must be 0–5, got {level}") + gdf = _read(iso.upper(), level) + # to_crs ensures WGS-84 (GeoJSON spec requires geographic coordinates) + return JSONResponse(content=gdf.to_crs("EPSG:4326").__geo_interface__) diff --git a/services/gadm/requirements.txt b/services/gadm/requirements.txt new file mode 100644 index 0000000..1e48c37 --- /dev/null +++ b/services/gadm/requirements.txt @@ -0,0 +1,5 @@ +fastapi>=0.110 +uvicorn[standard]>=0.27 +httpx>=0.27 +geopandas>=0.14 +pyogrio>=0.7 diff --git a/services/gadm/test_client.py b/services/gadm/test_client.py new file mode 100644 index 0000000..f7fef9b --- /dev/null +++ b/services/gadm/test_client.py @@ -0,0 +1,125 @@ +""" +Test client for gadm-service. + +Usage: + python test_client.py [--url http://localhost:8101] + +Uses Luxembourg (LUX) — tiny shapefile (~1 MB) — for tests that trigger a +real download. Checks GeoJSON structure, property presence, and error cases. +""" + +import argparse +import sys + +import httpx + +BASE = "http://localhost:8101" +TEST_ISO = "LUX" # Luxembourg — small file, fast download + + +def get(path: str, timeout: float = 120) -> httpx.Response: + return httpx.get(f"{BASE}{path}", timeout=timeout) + + +def check(label: str, cond: bool, detail: str = "") -> None: + status = "PASS" if cond else "FAIL" + print(f" [{status}] {label}" + (f" — {detail}" if detail else "")) + if not cond: + sys.exit(1) + + +def test_health(): + print("── /health ─────────────────────────────────────────────") + r = get("/health") + check("HTTP 200", r.status_code == 200) + check("status == ok", r.json().get("status") == "ok") + + +def test_boundaries_level0(): + print(f"── GET /boundaries/{TEST_ISO}/0 (country outline) ─────") + r = get(f"/boundaries/{TEST_ISO}/0") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + fc = r.json() + check("type == FeatureCollection", fc.get("type") == "FeatureCollection") + features = fc.get("features", []) + check("exactly 1 feature at level 0", len(features) == 1, f"got {len(features)}") + props = features[0].get("properties", {}) + check("nodeid present", "nodeid" in props) + check("name present", "name" in props) + check("gid present", "gid" in props) + geom = features[0].get("geometry", {}) + check("geometry present", bool(geom)) + check("geometry type is Polygon/MultiPolygon", geom.get("type") in ("Polygon", "MultiPolygon")) + + +def test_boundaries_level1(): + print(f"── GET /boundaries/{TEST_ISO}/1 ────────────────────────") + r = get(f"/boundaries/{TEST_ISO}/1") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + features = r.json().get("features", []) + check("at least 1 feature", len(features) >= 1, f"got {len(features)}") + for f in features: + props = f.get("properties", {}) + check("each feature has nodeid", "nodeid" in props) + check("each feature has name", "name" in props) + break # spot-check first feature only + + +def test_boundaries_level2(): + print(f"── GET /boundaries/{TEST_ISO}/2 ────────────────────────") + r = get(f"/boundaries/{TEST_ISO}/2") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + features = r.json().get("features", []) + check("more features at level 2 than level 1", len(features) > 1, f"got {len(features)}") + + +def test_lowercase_iso(): + print(f"── GET /boundaries/lux/1 (lowercase ISO) ───────────────") + r = get("/boundaries/lux/1") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + + +def test_second_request_is_cached(): + print(f"── Second request uses cache (fast) ────────────────────") + import time + t0 = time.monotonic() + r = get(f"/boundaries/{TEST_ISO}/1") + elapsed = time.monotonic() - t0 + check("HTTP 200", r.status_code == 200) + check("responded in < 10s (cache hit)", elapsed < 10, f"{elapsed:.2f}s") + + +def test_invalid_level(): + print("── GET /boundaries/LUX/9 (invalid level) ───────────────") + r = get("/boundaries/LUX/9") + check("HTTP 400", r.status_code == 400, f"got {r.status_code}") + + +def test_unknown_iso(): + print("── GET /boundaries/ZZZ/1 (unknown ISO) ─────────────────") + r = get("/boundaries/ZZZ/1") + check("HTTP 404", r.status_code == 404, f"got {r.status_code}") + + +def main(): + global BASE + parser = argparse.ArgumentParser() + parser.add_argument("--url", default=BASE) + args = parser.parse_args() + BASE = args.url.rstrip("/") + + print(f"\nRunning gadm-service tests against {BASE}\n") + print(f"Note: first run downloads {TEST_ISO} shapefile — may take a moment.\n") + test_health() + test_boundaries_level0() + test_boundaries_level1() + test_boundaries_level2() + test_lowercase_iso() + test_second_request_is_cached() + test_invalid_level() + test_unknown_iso() + print("\nAll tests passed.") + + +if __name__ == "__main__": + main() From 4427a2ea71d0779024bd66f6cda7c1a5b0cae197 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 11:22:13 -0700 Subject: [PATCH 03/37] feat: add gadm-service web client (client.html) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single-file Leaflet map — enter ISO + admin level, fetches from gadm-service, renders GeoJSON polygons with hover highlight and click-to-inspect properties. Open directly in a browser; no build step. Co-Authored-By: Claude Sonnet 4.6 --- services/gadm/client.html | 150 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 services/gadm/client.html diff --git a/services/gadm/client.html b/services/gadm/client.html new file mode 100644 index 0000000..1382196 --- /dev/null +++ b/services/gadm/client.html @@ -0,0 +1,150 @@ + + + + + gadm-service explorer + + + + + + + + + + +
+

gadm-service

+ + + + + + + + ready +
+ +
+ +
+

+
+
+ + + + From ec238fc248a4be6e2230549bd052c50860705c3c Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 11:26:40 -0700 Subject: [PATCH 04/37] fix: add CORS middleware to all services (allow file:// and cross-origin clients) Without this, browsers block fetch() calls from file:// pages or any origin that differs from the service host. All services now send Access-Control-Allow-Origin: * for GET requests. Co-Authored-By: Claude Sonnet 4.6 --- services/gadm/main.py | 2 ++ services/unwpp/main.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/services/gadm/main.py b/services/gadm/main.py index b480cfd..58e780a 100644 --- a/services/gadm/main.py +++ b/services/gadm/main.py @@ -17,6 +17,7 @@ import geopandas as gpd import httpx from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") @@ -84,6 +85,7 @@ def _read(iso: str, level: int) -> gpd.GeoDataFrame: app = FastAPI(title="gadm-service") +app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["GET"], allow_headers=["*"]) @app.get("/health") diff --git a/services/unwpp/main.py b/services/unwpp/main.py index bfd30e8..570f443 100644 --- a/services/unwpp/main.py +++ b/services/unwpp/main.py @@ -18,6 +18,7 @@ import numpy as np import pandas as pd from fastapi import FastAPI, HTTPException, Query +from fastapi.middleware.cors import CORSMiddleware logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") logger = logging.getLogger(__name__) @@ -78,6 +79,7 @@ async def lifespan(app: FastAPI): app = FastAPI(title="unwpp-service", lifespan=lifespan) +app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["GET"], allow_headers=["*"]) @app.get("/health") From 4ebc7143929f0c98ebfa6d6332549a87195fcf89 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 11:32:04 -0700 Subject: [PATCH 05/37] feat: add geoboundaries-service (Phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /health GET /boundaries/{ISO}/{level} → GeoJSON FeatureCollection Same endpoint contract as gadm-service. Downloads per-(ISO, level) zip from geoBoundaries v6.0.0 on GitHub; properties: nodeid, name, gid (from shapeID). CORS enabled. test_client.py: 14 checks — structure, cache speed, error cases. Co-Authored-By: Claude Sonnet 4.6 --- services/Makefile | 32 ++++++- services/geoboundaries/Dockerfile | 12 +++ services/geoboundaries/main.py | 92 +++++++++++++++++++ services/geoboundaries/requirements.txt | 5 ++ services/geoboundaries/test_client.py | 115 ++++++++++++++++++++++++ 5 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 services/geoboundaries/Dockerfile create mode 100644 services/geoboundaries/main.py create mode 100644 services/geoboundaries/requirements.txt create mode 100644 services/geoboundaries/test_client.py diff --git a/services/Makefile b/services/Makefile index 59d0e4a..f18a771 100644 --- a/services/Makefile +++ b/services/Makefile @@ -53,11 +53,39 @@ logs-gadm: test-gadm: $(PYTHON) gadm/test_client.py --url http://localhost:8101 +# ── geoboundaries-service (port 8102) ───────────────────────────────────────── + +build-geoboundaries: + docker build -t laser-geoboundaries-service geoboundaries/ + +run-geoboundaries: build-geoboundaries + @mkdir -p $(CACHE_ROOT)/geoboundaries + docker run -d \ + --name laser-geoboundaries-service \ + -p 8102:8000 \ + -v $(CACHE_ROOT)/geoboundaries:/cache \ + laser-geoboundaries-service + @echo "geoboundaries-service running on http://localhost:8102" + +stop-geoboundaries: + docker stop laser-geoboundaries-service 2>/dev/null || true + docker rm laser-geoboundaries-service 2>/dev/null || true + +restart-geoboundaries: stop-geoboundaries run-geoboundaries + +logs-geoboundaries: + docker logs -f laser-geoboundaries-service + +test-geoboundaries: + $(PYTHON) geoboundaries/test_client.py --url http://localhost:8102 + # ── all ──────────────────────────────────────────────────────────────────────── -run-all: run-unwpp run-gadm -stop-all: stop-unwpp stop-gadm +run-all: run-unwpp run-gadm run-geoboundaries +stop-all: stop-unwpp stop-gadm stop-geoboundaries .PHONY: build-unwpp run-unwpp stop-unwpp restart-unwpp logs-unwpp test-unwpp \ build-gadm run-gadm stop-gadm restart-gadm logs-gadm test-gadm \ + build-geoboundaries run-geoboundaries stop-geoboundaries \ + restart-geoboundaries logs-geoboundaries test-geoboundaries \ run-all stop-all diff --git a/services/geoboundaries/Dockerfile b/services/geoboundaries/Dockerfile new file mode 100644 index 0000000..377fddd --- /dev/null +++ b/services/geoboundaries/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY main.py . + +ENV CACHE_DIR=/cache + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/services/geoboundaries/main.py b/services/geoboundaries/main.py new file mode 100644 index 0000000..475c688 --- /dev/null +++ b/services/geoboundaries/main.py @@ -0,0 +1,92 @@ +""" +geoboundaries-service — serves geoBoundaries v6.0.0 admin boundary GeoJSON. + +GET /health +GET /boundaries/{ISO}/{admin_level} → GeoJSON FeatureCollection + +Each (ISO, level) combination is a separate zip on GitHub. Downloads on first +request and caches. Same endpoint contract as gadm-service. +""" + +import logging +import os +import shutil +import tempfile +from pathlib import Path + +import geopandas as gpd +import httpx +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + +CACHE_DIR = Path(os.environ.get("CACHE_DIR", Path.home() / ".laser" / "cache" / "geoboundaries")) + +_GB_BASE = "https://github.com/wmgeolab/geoBoundaries/raw/refs/tags/v6.0.0/releaseData/gbOpen" + + +def _zip_path(iso: str, level: int) -> Path: + return CACHE_DIR / iso / f"ADM{level}" / f"geoBoundaries-{iso}-ADM{level}-all.zip" + + +def _download(iso: str, level: int) -> Path: + dest = _zip_path(iso, level) + if dest.exists(): + logger.info("Cache hit: %s", dest) + return dest + + dest.parent.mkdir(parents=True, exist_ok=True) + url = f"{_GB_BASE}/{iso}/ADM{level}/geoBoundaries-{iso}-ADM{level}-all.zip" + logger.info("Downloading %s ...", url) + + tmp = Path(tempfile.mktemp(dir=dest.parent, suffix=".tmp")) + try: + with httpx.stream("GET", url, follow_redirects=True, timeout=600) as r: + if r.status_code == 404: + raise HTTPException(404, f"geoBoundaries has no data for {iso!r} ADM{level}") + r.raise_for_status() + with tmp.open("wb") as f: + for chunk in r.iter_bytes(chunk_size=65_536): + f.write(chunk) + shutil.move(str(tmp), str(dest)) + except Exception: + tmp.unlink(missing_ok=True) + raise + + logger.info("Saved %s (%.1f MB)", dest, dest.stat().st_size / 1e6) + return dest + + +def _read(iso: str, level: int) -> gpd.GeoDataFrame: + zip_path = _download(iso, level) + shp_name = f"geoBoundaries-{iso}-ADM{level}.shp" + logger.info("Reading %s ...", shp_name) + + try: + gdf = gpd.read_file(f"zip://{zip_path}!{shp_name}", engine="pyogrio") + except Exception as exc: + raise HTTPException(404, f"Could not read {shp_name} from zip: {exc}") from exc + + gdf["nodeid"] = range(len(gdf)) + gdf["name"] = gdf["shapeName"] + return gdf[["nodeid", "name", "shapeID", "geometry"]].rename(columns={"shapeID": "gid"}) + + +app = FastAPI(title="geoboundaries-service") +app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["GET"], allow_headers=["*"]) + + +@app.get("/health") +def health(): + return {"status": "ok"} + + +@app.get("/boundaries/{iso}/{level}") +def boundaries(iso: str, level: int): + if not 0 <= level <= 5: + raise HTTPException(400, f"admin_level must be 0–5, got {level}") + gdf = _read(iso.upper(), level) + return JSONResponse(content=gdf.to_crs("EPSG:4326").__geo_interface__) diff --git a/services/geoboundaries/requirements.txt b/services/geoboundaries/requirements.txt new file mode 100644 index 0000000..1e48c37 --- /dev/null +++ b/services/geoboundaries/requirements.txt @@ -0,0 +1,5 @@ +fastapi>=0.110 +uvicorn[standard]>=0.27 +httpx>=0.27 +geopandas>=0.14 +pyogrio>=0.7 diff --git a/services/geoboundaries/test_client.py b/services/geoboundaries/test_client.py new file mode 100644 index 0000000..b3aeac4 --- /dev/null +++ b/services/geoboundaries/test_client.py @@ -0,0 +1,115 @@ +""" +Test client for geoboundaries-service. + +Usage: + python test_client.py [--url http://localhost:8102] + +Uses Luxembourg (LUX) for download tests — small file, fast. +Same structural checks as gadm-service test client; property names +differ (gid comes from shapeID, not GID_N). +""" + +import argparse +import sys +import time + +import httpx + +BASE = "http://localhost:8102" +TEST_ISO = "LUX" + + +def get(path: str, timeout: float = 120) -> httpx.Response: + return httpx.get(f"{BASE}{path}", timeout=timeout) + + +def check(label: str, cond: bool, detail: str = "") -> None: + status = "PASS" if cond else "FAIL" + print(f" [{status}] {label}" + (f" — {detail}" if detail else "")) + if not cond: + sys.exit(1) + + +def test_health(): + print("── /health ─────────────────────────────────────────────") + r = get("/health") + check("HTTP 200", r.status_code == 200) + check("status == ok", r.json().get("status") == "ok") + + +def test_boundaries_level0(): + print(f"── GET /boundaries/{TEST_ISO}/0 (country outline) ─────") + r = get(f"/boundaries/{TEST_ISO}/0") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + fc = r.json() + check("type == FeatureCollection", fc.get("type") == "FeatureCollection") + features = fc.get("features", []) + check("exactly 1 feature at level 0", len(features) == 1, f"got {len(features)}") + props = features[0].get("properties", {}) + check("nodeid present", "nodeid" in props) + check("name present", "name" in props) + check("gid present", "gid" in props) + geom = features[0].get("geometry", {}) + check("geometry present", bool(geom)) + check("geometry is Polygon/MultiPolygon", geom.get("type") in ("Polygon", "MultiPolygon")) + + +def test_boundaries_level1(): + print(f"── GET /boundaries/{TEST_ISO}/1 ────────────────────────") + r = get(f"/boundaries/{TEST_ISO}/1") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + features = r.json().get("features", []) + check("at least 1 feature", len(features) >= 1, f"got {len(features)}") + props = features[0].get("properties", {}) + check("nodeid present", "nodeid" in props) + check("name present", "name" in props) + + +def test_lowercase_iso(): + print(f"── GET /boundaries/lux/1 (lowercase ISO) ───────────────") + r = get("/boundaries/lux/1") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + + +def test_cache_speed(): + print(f"── Second request uses cache (fast) ────────────────────") + t0 = time.monotonic() + r = get(f"/boundaries/{TEST_ISO}/1") + elapsed = time.monotonic() - t0 + check("HTTP 200", r.status_code == 200) + check("responded in < 10s (cache hit)", elapsed < 10, f"{elapsed:.2f}s") + + +def test_invalid_level(): + print("── GET /boundaries/LUX/9 (invalid level) ───────────────") + r = get("/boundaries/LUX/9") + check("HTTP 400", r.status_code == 400, f"got {r.status_code}") + + +def test_unknown_iso(): + print("── GET /boundaries/ZZZ/1 (unknown ISO) ─────────────────") + r = get("/boundaries/ZZZ/1") + check("HTTP 404", r.status_code == 404, f"got {r.status_code}") + + +def main(): + global BASE + parser = argparse.ArgumentParser() + parser.add_argument("--url", default=BASE) + args = parser.parse_args() + BASE = args.url.rstrip("/") + + print(f"\nRunning geoboundaries-service tests against {BASE}\n") + print(f"Note: first run downloads {TEST_ISO} shapefiles — may take a moment.\n") + test_health() + test_boundaries_level0() + test_boundaries_level1() + test_lowercase_iso() + test_cache_speed() + test_invalid_level() + test_unknown_iso() + print("\nAll tests passed.") + + +if __name__ == "__main__": + main() From 2c7c789f1b63149e448a60db013055931c9c4515 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 11:36:48 -0700 Subject: [PATCH 06/37] feat: add unocha-service (Phase 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /health (reports gdb_ready: true/false during startup) GET /boundaries/{ISO}/{level} → GeoJSON FeatureCollection Downloads the single ~1-2 GB UNOCHA global GDB zip from HDX on startup, extracts it, then serves filtered per-country/level slices. First request for an (ISO, level) pair reads the GDB layer and caches the GeoDataFrame in memory; all subsequent requests are served from the in-memory cache. Returns 503 if startup download/extraction is still in progress. Supports admin levels 0-3 (UNOCHA coverage). CORS enabled. test_client.py: --wait flag polls /health until gdb_ready before testing. Co-Authored-By: Claude Sonnet 4.6 --- services/Makefile | 32 ++++++- services/unocha/Dockerfile | 12 +++ services/unocha/main.py | 145 ++++++++++++++++++++++++++++++ services/unocha/requirements.txt | 5 ++ services/unocha/test_client.py | 150 +++++++++++++++++++++++++++++++ 5 files changed, 342 insertions(+), 2 deletions(-) create mode 100644 services/unocha/Dockerfile create mode 100644 services/unocha/main.py create mode 100644 services/unocha/requirements.txt create mode 100644 services/unocha/test_client.py diff --git a/services/Makefile b/services/Makefile index f18a771..50fa752 100644 --- a/services/Makefile +++ b/services/Makefile @@ -79,13 +79,41 @@ logs-geoboundaries: test-geoboundaries: $(PYTHON) geoboundaries/test_client.py --url http://localhost:8102 +# ── unocha-service (port 8103) ──────────────────────────────────────────────── + +build-unocha: + docker build -t laser-unocha-service unocha/ + +run-unocha: build-unocha + @mkdir -p $(CACHE_ROOT)/unocha + docker run -d \ + --name laser-unocha-service \ + -p 8103:8000 \ + -v $(CACHE_ROOT)/unocha:/cache \ + laser-unocha-service + @echo "unocha-service running on http://localhost:8103" + @echo "Note: first run downloads ~1-2 GB global GDB — use 'make logs-unocha' to monitor." + +stop-unocha: + docker stop laser-unocha-service 2>/dev/null || true + docker rm laser-unocha-service 2>/dev/null || true + +restart-unocha: stop-unocha run-unocha + +logs-unocha: + docker logs -f laser-unocha-service + +test-unocha: + $(PYTHON) unocha/test_client.py --url http://localhost:8103 --wait + # ── all ──────────────────────────────────────────────────────────────────────── -run-all: run-unwpp run-gadm run-geoboundaries -stop-all: stop-unwpp stop-gadm stop-geoboundaries +run-all: run-unwpp run-gadm run-geoboundaries run-unocha +stop-all: stop-unwpp stop-gadm stop-geoboundaries stop-unocha .PHONY: build-unwpp run-unwpp stop-unwpp restart-unwpp logs-unwpp test-unwpp \ build-gadm run-gadm stop-gadm restart-gadm logs-gadm test-gadm \ build-geoboundaries run-geoboundaries stop-geoboundaries \ restart-geoboundaries logs-geoboundaries test-geoboundaries \ + build-unocha run-unocha stop-unocha restart-unocha logs-unocha test-unocha \ run-all stop-all diff --git a/services/unocha/Dockerfile b/services/unocha/Dockerfile new file mode 100644 index 0000000..377fddd --- /dev/null +++ b/services/unocha/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY main.py . + +ENV CACHE_DIR=/cache + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/services/unocha/main.py b/services/unocha/main.py new file mode 100644 index 0000000..f98eced --- /dev/null +++ b/services/unocha/main.py @@ -0,0 +1,145 @@ +""" +unocha-service — serves UNOCHA global admin boundary GeoJSON. + +GET /health +GET /boundaries/{ISO}/{admin_level} → GeoJSON FeatureCollection + +On startup, downloads the single ~1-2 GB global GDB zip from HDX (if not +already cached) and extracts it. All country/level requests are served by +reading the appropriate GDB layer, filtering by iso3, and caching the result +in memory so each (ISO, level) pair is only loaded from disk once. +""" + +import logging +import os +import shutil +import tempfile +import warnings +import zipfile +from contextlib import asynccontextmanager +from pathlib import Path + +import geopandas as gpd +import httpx +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + +CACHE_DIR = Path(os.environ.get("CACHE_DIR", Path.home() / ".laser" / "cache" / "unocha")) + +_ZIP_NAME = "global_admin_boundaries_matched_latest.gdb.zip" +_GDB_NAME = "global_admin_boundaries_matched_latest.gdb" +_HDX_URL = ( + "https://data.humdata.org/dataset/70f1cb54-a30c-43b2-a751-44e77d8f5ade" + "/resource/733a9d4c-4e70-4f67-a5af-4138922cf43f/download/" + + _ZIP_NAME +) + +# In-memory cache: (iso, level) → GeoDataFrame +_cache: dict[tuple[str, int], gpd.GeoDataFrame] = {} +_gdb_ready = False + + +def _zip_path() -> Path: + return CACHE_DIR / _ZIP_NAME + + +def _gdb_path() -> Path: + return CACHE_DIR / _GDB_NAME + + +def _download_zip() -> None: + dest = _zip_path() + if dest.exists(): + logger.info("Cache hit: %s", dest) + return + CACHE_DIR.mkdir(parents=True, exist_ok=True) + logger.info("Downloading UNOCHA global GDB (~1-2 GB) — this takes a few minutes ...") + tmp = Path(tempfile.mktemp(dir=CACHE_DIR, suffix=".tmp")) + try: + with httpx.stream("GET", _HDX_URL, follow_redirects=True, timeout=1800) as r: + r.raise_for_status() + with tmp.open("wb") as f: + downloaded = 0 + for chunk in r.iter_bytes(chunk_size=1_048_576): # 1 MB chunks + f.write(chunk) + downloaded += len(chunk) + if downloaded % (100 * 1_048_576) == 0: + logger.info(" ... %.0f MB downloaded", downloaded / 1e6) + shutil.move(str(tmp), str(dest)) + except Exception: + tmp.unlink(missing_ok=True) + raise + logger.info("Saved %s (%.0f MB)", dest, dest.stat().st_size / 1e6) + + +def _extract_gdb() -> None: + gdb = _gdb_path() + if gdb.exists(): + logger.info("GDB already extracted: %s", gdb) + return + logger.info("Extracting GDB from zip (may take a minute) ...") + with zipfile.ZipFile(_zip_path(), "r") as zf: + zf.extractall(CACHE_DIR) + logger.info("Extraction complete: %s", gdb) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global _gdb_ready + CACHE_DIR.mkdir(parents=True, exist_ok=True) + _download_zip() + _extract_gdb() + _gdb_ready = True + logger.info("UNOCHA GDB ready — accepting requests.") + yield + + +def _read(iso: str, level: int) -> gpd.GeoDataFrame: + key = (iso, level) + if key in _cache: + return _cache[key] + + layer = f"admin{level}" + logger.info("Reading GDB layer %r for %s ...", layer, iso) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=RuntimeWarning) + full = gpd.read_file(_gdb_path(), layer=layer, engine="pyogrio") + + country = full[full["iso3"] == iso].copy() + if country.empty: + raise HTTPException(404, f"No UNOCHA data for ISO {iso!r} at admin level {level}") + + pcode_col = f"adm{level}_pcode" + name_col = f"adm{level}_name" + + country["nodeid"] = range(len(country)) + country["name"] = country[name_col] if level < 4 else country["adm3_name"] + country["adm4_name"] + country = country[["nodeid", "name", pcode_col, "geometry"]].rename( + columns={pcode_col: "gid"} + ) + _cache[key] = country + logger.info("Cached %s / admin%d — %d features", iso, level, len(country)) + return country + + +app = FastAPI(title="unocha-service", lifespan=lifespan) +app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["GET"], allow_headers=["*"]) + + +@app.get("/health") +def health(): + return {"status": "ok" if _gdb_ready else "warming", "gdb_ready": _gdb_ready} + + +@app.get("/boundaries/{iso}/{level}") +def boundaries(iso: str, level: int): + if not _gdb_ready: + raise HTTPException(503, "GDB not ready yet — startup download/extraction in progress") + if not 0 <= level <= 3: + raise HTTPException(400, f"UNOCHA supports admin levels 0–3, got {level}") + gdf = _read(iso.upper(), level) + return JSONResponse(content=gdf.to_crs("EPSG:4326").__geo_interface__) diff --git a/services/unocha/requirements.txt b/services/unocha/requirements.txt new file mode 100644 index 0000000..1e48c37 --- /dev/null +++ b/services/unocha/requirements.txt @@ -0,0 +1,5 @@ +fastapi>=0.110 +uvicorn[standard]>=0.27 +httpx>=0.27 +geopandas>=0.14 +pyogrio>=0.7 diff --git a/services/unocha/test_client.py b/services/unocha/test_client.py new file mode 100644 index 0000000..29170f1 --- /dev/null +++ b/services/unocha/test_client.py @@ -0,0 +1,150 @@ +""" +Test client for unocha-service. + +Usage: + python test_client.py [--url http://localhost:8103] [--wait] + +The service downloads a 1-2 GB global GDB on first startup. Use --wait to +poll /health until gdb_ready=true before running tests (useful in CI or +after a fresh container start). Without --wait, tests run immediately and +will get 503s if the GDB isn't ready yet. +""" + +import argparse +import sys +import time + +import httpx + +BASE = "http://localhost:8103" +TEST_ISO = "NGA" # Nigeria — well-covered in UNOCHA data + + +def get(path: str, timeout: float = 120) -> httpx.Response: + return httpx.get(f"{BASE}{path}", timeout=timeout) + + +def check(label: str, cond: bool, detail: str = "") -> None: + status = "PASS" if cond else "FAIL" + print(f" [{status}] {label}" + (f" — {detail}" if detail else "")) + if not cond: + sys.exit(1) + + +def wait_for_ready(timeout_s: int = 1800) -> None: + print(f"Waiting for GDB to be ready (up to {timeout_s}s) ...") + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + try: + r = httpx.get(f"{BASE}/health", timeout=5) + body = r.json() + if body.get("gdb_ready"): + print(" GDB ready.\n") + return + status = body.get("status", "?") + print(f" status={status} — waiting ...") + except Exception: + print(" service not yet reachable — waiting ...") + time.sleep(10) + print(" TIMED OUT waiting for GDB ready") + sys.exit(1) + + +def test_health(): + print("── /health ─────────────────────────────────────────────") + r = get("/health") + check("HTTP 200", r.status_code == 200) + body = r.json() + check("gdb_ready == true", body.get("gdb_ready") is True, str(body)) + + +def test_boundaries_level0(): + print(f"── GET /boundaries/{TEST_ISO}/0 ────────────────────────") + r = get(f"/boundaries/{TEST_ISO}/0") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + fc = r.json() + check("type == FeatureCollection", fc.get("type") == "FeatureCollection") + features = fc.get("features", []) + check("exactly 1 feature at level 0", len(features) == 1, f"got {len(features)}") + props = features[0].get("properties", {}) + check("nodeid present", "nodeid" in props) + check("name present", "name" in props) + check("gid present", "gid" in props) + geom = features[0].get("geometry", {}) + check("geometry present", bool(geom)) + check("geometry is Polygon/MultiPolygon", geom.get("type") in ("Polygon", "MultiPolygon")) + + +def test_boundaries_level1(): + print(f"── GET /boundaries/{TEST_ISO}/1 ────────────────────────") + r = get(f"/boundaries/{TEST_ISO}/1") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + features = r.json().get("features", []) + check("multiple features at level 1", len(features) > 1, f"got {len(features)}") + props = features[0].get("properties", {}) + check("nodeid present", "nodeid" in props) + check("name present", "name" in props) + + +def test_cache_speed(): + print(f"── Second request uses in-memory cache (fast) ──────────") + t0 = time.monotonic() + r = get(f"/boundaries/{TEST_ISO}/1") + elapsed = time.monotonic() - t0 + check("HTTP 200", r.status_code == 200) + check("responded in < 5s (memory cache)", elapsed < 5, f"{elapsed:.2f}s") + + +def test_lowercase_iso(): + print(f"── GET /boundaries/nga/1 (lowercase) ───────────────────") + r = get("/boundaries/nga/1") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + + +def test_invalid_level(): + print("── GET /boundaries/NGA/5 (unsupported level) ───────────") + r = get("/boundaries/NGA/5") + check("HTTP 400", r.status_code == 400, f"got {r.status_code}") + + +def test_unknown_iso(): + print("── GET /boundaries/ZZZ/1 (unknown ISO) ─────────────────") + r = get("/boundaries/ZZZ/1") + check("HTTP 404", r.status_code == 404, f"got {r.status_code}") + + +def test_503_before_ready(): + print("── /health returns warming status when not ready ────────") + # Can't easily test 503 on a live ready service; just confirm health + # reports gdb_ready correctly + r = get("/health") + check("HTTP 200", r.status_code == 200) + check("gdb_ready key present", "gdb_ready" in r.json()) + + +def main(): + global BASE + parser = argparse.ArgumentParser() + parser.add_argument("--url", default=BASE) + parser.add_argument("--wait", action="store_true", + help="Poll /health until gdb_ready before running tests") + args = parser.parse_args() + BASE = args.url.rstrip("/") + + print(f"\nRunning unocha-service tests against {BASE}\n") + if args.wait: + wait_for_ready() + + test_health() + test_boundaries_level0() + test_boundaries_level1() + test_cache_speed() + test_lowercase_iso() + test_invalid_level() + test_unknown_iso() + test_503_before_ready() + print("\nAll tests passed.") + + +if __name__ == "__main__": + main() From 47f5a080be49076841680c6c7d50243926c56675 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 11:49:15 -0700 Subject: [PATCH 07/37] feat: add worldpop-service (Phase 5) POST /prewarm/{iso} downloads WorldPop UN-adjusted raster on demand; POST /aggregate/{iso} accepts a GeoJSON FeatureCollection and returns {nodeid: population} by summing raster pixels per polygon via rasterio. Requires libexpat1 in Docker (rasterio shared lib dependency). Co-Authored-By: Claude Sonnet 4.6 --- services/Makefile | 32 ++++++- services/worldpop/Dockerfile | 13 +++ services/worldpop/main.py | 141 ++++++++++++++++++++++++++++ services/worldpop/requirements.txt | 7 ++ services/worldpop/test_client.py | 146 +++++++++++++++++++++++++++++ 5 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 services/worldpop/Dockerfile create mode 100644 services/worldpop/main.py create mode 100644 services/worldpop/requirements.txt create mode 100644 services/worldpop/test_client.py diff --git a/services/Makefile b/services/Makefile index 50fa752..8426406 100644 --- a/services/Makefile +++ b/services/Makefile @@ -106,14 +106,42 @@ logs-unocha: test-unocha: $(PYTHON) unocha/test_client.py --url http://localhost:8103 --wait +# ── worldpop-service (port 8104) ────────────────────────────────────────────── + +build-worldpop: + docker build -t laser-worldpop-service worldpop/ + +run-worldpop: build-worldpop + @mkdir -p $(CACHE_ROOT)/worldpop + docker run -d \ + --name laser-worldpop-service \ + -p 8104:8000 \ + -v $(CACHE_ROOT)/worldpop:/cache \ + laser-worldpop-service + @echo "worldpop-service running on http://localhost:8104" + @echo "Note: rasters are downloaded on demand via POST /prewarm/{iso} or POST /aggregate/{iso}." + +stop-worldpop: + docker stop laser-worldpop-service 2>/dev/null || true + docker rm laser-worldpop-service 2>/dev/null || true + +restart-worldpop: stop-worldpop run-worldpop + +logs-worldpop: + docker logs -f laser-worldpop-service + +test-worldpop: + $(PYTHON) worldpop/test_client.py --url http://localhost:8104 + # ── all ──────────────────────────────────────────────────────────────────────── -run-all: run-unwpp run-gadm run-geoboundaries run-unocha -stop-all: stop-unwpp stop-gadm stop-geoboundaries stop-unocha +run-all: run-unwpp run-gadm run-geoboundaries run-unocha run-worldpop +stop-all: stop-unwpp stop-gadm stop-geoboundaries stop-unocha stop-worldpop .PHONY: build-unwpp run-unwpp stop-unwpp restart-unwpp logs-unwpp test-unwpp \ build-gadm run-gadm stop-gadm restart-gadm logs-gadm test-gadm \ build-geoboundaries run-geoboundaries stop-geoboundaries \ restart-geoboundaries logs-geoboundaries test-geoboundaries \ build-unocha run-unocha stop-unocha restart-unocha logs-unocha test-unocha \ + build-worldpop run-worldpop stop-worldpop restart-worldpop logs-worldpop test-worldpop \ run-all stop-all diff --git a/services/worldpop/Dockerfile b/services/worldpop/Dockerfile new file mode 100644 index 0000000..0c1fe0d --- /dev/null +++ b/services/worldpop/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN apt-get update && apt-get install -y --no-install-recommends libexpat1 && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir -r requirements.txt + +COPY main.py . + +ENV CACHE_DIR=/cache + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/services/worldpop/main.py b/services/worldpop/main.py new file mode 100644 index 0000000..f23278a --- /dev/null +++ b/services/worldpop/main.py @@ -0,0 +1,141 @@ +""" +worldpop-service — aggregates WorldPop population raster into polygon areas. + +GET /health +POST /prewarm/{iso}?year=2020 → {"status": "ok", "iso": "...", "year": ...} +POST /aggregate/{iso}?year=2020 → {nodeid: population, ...} + body: GeoJSON FeatureCollection; each feature must have "nodeid" in properties + +Rasters are downloaded on demand and cached on disk. The WorldPop constrained +UN-adjusted dataset is used (years 2000–2020). +""" + +import logging +import os +import shutil +import tempfile +from contextlib import asynccontextmanager +from pathlib import Path + +import numpy as np +import geopandas as gpd +import httpx +import rasterio +import rasterio.mask +from fastapi import Body, FastAPI, HTTPException, Query +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from shapely.geometry import mapping + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + +CACHE_DIR = Path(os.environ.get("CACHE_DIR", Path.home() / ".laser" / "cache" / "worldpop")) +_YEAR_DEFAULT = 2020 + + +def _raster_path(iso: str, year: int) -> Path: + return CACHE_DIR / f"{iso.lower()}_ppp_{year}_UNadj.tif" + + +def _worldpop_url(iso: str, year: int) -> str: + return ( + f"https://data.worldpop.org/GIS/Population/" + f"Global_2000_2020/{year}/{iso.upper()}/" + f"{iso.lower()}_ppp_{year}_UNadj.tif" + ) + + +def _download_raster(iso: str, year: int) -> Path: + dest = _raster_path(iso, year) + if dest.exists(): + logger.info("Cache hit: %s", dest) + return dest + CACHE_DIR.mkdir(parents=True, exist_ok=True) + url = _worldpop_url(iso, year) + logger.info("Downloading WorldPop raster for %s year=%d ...", iso, year) + tmp = Path(tempfile.mktemp(dir=CACHE_DIR, suffix=".tmp")) + try: + with httpx.stream("GET", url, follow_redirects=True, timeout=600) as r: + if r.status_code == 404: + raise HTTPException(404, f"No WorldPop data for ISO {iso!r} year={year}") + r.raise_for_status() + with tmp.open("wb") as f: + downloaded = 0 + for chunk in r.iter_bytes(chunk_size=1_048_576): + f.write(chunk) + downloaded += len(chunk) + if downloaded % (50 * 1_048_576) == 0: + logger.info(" ... %.0f MB downloaded", downloaded / 1e6) + shutil.move(str(tmp), str(dest)) + except Exception: + tmp.unlink(missing_ok=True) + raise + logger.info("Saved %s (%.0f MB)", dest, dest.stat().st_size / 1e6) + return dest + + +@asynccontextmanager +async def lifespan(app: FastAPI): + CACHE_DIR.mkdir(parents=True, exist_ok=True) + logger.info("worldpop-service ready — rasters downloaded on demand via /prewarm or /aggregate.") + yield + + +app = FastAPI(title="worldpop-service", lifespan=lifespan) +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["GET", "POST"], + allow_headers=["*"], +) + + +@app.get("/health") +def health(): + return {"status": "ok"} + + +@app.post("/prewarm/{iso}") +def prewarm( + iso: str, + year: int = Query(_YEAR_DEFAULT, ge=2000, le=2020), +): + _download_raster(iso.upper(), year) + return {"status": "ok", "iso": iso.upper(), "year": year} + + +@app.post("/aggregate/{iso}") +def aggregate( + iso: str, + year: int = Query(_YEAR_DEFAULT, ge=2000, le=2020), + body: dict = Body(...), +): + if body.get("type") != "FeatureCollection": + raise HTTPException(400, "Body must be a GeoJSON FeatureCollection") + features = body.get("features", []) + if not features: + raise HTTPException(400, "FeatureCollection has no features") + + raster_path = _download_raster(iso.upper(), year) + + gdf = gpd.GeoDataFrame.from_features(features, crs="EPSG:4326") + + logger.info("Aggregating %s/%d over %d features ...", iso.upper(), year, len(gdf)) + result = {} + with rasterio.open(str(raster_path)) as src: + nodata = src.nodata + for feat, geom in zip(features, gdf.geometry): + try: + out_image, _ = rasterio.mask.mask(src, [mapping(geom)], crop=True) + data = out_image[0].astype(np.float64) + if nodata is not None: + data[data == nodata] = np.nan + total = float(np.nansum(data)) + except Exception: + total = 0.0 + nodeid = feat["properties"]["nodeid"] + result[int(nodeid)] = total + + logger.info("Aggregated %d features for %s/%d", len(result), iso.upper(), year) + return JSONResponse(content=result) diff --git a/services/worldpop/requirements.txt b/services/worldpop/requirements.txt new file mode 100644 index 0000000..63fcbb2 --- /dev/null +++ b/services/worldpop/requirements.txt @@ -0,0 +1,7 @@ +fastapi>=0.110 +uvicorn[standard]>=0.27 +httpx>=0.27 +geopandas>=0.14 +pyogrio>=0.7 +rasterio>=1.3 +numpy>=1.24 diff --git a/services/worldpop/test_client.py b/services/worldpop/test_client.py new file mode 100644 index 0000000..9a9e506 --- /dev/null +++ b/services/worldpop/test_client.py @@ -0,0 +1,146 @@ +""" +Test client for worldpop-service. + +Usage: + python test_client.py [--url http://localhost:8104] [--wait] + +The service downloads WorldPop constrained rasters on demand. Use --wait to +poll /health before running tests. The first run for a new ISO downloads the +raster (seconds to minutes depending on country size). +""" + +import argparse +import sys +import time + +import httpx + +BASE = "http://localhost:8104" +TEST_ISO = "LUX" # Luxembourg — small country, small raster (~1 MB) + +# Bounding box polygon covering Luxembourg (WGS84) +_LUX_RING = [[5.7, 49.4], [6.5, 49.4], [6.5, 50.2], [5.7, 50.2], [5.7, 49.4]] +TEST_GEOJSON = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"nodeid": 0, "name": "Luxembourg"}, + "geometry": {"type": "Polygon", "coordinates": [_LUX_RING]}, + } + ], +} + + +def get(path: str, timeout: float = 30) -> httpx.Response: + return httpx.get(f"{BASE}{path}", timeout=timeout) + + +def post(path: str, body: dict, timeout: float = 120) -> httpx.Response: + return httpx.post(f"{BASE}{path}", json=body, timeout=timeout) + + +def check(label: str, cond: bool, detail: str = "") -> None: + status = "PASS" if cond else "FAIL" + print(f" [{status}] {label}" + (f" — {detail}" if detail else "")) + if not cond: + sys.exit(1) + + +def wait_for_ready(timeout_s: int = 120) -> None: + print(f"Waiting for service to be ready (up to {timeout_s}s) ...") + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + try: + r = httpx.get(f"{BASE}/health", timeout=5) + if r.status_code == 200: + print(" Service ready.\n") + return + except Exception: + print(" Service not yet reachable — waiting ...") + time.sleep(5) + print(" TIMED OUT waiting for service") + sys.exit(1) + + +def test_health(): + print("── /health ─────────────────────────────────────────────") + r = get("/health") + check("HTTP 200", r.status_code == 200) + body = r.json() + check("status == ok", body.get("status") == "ok", str(body)) + + +def test_prewarm(): + print(f"── POST /prewarm/{TEST_ISO} (downloads raster) ─────────") + r = post(f"/prewarm/{TEST_ISO}", {}, timeout=600) + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + body = r.json() + check("status == ok", body.get("status") == "ok", str(body)) + check("iso present", "iso" in body) + check("year present", "year" in body) + + +def test_aggregate(): + print(f"── POST /aggregate/{TEST_ISO} ─────────────────────────") + r = post(f"/aggregate/{TEST_ISO}", TEST_GEOJSON, timeout=60) + check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + result = r.json() + check("nodeid 0 in result", "0" in result, str(list(result.keys())[:5])) + pop = result["0"] + check("population > 100_000", pop > 100_000, f"got {pop:.0f}") + check("population < 1_500_000", pop < 1_500_000, f"got {pop:.0f}") + + +def test_cache_speed(): + print(f"── Second /aggregate/{TEST_ISO} uses cached raster ─────") + t0 = time.monotonic() + r = post(f"/aggregate/{TEST_ISO}", TEST_GEOJSON, timeout=60) + elapsed = time.monotonic() - t0 + check("HTTP 200", r.status_code == 200) + check("responded in < 10s (cached raster)", elapsed < 10, f"{elapsed:.2f}s") + + +def test_unknown_iso(): + print("── POST /aggregate/ZZZ (unknown ISO) ────────────────────") + r = post("/aggregate/ZZZ", TEST_GEOJSON, timeout=60) + check("HTTP 404", r.status_code == 404, f"got {r.status_code}") + + +def test_bad_body(): + print("── POST /aggregate with non-FeatureCollection body ──────") + r = post(f"/aggregate/{TEST_ISO}", {"type": "Point", "coordinates": [6.1, 49.8]}, timeout=30) + check("HTTP 400", r.status_code == 400, f"got {r.status_code}") + + +def test_empty_features(): + print("── POST /aggregate with empty features array ─────────────") + r = post(f"/aggregate/{TEST_ISO}", {"type": "FeatureCollection", "features": []}, timeout=30) + check("HTTP 400", r.status_code == 400, f"got {r.status_code}") + + +def main(): + global BASE + parser = argparse.ArgumentParser() + parser.add_argument("--url", default=BASE) + parser.add_argument("--wait", action="store_true", + help="Wait for service to be ready before running tests") + args = parser.parse_args() + BASE = args.url.rstrip("/") + + print(f"\nRunning worldpop-service tests against {BASE}\n") + if args.wait: + wait_for_ready() + + test_health() + test_prewarm() + test_aggregate() + test_cache_speed() + test_unknown_iso() + test_bad_body() + test_empty_features() + print("\nAll tests passed.") + + +if __name__ == "__main__": + main() From 552e3d44e0bca5b862f5055bd58862f0d9745b28 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 12:05:15 -0700 Subject: [PATCH 08/37] feat: worldpop client.html + download lock + boundaries-only mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - client.html: Leaflet choropleth that chains shapes service → worldpop aggregate; mode toggle for "Boundaries only" (no worldpop call) useful for testing shapes independently - main.py: per-(iso,year) threading.Lock prevents duplicate concurrent raster downloads; updated docstring Co-Authored-By: Claude Sonnet 4.6 --- services/worldpop/client.html | 290 ++++++++++++++++++++++++++++++++++ services/worldpop/main.py | 66 +++++--- 2 files changed, 333 insertions(+), 23 deletions(-) create mode 100644 services/worldpop/client.html diff --git a/services/worldpop/client.html b/services/worldpop/client.html new file mode 100644 index 0000000..e744aed --- /dev/null +++ b/services/worldpop/client.html @@ -0,0 +1,290 @@ + + + + + WorldPop Population Choropleth + + + + + + +
+ + + + + + + + + +
+ +
+ + + + diff --git a/services/worldpop/main.py b/services/worldpop/main.py index f23278a..35cad54 100644 --- a/services/worldpop/main.py +++ b/services/worldpop/main.py @@ -6,14 +6,16 @@ POST /aggregate/{iso}?year=2020 → {nodeid: population, ...} body: GeoJSON FeatureCollection; each feature must have "nodeid" in properties -Rasters are downloaded on demand and cached on disk. The WorldPop constrained -UN-adjusted dataset is used (years 2000–2020). +Rasters are downloaded on demand and cached on disk. The WorldPop UN-adjusted +dataset (Global_2000_2020) is used. Per-(iso,year) locks prevent duplicate +concurrent downloads. """ import logging import os import shutil import tempfile +import threading from contextlib import asynccontextmanager from pathlib import Path @@ -33,6 +35,10 @@ CACHE_DIR = Path(os.environ.get("CACHE_DIR", Path.home() / ".laser" / "cache" / "worldpop")) _YEAR_DEFAULT = 2020 +# Per-(iso, year) lock prevents duplicate concurrent downloads of the same raster. +_lock_registry_mu: threading.Lock = threading.Lock() +_download_locks: dict[tuple[str, int], threading.Lock] = {} + def _raster_path(iso: str, year: int) -> Path: return CACHE_DIR / f"{iso.lower()}_ppp_{year}_UNadj.tif" @@ -51,27 +57,41 @@ def _download_raster(iso: str, year: int) -> Path: if dest.exists(): logger.info("Cache hit: %s", dest) return dest - CACHE_DIR.mkdir(parents=True, exist_ok=True) - url = _worldpop_url(iso, year) - logger.info("Downloading WorldPop raster for %s year=%d ...", iso, year) - tmp = Path(tempfile.mktemp(dir=CACHE_DIR, suffix=".tmp")) - try: - with httpx.stream("GET", url, follow_redirects=True, timeout=600) as r: - if r.status_code == 404: - raise HTTPException(404, f"No WorldPop data for ISO {iso!r} year={year}") - r.raise_for_status() - with tmp.open("wb") as f: - downloaded = 0 - for chunk in r.iter_bytes(chunk_size=1_048_576): - f.write(chunk) - downloaded += len(chunk) - if downloaded % (50 * 1_048_576) == 0: - logger.info(" ... %.0f MB downloaded", downloaded / 1e6) - shutil.move(str(tmp), str(dest)) - except Exception: - tmp.unlink(missing_ok=True) - raise - logger.info("Saved %s (%.0f MB)", dest, dest.stat().st_size / 1e6) + + # Acquire a per-(iso, year) lock so concurrent requests wait rather than + # each downloading the same raster simultaneously. + key = (iso, year) + with _lock_registry_mu: + if key not in _download_locks: + _download_locks[key] = threading.Lock() + lock = _download_locks[key] + + with lock: + if dest.exists(): # another thread finished while we waited + logger.info("Cache hit (post-lock): %s", dest) + return dest + + CACHE_DIR.mkdir(parents=True, exist_ok=True) + url = _worldpop_url(iso, year) + logger.info("Downloading WorldPop raster for %s year=%d ...", iso, year) + tmp = Path(tempfile.mktemp(dir=CACHE_DIR, suffix=".tmp")) + try: + with httpx.stream("GET", url, follow_redirects=True, timeout=600) as r: + if r.status_code == 404: + raise HTTPException(404, f"No WorldPop data for ISO {iso!r} year={year}") + r.raise_for_status() + with tmp.open("wb") as f: + downloaded = 0 + for chunk in r.iter_bytes(chunk_size=1_048_576): + f.write(chunk) + downloaded += len(chunk) + if downloaded % (50 * 1_048_576) == 0: + logger.info(" ... %.0f MB downloaded", downloaded / 1e6) + shutil.move(str(tmp), str(dest)) + except Exception: + tmp.unlink(missing_ok=True) + raise + logger.info("Saved %s (%.0f MB)", dest, dest.stat().st_size / 1e6) return dest From 1f4b5b60e1fbc7333bd856271d651ddf880ba958 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 15:36:29 -0700 Subject: [PATCH 09/37] feat: add worldpop prewarm script and improve client status message - prewarm_countries.py: parallel AKS init-job script; default list of ~50 humanitarian ISOs; --countries FILE, --workers, --year flags - client.html: clarify status message during first-run raster download Co-Authored-By: Claude Sonnet 4.6 --- services/worldpop/client.html | 2 +- services/worldpop/prewarm_countries.py | 93 ++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 services/worldpop/prewarm_countries.py diff --git a/services/worldpop/client.html b/services/worldpop/client.html index e744aed..7f5caf1 100644 --- a/services/worldpop/client.html +++ b/services/worldpop/client.html @@ -228,7 +228,7 @@ } // ── Step 2: aggregate population via worldpop ───────────────────────────── - status.textContent = `${fc.features.length} features loaded. Aggregating population…`; + status.textContent = `${fc.features.length} features loaded. Requesting population (first run downloads raster — may take minutes)…`; const wpResp = await fetch(`${wpUrl}/aggregate/${iso}?year=${year}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, diff --git a/services/worldpop/prewarm_countries.py b/services/worldpop/prewarm_countries.py new file mode 100644 index 0000000..140db91 --- /dev/null +++ b/services/worldpop/prewarm_countries.py @@ -0,0 +1,93 @@ +""" +Prewarm worldpop-service raster cache for a list of countries. + +Usage: + python prewarm_countries.py [--url http://localhost:8104] [--year 2020] + [--workers 4] [--countries FILE] + +Designed to run as a Kubernetes init Job that pre-populates the PVC before +worldpop-service pods start serving traffic. Each ISO triggers a +POST /prewarm/{iso} call; the service downloads and caches the raster. + +Default country list covers common LASER / humanitarian contexts (~50 ISOs). +Pass --countries to override with a newline-delimited file of ISO3 codes. +""" + +import argparse +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +import httpx + +# fmt: off +DEFAULT_COUNTRIES = [ + # Sub-Saharan Africa + "NGA", "ETH", "COD", "TZA", "KEN", "UGA", "MOZ", "GHA", "MDG", "CMR", + "CIV", "NER", "BFA", "MLI", "SEN", "ZMB", "ZWE", "SOM", "SSD", "SDN", + "AGO", "RWA", "BDI", "TCD", "GIN", "SLE", "LBR", "MWI", "NAM", "BWA", + # North Africa / Middle East + "EGY", "DZA", "MAR", "TUN", "LBY", "SYR", "IRQ", "YEM", "AFG", "PAK", + # South / Southeast Asia + "IND", "BGD", "NPL", "MMR", "KHM", "LAO", "PHL", "IDN", + # Latin America + "BRA", "COL", "PER", "BOL", "HTI", +] +# fmt: on + + +def prewarm_one(url: str, iso: str, year: int) -> tuple[str, bool, str]: + try: + r = httpx.post(f"{url}/prewarm/{iso}?year={year}", timeout=1800) + if r.status_code == 200: + return iso, True, "" + return iso, False, f"HTTP {r.status_code}: {r.text[:120]}" + except Exception as exc: + return iso, False, str(exc) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Pre-warm WorldPop raster cache.") + parser.add_argument("--url", default="http://localhost:8104", + help="worldpop-service base URL") + parser.add_argument("--year", type=int, default=2020, + help="WorldPop year (2000–2020, default 2020)") + parser.add_argument("--workers", type=int, default=4, + help="Parallel download workers (default 4)") + parser.add_argument("--countries", metavar="FILE", + help="Newline-delimited file of ISO3 codes (overrides default list)") + args = parser.parse_args() + + base_url = args.url.rstrip("/") + + if args.countries: + with open(args.countries) as f: + isos = [ln.strip().upper() for ln in f if ln.strip() and not ln.startswith("#")] + else: + isos = DEFAULT_COUNTRIES + + print(f"Pre-warming {len(isos)} countries at {base_url} (year={args.year}, workers={args.workers})\n") + + t0 = time.monotonic() + ok, failed = [], [] + + with ThreadPoolExecutor(max_workers=args.workers) as pool: + futures = {pool.submit(prewarm_one, base_url, iso, args.year): iso for iso in isos} + for fut in as_completed(futures): + iso, success, msg = fut.result() + if success: + ok.append(iso) + print(f" [OK] {iso}") + else: + failed.append(iso) + print(f" [FAIL] {iso} — {msg}") + + elapsed = time.monotonic() - t0 + print(f"\n{len(ok)}/{len(isos)} countries cached in {elapsed:.0f}s") + if failed: + print(f"Failed: {', '.join(failed)}") + sys.exit(1) + + +if __name__ == "__main__": + main() From 7a30f07feb5573e9d9d40419ca61207e02d0b8a1 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 15:53:38 -0700 Subject: [PATCH 10/37] =?UTF-8?q?feat:=20add=20services/generate.py=20?= =?UTF-8?q?=E2=80=94=20service-based=20laser-init=20data=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calls gadm/geoboundaries/unocha, worldpop, and unwpp services to produce the same data files as laser-init (gpkg, cxr.csv, age_dist.csv, life_exp.csv, config.yaml, provenance.json). Validated for ETH admin2 2010-2020 with GADM shapes: 79 features, 87.5M total population. Model scripts and validation plots still require the laser-init Load phase. Co-Authored-By: Claude Sonnet 4.6 --- services/generate.py | 267 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 services/generate.py diff --git a/services/generate.py b/services/generate.py new file mode 100644 index 0000000..e351348 --- /dev/null +++ b/services/generate.py @@ -0,0 +1,267 @@ +""" +generate.py — service-based equivalent of the laser-init Extract + Transform phases. + +Calls the geodata microservices to produce the same data files as `laser-init`, +without downloading anything directly. The Load phase (config.yaml, model scripts, +validation plots) can then be run by passing the output directory to the existing +laser-init CLI once it supports a --data-dir option, or by running the loader +directly. + +Produces (equivalent to laser-init output): + {ISO}_admin{level}.gpkg — GeoPackage (nodeid, name, population, geometry) + cxr.csv — Crude birth/death rates (Time, CBR, CDR) + age_dist.csv — Age distribution (AgeGrpStart, PopTotal) + life_exp.csv — Life expectancy curve (cumulative_deaths) + config.yaml — Model configuration (default parameters) + provenance.json — Data sources and timestamps + +Usage: + python generate.py ETH 2 2010 2020 --shape-source gadm + python generate.py NGA 1 2015 2025 --shape-source unocha --output-dir NGA/2015 + +Default service URLs (override with --*-url flags): + shapes: gadm=localhost:8101 geoboundaries=localhost:8102 unocha=localhost:8103 + worldpop: localhost:8104 + unwpp: localhost:8100 +""" + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +import geopandas as gpd +import httpx +import pandas as pd +import yaml +from shapely.geometry import shape + +SHAPE_SOURCE_PORTS = { + "gadm": 8101, + "geoboundaries": 8102, + "unocha": 8103, +} + + +# ── HTTP helpers ─────────────────────────────────────────────────────────────── + +def _get(url: str, timeout: float = 60) -> dict: + r = httpx.get(url, timeout=timeout) + r.raise_for_status() + return r.json() + + +def _post(url: str, body: dict, timeout: float = 1800) -> dict: + r = httpx.post(url, json=body, timeout=timeout) + r.raise_for_status() + return r.json() + + +# ── Service calls ────────────────────────────────────────────────────────────── + +def fetch_shapes(shapes_url: str, iso: str, level: int) -> dict: + url = f"{shapes_url}/boundaries/{iso}/{level}" + print(f" shapes ← {url}") + fc = _get(url) + print(f" {len(fc['features'])} features") + return fc + + +def fetch_population(wp_url: str, iso: str, year: int, fc: dict) -> dict: + url = f"{wp_url}/aggregate/{iso}?year={year}" + print(f" worldpop← {url} (first run downloads raster — may take minutes)") + pop = _post(url, fc) + total = sum(pop.values()) + print(f" {len(pop)} features, total pop {total:,.0f}") + return pop + + +def fetch_demographics(unwpp_url: str, iso: str, start_year: int, end_year: int) -> dict: + url = f"{unwpp_url}/demographics/{iso}?start_year={start_year}&end_year={end_year}" + print(f" unwpp ← {url}") + demo = _get(url) + print(f" {len(demo['cxr'])} years CBR/CDR, " + f"{len(demo['age_dist'])} age groups, " + f"{len(demo['life_exp'])} life-exp rows") + return demo + + +# ── Build outputs ────────────────────────────────────────────────────────────── + +def build_gdf(fc: dict, pop: dict) -> gpd.GeoDataFrame: + rows = [] + for f in fc["features"]: + p = f["properties"] + nodeid = int(p["nodeid"]) + rows.append({ + "nodeid": nodeid, + "name": p.get("name", ""), + "population": float(pop.get(str(nodeid), 0.0)), + "geometry": shape(f["geometry"]), + }) + gdf = gpd.GeoDataFrame(rows, crs="EPSG:4326") + return gdf.sort_values("nodeid").reset_index(drop=True) + + +def write_gpkg(gdf: gpd.GeoDataFrame, output_dir: Path, iso: str, level: int) -> Path: + dest = output_dir / f"{iso}_admin{level}.gpkg" + gdf.to_file(dest, driver="GPKG") + print(f" → {dest.name} ({len(gdf)} features, total pop {gdf.population.sum():,.0f})") + return dest + + +def write_cxr(demo: dict, output_dir: Path) -> Path: + dest = output_dir / "cxr.csv" + pd.DataFrame( + [{"Time": r["year"], "CBR": r["CBR"], "CDR": r["CDR"]} for r in demo["cxr"]] + ).to_csv(dest, index=False) + print(f" → {dest.name} ({len(demo['cxr'])} rows)") + return dest + + +def write_age_dist(demo: dict, output_dir: Path) -> Path: + dest = output_dir / "age_dist.csv" + pd.DataFrame( + [{"AgeGrpStart": r["age_start"], "PopTotal": r["pop_total"]} for r in demo["age_dist"]] + ).to_csv(dest, index=False) + print(f" → {dest.name} ({len(demo['age_dist'])} rows)") + return dest + + +def write_life_exp(demo: dict, output_dir: Path) -> Path: + dest = output_dir / "life_exp.csv" + pd.DataFrame( + [{"cumulative_deaths": r["cumulative_deaths"]} for r in demo["life_exp"]] + ).to_csv(dest, index=False) + print(f" → {dest.name} ({len(demo['life_exp'])} rows)") + return dest + + +def write_config(output_dir: Path, iso: str, level: int) -> Path: + dest = output_dir / "config.yaml" + cfg = { + "data-dir": str(output_dir.resolve()), + "datafiles": { + "shape-data": f"{iso}_admin{level}.gpkg", + "cxr-data": "cxr.csv", + "pop-data": "age_dist.csv", + "exp-data": "life_exp.csv", + }, + "simulation": { + "nyears": 10, + "r0": 2.5, + "exposed-duration-shape": 4.5, + "exposed-duration-scale": 1.0, + "infectious-duration-mean": 7.0, + "naive-population": True, + "gravity_k": 500, + "gravity_a": 1, + "gravity_b": 1, + "gravity_c": 2, + }, + } + with dest.open("w") as f: + yaml.dump(cfg, f, default_flow_style=False, sort_keys=False) + print(f" → {dest.name}") + return dest + + +def write_provenance( + output_dir: Path, iso: str, level: int, + shapes_url: str, wp_url: str, unwpp_url: str, + shape_source: str, raster_year: int, +) -> Path: + dest = output_dir / "provenance.json" + ts = datetime.now(timezone.utc).isoformat() + prov = { + f"{iso}_admin{level}.gpkg": { + "shape_source": shape_source, + "shapes_service": shapes_url, + "worldpop_service": wp_url, + "worldpop_year": raster_year, + "timestamp": ts, + }, + "cxr.csv": {"unwpp_service": unwpp_url, "timestamp": ts}, + "age_dist.csv": {"unwpp_service": unwpp_url, "timestamp": ts}, + "life_exp.csv": {"unwpp_service": unwpp_url, "timestamp": ts}, + } + dest.write_text(json.dumps(prov, indent=2)) + print(f" → {dest.name}") + return dest + + +# ── CLI ──────────────────────────────────────────────────────────────────────── + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate laser-init data files via geodata microservices.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("country", help="ISO-3 country code (e.g. ETH)") + parser.add_argument("level", type=int, help="Admin level (0–3)") + parser.add_argument("start_year", type=int, help="Start year (e.g. 2010)") + parser.add_argument("end_year", type=int, help="End year (e.g. 2020)") + parser.add_argument("--shape-source", + choices=["gadm", "geoboundaries", "unocha"], default="unocha", + help="Boundary data source (default: unocha)") + parser.add_argument("--output-dir", type=Path, default=None, + help="Output directory (default: ./{ISO}/{start_year})") + parser.add_argument("--shapes-url", default=None, + help="Shape service base URL (auto-selected from --shape-source)") + parser.add_argument("--worldpop-url", default="http://127.0.0.1:8104") + parser.add_argument("--unwpp-url", default="http://127.0.0.1:8100") + parser.add_argument("--raster-year", type=int, default=None, + help="WorldPop raster year (default: start_year clamped to 2020)") + args = parser.parse_args() + + iso = args.country.upper() + level = args.level + start_year = args.start_year + end_year = args.end_year + shape_src = args.shape_source + raster_year = args.raster_year or min(start_year, 2020) + output_dir = args.output_dir or (Path(iso) / str(start_year)) + + shapes_url = (args.shapes_url or + f"http://127.0.0.1:{SHAPE_SOURCE_PORTS[shape_src]}").rstrip("/") + wp_url = args.worldpop_url.rstrip("/") + unwpp_url = args.unwpp_url.rstrip("/") + + output_dir.mkdir(parents=True, exist_ok=True) + + print(f"\nlaser-init (services) — {iso} admin{level} {start_year}–{end_year}" + f" shape-source={shape_src}") + print() + + try: + fc = fetch_shapes(shapes_url, iso, level) + pop = fetch_population(wp_url, iso, raster_year, fc) + demo = fetch_demographics(unwpp_url, iso, start_year, end_year) + except httpx.HTTPStatusError as exc: + print(f"\nHTTP {exc.response.status_code} from {exc.request.url}") + print(exc.response.text[:300]) + sys.exit(1) + except Exception as exc: + print(f"\nError: {exc}") + sys.exit(1) + + print() + gdf = build_gdf(fc, pop) + write_gpkg(gdf, output_dir, iso, level) + write_cxr(demo, output_dir) + write_age_dist(demo, output_dir) + write_life_exp(demo, output_dir) + write_config(output_dir, iso, level) + write_provenance(output_dir, iso, level, + shapes_url, wp_url, unwpp_url, shape_src, raster_year) + + print(f"\nDone — {output_dir}/") + print("Next: run `laser-init {iso} {level} {start} {end} --shape-source {src}` " + "on the same output dir to add model scripts and validation plots,\n" + " or use the laser-init Load phase directly once it accepts pre-built data.".format( + iso=iso, level=level, start=start_year, end=end_year, src=shape_src)) + + +if __name__ == "__main__": + main() From 55692ac13d4676ffdb1fadf8830f267871f722bc Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 16:18:33 -0700 Subject: [PATCH 11/37] feat: generate.py --emit-scripts calls laser-init Load phase After producing data files via services, --emit-scripts imports AbmLoader and write_plots from laser.init (falling back to ../src if not installed) to emit config.yaml, seir.py, plot.py, PNGs, and report.pdf. Full workflow for ETH admin2 2010-2020 with GADM validated end-to-end. User runs: python generate.py ETH 2 2010 2020 --shape-source gadm --emit-scripts Then: cd ETH/2010 && python seir.py Co-Authored-By: Claude Sonnet 4.6 --- services/generate.py | 105 ++++++++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 25 deletions(-) diff --git a/services/generate.py b/services/generate.py index e351348..5105e42 100644 --- a/services/generate.py +++ b/services/generate.py @@ -139,26 +139,28 @@ def write_life_exp(demo: dict, output_dir: Path) -> Path: def write_config(output_dir: Path, iso: str, level: int) -> Path: + """Write a standalone config.yaml (used when --emit-scripts is not set).""" dest = output_dir / "config.yaml" + # Key names match what AbmLoader and the model scripts expect (underscores). cfg = { - "data-dir": str(output_dir.resolve()), + "data_dir": str(output_dir.resolve()), "datafiles": { - "shape-data": f"{iso}_admin{level}.gpkg", - "cxr-data": "cxr.csv", - "pop-data": "age_dist.csv", - "exp-data": "life_exp.csv", + "shape_data": f"{iso}_admin{level}.gpkg", + "cxr_data": "cxr.csv", + "pop_data": "age_dist.csv", + "exp_data": "life_exp.csv", }, "simulation": { - "nyears": 10, - "r0": 2.5, - "exposed-duration-shape": 4.5, - "exposed-duration-scale": 1.0, - "infectious-duration-mean": 7.0, - "naive-population": True, - "gravity_k": 500, - "gravity_a": 1, - "gravity_b": 1, - "gravity_c": 2, + "nyears": 10, + "r0": 2.5, + "exposed_duration_shape": 4.5, + "exposed_duration_scale": 1.0, + "infectious_duration_mean": 7.0, + "naive_population": True, + "gravity_k": 500, + "gravity_a": 1, + "gravity_b": 1, + "gravity_c": 2, }, } with dest.open("w") as f: @@ -167,6 +169,49 @@ def write_config(output_dir: Path, iso: str, level: int) -> Path: return dest +def _emit_scripts( + output_dir: Path, + iso: str, + level: int, + gpkg: Path, + cxr: Path, + age_dist: Path, + life_exp: Path, + model: str, +) -> None: + """Call the laser-init Load phase to emit model scripts and validation plots.""" + import sys as _sys + try: + from laser.init.loaders.abm import AbmLoader + from laser.init.cli import write_plots + except ImportError: + # generate.py lives in services/ inside the laser-init repo; try ../src + _src = Path(__file__).resolve().parent.parent / "src" + if not _src.exists(): + raise RuntimeError( + "laser.init not found. Install it: pip install -e /path/to/laser-init" + ) + _sys.path.insert(0, str(_src)) + from laser.init.loaders.abm import AbmLoader + from laser.init.cli import write_plots + + print("\nEmitting model scripts via laser-init Load phase ...") + AbmLoader().emit_script( + mode="ABM", + model=model, + shape_filename=gpkg, + cxr_filename=cxr, + pop_filename=age_dist, + exp_filename=life_exp, + output_dir=output_dir, + ) + print(f" → config.yaml, {model.lower()}.py, plot.py") + + print("Writing validation plots ...") + write_plots(gpkg, cxr, age_dist, life_exp, output_dir) + print(" → choropleth.png, cbr_cdr.png, age_distribution.png, life_expectancy.png, report.pdf") + + def write_provenance( output_dir: Path, iso: str, level: int, shapes_url: str, wp_url: str, unwpp_url: str, @@ -213,6 +258,11 @@ def main() -> None: parser.add_argument("--unwpp-url", default="http://127.0.0.1:8100") parser.add_argument("--raster-year", type=int, default=None, help="WorldPop raster year (default: start_year clamped to 2020)") + parser.add_argument("--model", choices=["SI", "SIR", "SEIR"], default="SEIR", + help="Model type for emitted script (default: SEIR)") + parser.add_argument("--emit-scripts", action="store_true", + help="Also emit model scripts and validation plots via laser-init " + "Load phase (requires laser-init to be installed)") args = parser.parse_args() iso = args.country.upper() @@ -247,20 +297,25 @@ def main() -> None: sys.exit(1) print() - gdf = build_gdf(fc, pop) - write_gpkg(gdf, output_dir, iso, level) - write_cxr(demo, output_dir) - write_age_dist(demo, output_dir) - write_life_exp(demo, output_dir) - write_config(output_dir, iso, level) + gdf = build_gdf(fc, pop) + gpkg = write_gpkg(gdf, output_dir, iso, level) + cxr = write_cxr(demo, output_dir) + age_dist = write_age_dist(demo, output_dir) + life_exp = write_life_exp(demo, output_dir) write_provenance(output_dir, iso, level, shapes_url, wp_url, unwpp_url, shape_src, raster_year) + if args.emit_scripts: + _emit_scripts(output_dir, iso, level, gpkg, cxr, age_dist, life_exp, args.model) + else: + write_config(output_dir, iso, level) + print(f"\nDone — {output_dir}/") + print(f"Run with --emit-scripts to also generate {args.model.lower()}.py, " + f"plot.py, config.yaml, and validation plots.") + return + print(f"\nDone — {output_dir}/") - print("Next: run `laser-init {iso} {level} {start} {end} --shape-source {src}` " - "on the same output dir to add model scripts and validation plots,\n" - " or use the laser-init Load phase directly once it accepts pre-built data.".format( - iso=iso, level=level, start=start_year, end=end_year, src=shape_src)) + print(f"To run the model:\n cd {output_dir} && python {args.model.lower()}.py") if __name__ == "__main__": From 6d799f820e1d41563303656d09871fe0313b0cd4 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 16:20:37 -0700 Subject: [PATCH 12/37] refactor: simplify _emit_scripts after installing laser-init as package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove sys.path fallback — laser.init is now importable directly. install: pip install -e /path/to/laser-init Co-Authored-By: Claude Sonnet 4.6 --- services/generate.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/services/generate.py b/services/generate.py index 5105e42..7b8585f 100644 --- a/services/generate.py +++ b/services/generate.py @@ -180,20 +180,8 @@ def _emit_scripts( model: str, ) -> None: """Call the laser-init Load phase to emit model scripts and validation plots.""" - import sys as _sys - try: - from laser.init.loaders.abm import AbmLoader - from laser.init.cli import write_plots - except ImportError: - # generate.py lives in services/ inside the laser-init repo; try ../src - _src = Path(__file__).resolve().parent.parent / "src" - if not _src.exists(): - raise RuntimeError( - "laser.init not found. Install it: pip install -e /path/to/laser-init" - ) - _sys.path.insert(0, str(_src)) - from laser.init.loaders.abm import AbmLoader - from laser.init.cli import write_plots + from laser.init.loaders.abm import AbmLoader + from laser.init.cli import write_plots print("\nEmitting model scripts via laser-init Load phase ...") AbmLoader().emit_script( From 89eef3e8146e8e629442a4dda711c426d1c49781 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 17:15:35 -0700 Subject: [PATCH 13/37] fix: round WorldPop float populations to integers in build_gdf WorldPop pixel sums are float; LASER is agent-based and requires exact integer counts. np.repeat silently truncates floats causing agent count mismatch and broadcast errors in the model. Co-Authored-By: Claude Sonnet 4.6 --- services/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/generate.py b/services/generate.py index 7b8585f..e9780b9 100644 --- a/services/generate.py +++ b/services/generate.py @@ -97,7 +97,7 @@ def build_gdf(fc: dict, pop: dict) -> gpd.GeoDataFrame: rows.append({ "nodeid": nodeid, "name": p.get("name", ""), - "population": float(pop.get(str(nodeid), 0.0)), + "population": round(float(pop.get(str(nodeid), 0.0))), "geometry": shape(f["geometry"]), }) gdf = gpd.GeoDataFrame(rows, crs="EPSG:4326") From 215217439825c1b9b951860a6ee39ea65efe5b4d Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 17:39:44 -0700 Subject: [PATCH 14/37] feat: AKS deployment manifests for geodata microservices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds services/AKS/ with: - geodata-services.yaml: PVCs (2–100 Gi), Deployments, and LoadBalancer Services for all 5 services in the existing laser-ai namespace - worldpop-prewarm-job.yaml: one-shot Job to pre-warm raster cache post-deploy - DEPLOYMENT.md: push → deploy → prewarm → get IPs → use with generate.py Makefile: push-{service}/push-all-geodata and k8s-deploy/k8s-status/k8s-ips/ k8s-prewarm/k8s-delete/k8s-delete-all targets with REGISTRY/TAG/KUBECONFIG vars. worldpop Dockerfile: include prewarm_countries.py so the Job reuses the same image. Co-Authored-By: Claude Sonnet 4.6 --- services/AKS/DEPLOYMENT.md | 135 ++++++++ services/AKS/geodata-services.yaml | 438 +++++++++++++++++++++++++ services/AKS/worldpop-prewarm-job.yaml | 52 +++ services/Makefile | 72 +++- services/worldpop/Dockerfile | 2 +- 5 files changed, 696 insertions(+), 3 deletions(-) create mode 100644 services/AKS/DEPLOYMENT.md create mode 100644 services/AKS/geodata-services.yaml create mode 100644 services/AKS/worldpop-prewarm-job.yaml diff --git a/services/AKS/DEPLOYMENT.md b/services/AKS/DEPLOYMENT.md new file mode 100644 index 0000000..006dc24 --- /dev/null +++ b/services/AKS/DEPLOYMENT.md @@ -0,0 +1,135 @@ +# Geodata Services — AKS Deployment Guide + +All five geodata microservices deploy into the existing `laser-ai` namespace. +Each gets its own LoadBalancer IP (matching the jenner-mcp pattern). +Each service mounts a PVC at `/cache` for persistent raster/shapefile storage +that survives pod restarts and redeployments. + +## Services + +| Name | Image | Port | Cache PVC | Notes | +|---|---|---|---|---| +| `laser-unwpp-svc` | `laser-unwpp-service` | 80 | 2 Gi | WPP CSVs; downloads on startup | +| `laser-gadm-svc` | `laser-gadm-service` | 80 | 20 Gi | GADM GDB per country; on-demand | +| `laser-geoboundaries-svc` | `laser-geoboundaries-service` | 80 | 10 Gi | geoBoundaries ZIPs; on-demand | +| `laser-unocha-svc` | `laser-unocha-service` | 80 | 5 Gi | UNOCHA global GDB (~2 GB); on first request | +| `laser-worldpop-svc` | `laser-worldpop-service` | 80 | 100 Gi | WorldPop rasters; on-demand or pre-warmed | + +Registry: `idm-docker-staging.packages.idmod.org/` + +--- + +## Quick Start + +### 1. Push images + +```bash +cd services/ +make push-all-geodata TAG=v0.5.1a +``` + +### 2. Deploy PVCs, Deployments, and Services + +```bash +make k8s-deploy +# or directly: +kubectl apply -f AKS/geodata-services.yaml --kubeconfig AKS/kube.conf +``` + +### 3. Wait for all pods to be Ready + +```bash +make k8s-status +# or: +kubectl get deployment,svc,pvc -n laser-ai -l app.kubernetes.io/part-of=laser-geodata \ + --kubeconfig AKS/kube.conf +``` + +### 4. Pre-warm the WorldPop raster cache (optional but recommended) + +Wait for `laser-worldpop-svc` to be Ready, then: + +```bash +make k8s-prewarm +# or directly: +kubectl apply -f AKS/worldpop-prewarm-job.yaml --kubeconfig AKS/kube.conf + +# Monitor progress (~50 countries, 4 parallel workers): +kubectl logs -f job/laser-worldpop-prewarm -n laser-ai --kubeconfig AKS/kube.conf +``` + +The Job downloads ~50 commonly-used country rasters (~2–500 MB each) in parallel. +Total time ~15–30 min depending on network. Completes once, then stays in a +terminal state (safe to leave or delete). + +--- + +## Get service IPs + +After deployment, retrieve LoadBalancer IPs: + +```bash +kubectl get svc -n laser-ai -l app.kubernetes.io/part-of=laser-geodata \ + --kubeconfig AKS/kube.conf \ + -o custom-columns='NAME:.metadata.name,IP:.status.loadBalancer.ingress[0].ip' +``` + +Use those IPs in `generate.py`: + +```bash +python services/generate.py NGA 1 2015 2020 \ + --shape-source unocha \ + --shapes-url http:// \ + --worldpop-url http:// \ + --unwpp-url http:// \ + --emit-scripts +``` + +--- + +## Update a service + +```bash +make push-worldpop TAG=v0.5.2a +kubectl set image deployment/laser-worldpop-svc \ + laser-worldpop-svc=idm-docker-staging.packages.idmod.org/laser-worldpop-service:v0.5.2a \ + -n laser-ai --kubeconfig AKS/kube.conf +kubectl rollout status deployment/laser-worldpop-svc -n laser-ai --kubeconfig AKS/kube.conf +``` + +## Rollback + +```bash +kubectl rollout undo deployment/laser-worldpop-svc -n laser-ai --kubeconfig AKS/kube.conf +``` + +## Tail logs + +```bash +kubectl logs -f deployment/laser-worldpop-svc -n laser-ai --kubeconfig AKS/kube.conf +``` + +## Tear down (preserves PVCs and cached data) + +```bash +kubectl delete deployments,services \ + laser-unwpp-svc laser-gadm-svc laser-geoboundaries-svc laser-unocha-svc laser-worldpop-svc \ + -n laser-ai --kubeconfig AKS/kube.conf +``` + +To also delete the cached data (irreversible): + +```bash +kubectl delete pvc \ + laser-unwpp-cache laser-gadm-cache laser-geoboundaries-cache \ + laser-unocha-cache laser-worldpop-cache \ + -n laser-ai --kubeconfig AKS/kube.conf +``` + +--- + +## Image Tagging Convention + +Matches the jenner-mcp pattern: `v0..` + +Examples: `v0.5.1a`, `v0.5.7b` diff --git a/services/AKS/geodata-services.yaml b/services/AKS/geodata-services.yaml new file mode 100644 index 0000000..dfeda46 --- /dev/null +++ b/services/AKS/geodata-services.yaml @@ -0,0 +1,438 @@ +# geodata-services.yaml — PVCs + Deployments + LoadBalancer Services +# +# Deploys all 5 geodata microservices into the laser-ai namespace. +# Each service gets its own LoadBalancer IP (matching the jenner-mcp pattern). +# Each service mounts a PVC at /cache for persistent raster/shapefile storage. +# +# Deploy: +# kubectl apply -f AKS/geodata-services.yaml --kubeconfig AKS/kube.conf +# +# Push images first: +# make push-all-geodata TAG=v0.5.1a (from services/ directory) + +# ── PersistentVolumeClaims ───────────────────────────────────────────────────── + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: laser-unwpp-cache + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + accessModes: [ReadWriteOnce] + storageClassName: managed-csi + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: laser-gadm-cache + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + accessModes: [ReadWriteOnce] + storageClassName: managed-csi + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: laser-geoboundaries-cache + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + accessModes: [ReadWriteOnce] + storageClassName: managed-csi + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: laser-unocha-cache + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + accessModes: [ReadWriteOnce] + storageClassName: managed-csi + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: laser-worldpop-cache + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + accessModes: [ReadWriteOnce] + storageClassName: managed-csi + resources: + requests: + storage: 100Gi + +# ── unwpp-service (UN World Population Prospects) ───────────────────────────── + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: laser-unwpp-svc + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + replicas: 1 + revisionHistoryLimit: 3 + selector: + matchLabels: + app: laser-unwpp-svc + template: + metadata: + labels: + app: laser-unwpp-svc + app.kubernetes.io/part-of: laser-geodata + spec: + imagePullSecrets: + - name: artifactory-login + containers: + - name: laser-unwpp-svc + image: idm-docker-staging.packages.idmod.org/laser-unwpp-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8000 + env: + - name: CACHE_DIR + value: /cache + resources: + requests: + cpu: "250m" + memory: "512Mi" + limits: + cpu: "1" + memory: "1Gi" + volumeMounts: + - name: cache + mountPath: /cache + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + volumes: + - name: cache + persistentVolumeClaim: + claimName: laser-unwpp-cache +--- +apiVersion: v1 +kind: Service +metadata: + name: laser-unwpp-svc + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + type: LoadBalancer + ports: + - name: http + port: 80 + targetPort: 8000 + selector: + app: laser-unwpp-svc + +# ── gadm-service ────────────────────────────────────────────────────────────── + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: laser-gadm-svc + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + replicas: 1 + revisionHistoryLimit: 3 + selector: + matchLabels: + app: laser-gadm-svc + template: + metadata: + labels: + app: laser-gadm-svc + app.kubernetes.io/part-of: laser-geodata + spec: + imagePullSecrets: + - name: artifactory-login + containers: + - name: laser-gadm-svc + image: idm-docker-staging.packages.idmod.org/laser-gadm-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8000 + env: + - name: CACHE_DIR + value: /cache + resources: + requests: + cpu: "250m" + memory: "1Gi" + limits: + cpu: "2" + memory: "2Gi" + volumeMounts: + - name: cache + mountPath: /cache + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + volumes: + - name: cache + persistentVolumeClaim: + claimName: laser-gadm-cache +--- +apiVersion: v1 +kind: Service +metadata: + name: laser-gadm-svc + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + type: LoadBalancer + ports: + - name: http + port: 80 + targetPort: 8000 + selector: + app: laser-gadm-svc + +# ── geoboundaries-service ───────────────────────────────────────────────────── + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: laser-geoboundaries-svc + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + replicas: 1 + revisionHistoryLimit: 3 + selector: + matchLabels: + app: laser-geoboundaries-svc + template: + metadata: + labels: + app: laser-geoboundaries-svc + app.kubernetes.io/part-of: laser-geodata + spec: + imagePullSecrets: + - name: artifactory-login + containers: + - name: laser-geoboundaries-svc + image: idm-docker-staging.packages.idmod.org/laser-geoboundaries-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8000 + env: + - name: CACHE_DIR + value: /cache + resources: + requests: + cpu: "250m" + memory: "512Mi" + limits: + cpu: "2" + memory: "2Gi" + volumeMounts: + - name: cache + mountPath: /cache + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + volumes: + - name: cache + persistentVolumeClaim: + claimName: laser-geoboundaries-cache +--- +apiVersion: v1 +kind: Service +metadata: + name: laser-geoboundaries-svc + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + type: LoadBalancer + ports: + - name: http + port: 80 + targetPort: 8000 + selector: + app: laser-geoboundaries-svc + +# ── unocha-service ──────────────────────────────────────────────────────────── + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: laser-unocha-svc + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + replicas: 1 + revisionHistoryLimit: 3 + selector: + matchLabels: + app: laser-unocha-svc + template: + metadata: + labels: + app: laser-unocha-svc + app.kubernetes.io/part-of: laser-geodata + spec: + imagePullSecrets: + - name: artifactory-login + containers: + - name: laser-unocha-svc + image: idm-docker-staging.packages.idmod.org/laser-unocha-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8000 + env: + - name: CACHE_DIR + value: /cache + resources: + requests: + cpu: "250m" + memory: "1Gi" + limits: + cpu: "2" + memory: "3Gi" + volumeMounts: + - name: cache + mountPath: /cache + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + volumes: + - name: cache + persistentVolumeClaim: + claimName: laser-unocha-cache +--- +apiVersion: v1 +kind: Service +metadata: + name: laser-unocha-svc + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + type: LoadBalancer + ports: + - name: http + port: 80 + targetPort: 8000 + selector: + app: laser-unocha-svc + +# ── worldpop-service ────────────────────────────────────────────────────────── + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: laser-worldpop-svc + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + replicas: 1 + revisionHistoryLimit: 3 + selector: + matchLabels: + app: laser-worldpop-svc + template: + metadata: + labels: + app: laser-worldpop-svc + app.kubernetes.io/part-of: laser-geodata + spec: + imagePullSecrets: + - name: artifactory-login + containers: + - name: laser-worldpop-svc + image: idm-docker-staging.packages.idmod.org/laser-worldpop-service:latest + imagePullPolicy: Always + ports: + - containerPort: 8000 + env: + - name: CACHE_DIR + value: /cache + resources: + requests: + cpu: "500m" + memory: "2Gi" + limits: + cpu: "4" + memory: "4Gi" + volumeMounts: + - name: cache + mountPath: /cache + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + volumes: + - name: cache + persistentVolumeClaim: + claimName: laser-worldpop-cache +--- +apiVersion: v1 +kind: Service +metadata: + name: laser-worldpop-svc + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + type: LoadBalancer + ports: + - name: http + port: 80 + targetPort: 8000 + selector: + app: laser-worldpop-svc diff --git a/services/AKS/worldpop-prewarm-job.yaml b/services/AKS/worldpop-prewarm-job.yaml new file mode 100644 index 0000000..2ea52a4 --- /dev/null +++ b/services/AKS/worldpop-prewarm-job.yaml @@ -0,0 +1,52 @@ +# worldpop-prewarm-job.yaml — one-shot Job that pre-populates the worldpop PVC. +# +# Run AFTER laser-worldpop-svc is Ready. The Job calls POST /prewarm/{iso} for +# each country in the default humanitarian list (~50 ISOs, 4 workers). +# Re-applying is safe: completed jobs are not re-run; delete and re-apply to retry. +# +# Apply: +# kubectl apply -f AKS/worldpop-prewarm-job.yaml --kubeconfig AKS/kube.conf +# +# Monitor: +# kubectl logs -f job/laser-worldpop-prewarm -n laser-ai --kubeconfig AKS/kube.conf +# +# Delete once complete (optional): +# kubectl delete job laser-worldpop-prewarm -n laser-ai --kubeconfig AKS/kube.conf + +apiVersion: batch/v1 +kind: Job +metadata: + name: laser-worldpop-prewarm + namespace: laser-ai + labels: + app.kubernetes.io/part-of: laser-geodata +spec: + # Do not restart on success; retry up to 2× on failure. + backoffLimit: 2 + template: + spec: + imagePullSecrets: + - name: artifactory-login + restartPolicy: OnFailure + containers: + - name: prewarm + image: idm-docker-staging.packages.idmod.org/laser-worldpop-service:latest + imagePullPolicy: Always + # Override the default uvicorn CMD with the prewarm script. + command: ["python", "prewarm_countries.py"] + args: + - "--url" + - "http://laser-worldpop-svc" + - "--year" + - "2020" + - "--workers" + - "4" + resources: + requests: + cpu: "250m" + memory: "256Mi" + limits: + cpu: "1" + memory: "512Mi" + # activeDeadlineSeconds: 7200 on the Job spec handles overall timeout. + activeDeadlineSeconds: 7200 diff --git a/services/Makefile b/services/Makefile index 8426406..bdc8676 100644 --- a/services/Makefile +++ b/services/Makefile @@ -133,15 +133,83 @@ logs-worldpop: test-worldpop: $(PYTHON) worldpop/test_client.py --url http://localhost:8104 -# ── all ──────────────────────────────────────────────────────────────────────── +# ── all (local Docker) ──────────────────────────────────────────────────────── run-all: run-unwpp run-gadm run-geoboundaries run-unocha run-worldpop stop-all: stop-unwpp stop-gadm stop-geoboundaries stop-unocha stop-worldpop +# ── AKS: build + push to registry ──────────────────────────────────────────── +# Usage: make push-all-geodata TAG=v0.5.1a +# make push-worldpop TAG=v0.5.2a + +REGISTRY ?= idm-docker-staging.packages.idmod.org +TAG ?= latest +KUBECONFIG ?= AKS/kube.conf +KUBECTL = kubectl --kubeconfig $(KUBECONFIG) +NAMESPACE = laser-ai + +push-unwpp: + docker build -t $(REGISTRY)/laser-unwpp-service:$(TAG) unwpp/ + docker push $(REGISTRY)/laser-unwpp-service:$(TAG) + +push-gadm: + docker build -t $(REGISTRY)/laser-gadm-service:$(TAG) gadm/ + docker push $(REGISTRY)/laser-gadm-service:$(TAG) + +push-geoboundaries: + docker build -t $(REGISTRY)/laser-geoboundaries-service:$(TAG) geoboundaries/ + docker push $(REGISTRY)/laser-geoboundaries-service:$(TAG) + +push-unocha: + docker build -t $(REGISTRY)/laser-unocha-service:$(TAG) unocha/ + docker push $(REGISTRY)/laser-unocha-service:$(TAG) + +push-worldpop: + docker build -t $(REGISTRY)/laser-worldpop-service:$(TAG) worldpop/ + docker push $(REGISTRY)/laser-worldpop-service:$(TAG) + +push-all-geodata: push-unwpp push-gadm push-geoboundaries push-unocha push-worldpop + +# ── AKS: deploy / status / manage ──────────────────────────────────────────── + +k8s-deploy: + $(KUBECTL) apply -f AKS/geodata-services.yaml -n $(NAMESPACE) + +k8s-status: + $(KUBECTL) get deployment,svc,pvc -n $(NAMESPACE) \ + -l app.kubernetes.io/part-of=laser-geodata + +k8s-ips: + $(KUBECTL) get svc -n $(NAMESPACE) \ + -l app.kubernetes.io/part-of=laser-geodata \ + -o custom-columns='NAME:.metadata.name,IP:.status.loadBalancer.ingress[0].ip' + +k8s-prewarm: + $(KUBECTL) apply -f AKS/worldpop-prewarm-job.yaml -n $(NAMESPACE) + +k8s-logs-%: + $(KUBECTL) logs -f deployment/laser-$*-svc -n $(NAMESPACE) + +# Tear down Deployments + Services but preserve PVCs (cached data survives). +k8s-delete: + $(KUBECTL) delete deployment,service \ + laser-unwpp-svc laser-gadm-svc laser-geoboundaries-svc \ + laser-unocha-svc laser-worldpop-svc \ + -n $(NAMESPACE) --ignore-not-found + +# Also wipe PVCs (destroys all cached rasters/shapefiles — use with care). +k8s-delete-all: k8s-delete + $(KUBECTL) delete pvc \ + laser-unwpp-cache laser-gadm-cache laser-geoboundaries-cache \ + laser-unocha-cache laser-worldpop-cache \ + -n $(NAMESPACE) --ignore-not-found + .PHONY: build-unwpp run-unwpp stop-unwpp restart-unwpp logs-unwpp test-unwpp \ build-gadm run-gadm stop-gadm restart-gadm logs-gadm test-gadm \ build-geoboundaries run-geoboundaries stop-geoboundaries \ restart-geoboundaries logs-geoboundaries test-geoboundaries \ build-unocha run-unocha stop-unocha restart-unocha logs-unocha test-unocha \ build-worldpop run-worldpop stop-worldpop restart-worldpop logs-worldpop test-worldpop \ - run-all stop-all + run-all stop-all \ + push-unwpp push-gadm push-geoboundaries push-unocha push-worldpop push-all-geodata \ + k8s-deploy k8s-status k8s-ips k8s-prewarm k8s-delete k8s-delete-all diff --git a/services/worldpop/Dockerfile b/services/worldpop/Dockerfile index 0c1fe0d..cd2b40c 100644 --- a/services/worldpop/Dockerfile +++ b/services/worldpop/Dockerfile @@ -6,7 +6,7 @@ COPY requirements.txt . RUN apt-get update && apt-get install -y --no-install-recommends libexpat1 && rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir -r requirements.txt -COPY main.py . +COPY main.py prewarm_countries.py ./ ENV CACHE_DIR=/cache From 2493960439fe4dea2c49991f2663bfb0c8846074 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 20:34:17 -0700 Subject: [PATCH 15/37] fix: retry worldpop aggregate on Azure LB idle-timeout connection reset The Azure LoadBalancer has a 4-min TCP idle timeout. A slow raster download (NGA = 421 MB, ~8 min) holds the HTTP connection open with no data flowing, triggering a reset. The download continues server-side; retrying hits the cache. Co-Authored-By: Claude Sonnet 4.6 --- services/generate.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/services/generate.py b/services/generate.py index e9780b9..2fb7b03 100644 --- a/services/generate.py +++ b/services/generate.py @@ -28,6 +28,7 @@ import argparse import json import sys +import time from datetime import datetime, timezone from pathlib import Path @@ -71,7 +72,20 @@ def fetch_shapes(shapes_url: str, iso: str, level: int) -> dict: def fetch_population(wp_url: str, iso: str, year: int, fc: dict) -> dict: url = f"{wp_url}/aggregate/{iso}?year={year}" print(f" worldpop← {url} (first run downloads raster — may take minutes)") - pop = _post(url, fc) + # The Azure LB has a 4-min idle timeout; a slow raster download causes a + # connection reset mid-request. Retry: the download continues server-side + # and subsequent calls hit the cache quickly. + for attempt in range(1, 11): + try: + pop = _post(url, fc) + break + except (httpx.RemoteProtocolError, httpx.ReadError, httpx.ConnectError) as exc: + if attempt == 10: + raise + wait = 60 + print(f" worldpop connection reset (raster download in progress) " + f"— retry {attempt}/10 in {wait}s ...") + time.sleep(wait) total = sum(pop.values()) print(f" {len(pop)} features, total pop {total:,.0f}") return pop From 234e92baf75b2425705de184ae56307687d0466f Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Fri, 1 May 2026 21:43:08 -0700 Subject: [PATCH 16/37] feat: update client.html to use AKS service IPs Replace localhost port-based URLs with AKS LoadBalancer IPs. Source selector now swaps full URLs (each service has its own IP on port 80). Defaults: shapes (GADM): http://48.200.52.126 shapes (geoBoundaries): http://4.149.210.205 shapes (UNOCHA): http://40.91.121.206 worldpop: http://4.155.140.158 Co-Authored-By: Claude Sonnet 4.6 --- services/worldpop/client.html | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/services/worldpop/client.html b/services/worldpop/client.html index 7f5caf1..39bfca4 100644 --- a/services/worldpop/client.html +++ b/services/worldpop/client.html @@ -74,16 +74,16 @@
- +
@@ -212,6 +227,7 @@ const status = document.getElementById('status'); btn.disabled = true; + btn.classList.add('loading'); setStatus('Fetching shapes…'); // Wrapper: fetch with a descriptive error that includes the URL, HTTP status, @@ -265,17 +281,70 @@ return; } - // ── Step 2: aggregate population via worldpop ───────────────────────────── - setStatus(`✓ ${fc.features.length} shapes. Requesting WorldPop population (may take a minute if raster not cached)…`); + // ── Step 2: guard against slow aggregation ──────────────────────────────── + // Countries whose uncompressed raster exceeds 2 GB use windowed disk reads + // instead of in-memory mode. For those countries, aggregating hundreds of + // polygons takes hours. Warn and bail out early. + const _LARGE_RASTER_ISOS = new Set([ + 'BRA','RUS','CAN','USA','CHN','AUS','IND','ARG','KAZ','DZA', + 'COD','SAU','MEX','IDN','SDN','LBY','IRN','MNG','PER','TCD', + 'ANG','MAL','ETH','MLI','NER','COL','BOL','MRT','EGY','TZA', + ]); + const _MAX_WINDOWED_POLYGONS = 500; + if (_LARGE_RASTER_ISOS.has(iso.toUpperCase()) && fc.features.length > _MAX_WINDOWED_POLYGONS) { + setStatus( + `⚠ ${fc.features.length} polygons for ${iso.toUpperCase()} — WorldPop aggregation at this admin level would take many hours in the browser.\n\n` + + `Use the command line instead:\n` + + ` python generate.py --iso ${iso.toUpperCase()} --level ${level} --year ${year}`, + true + ); + // Still render the boundary map so the user can see the shapes. + if (geojsonLayer) map.removeLayer(geojsonLayer); + if (legendControl) { map.removeControl(legendControl); legendControl = null; } + geojsonLayer = L.geoJSON(fc, { + style: { fillColor: '#4a90d9', fillOpacity: 0.35, color: '#2563eb', weight: 1 }, + onEachFeature: (feature, layer) => { + const name = feature.properties.name || feature.properties.NAME || ''; + layer.bindTooltip(`${name}`, { sticky: true }); + layer.on('mouseover', function () { this.setStyle({ weight: 2, color: '#111' }); }); + layer.on('mouseout', function () { geojsonLayer.resetStyle(this); }); + }, + }).addTo(map); + map.fitBounds(geojsonLayer.getBounds()); + btn.disabled = false; + btn.classList.remove('loading'); + return; + } + + // ── Step 3: aggregate population via worldpop ───────────────────────────── + // Truncate coordinates to 3 decimal places (~100 m) before sending — + // WorldPop rasters are ~1 km resolution so this loses nothing meaningful + // but cuts payload size 5–10× for countries with detailed borders (BRA, IDN…). + function truncateCoords(coords, dp) { + if (typeof coords[0] === 'number') return coords.map(v => +v.toFixed(dp)); + return coords.map(c => truncateCoords(c, dp)); + } + const fcSlim = { + type: 'FeatureCollection', + features: fc.features.map(f => ({ + ...f, + geometry: { + ...f.geometry, + coordinates: truncateCoords(f.geometry.coordinates, 3), + }, + })), + }; + + setStatus(`✓ ${fc.features.length} shapes. Requesting WorldPop population (may take a few minutes for large countries)…`); const wpEndpoint = `${wpUrl}/aggregate/${iso}?year=${year}`; const wpResp = await apiFetch('WorldPop', wpEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(fc), + body: JSON.stringify(fcSlim), }); const popByNode = await wpResp.json(); - // ── Step 3: attach population, compute range + quantile colour fn ───────── + // ── Step 4: attach population, compute range + quantile colour fn ───────── let minPop = Infinity, maxPop = -Infinity, totalPop = 0; fc.features.forEach(f => { const pop = popByNode[String(f.properties.nodeid)] ?? 0; @@ -286,7 +355,7 @@ }); const colorFor = buildQuantileColorFn(popByNode); - // ── Step 4: render choropleth ───────────────────────────────────────────── + // ── Step 5: render choropleth ───────────────────────────────────────────── if (geojsonLayer) map.removeLayer(geojsonLayer); geojsonLayer = L.geoJSON(fc, { style: feature => ({ @@ -317,6 +386,7 @@ console.error(err); } finally { btn.disabled = false; + btn.classList.remove('loading'); } }); diff --git a/services/worldpop/main.py b/services/worldpop/main.py index 5ebacda..e2d6778 100644 --- a/services/worldpop/main.py +++ b/services/worldpop/main.py @@ -28,7 +28,7 @@ import rasterio.windows from fastapi import Body, FastAPI, HTTPException, Query from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, StreamingResponse from shapely.geometry import mapping logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") @@ -38,6 +38,10 @@ _YEAR_DEFAULT = 2020 _MAX_RASTER_MB = 2000 # on-demand size limit; larger rasters must use generate.py +# 512 MB GDAL block cache — default is 64 MB which is too small when reading many +# per-polygon windows from a large compressed GeoTIFF (constant cache thrash). +os.environ.setdefault("GDAL_CACHEMAX", "512") + # Per-(iso, year) lock prevents duplicate concurrent downloads of the same raster. _lock_registry_mu: threading.Lock = threading.Lock() _download_locks: dict[tuple[str, int], threading.Lock] = {} @@ -152,40 +156,52 @@ def prewarm( return {"status": "ok", "iso": iso.upper(), "year": year} -@app.post("/aggregate/{iso}") -def aggregate( - iso: str, - year: int = Query(_YEAR_DEFAULT, ge=2000, le=2020), - body: dict = Body(...), -): - if body.get("type") != "FeatureCollection": - raise HTTPException(400, "Body must be a GeoJSON FeatureCollection") - features = body.get("features", []) - if not features: - raise HTTPException(400, "FeatureCollection has no features") +# Maximum pixels read from disk per strip in windowed mode. Each strip is one +# horizontal band of a polygon's bounding-box window. Keeping this at 8 M pixels +# caps per-strip peak memory at ~64 MB (float32 data + bool mask). +_MAX_STRIP_PIXELS = 8_000_000 + - raster_path = _download_raster(iso.upper(), year) +def _sum_polygon_windowed(src, win, transform, geom, nodata) -> float: + """Sum raster pixels inside geom using horizontal strips to bound memory. - raster_mb = raster_path.stat().st_size // (1024 * 1024) - if raster_mb > _MAX_RASTER_MB: - raise HTTPException( - 422, - f"{iso.upper()} raster is {raster_mb} MB — too large to aggregate in real time " - f"(limit {_MAX_RASTER_MB} MB; would exceed the load-balancer timeout). " - f"Use generate.py instead, which retries automatically." + Used when the polygon's bounding-box window is too large to read at once + (e.g. Amazonian states in BRA). Reads _MAX_STRIP_PIXELS pixels at a time. + """ + height = int(win.height) + width = int(win.width) + strip_rows = max(1, _MAX_STRIP_PIXELS // max(width, 1)) + total = 0.0 + for row_start in range(0, height, strip_rows): + strip_h = min(strip_rows, height - row_start) + strip_win = rasterio.windows.Window( + int(win.col_off), int(win.row_off) + row_start, width, strip_h + ) + strip_data = src.read(1, window=strip_win) + strip_transform = rasterio.windows.transform(strip_win, transform) + strip_mask = rasterio.features.geometry_mask( + [mapping(geom)], + out_shape=(strip_h, width), + transform=strip_transform, + invert=True, ) + vals = strip_data[strip_mask].astype(np.float64) + if nodata is not None: + vals = vals[vals != nodata] + total += float(np.sum(vals)) + return total - gdf = gpd.GeoDataFrame.from_features(features, crs="EPSG:4326") - logger.info("Aggregating %s/%d over %d features ...", iso.upper(), year, len(gdf)) +def _aggregate_stream(iso: str, year: int, features: list, gdf): + """Generator that yields a JSON object one key:value at a time. - # For countries whose raster fits in memory (< 2 GB uncompressed float32), - # read the whole array once and slice per polygon — fast, one disk read. - # For very large countries (CAN ~21 GB, RUS, AUS, …) read only each - # polygon's bounding-box window from disk to avoid OOM. + Streaming keeps data flowing to the client throughout computation so the + Azure Load Balancer (4-min idle timeout) never sees a silent connection, + even when aggregating large-country rasters like BRA or IDN. + """ + raster_path = _raster_path(iso, year) _2GB = 2 * 1024 ** 3 - result = {} with rasterio.open(str(raster_path)) as src: nodata = src.nodata transform = src.transform @@ -193,14 +209,27 @@ def aggregate( uncompressed = src.width * src.height * 4 # float32 bytes if uncompressed < _2GB: - data = src.read(1) # float32 — read entire raster once + data = src.read(1) logger.info("In-memory mode (%.0f MB uncompressed)", uncompressed / 1e6) + ordered = list(zip(features, gdf.geometry)) else: data = None logger.info("Windowed mode (%.0f MB uncompressed — too large for in-memory)", uncompressed / 1e6) + # Sort by raster row then column so consecutive polygons share GDAL + # tile-cache entries — critical for level-2 queries with hundreds of + # small polygons spread across a large compressed raster. + def _scan_key(feat_geom): + _, geom = feat_geom + win = rasterio.windows.from_bounds( + *geom.bounds, transform=transform + ).round_offsets() + return (int(win.row_off), int(win.col_off)) + ordered = sorted(zip(features, gdf.geometry), key=_scan_key) + logger.info("Sorted %d features by raster scan order", len(ordered)) - for feat, geom in zip(features, gdf.geometry): + first = True + for feat, geom in ordered: nodeid = int(feat["properties"]["nodeid"]) try: win = rasterio.windows.from_bounds( @@ -211,30 +240,55 @@ def aggregate( height = int(win.height) width = int(win.width) if height <= 0 or width <= 0: - result[nodeid] = 0.0 - continue - - if data is not None: + pop = 0.0 + elif data is not None: row_off = int(win.row_off) col_off = int(win.col_off) window_data = data[row_off:row_off + height, col_off:col_off + width] + window_transform = rasterio.windows.transform(win, transform) + geom_mask = rasterio.features.geometry_mask( + [mapping(geom)], + out_shape=(height, width), + transform=window_transform, + invert=True, + ) + vals = window_data[geom_mask].astype(np.float64) + if nodata is not None: + vals = vals[vals != nodata] + pop = float(np.sum(vals)) else: - window_data = src.read(1, window=win) - - window_transform = rasterio.windows.transform(win, transform) - geom_mask = rasterio.features.geometry_mask( - [mapping(geom)], - out_shape=(height, width), - transform=window_transform, - invert=True, - ) - - vals = window_data[geom_mask].astype(np.float64) - if nodata is not None: - vals = vals[vals != nodata] - result[nodeid] = float(np.sum(vals)) + # Windowed mode: read in strips to bound peak memory. + pop = _sum_polygon_windowed(src, win, transform, geom, nodata) except Exception: - result[nodeid] = 0.0 + logger.exception("Error aggregating nodeid=%d for %s", nodeid, iso) + pop = 0.0 + + prefix = b"{" if first else b"," + first = False + yield prefix + f'"{nodeid}":{pop}'.encode() - logger.info("Aggregated %d features for %s/%d", len(result), iso.upper(), year) - return JSONResponse(content=result) + yield b"}" if not first else b"{}" + logger.info("Aggregated %d features for %s/%d", len(features), iso, year) + + +@app.post("/aggregate/{iso}") +def aggregate( + iso: str, + year: int = Query(_YEAR_DEFAULT, ge=2000, le=2020), + body: dict = Body(...), +): + if body.get("type") != "FeatureCollection": + raise HTTPException(400, "Body must be a GeoJSON FeatureCollection") + features = body.get("features", []) + if not features: + raise HTTPException(400, "FeatureCollection has no features") + + _download_raster(iso.upper(), year) + + gdf = gpd.GeoDataFrame.from_features(features, crs="EPSG:4326") + logger.info("Aggregating %s/%d over %d features ...", iso.upper(), year, len(gdf)) + + return StreamingResponse( + _aggregate_stream(iso.upper(), year, features, gdf), + media_type="application/json", + ) diff --git a/services/worldpop/test_client.py b/services/worldpop/test_client.py index 9a9e506..2f1261d 100644 --- a/services/worldpop/test_client.py +++ b/services/worldpop/test_client.py @@ -3,10 +3,13 @@ Usage: python test_client.py [--url http://localhost:8104] [--wait] + python test_client.py --url http://4.155.140.158 --large-country BRA -The service downloads WorldPop constrained rasters on demand. Use --wait to -poll /health before running tests. The first run for a new ISO downloads the -raster (seconds to minutes depending on country size). +The service downloads WorldPop rasters on demand. Use --wait to poll /health +before running tests. The first run for a new ISO downloads the raster. + +Default ISO is LUX (Luxembourg, ~1 MB raster — fast, always downloaded on demand). +Pass --large-country to also test a large pre-warmed country (e.g. BRA, NGA). """ import argparse @@ -16,7 +19,7 @@ import httpx BASE = "http://localhost:8104" -TEST_ISO = "LUX" # Luxembourg — small country, small raster (~1 MB) +TEST_ISO = "LUX" # Bounding box polygon covering Luxembourg (WGS84) _LUX_RING = [[5.7, 49.4], [6.5, 49.4], [6.5, 50.2], [5.7, 50.2], [5.7, 49.4]] @@ -31,12 +34,32 @@ ], } +# Simple 4-polygon GeoJSON for NGA that exercises the multi-polygon path +# (bounding boxes of 4 rough quadrants of Nigeria, WGS84) +_NGA_QUADS = [ + {"nodeid": 0, "name": "NW", "ring": [[3.0,9.5],[9.5,9.5],[9.5,14.0],[3.0,14.0],[3.0,9.5]]}, + {"nodeid": 1, "name": "NE", "ring": [[9.5,9.5],[15.0,9.5],[15.0,14.0],[9.5,14.0],[9.5,9.5]]}, + {"nodeid": 2, "name": "SW", "ring": [[3.0,4.0],[9.5,4.0],[9.5,9.5],[3.0,9.5],[3.0,4.0]]}, + {"nodeid": 3, "name": "SE", "ring": [[9.5,4.0],[15.0,4.0],[15.0,9.5],[9.5,9.5],[9.5,4.0]]}, +] +NGA_GEOJSON = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"nodeid": q["nodeid"], "name": q["name"]}, + "geometry": {"type": "Polygon", "coordinates": [q["ring"]]}, + } + for q in _NGA_QUADS + ], +} + def get(path: str, timeout: float = 30) -> httpx.Response: return httpx.get(f"{BASE}{path}", timeout=timeout) -def post(path: str, body: dict, timeout: float = 120) -> httpx.Response: +def post(path: str, body: dict, timeout: float = 300) -> httpx.Response: return httpx.post(f"{BASE}{path}", json=body, timeout=timeout) @@ -84,7 +107,7 @@ def test_prewarm(): def test_aggregate(): print(f"── POST /aggregate/{TEST_ISO} ─────────────────────────") r = post(f"/aggregate/{TEST_ISO}", TEST_GEOJSON, timeout=60) - check("HTTP 200", r.status_code == 200, f"got {r.status_code}") + check("HTTP 200", r.status_code == 200, f"got {r.status_code}: {r.text[:200]}") result = r.json() check("nodeid 0 in result", "0" in result, str(list(result.keys())[:5])) pop = result["0"] @@ -101,6 +124,89 @@ def test_cache_speed(): check("responded in < 10s (cached raster)", elapsed < 10, f"{elapsed:.2f}s") +def test_nga_multipolygon(): + print("── POST /aggregate/NGA (4 quads, multi-polygon) ─────────") + t0 = time.monotonic() + r = post("/aggregate/NGA", NGA_GEOJSON, timeout=120) + elapsed = time.monotonic() - t0 + check("HTTP 200", r.status_code == 200, f"got {r.status_code}: {r.text[:200]}") + result = r.json() + check("4 nodeids returned", len(result) == 4, f"got {len(result)}: {list(result.keys())}") + total = sum(result.values()) + check("total NGA pop plausible (150M–230M)", 150_000_000 < total < 230_000_000, + f"got {total:.0f}") + check("each quad > 0", all(v > 0 for v in result.values()), + str({k: f"{v:.0f}" for k, v in result.items()})) + print(f" Quads: { {k: f'{v/1e6:.1f}M' for k, v in result.items()} }") + print(f" Total: {total/1e6:.1f}M ({elapsed:.1f}s)") + + +def test_large_country(iso: str, shapes_url: str): + """Fetch real admin-1 shapes from GADM and aggregate with worldpop. + + This is the critical test: large countries (BRA, IDN) have states whose + raster windows exceed 8 M pixels and must be downsampled via Resampling.sum. + """ + print(f"── POST /aggregate/{iso} (real admin-1 shapes from GADM) ──") + + # Fetch shapes + print(f" Fetching {iso}/1 from {shapes_url} ...") + try: + rs = httpx.get(f"{shapes_url}/boundaries/{iso}/1", timeout=120) + except Exception as exc: + print(f" [SKIP] Could not reach GADM at {shapes_url}: {exc}") + return + check(f"GADM HTTP 200 for {iso}", rs.status_code == 200, + f"got {rs.status_code}") + fc = rs.json() + n_features = len(fc["features"]) + print(f" {n_features} features, raw body {len(rs.content)/1024:.0f} KB") + + # Truncate coordinates (same as client.html) + def trunc(coords, dp=3): + if isinstance(coords[0], (int, float)): + return [round(v, dp) for v in coords] + return [trunc(c, dp) for c in coords] + + fc_slim = { + "type": "FeatureCollection", + "features": [ + {**f, "geometry": {**f["geometry"], + "coordinates": trunc(f["geometry"]["coordinates"])}} + for f in fc["features"] + ], + } + import json + body_bytes = json.dumps(fc_slim).encode() + print(f" Slim body: {len(body_bytes)/1024:.0f} KB") + + # Aggregate + print(f" POSTing to {BASE}/aggregate/{iso}?year=2020 ...") + t0 = time.monotonic() + try: + r = httpx.post( + f"{BASE}/aggregate/{iso}?year=2020", + content=body_bytes, + headers={"Content-Type": "application/json"}, + timeout=600, + ) + except Exception as exc: + elapsed = time.monotonic() - t0 + print(f" [FAIL] Exception after {elapsed:.1f}s: {type(exc).__name__}: {exc}") + sys.exit(1) + + elapsed = time.monotonic() - t0 + check("HTTP 200", r.status_code == 200, + f"got {r.status_code}: {r.text[:300]}") + result = r.json() + check(f"{n_features} nodeids returned", len(result) == n_features, + f"got {len(result)}") + total = sum(result.values()) + print(f" {len(result)} regions, total pop {total/1e6:.1f}M ({elapsed:.1f}s)") + check("all nodeids present", len(result) == n_features) + check("total population > 0", total > 0, f"got {total:.0f}") + + def test_unknown_iso(): print("── POST /aggregate/ZZZ (unknown ISO) ────────────────────") r = post("/aggregate/ZZZ", TEST_GEOJSON, timeout=60) @@ -122,9 +228,14 @@ def test_empty_features(): def main(): global BASE parser = argparse.ArgumentParser() - parser.add_argument("--url", default=BASE) + parser.add_argument("--url", default=BASE, help="worldpop-service base URL") parser.add_argument("--wait", action="store_true", help="Wait for service to be ready before running tests") + parser.add_argument("--large-country", metavar="ISO", + help="Also test a large pre-warmed country (e.g. BRA). " + "Requires --gadm-url.") + parser.add_argument("--gadm-url", default="http://48.200.52.126", + help="GADM service base URL (for --large-country shapes)") args = parser.parse_args() BASE = args.url.rstrip("/") @@ -136,6 +247,9 @@ def main(): test_prewarm() test_aggregate() test_cache_speed() + test_nga_multipolygon() + if args.large_country: + test_large_country(args.large_country.upper(), args.gadm_url) test_unknown_iso() test_bad_body() test_empty_features() From 9cace3073ab5b0043cf12c2219484d96ac4bb736 Mon Sep 17 00:00:00 2001 From: Jonathan Bloedow Date: Tue, 5 May 2026 08:10:12 -0700 Subject: [PATCH 27/37] More obvious progress spinner. --- services/worldpop/client.html | 226 ++++++++++++++++++++++++++++++++-- 1 file changed, 218 insertions(+), 8 deletions(-) diff --git a/services/worldpop/client.html b/services/worldpop/client.html index 56976c3..89d8be0 100644 --- a/services/worldpop/client.html +++ b/services/worldpop/client.html @@ -49,6 +49,8 @@ #load-btn:disabled { background: #93c5fd; cursor: default; } @keyframes spin { to { transform: rotate(360deg); } } + + /* Small spinner inside the Load button */ #spinner { width: 12px; height: 12px; border: 2px solid rgba(255,255,255,.4); @@ -56,8 +58,34 @@ border-radius: 50%; animation: spin .7s linear infinite; display: none; + flex-shrink: 0; + } + #load-btn.loading #spinner { display: inline-block; } + + /* Centered overlay spinner on the map */ + #map-spinner { + display: none; + position: absolute; + top: 50%; left: 50%; + transform: translate(-50%, -50%); + z-index: 1000; + background: rgba(255,255,255,.85); + border-radius: 12px; + padding: 18px 24px; + text-align: center; + box-shadow: 0 2px 12px rgba(0,0,0,.2); + pointer-events: none; + } + #map-spinner .ring { + width: 40px; height: 40px; + border: 4px solid #dbeafe; + border-top-color: #1d4ed8; + border-radius: 50%; + animation: spin .8s linear infinite; + margin: 0 auto 10px; } - #load-btn.loading #spinner { display: block; } + #map-spinner .label { font-size: 12px; color: #555; } + #map-spinner.visible { display: block; } #status { font-size: 12px; color: #555; align-self: center; max-width: 480px; white-space: pre-wrap; } @@ -82,6 +110,10 @@ margin-bottom: 3px; } .wp-legend .labels { display: flex; justify-content: space-between; color: #555; } + + /* map wrapper so the overlay can use position:absolute */ + #map-wrap { flex: 1; position: relative; } + #map { width: 100%; height: 100%; } @@ -100,8 +132,164 @@ -