docker-compose.yml

# Docker Compose for local development and testing
# Run: docker-compose up --build
#
# Make sure to create a .env file with your API keys (copy from .env.example)


services:
  data-concierge:
    build:
      context: .
      dockerfile: Dockerfile
    ports:
      - "8080:8080"
    env_file:
      - .env  # Load all variables from .env file
    environment:
      # Server config (override defaults)
      - PORT=8080
      - HOST=0.0.0.0
      - ENVIRONMENT=development

      # Required: Anthropic API Key
      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}

      # Federal Data Source API Keys (for live data)
      - BLS_API_KEY=${BLS_API_KEY:-}
      - CENSUS_API_KEY=${CENSUS_API_KEY:-}
      - BEA_API_KEY=${BEA_API_KEY:-}
      - FRED_API_KEY=${FRED_API_KEY:-}

      # Data Commons
      - DATA_COMMONS_API_KEY=${DATA_COMMONS_API_KEY:-}

      # CKAN
      - CKAN_URL=${CKAN_URL:-https://data.dathere.com}
      - CKAN_API_KEY=${CKAN_API_KEY:-}

      # Pinecone (for CKAN semantic search)
      - PINECONE_API_KEY=${PINECONE_API_KEY:-}
      - PINECONE_INDEX_NAME=${PINECONE_INDEX_NAME:-ckan-dathere-index}
      - PINECONE_NAMESPACE=${PINECONE_NAMESPACE:-ckan-namespace}

      # Redis (optional caching)
      - REDIS_HOST=${REDIS_HOST:-}
      - REDIS_PORT=${REDIS_PORT:-6379}
      - REDIS_PASSWORD=${REDIS_PASSWORD:-}

    volumes:
      # Persist notebooks and data between restarts
      - ./notebooks:/app/notebooks
      - ./data:/app/data

    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

    restart: unless-stopped

  # ---------------------------------------------------------------------
  # Fair Store (#133): a local CKAN holding verified metadata, data
  # schematics and ontologies.
  #
  # This is the structured, deterministic half of the hybrid storage
  # design intent: CKAN owns canonical dataset
  # records, schemas, column-level data dictionaries, provenance and
  # verification status, and exposes them over an API other programs can
  # call. Pinecone keeps the fuzzy half — embeddings for semantic recall —
  # and points back at CKAN identifiers rather than duplicating the truth.
  #
  # Start it with the fairstore profile so a plain `docker compose up`
  # stays lightweight:
  #     docker compose --profile fairstore up
  # then open http://localhost:5001 (host 5001 -> container 5000; macOS
  # reserves 5000 for AirPlay).
  #
  # First run: create a sysadmin and an API token (commands under "CKAN Fair
  # Store mirror" in README.md), load every registered portal with
  #     python -m scripts.populate_fairstore --site all --apply
  # then set CKAN_URL / CKAN_API_KEY for the app.
  # ---------------------------------------------------------------------
  fairstore:
    profiles: ["fairstore"]
    build:
      context: ./docker/fairstore
      dockerfile: Dockerfile
    image: data-concierge-fairstore:2.11.6
    platform: linux/amd64
    depends_on:
      fairstore-db:
        condition: service_healthy
      fairstore-solr:
        condition: service_started
      fairstore-redis:
        condition: service_started
    ports:
      - "${FAIRSTORE_PORT:-5001}:5000"
    environment:
      # The URL browsers reach CKAN at. CKAN builds upload and DataStore dump
      # links from it, so it must be the host port, not the container's 5000.
      - CKAN_SITE_URL=http://localhost:${FAIRSTORE_PORT:-5001}
      # Credentials must match the users the postgres image's init scripts
      # create (see fairstore-db below): ckan / ckan_datastore_write /
      # ckan_datastore_read.
      - CKAN_SQLALCHEMY_URL=postgresql://ckan:${FAIRSTORE_DB_PASSWORD:-ckan}@fairstore-db/ckan
      - CKAN_DATASTORE_WRITE_URL=postgresql://ckan_datastore_write:${FAIRSTORE_DATASTORE_PASSWORD:-datastore}@fairstore-db/datastore
      - CKAN_DATASTORE_READ_URL=postgresql://ckan_datastore_read:${FAIRSTORE_DATASTORE_PASSWORD:-datastore}@fairstore-db/datastore
      - CKAN_SOLR_URL=http://fairstore-solr:8983/solr/ckan
      - CKAN_REDIS_URL=redis://fairstore-redis:6379/1
      # Pins the session and API-token signing secrets (docker/fairstore/
      # pin-secrets.sh). Unset, each new container signs with fresh random
      # secrets and every issued API token stops working.
      - CKAN_SECRET_KEY=${FAIRSTORE_SECRET_KEY:-}
      # hierarchy_display must load before hierarchy_form. Organizations that
      # represent source portals can then be parents of the organizations
      # mirrored from those portals.
      - CKAN__PLUGINS=envvars datastore datatables_view text_view image_view hierarchy_display hierarchy_form
      # qsv profiles (stats, frequency, data dictionary) are DataStore tables,
      # shown as sortable tables; describegpt's JSON output is shown inline.
      - CKAN__VIEWS__DEFAULT_VIEWS=datatables_view text_view image_view
      # The 2.11.6 image's ckan.ini sets ckan.uploads_enabled to an empty
      # value, which turns file uploads off in the web forms.
      - CKAN__UPLOADS_ENABLED=true
      # CKAN exempts extension endpoints (the DataStore dictionary editor, the
      # DataTables AJAX feed) from CSRF checks by default, and SameSite=Lax
      # does not cover them here: sslip.io is not a public suffix, so every
      # other *.sslip.io site counts as same-site.
      - CKAN__CSRF_PROTECTION__IGNORE_EXTENSIONS=false
    volumes:
      - fairstore_data:/var/lib/ckan
    restart: unless-stopped

  fairstore-db:
    profiles: ["fairstore"]
    # Plain Postgres, not ckan/ckan-postgres-dev — that image is a CKAN test
    # fixture with hardcoded ckan_test/datastore_test names that ignores
    # config. init-db.sh creates exactly the users and databases CKAN needs.
    image: postgres:16-alpine
    environment:
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=${FAIRSTORE_POSTGRES_PASSWORD:-postgres}
      - POSTGRES_DB=postgres
      # Read by init-db.sh on first start; must match the URLs above.
      - CKAN_DB_PASSWORD=${FAIRSTORE_DB_PASSWORD:-ckan}
      - DATASTORE_DB_PASSWORD=${FAIRSTORE_DATASTORE_PASSWORD:-datastore}
    volumes:
      - fairstore_db:/var/lib/postgresql/data
      - ./docker/fairstore/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 10s
      timeout: 5s
      retries: 10

  fairstore-solr:
    profiles: ["fairstore"]
    image: ckan/ckan-solr:2.11-solr9
    platform: linux/amd64
    volumes:
      - fairstore_solr:/var/solr
    restart: unless-stopped

  # CKAN 2.11 logs a critical error on every start without Redis, and its
  # background jobs queue lives there.
  fairstore-redis:
    profiles: ["fairstore"]
    image: redis:7-alpine
    restart: unless-stopped

  # Optional: Redis for caching (uncomment if needed)
  # redis:
  #   image: redis:7-alpine
  #   ports:
  #     - "6379:6379"
  #   volumes:
  #     - redis_data:/data
  #   healthcheck:
  #     test: ["CMD", "redis-cli", "ping"]
  #     interval: 10s
  #     timeout: 5s
  #     retries: 5

volumes:
  fairstore_data:
  fairstore_db:
  fairstore_solr:
  # redis_data: