From f1ab2834c30cb03aab90ec61888f2b3ec027b193 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Thu, 13 Aug 2026 17:12:56 -0400 Subject: [PATCH 01/29] FECFILE-3332: Set min pool size to 0 to avoid hoarding stale connections. --- docker-compose.yml | 8 ++++---- manifests/manifest-dev.yml | 2 +- manifests/manifest-prod.yml | 2 +- manifests/manifest-stage.yml | 2 +- manifests/manifest-test.yml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 62fb3b3815..5de755e5e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,8 +65,8 @@ services: INCLUDE_SILK: "${INCLUDE_SILK}" DJANGO_APPLICATION: FECFILE_CELERY_WORKER E2E_TEST: "${E2E_TEST:-False}" # E2E-only endpoints disabled by default, set to True to enable - DB_POOL_MIN_SIZE: 10 - DB_POOL_MAX_SIZE: 10 + DB_POOL_MIN_SIZE: 0 + DB_POOL_MAX_SIZE: 8 DB_POOL_MAX_IDLE: 600 CELERY_WORKER_CONCURRENCY: 10 MOCK_EFO_DOT_FEC_SUBMISSION_DURATION_SECONDS: 0 @@ -187,8 +187,8 @@ services: INCLUDE_SILK: "${INCLUDE_SILK}" DJANGO_APPLICATION: FECFILE_API E2E_TEST: "${E2E_TEST:-False}" - DB_POOL_MIN_SIZE: 8 - DB_POOL_MAX_SIZE: 20 + DB_POOL_MIN_SIZE: 0 + DB_POOL_MAX_SIZE: 8 DB_POOL_MAX_IDLE: 600 MOCK_EFO_DOT_FEC_SUBMISSION_DURATION_SECONDS: 0 diff --git a/manifests/manifest-dev.yml b/manifests/manifest-dev.yml index 5986cb8761..9b58b8f596 100644 --- a/manifests/manifest-dev.yml +++ b/manifests/manifest-dev.yml @@ -24,7 +24,7 @@ defaults: &defaults STAGE_OPEN_FEC_API: https://api-stage.open.fec.gov/v1/ FEC_AGENCY_ID: FEC SESSION_COOKIE_AGE: 3600 # Value in seconds (1 hour) - DB_POOL_MIN_SIZE: 8 + DB_POOL_MIN_SIZE: 0 DB_POOL_MAX_SIZE: 8 DB_POOL_MAX_IDLE: 600 BP_PIP_VERSION: latest diff --git a/manifests/manifest-prod.yml b/manifests/manifest-prod.yml index 3f2a7b5a97..125c9cf376 100644 --- a/manifests/manifest-prod.yml +++ b/manifests/manifest-prod.yml @@ -25,7 +25,7 @@ defaults: &defaults EFO_FILING_API: https://efoservices.fec.gov FEC_AGENCY_ID: FEC SESSION_COOKIE_AGE: 3600 # Value in seconds (1 hour) - DB_POOL_MIN_SIZE: 8 + DB_POOL_MIN_SIZE: 0 DB_POOL_MAX_SIZE: 8 DB_POOL_MAX_IDLE: 600 BP_PIP_VERSION: latest diff --git a/manifests/manifest-stage.yml b/manifests/manifest-stage.yml index c072e9194c..177967b15e 100644 --- a/manifests/manifest-stage.yml +++ b/manifests/manifest-stage.yml @@ -25,7 +25,7 @@ defaults: &defaults EFO_FILING_API: https://efoservices.stage.efo.fec.gov FEC_AGENCY_ID: FEC SESSION_COOKIE_AGE: 3600 # Value in seconds (1 hour) - DB_POOL_MIN_SIZE: 8 + DB_POOL_MIN_SIZE: 0 DB_POOL_MAX_SIZE: 8 DB_POOL_MAX_IDLE: 600 BP_PIP_VERSION: latest diff --git a/manifests/manifest-test.yml b/manifests/manifest-test.yml index 73da704926..847181d7d0 100644 --- a/manifests/manifest-test.yml +++ b/manifests/manifest-test.yml @@ -25,7 +25,7 @@ defaults: &defaults EFO_FILING_API: https://efoservices.stage.efo.fec.gov FEC_AGENCY_ID: FEC SESSION_COOKIE_AGE: 3600 # Value in seconds (1 hour) - DB_POOL_MIN_SIZE: 8 + DB_POOL_MIN_SIZE: 0 DB_POOL_MAX_SIZE: 8 DB_POOL_MAX_IDLE: 600 BP_PIP_VERSION: latest From e4e1a4de12de71d5a9cd5a17bb9dbf5b103c50d9 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Thu, 13 Aug 2026 17:16:48 -0400 Subject: [PATCH 02/29] FECFILE-3332: Set min pool size to 0 to avoid hoarding stale connections. --- django-backend/fecfiler/settings/base.py | 28 +++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 93f645bd24..353d02bf2b 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -174,19 +174,41 @@ def custom_silk_filter(request): database = dj_database_url.config() database.setdefault("OPTIONS", {}) database["OPTIONS"]["application_name"] = f"{APPLICATION_NAME}_{APPLICATION_INDEX}" +database["CONN_HEALTH_CHECKS"] = True +database["CONN_MAX_AGE"] = 0 + # psycopg pool settings: # https://www.psycopg.org/psycopg3/docs/api/pool.html#psycopg_pool.ConnectionPool # this works for celery workers as well # https://docs.celeryq.dev/en/main/django/first-steps-with-django.html#django-connection-pool database["OPTIONS"]["pool"] = { # min_size: psycopg default is 4 - "min_size": int(env.get_credential("DB_POOL_MIN_SIZE", "4")), + "min_size": int(env.get_credential("DB_POOL_MIN_SIZE", "0")), # max_size: default to no overflow "max_size": int(env.get_credential("DB_POOL_MAX_SIZE", "4")), - # max_idle: psycopg default 10 minutes - "max_idle": int(env.get_credential("DB_POOL_MAX_IDLE", "600")), + # max_idle: psycopg default is 10m, prune before 350s AWS timeout + "max_idle": int(env.get_credential("DB_POOL_MAX_IDLE", "300")), + # Retire conns older than 30m + "max_lifetime": int(env.get_credential("DB_POOL_MAX_LIFETIME", "1800")), + # getconn() should give up on trying to get a connection faster + "timeout": float(env.get_credential("DB_POOL_TIMEOUT", "10.0")), + # controls how long AttemptWithBackoff will retry before resetting _growing and _nconns + "reconnect_timeout": float(env.get_credential("DB_POOL_RECONNECT_TIMEOUT", "15.0")), } +# fail fast on unreachable db instead of blocking for default 30s +database["OPTIONS"]["connect_timeout"] = int( + env.get_credential("DB_CONNECT_TIMEOUT", "3") +) + +# TCP Keepalives prevent firewalls / NAT gateways from silently dropping idle connections +database["OPTIONS"].update({ + "keepalives": 1, + "keepalives_idle": 30, + "keepalives_interval": 10, + "keepalives_count": 5, +}) + # Database DATABASES = {"default": database} From e4d560e2e40b67c6bb468cf818419c8d7afe23a0 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Thu, 13 Aug 2026 17:29:48 -0400 Subject: [PATCH 03/29] FECFILE-3332: Removed unnecessary explicit config to let OOTB defaults rule. --- django-backend/fecfiler/settings/base.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 353d02bf2b..6a174faa8a 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -175,7 +175,6 @@ def custom_silk_filter(request): database.setdefault("OPTIONS", {}) database["OPTIONS"]["application_name"] = f"{APPLICATION_NAME}_{APPLICATION_INDEX}" database["CONN_HEALTH_CHECKS"] = True -database["CONN_MAX_AGE"] = 0 # psycopg pool settings: # https://www.psycopg.org/psycopg3/docs/api/pool.html#psycopg_pool.ConnectionPool @@ -188,27 +187,8 @@ def custom_silk_filter(request): "max_size": int(env.get_credential("DB_POOL_MAX_SIZE", "4")), # max_idle: psycopg default is 10m, prune before 350s AWS timeout "max_idle": int(env.get_credential("DB_POOL_MAX_IDLE", "300")), - # Retire conns older than 30m - "max_lifetime": int(env.get_credential("DB_POOL_MAX_LIFETIME", "1800")), - # getconn() should give up on trying to get a connection faster - "timeout": float(env.get_credential("DB_POOL_TIMEOUT", "10.0")), - # controls how long AttemptWithBackoff will retry before resetting _growing and _nconns - "reconnect_timeout": float(env.get_credential("DB_POOL_RECONNECT_TIMEOUT", "15.0")), } -# fail fast on unreachable db instead of blocking for default 30s -database["OPTIONS"]["connect_timeout"] = int( - env.get_credential("DB_CONNECT_TIMEOUT", "3") -) - -# TCP Keepalives prevent firewalls / NAT gateways from silently dropping idle connections -database["OPTIONS"].update({ - "keepalives": 1, - "keepalives_idle": 30, - "keepalives_interval": 10, - "keepalives_count": 5, -}) - # Database DATABASES = {"default": database} From 82e6ce31a2de9677fc6fe59831ab40fac421907c Mon Sep 17 00:00:00 2001 From: Max Zaremba Date: Mon, 17 Aug 2026 09:16:40 -0400 Subject: [PATCH 04/29] Added environment variables to define gunicorn workers and threads --- Dockerfile-e2e | 2 +- bin/entrypoint.sh | 2 +- bin/run-api.sh | 3 ++- docker-compose.yml | 28 +++++++++++++++------------- manifests/manifest-dev.yml | 2 ++ manifests/manifest-prod.yml | 2 ++ manifests/manifest-stage.yml | 2 ++ manifests/manifest-test.yml | 2 ++ 8 files changed, 27 insertions(+), 16 deletions(-) diff --git a/Dockerfile-e2e b/Dockerfile-e2e index 00b11422d9..fbbd79b51d 100644 --- a/Dockerfile-e2e +++ b/Dockerfile-e2e @@ -13,4 +13,4 @@ RUN useradd nxgu --no-create-home --home /opt/nxg_fec_e2e && chown -R nxgu:nxgu USER nxgu EXPOSE 8080 -ENTRYPOINT ["/bin/sh", "-c", "python manage.py migrate && python manage.py loaddata fixtures/user-data.json && python manage.py load_mocked_committee_data && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 --threads=8 --reload"] +ENTRYPOINT ["/bin/sh", "-c", "python manage.py migrate && python manage.py loaddata fixtures/user-data.json && python manage.py load_mocked_committee_data && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w "$GUNICORN_WORKERS" --threads="$GUNICORN_THREADS" --reload"] diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 60beee9690..b74c212c2c 100644 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -22,4 +22,4 @@ python manage.py load_mocked_committee_data # Start the Gunicorn server echo "Starting Gunicorn server..." -exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 --threads=8 --reload \ No newline at end of file +exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w "$GUNICORN_WORKERS" --threads="$GUNICORN_THREADS" --reload diff --git a/bin/run-api.sh b/bin/run-api.sh index e13824479d..cdb2123be6 100755 --- a/bin/run-api.sh +++ b/bin/run-api.sh @@ -3,4 +3,5 @@ cd django-backend echo "------ Starting APP ------" # Run application -./manage.py collectstatic --noinput --traceback --verbosity 3 && exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 --threads=8 +./manage.py collectstatic --noinput --traceback --verbosity 3 && exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w "$GUNICORN_WORKERS" --threads="$GUNICORN_THREADS" + diff --git a/docker-compose.yml b/docker-compose.yml index 62fb3b3815..200562cd7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,14 +25,14 @@ services: celery-worker: build: - context: './' - dockerfile: '${WORKER_DOCKERFILE:-Worker_Dockerfile}' + context: "./" + dockerfile: "${WORKER_DOCKERFILE:-Worker_Dockerfile}" image: fecfile-celery-worker container_name: fecfile-celery-worker volumes: - ./django-backend:/opt/nxg_fec extra_hosts: - - 'host.docker.internal:host-gateway' + - "host.docker.internal:host-gateway" depends_on: - db - redis @@ -68,6 +68,8 @@ services: DB_POOL_MIN_SIZE: 10 DB_POOL_MAX_SIZE: 10 DB_POOL_MAX_IDLE: 600 + GUNICORN_WORKERS: 9 + GUNICORN_THREADS: 8 CELERY_WORKER_CONCURRENCY: 10 MOCK_EFO_DOT_FEC_SUBMISSION_DURATION_SECONDS: 0 FLAG__ENABLE_IMPORT: True @@ -80,15 +82,15 @@ services: scheduler: build: - context: './' - dockerfile: '${SCHEDULER_DOCKERFILE:-Scheduler_Dockerfile}' + context: "./" + dockerfile: "${SCHEDULER_DOCKERFILE:-Scheduler_Dockerfile}" image: fecfile-celery-beat container_name: fecfile-scheduler command: celery -A fecfiler beat -l info volumes: - ./django-backend:/opt/nxg_fec extra_hosts: - - 'host.docker.internal:host-gateway' + - "host.docker.internal:host-gateway" depends_on: - db - redis @@ -128,14 +130,14 @@ services: api: build: - context: './' - dockerfile: '${API_DOCKERFILE:-Dockerfile}' + context: "./" + dockerfile: "${API_DOCKERFILE:-Dockerfile}" image: fecfile-api container_name: fecfile-api volumes: - ./django-backend:/opt/nxg_fec extra_hosts: - - 'host.docker.internal:host-gateway' + - "host.docker.internal:host-gateway" depends_on: - db - redis @@ -224,14 +226,14 @@ services: - ./:/mnt/locust # command: -f /mnt/locust/performance-testing/locust_run.py --master -H https://dev-api.fecfile.fec.gov command: -f /mnt/locust/performance-testing/locust_run.py --master -H http://fecfile-api-proxy:8080 -r .5 - profiles: [ locust ] + profiles: [locust] locust-follower: image: locustio/locust volumes: - ./:/mnt/locust command: -f /mnt/locust/performance-testing/locust_run.py --worker --master-host locust-leader -L DEBUG - profiles: [ locust ] + profiles: [locust] locust-leader-app: image: locustio/locust @@ -240,11 +242,11 @@ services: volumes: - ./:/mnt/locust command: -f /mnt/locust/performance-testing/locust_run_app.py --master -H https://dev.fecfile.fec.gov - profiles: [ locust-app ] + profiles: [locust-app] locust-follower-app: image: locustio/locust volumes: - ./:/mnt/locust command: -f /mnt/locust/performance-testing/locust_run_app.py --worker --master-host locust-leader-app -L DEBUG - profiles: [ locust-app ] + profiles: [locust-app] diff --git a/manifests/manifest-dev.yml b/manifests/manifest-dev.yml index 5986cb8761..0fa9f8c238 100644 --- a/manifests/manifest-dev.yml +++ b/manifests/manifest-dev.yml @@ -58,6 +58,8 @@ applications: DB_POOL_MIN_SIZE: 10 DB_POOL_MAX_SIZE: 10 DB_POOL_MAX_IDLE: 600 + GUNICORN_WORKERS: 9 + GUNICORN_THREADS: 8 CELERY_WORKER_CONCURRENCY: 10 instances: 2 no-route: true diff --git a/manifests/manifest-prod.yml b/manifests/manifest-prod.yml index 3f2a7b5a97..34665d4a45 100644 --- a/manifests/manifest-prod.yml +++ b/manifests/manifest-prod.yml @@ -58,6 +58,8 @@ applications: DB_POOL_MIN_SIZE: 50 DB_POOL_MAX_SIZE: 50 DB_POOL_MAX_IDLE: 600 + GUNICORN_WORKERS: 9 + GUNICORN_THREADS: 8 CELERY_WORKER_CONCURRENCY: 50 no-route: true health-check-type: process diff --git a/manifests/manifest-stage.yml b/manifests/manifest-stage.yml index c072e9194c..ec2785e37d 100644 --- a/manifests/manifest-stage.yml +++ b/manifests/manifest-stage.yml @@ -58,6 +58,8 @@ applications: DB_POOL_MIN_SIZE: 10 DB_POOL_MAX_SIZE: 10 DB_POOL_MAX_IDLE: 600 + GUNICORN_WORKERS: 9 + GUNICORN_THREADS: 8 CELERY_WORKER_CONCURRENCY: 10 instances: 2 no-route: true diff --git a/manifests/manifest-test.yml b/manifests/manifest-test.yml index 73da704926..f87b09ba70 100644 --- a/manifests/manifest-test.yml +++ b/manifests/manifest-test.yml @@ -58,6 +58,8 @@ applications: DB_POOL_MIN_SIZE: 50 DB_POOL_MAX_SIZE: 50 DB_POOL_MAX_IDLE: 600 + GUNICORN_WORKERS: 9 + GUNICORN_THREADS: 8 CELERY_WORKER_CONCURRENCY: 50 instances: 2 no-route: true From a787d37a084352da23df8c73fe9da60ddec02688 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Fri, 21 Aug 2026 09:48:03 -0400 Subject: [PATCH 05/29] FECFILE-3346: Update gitpython to 3.1.58 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8fe26c114b..9e817a6b70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ drf-spectacular-sidecar==2026.4.14 Faker==37.6.0 git+https://github.com/fecgov/fecfile-validate@8384e8c2747e64ebacd85031dcb9d214cfa513c5#egg=fecfile_validate&subdirectory=fecfile_validate_python github3.py==4.0.1 -GitPython==3.1.55 +GitPython==3.1.58 gunicorn==23.0.0 invoke==2.2.0 jwcrypto==1.5.7 From e05bda92aa0299c38ccd4adb54bf40897262a2c2 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 24 Aug 2026 10:05:12 -0400 Subject: [PATCH 06/29] update sqlparse --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 16ec6a08e4..43e5df5440 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,6 @@ zeep==4.3.3 whitenoise==6.12.0 # Pinned -sqlparse==0.5.5 +sqlparse==0.6.0 urllib3==2.7.0 lxml==6.1.0 \ No newline at end of file From e377d403b9a64deb1de5b42265d1857dd679fef2 Mon Sep 17 00:00:00 2001 From: Max Zaremba Date: Mon, 24 Aug 2026 16:55:57 -0400 Subject: [PATCH 07/29] Removed quotation marks around environment variables --- Dockerfile-e2e | 2 +- bin/entrypoint.sh | 2 +- bin/run-api.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile-e2e b/Dockerfile-e2e index fbbd79b51d..d024d82ba5 100644 --- a/Dockerfile-e2e +++ b/Dockerfile-e2e @@ -13,4 +13,4 @@ RUN useradd nxgu --no-create-home --home /opt/nxg_fec_e2e && chown -R nxgu:nxgu USER nxgu EXPOSE 8080 -ENTRYPOINT ["/bin/sh", "-c", "python manage.py migrate && python manage.py loaddata fixtures/user-data.json && python manage.py load_mocked_committee_data && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w "$GUNICORN_WORKERS" --threads="$GUNICORN_THREADS" --reload"] +ENTRYPOINT ["/bin/sh", "-c", "python manage.py migrate && python manage.py loaddata fixtures/user-data.json && python manage.py load_mocked_committee_data && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w $GUNICORN_WORKERS --threads=$GUNICORN_THREADS --reload"] diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index b74c212c2c..48a0d358e3 100644 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -22,4 +22,4 @@ python manage.py load_mocked_committee_data # Start the Gunicorn server echo "Starting Gunicorn server..." -exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w "$GUNICORN_WORKERS" --threads="$GUNICORN_THREADS" --reload +exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w $GUNICORN_WORKERS --threads=$GUNICORN_THREADS --reload diff --git a/bin/run-api.sh b/bin/run-api.sh index cdb2123be6..17b68a49d5 100755 --- a/bin/run-api.sh +++ b/bin/run-api.sh @@ -3,5 +3,5 @@ cd django-backend echo "------ Starting APP ------" # Run application -./manage.py collectstatic --noinput --traceback --verbosity 3 && exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w "$GUNICORN_WORKERS" --threads="$GUNICORN_THREADS" +./manage.py collectstatic --noinput --traceback --verbosity 3 && exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w $GUNICORN_WORKERS --threads=$GUNICORN_THREADS From 5f71f5a139ce5c707a0a7e8276fb568059eab5f5 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 25 Aug 2026 09:12:41 -0400 Subject: [PATCH 08/29] fosec-133 process log tokens --- django-backend/fecfiler/settings/base.py | 86 +++++++++++++++++------- 1 file changed, 63 insertions(+), 23 deletions(-) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 6a174faa8a..7ce9413692 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -7,7 +7,9 @@ import structlog import logging import sys +import re +from copy import deepcopy from enum import Enum from .env import env from fecfiler.shared.utilities import get_float_from_string, get_boolean_from_string @@ -292,33 +294,45 @@ def filter(self, record): def get_logging_config(log_format=LINE): stream_handler = "logging.StreamHandler" + json_formatter = { + "()": structlog.stdlib.ProcessorFormatter, + "processors": [ + structlog.stdlib.ProcessorFormatter.remove_processors_meta, + structlog.processors.ExceptionRenderer( + structlog.processors.ExceptionDictTransformer(show_locals=False) + ), + structlog.processors.JSONRenderer(), + ], + } + psycopg_json_formatter = deepcopy(json_formatter) + psycopg_json_formatter["processors"] = [process_log_tokens] + psycopg_json_formatter[ + "processors" + ] + plain_console_formatter = { + "()": structlog.stdlib.ProcessorFormatter, + "processors": [ + structlog.stdlib.ProcessorFormatter.remove_processors_meta, + structlog.processors.dict_tracebacks, + structlog.dev.ConsoleRenderer( + colors=True, exception_formatter=structlog.dev.rich_traceback + ), + ], + "foreign_pre_chain": [ + structlog.contextvars.merge_contextvars, + ], + } + psycopg_plain_console_formatter = deepcopy(plain_console_formatter) + psycopg_plain_console_formatter["processors"] = [ + process_log_tokens + ] + psycopg_plain_console_formatter["processors"] logging_config = { "version": 1, "disable_existing_loggers": False, "formatters": { - "json_formatter": { - "()": structlog.stdlib.ProcessorFormatter, - "processors": [ - structlog.stdlib.ProcessorFormatter.remove_processors_meta, - structlog.processors.ExceptionRenderer( - structlog.processors.ExceptionDictTransformer(show_locals=False) - ), - structlog.processors.JSONRenderer(), - ], - }, - "plain_console": { - "()": structlog.stdlib.ProcessorFormatter, - "processors": [ - structlog.stdlib.ProcessorFormatter.remove_processors_meta, - structlog.processors.dict_tracebacks, - structlog.dev.ConsoleRenderer( - colors=True, exception_formatter=structlog.dev.rich_traceback - ), - ], - "foreign_pre_chain": [ - structlog.contextvars.merge_contextvars, - ], - }, + "json_formatter": json_formatter, + "psycopg_json_formatter": psycopg_json_formatter, + "plain_console": plain_console_formatter, + "psycopg_plain_console": psycopg_plain_console_formatter, "key_value": { "()": structlog.stdlib.ProcessorFormatter, "processors": [ @@ -352,6 +366,11 @@ def get_logging_config(log_format=LINE): "formatter": "plain_console", "stream": sys.stderr, }, + "psycopg_console": { + "class": stream_handler, + "formatter": "psycopg_plain_console", + "stream": sys.stdout, + }, "cloud": { "class": stream_handler, "formatter": "json_formatter", @@ -364,6 +383,11 @@ def get_logging_config(log_format=LINE): "formatter": "json_formatter", "stream": sys.stderr, }, + "psycopg_cloud": { + "class": stream_handler, + "formatter": "psycopg_json_formatter", + "stream": sys.stdout, + }, }, } @@ -377,6 +401,11 @@ def get_logging_config(log_format=LINE): "handlers": ["console", "console_error"], "level": "DEBUG", }, + "psycopg": { + "handlers": [ + "psycopg_console", + ], + }, } if ENABLE_PL_SQL_LOGGING is True: logging_config["loggers"]["django.db.backends"] = { @@ -393,6 +422,11 @@ def get_logging_config(log_format=LINE): "handlers": ["cloud", "cloud_error"], "level": "INFO", }, + "psycopg": { + "handlers": [ + "psycopg_cloud", + ], + }, } return logging_config @@ -404,6 +438,12 @@ def add_migration_logs(logger: logging.Logger, method_name: str, event_dict): return event_dict +def process_log_tokens(logger, method_name, event_dict): + event_dict["event"] = re.sub(r'user ".*?"', 'user ""', event_dict["event"]) + event_dict["event"] = re.sub(r'database ".*?"', 'database ""', event_dict["event"]) + return event_dict + + def get_logging_processors(): """ get structlog processors From 083d46c0d2f3515a107ac91ddb524462639463d4 Mon Sep 17 00:00:00 2001 From: Laura Beaufort Date: Tue, 25 Aug 2026 11:42:07 -0400 Subject: [PATCH 09/29] Update cryptography package --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6ed14fdfe..38bdc8eb93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ boto3==1.35.72 celery==5.6.3 cfenv==0.5.3 -cryptography==48.0.1 +cryptography==50.0.0 dj_database_url==2.3.0 django-deprecate-fields==0.2.3 django-migration-linter==5.2.0 From 96223765fd4cf101207970a0c464258f37bf4290 Mon Sep 17 00:00:00 2001 From: Laura Beaufort Date: Tue, 25 Aug 2026 13:16:40 -0400 Subject: [PATCH 10/29] Update gitpython to latest --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 38bdc8eb93..d76901fa90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ drf-spectacular-sidecar==2026.4.14 Faker==37.6.0 git+https://github.com/fecgov/fecfile-validate@8384e8c2747e64ebacd85031dcb9d214cfa513c5#egg=fecfile_validate&subdirectory=fecfile_validate_python github3.py==4.0.1 -GitPython==3.1.58 +GitPython==3.1.59 gunicorn==23.0.0 invoke==2.2.0 jwcrypto==1.5.7 From c9ecced9f22068d2c65d35d56303fb3bf26e4bb6 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 25 Aug 2026 13:20:03 -0400 Subject: [PATCH 11/29] 3293 3306 F3 changes --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6ed14fdfe..9d8625e662 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ djangorestframework==3.17.1 drf-spectacular==0.29.0 drf-spectacular-sidecar==2026.4.14 Faker==37.6.0 -git+https://github.com/fecgov/fecfile-validate@8384e8c2747e64ebacd85031dcb9d214cfa513c5#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@3822e1d34fc5b8e01db52487e3f5b750146fb16b#egg=fecfile_validate&subdirectory=fecfile_validate_python github3.py==4.0.1 GitPython==3.1.58 gunicorn==23.0.0 From ba30b1e3e77e6f68eff2b1452fe8f68fbdad8e0f Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 25 Aug 2026 15:54:50 -0400 Subject: [PATCH 12/29] fosec-133 CR comments --- django-backend/fecfiler/settings/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 7ce9413692..4b276eb701 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -439,8 +439,10 @@ def add_migration_logs(logger: logging.Logger, method_name: str, event_dict): def process_log_tokens(logger, method_name, event_dict): - event_dict["event"] = re.sub(r'user ".*?"', 'user ""', event_dict["event"]) - event_dict["event"] = re.sub(r'database ".*?"', 'database ""', event_dict["event"]) + event_dict["event"] = re.sub(r'user ".*?"', 'user "****"', event_dict["event"]) + event_dict["event"] = re.sub( + r'database ".*?"', 'database "****"', event_dict["event"] + ) return event_dict From 737665b0afffe568a0785864fdab0bc02d37b359 Mon Sep 17 00:00:00 2001 From: Max Zaremba Date: Tue, 25 Aug 2026 21:02:14 -0400 Subject: [PATCH 13/29] Moved environment varibles from celery-worker to api --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 200562cd7f..55d199102d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,8 +68,6 @@ services: DB_POOL_MIN_SIZE: 10 DB_POOL_MAX_SIZE: 10 DB_POOL_MAX_IDLE: 600 - GUNICORN_WORKERS: 9 - GUNICORN_THREADS: 8 CELERY_WORKER_CONCURRENCY: 10 MOCK_EFO_DOT_FEC_SUBMISSION_DURATION_SECONDS: 0 FLAG__ENABLE_IMPORT: True @@ -192,6 +190,8 @@ services: DB_POOL_MIN_SIZE: 8 DB_POOL_MAX_SIZE: 20 DB_POOL_MAX_IDLE: 600 + GUNICORN_WORKERS: 9 + GUNICORN_THREADS: 8 MOCK_EFO_DOT_FEC_SUBMISSION_DURATION_SECONDS: 0 # ---- FEATURE FLAGS ---- From 8d8e62587d7c902df74c1b88d5c0d0ea482661e3 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Wed, 26 Aug 2026 14:10:53 -0400 Subject: [PATCH 14/29] FECFILE-1236: Added SES service to manifests. --- manifests/manifest-dev.yml | 1 + manifests/manifest-prod.yml | 1 + manifests/manifest-stage.yml | 1 + manifests/manifest-test.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/manifests/manifest-dev.yml b/manifests/manifest-dev.yml index 9b58b8f596..b74ed5d4aa 100644 --- a/manifests/manifest-dev.yml +++ b/manifests/manifest-dev.yml @@ -8,6 +8,7 @@ defaults: &defaults services: - fecfile-api-rds - fecfile-api-s3 + - fecfile-api-ses - fecfile-api-redis - fecfile-api-creds-dev env: &default-env diff --git a/manifests/manifest-prod.yml b/manifests/manifest-prod.yml index 125c9cf376..c7372288d6 100644 --- a/manifests/manifest-prod.yml +++ b/manifests/manifest-prod.yml @@ -8,6 +8,7 @@ defaults: &defaults services: - fecfile-api-rds - fecfile-api-s3 + - fecfile-api-ses - fecfile-api-redis - fecfile-api-creds-prod env: &default-env diff --git a/manifests/manifest-stage.yml b/manifests/manifest-stage.yml index 177967b15e..221c45efa2 100644 --- a/manifests/manifest-stage.yml +++ b/manifests/manifest-stage.yml @@ -8,6 +8,7 @@ defaults: &defaults services: - fecfile-api-rds - fecfile-api-s3 + - fecfile-api-ses - fecfile-api-redis - fecfile-api-creds-stage env: &default-env diff --git a/manifests/manifest-test.yml b/manifests/manifest-test.yml index 847181d7d0..a283e1a49a 100644 --- a/manifests/manifest-test.yml +++ b/manifests/manifest-test.yml @@ -8,6 +8,7 @@ defaults: &defaults services: - fecfile-api-rds - fecfile-api-s3 + - fecfile-api-ses - fecfile-api-redis - fecfile-api-creds-test env: &default-env From 1b5019608e491e26416cdb7778c5d36a7f2e1173 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Wed, 26 Aug 2026 14:17:33 -0400 Subject: [PATCH 15/29] FECFILE-1236: Renamed email notification function for clarity, switched to full name. --- django-backend/fecfiler/committee_accounts/views.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index ce8c2e8e4a..62ea251e46 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -242,15 +242,17 @@ def add_member(self, request): # if no Exception was returned, send email notification to the user if not isinstance(new_member, BaseException): + # fall back to email if name is unavailable + full_name = request.user.get_full_name() or request.user.email logger.info( - f"User {request.user.first_name} added {email} to committee " + f"User {full_name} added {email} to committee " f"{committee_id} as {role}" ) if FLAG__ENABLE_EMAIL: - self.sendAddUserToCommitteeEmail( + self.sendAddMemberEmailNotification( committee_id, email, - request.user.first_name, + full_name, role ) else: @@ -362,7 +364,7 @@ def list(self, request, *args, **kwargs): serializer = self.get_serializer(queryset, many=True) return Response(serializer.data) - def sendAddUserToCommitteeEmail(self, committee_id, email, first_name, role): + def sendAddMemberEmailNotification(self, committee_id, email, first_name, role): subject = f"[FECfile+] Invite to committee {committee_id}" # adjust links based on space From 395addb3753a88cc691e4dc80e3ed7244186fba1 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Wed, 26 Aug 2026 16:09:47 -0400 Subject: [PATCH 16/29] FECFILE-1236: Updated plaintext version. --- .../fecfiler/committee_accounts/views.py | 48 +++++++++++++++---- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 62ea251e46..b9fd755071 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -249,11 +249,15 @@ def add_member(self, request): f"{committee_id} as {role}" ) if FLAG__ENABLE_EMAIL: + committee_data = get_committee_account_data(committee_id) + committee_name = committee_data.get("name", None) + self.sendAddMemberEmailNotification( committee_id, + committee_name, email, full_name, - role + role, ) else: logger.error( @@ -364,23 +368,47 @@ def list(self, request, *args, **kwargs): serializer = self.get_serializer(queryset, many=True) return Response(serializer.data) - def sendAddMemberEmailNotification(self, committee_id, email, first_name, role): - subject = f"[FECfile+] Invite to committee {committee_id}" + def sendAddMemberEmailNotification(self, committee_id, committee_name, email, full_name, role): + subject = f"{full_name} has added you to a FECfile+ committee account" # adjust links based on space - if settings.SPACE == "prod": - envbit = "" + if settings.SPACE == "local": + fecfile_link = "http://localhost:4200/login" else: - envbit = f"{settings.SPACE}." + if not settings.SPACE or settings.SPACE == "prod": + envbit = "" + else: + envbit = f"{settings.SPACE}." + fecfile_link = f"https://{envbit}fecfile.fec.gov/login" body_text = ( - "ADDED TO FECfile+ COMMITTEE\n" + f"{full_name} has added you as a {role} in FECfile+ for the following committee:\n" "\n" - f"{first_name} has added you as a {role} " - f"to {committee_id}.\n" + f"Committee ID: {committee_id}\n" + f"Committee Name: {committee_name}\n" "\n" "You can access the committee account by signing in to FECfile+:\n" - f"https://{envbit}fecfile.fec.gov/" + f"{fecfile_link}\n" + "\n" + "Important: You must have a Login.gov account to sign in to FECfile+. " + "If you don't already have a Login.gov account for this email, " + "select \"Create an account\" to get started.\n" + "\n" + "==================================================================\n" + "\n" + "If you are receiving this email in error or have any questions, " + "please contact the FEC Electronic Filing Office " + "toll-free at (800) 424-9530 ext. 1307 or locally at (202) 694-1307.\n" + "\n" + f"FECfile+: {fecfile_link}\n" + "Contact us: " + "https://www.fec.gov/contact/" + "#filing-reports-and-amendments-reporting-specific-transactions\n" + "\n" + "FEC.gov: https://www.fec.gov/\n" + "Electronic filing overview: " + "https://www.fec.gov/help-candidates-and-committees/filing-reports/electronic-filing/\n" + "Privacy Policy: https://www.fec.gov/about/privacy-and-security-policy/" ) try: From ee5d34598c8d875a0fa20d1cdb664c4b9cbdcc07 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Wed, 26 Aug 2026 16:11:22 -0400 Subject: [PATCH 17/29] FECFILE-1236: Linted plaintext version. --- django-backend/fecfiler/committee_accounts/views.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index b9fd755071..d336f72e11 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -368,7 +368,9 @@ def list(self, request, *args, **kwargs): serializer = self.get_serializer(queryset, many=True) return Response(serializer.data) - def sendAddMemberEmailNotification(self, committee_id, committee_name, email, full_name, role): + def sendAddMemberEmailNotification( + self, committee_id, committee_name, email, full_name, role + ): subject = f"{full_name} has added you to a FECfile+ committee account" # adjust links based on space @@ -382,7 +384,8 @@ def sendAddMemberEmailNotification(self, committee_id, committee_name, email, fu fecfile_link = f"https://{envbit}fecfile.fec.gov/login" body_text = ( - f"{full_name} has added you as a {role} in FECfile+ for the following committee:\n" + f"{full_name} has added you as a {role} in FECfile+ " + "for the following committee:\n" "\n" f"Committee ID: {committee_id}\n" f"Committee Name: {committee_name}\n" @@ -407,7 +410,8 @@ def sendAddMemberEmailNotification(self, committee_id, committee_name, email, fu "\n" "FEC.gov: https://www.fec.gov/\n" "Electronic filing overview: " - "https://www.fec.gov/help-candidates-and-committees/filing-reports/electronic-filing/\n" + "https://www.fec.gov/help-candidates-and-committees/" + "filing-reports/electronic-filing/\n" "Privacy Policy: https://www.fec.gov/about/privacy-and-security-policy/" ) From 0338cd0b104d1c8fc7048a343d401f112a3c3231 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 26 Aug 2026 16:56:42 -0400 Subject: [PATCH 18/29] fosec-133 test logging exceptions --- .../fecfiler/settings/tests/test_logger.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 django-backend/fecfiler/settings/tests/test_logger.py diff --git a/django-backend/fecfiler/settings/tests/test_logger.py b/django-backend/fecfiler/settings/tests/test_logger.py new file mode 100644 index 0000000000..071a0afdbb --- /dev/null +++ b/django-backend/fecfiler/settings/tests/test_logger.py @@ -0,0 +1,61 @@ +import logging.config +from django.test import TestCase +from django.core.exceptions import ValidationError +import logging +from fecfiler import settings +import structlog +from io import StringIO +from contextlib import redirect_stdout, redirect_stderr + + +class LoggerTestCase(TestCase): + + # Local + + def test_info_logging_local(self): + buffer = StringIO() + with redirect_stdout(buffer): + logging.config.dictConfig(settings.get_logging_config("LINE")) + logger = structlog.get_logger() + logger.info("Test info log message") + output = buffer.getvalue() + self.assertIn("Test info log message", output) + + def test_exception_logging_local(self): + buffer = StringIO() + with redirect_stderr(buffer): + logging.config.dictConfig(settings.get_logging_config("LINE")) + logger = structlog.get_logger() + try: + raise ValidationError("Test exception") + except ValidationError as e: + logger.exception(e) + output = buffer.getvalue() + self.assertIn("Test exception", output) + + # Cloud + + def test_info_logging_cloud(self): + buffer = StringIO() + with redirect_stdout(buffer): + logging.config.dictConfig(settings.get_logging_config("NOT_LINE")) + logger = structlog.get_logger() + logger.info("Test info log message") + output = buffer.getvalue() + self.assertIn("Test info log message", output) + + def test_exception_logging_cloud(self): + buffer = StringIO() + with redirect_stderr(buffer): + logging.config.dictConfig(settings.get_logging_config("NOT_LINE")) + logger = structlog.get_logger() + try: + raise ValidationError("Test exception1") + except ValidationError as e: + try: + raise ValidationError("Test exception2") from e + except ValidationError as e2: + logger.exception(e2) + output = buffer.getvalue() + self.assertIn("Test exception1", output) + self.assertIn("Test exception2", output) From 67a821023f0ac24cce18240ec94f905d314e0c88 Mon Sep 17 00:00:00 2001 From: toddlees Date: Thu, 27 Aug 2026 10:10:03 -0400 Subject: [PATCH 19/29] Moving gunicorn env variables to be in defaults rather than ony api --- manifests/manifest-dev.yml | 4 ++-- manifests/manifest-prod.yml | 4 ++-- manifests/manifest-stage.yml | 4 ++-- manifests/manifest-test.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifests/manifest-dev.yml b/manifests/manifest-dev.yml index f6d9ace6fb..8324e4fdfc 100644 --- a/manifests/manifest-dev.yml +++ b/manifests/manifest-dev.yml @@ -26,6 +26,8 @@ defaults: &defaults SESSION_COOKIE_AGE: 3600 # Value in seconds (1 hour) DB_POOL_MIN_SIZE: 0 DB_POOL_MAX_SIZE: 8 + GUNICORN_WORKERS: 9 + GUNICORN_THREADS: 8 DB_POOL_MAX_IDLE: 600 BP_PIP_VERSION: latest LOG_FORMAT: KEY_VALUE @@ -58,8 +60,6 @@ applications: DB_POOL_MIN_SIZE: 10 DB_POOL_MAX_SIZE: 10 DB_POOL_MAX_IDLE: 600 - GUNICORN_WORKERS: 9 - GUNICORN_THREADS: 8 CELERY_WORKER_CONCURRENCY: 10 instances: 2 no-route: true diff --git a/manifests/manifest-prod.yml b/manifests/manifest-prod.yml index 7c8b3d587d..4a2ef42a72 100644 --- a/manifests/manifest-prod.yml +++ b/manifests/manifest-prod.yml @@ -27,6 +27,8 @@ defaults: &defaults SESSION_COOKIE_AGE: 3600 # Value in seconds (1 hour) DB_POOL_MIN_SIZE: 0 DB_POOL_MAX_SIZE: 8 + GUNICORN_WORKERS: 9 + GUNICORN_THREADS: 8 DB_POOL_MAX_IDLE: 600 BP_PIP_VERSION: latest LOG_FORMAT: KEY_VALUE @@ -58,8 +60,6 @@ applications: DB_POOL_MIN_SIZE: 50 DB_POOL_MAX_SIZE: 50 DB_POOL_MAX_IDLE: 600 - GUNICORN_WORKERS: 9 - GUNICORN_THREADS: 8 CELERY_WORKER_CONCURRENCY: 50 no-route: true health-check-type: process diff --git a/manifests/manifest-stage.yml b/manifests/manifest-stage.yml index 68807eb524..08e3a3e40f 100644 --- a/manifests/manifest-stage.yml +++ b/manifests/manifest-stage.yml @@ -27,6 +27,8 @@ defaults: &defaults SESSION_COOKIE_AGE: 3600 # Value in seconds (1 hour) DB_POOL_MIN_SIZE: 0 DB_POOL_MAX_SIZE: 8 + GUNICORN_WORKERS: 9 + GUNICORN_THREADS: 8 DB_POOL_MAX_IDLE: 600 BP_PIP_VERSION: latest LOG_FORMAT: KEY_VALUE @@ -58,8 +60,6 @@ applications: DB_POOL_MIN_SIZE: 10 DB_POOL_MAX_SIZE: 10 DB_POOL_MAX_IDLE: 600 - GUNICORN_WORKERS: 9 - GUNICORN_THREADS: 8 CELERY_WORKER_CONCURRENCY: 10 instances: 2 no-route: true diff --git a/manifests/manifest-test.yml b/manifests/manifest-test.yml index b87f169304..d7309923ea 100644 --- a/manifests/manifest-test.yml +++ b/manifests/manifest-test.yml @@ -27,6 +27,8 @@ defaults: &defaults SESSION_COOKIE_AGE: 3600 # Value in seconds (1 hour) DB_POOL_MIN_SIZE: 0 DB_POOL_MAX_SIZE: 8 + GUNICORN_WORKERS: 9 + GUNICORN_THREADS: 8 DB_POOL_MAX_IDLE: 600 BP_PIP_VERSION: latest LOG_FORMAT: KEY_VALUE @@ -58,8 +60,6 @@ applications: DB_POOL_MIN_SIZE: 50 DB_POOL_MAX_SIZE: 50 DB_POOL_MAX_IDLE: 600 - GUNICORN_WORKERS: 9 - GUNICORN_THREADS: 8 CELERY_WORKER_CONCURRENCY: 50 instances: 2 no-route: true From 09179b3c114dea1cd40bdd9ba0f720e851906ae5 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Thu, 27 Aug 2026 10:59:11 -0400 Subject: [PATCH 20/29] FECFILE-3391: Add SES service to bindings in manifests. --- manifests/manifest-dev.yml | 1 + manifests/manifest-prod.yml | 1 + manifests/manifest-stage.yml | 1 + manifests/manifest-test.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/manifests/manifest-dev.yml b/manifests/manifest-dev.yml index 8324e4fdfc..33a7160773 100644 --- a/manifests/manifest-dev.yml +++ b/manifests/manifest-dev.yml @@ -8,6 +8,7 @@ defaults: &defaults services: - fecfile-api-rds - fecfile-api-s3 + - fecfile-api-ses - fecfile-api-redis - fecfile-api-creds-dev env: &default-env diff --git a/manifests/manifest-prod.yml b/manifests/manifest-prod.yml index 4a2ef42a72..46ee16c65f 100644 --- a/manifests/manifest-prod.yml +++ b/manifests/manifest-prod.yml @@ -8,6 +8,7 @@ defaults: &defaults services: - fecfile-api-rds - fecfile-api-s3 + - fecfile-api-ses - fecfile-api-redis - fecfile-api-creds-prod env: &default-env diff --git a/manifests/manifest-stage.yml b/manifests/manifest-stage.yml index 08e3a3e40f..623c3bb46f 100644 --- a/manifests/manifest-stage.yml +++ b/manifests/manifest-stage.yml @@ -8,6 +8,7 @@ defaults: &defaults services: - fecfile-api-rds - fecfile-api-s3 + - fecfile-api-ses - fecfile-api-redis - fecfile-api-creds-stage env: &default-env diff --git a/manifests/manifest-test.yml b/manifests/manifest-test.yml index d7309923ea..abe9639427 100644 --- a/manifests/manifest-test.yml +++ b/manifests/manifest-test.yml @@ -8,6 +8,7 @@ defaults: &defaults services: - fecfile-api-rds - fecfile-api-s3 + - fecfile-api-ses - fecfile-api-redis - fecfile-api-creds-test env: &default-env From f8433d244075a89e88d32239f81770e78706bb3b Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Thu, 27 Aug 2026 12:15:03 -0400 Subject: [PATCH 21/29] FECFILE-1236: Moved plaintext email to template. --- .gitignore | 6 ++- .../committee_accounts/tests/test_views.py | 26 ++++++++++++ .../fecfiler/committee_accounts/views.py | 40 +++++-------------- .../emails/add_member_notification.txt | 20 ++++++++++ 4 files changed, 60 insertions(+), 32 deletions(-) create mode 100644 django-backend/static/templates/emails/add_member_notification.txt diff --git a/.gitignore b/.gitignore index 77dec0ca5f..8ccfe119da 100644 --- a/.gitignore +++ b/.gitignore @@ -56,7 +56,6 @@ testem.log django-backend/.cache/ django-backend/.local/ - ### Bower ### */bower_components *.bower-cache @@ -89,4 +88,7 @@ django-backend/celerybeat-schedule-wal #profiling static/ -*.prof \ No newline at end of file +*.prof + +# exceptions +!django-backend/static/ diff --git a/django-backend/fecfiler/committee_accounts/tests/test_views.py b/django-backend/fecfiler/committee_accounts/tests/test_views.py index bcbedea5c8..9106432e72 100644 --- a/django-backend/fecfiler/committee_accounts/tests/test_views.py +++ b/django-backend/fecfiler/committee_accounts/tests/test_views.py @@ -180,6 +180,32 @@ def test_add_membership_requires_correct_parameters(self): ) self.assertEqual(response.status_code, 400) + @patch("fecfiler.committee_accounts.views.send_email_notification") + @patch("fecfiler.committee_accounts.views.settings") + def test_add_member_email_template_renders_expected_values( + self, mock_settings, mock_send_email + ): + mock_settings.SPACE = "local" + + viewset = CommitteeMembershipViewSet() + viewset.sendAddMemberEmailNotification( + committee_id="C12345678", + committee_name="Example Committee", + email="invitee@fec.gov", + full_name="Alex Admin", + role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + ) + + self.assertEqual(mock_send_email.call_count, 1) + body_text = mock_send_email.call_args.kwargs["body_text"] + + self.assertIn("Alex Admin has added you as a", body_text) + self.assertIn("Committee ID: C12345678", body_text) + self.assertIn("Committee Name: Example Committee", body_text) + self.assertIn("http://localhost:4200/login", body_text) + self.assertNotIn("{{", body_text) + self.assertNotIn("}}", body_text) + def test_update_membership_forbidden(self): user = User.objects.get(id="fb20ffc3-285e-448e-9e56-9ca1fd43e7d3") response = self.send_viewset_put_request( diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index d336f72e11..de04bc877e 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -18,6 +18,7 @@ HttpResponseBadRequest, HttpResponseServerError, ) +from django.template.loader import render_to_string from django.core.exceptions import ValidationError from drf_spectacular.utils import extend_schema, OpenApiParameter from .serializers import CommitteeAccountSerializer, CommitteeMembershipSerializer @@ -383,36 +384,15 @@ def sendAddMemberEmailNotification( envbit = f"{settings.SPACE}." fecfile_link = f"https://{envbit}fecfile.fec.gov/login" - body_text = ( - f"{full_name} has added you as a {role} in FECfile+ " - "for the following committee:\n" - "\n" - f"Committee ID: {committee_id}\n" - f"Committee Name: {committee_name}\n" - "\n" - "You can access the committee account by signing in to FECfile+:\n" - f"{fecfile_link}\n" - "\n" - "Important: You must have a Login.gov account to sign in to FECfile+. " - "If you don't already have a Login.gov account for this email, " - "select \"Create an account\" to get started.\n" - "\n" - "==================================================================\n" - "\n" - "If you are receiving this email in error or have any questions, " - "please contact the FEC Electronic Filing Office " - "toll-free at (800) 424-9530 ext. 1307 or locally at (202) 694-1307.\n" - "\n" - f"FECfile+: {fecfile_link}\n" - "Contact us: " - "https://www.fec.gov/contact/" - "#filing-reports-and-amendments-reporting-specific-transactions\n" - "\n" - "FEC.gov: https://www.fec.gov/\n" - "Electronic filing overview: " - "https://www.fec.gov/help-candidates-and-committees/" - "filing-reports/electronic-filing/\n" - "Privacy Policy: https://www.fec.gov/about/privacy-and-security-policy/" + body_text = render_to_string( + "emails/add_member_notification.txt", + { + "full_name": full_name, + "role": role, + "committee_id": committee_id, + "committee_name": committee_name, + "fecfile_link": fecfile_link, + }, ) try: diff --git a/django-backend/static/templates/emails/add_member_notification.txt b/django-backend/static/templates/emails/add_member_notification.txt new file mode 100644 index 0000000000..a27a1250fc --- /dev/null +++ b/django-backend/static/templates/emails/add_member_notification.txt @@ -0,0 +1,20 @@ +{{ full_name }} has added you as a {{ role }} in FECfile+ for the following committee: + +Committee ID: {{ committee_id }} +Committee Name: {{ committee_name }} + +You can access the committee account by signing in to FECfile+: +{{ fecfile_link }} + +Important: You must have a Login.gov account to sign in to FECfile+. If you don't already have a Login.gov account for this email, select "Create an account" to get started. + +================================================================== + +If you are receiving this email in error or have any questions, please contact the FEC Electronic Filing Office toll-free at (800) 424-9530 ext. 1307 or locally at (202) 694-1307. + +FECfile+: {{ fecfile_link }} +Contact us: https://www.fec.gov/contact/#filing-reports-and-amendments-reporting-specific-transactions + +FEC.gov: https://www.fec.gov/ +Electronic filing overview: https://www.fec.gov/help-candidates-and-committees/filing-reports/electronic-filing/ +Privacy Policy: https://www.fec.gov/about/privacy-and-security-policy/ From a93af074d98d3b93cce282d857a3141394799b2d Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Thu, 27 Aug 2026 16:06:29 -0400 Subject: [PATCH 22/29] FECFILE-1236: Added HTML template. --- .../fecfiler/committee_accounts/views.py | 27 +- .../emails/add_member_notification.html | 282 ++++++++++++++++++ .../emails/add_member_notification.txt | 4 +- 3 files changed, 300 insertions(+), 13 deletions(-) create mode 100644 django-backend/static/templates/emails/add_member_notification.html diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index de04bc877e..bc0083b2cb 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -376,28 +376,33 @@ def sendAddMemberEmailNotification( # adjust links based on space if settings.SPACE == "local": - fecfile_link = "http://localhost:4200/login" + fecfile_link = "http://localhost:4200" else: if not settings.SPACE or settings.SPACE == "prod": envbit = "" else: envbit = f"{settings.SPACE}." - fecfile_link = f"https://{envbit}fecfile.fec.gov/login" - + fecfile_link = f"https://{envbit}fecfile.fec.gov" + + email_dict = { + "full_name": full_name, + "role": role, + "committee_id": committee_id, + "committee_name": committee_name, + "fecfile_link": fecfile_link, + } body_text = render_to_string( "emails/add_member_notification.txt", - { - "full_name": full_name, - "role": role, - "committee_id": committee_id, - "committee_name": committee_name, - "fecfile_link": fecfile_link, - }, + email_dict, + ) + body_html = render_to_string( + "emails/add_member_notification.html", + email_dict, ) try: send_email_notification( - to_email=email, subject=subject, body_text=body_text + to_email=email, subject=subject, body_text=body_text, body_html=body_html ) except Exception as e: logger.error( diff --git a/django-backend/static/templates/emails/add_member_notification.html b/django-backend/static/templates/emails/add_member_notification.html new file mode 100644 index 0000000000..a587f5cd28 --- /dev/null +++ b/django-backend/static/templates/emails/add_member_notification.html @@ -0,0 +1,282 @@ + + + + + + Added to FECfile+ Committee Account + + + + + + diff --git a/django-backend/static/templates/emails/add_member_notification.txt b/django-backend/static/templates/emails/add_member_notification.txt index a27a1250fc..000ca9b1cf 100644 --- a/django-backend/static/templates/emails/add_member_notification.txt +++ b/django-backend/static/templates/emails/add_member_notification.txt @@ -4,7 +4,7 @@ Committee ID: {{ committee_id }} Committee Name: {{ committee_name }} You can access the committee account by signing in to FECfile+: -{{ fecfile_link }} +{{ fecfile_link }}/login Important: You must have a Login.gov account to sign in to FECfile+. If you don't already have a Login.gov account for this email, select "Create an account" to get started. @@ -12,7 +12,7 @@ Important: You must have a Login.gov account to sign in to FECfile+. If you don' If you are receiving this email in error or have any questions, please contact the FEC Electronic Filing Office toll-free at (800) 424-9530 ext. 1307 or locally at (202) 694-1307. -FECfile+: {{ fecfile_link }} +FECfile+: {{ fecfile_link }}/login Contact us: https://www.fec.gov/contact/#filing-reports-and-amendments-reporting-specific-transactions FEC.gov: https://www.fec.gov/ From e7620140d68205588f8b0315c79b13762a530800 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Fri, 28 Aug 2026 15:35:13 -0400 Subject: [PATCH 23/29] FECFILE-1236: Refactoring HTML email notification template. --- .../emails/add_member_notification.html | 63 +++++++------------ 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/django-backend/static/templates/emails/add_member_notification.html b/django-backend/static/templates/emails/add_member_notification.html index a587f5cd28..cdeafba6fe 100644 --- a/django-backend/static/templates/emails/add_member_notification.html +++ b/django-backend/static/templates/emails/add_member_notification.html @@ -44,9 +44,8 @@ height: auto; } - a { - color: #0aacfa; - text-decoration: underline; + p { + margin: 0 0 20px; } .email-shell { @@ -88,7 +87,11 @@ } .email-content { - padding: 30px 60px 20px; + padding: 20px 60px 0px; + } + + .email-content .divider { + margin-bottom: 5px; } .email-title { @@ -99,10 +102,6 @@ color: #1a1a1a; } - .email-copy { - margin: 0 0 20px; - } - .meta-list { width: 100%; margin: 0 0 20px; @@ -130,14 +129,6 @@ text-decoration: underline; } - .email-content .divider { - margin: 20px 0; - } - - .support-note { - margin: 0; - } - .footer-primary, .footer-secondary { text-align: center; @@ -153,6 +144,11 @@ justify-content: center; } + .footer-secondary { + background-color: #e8e8e8; + padding-top: 15px; + } + .footer-links { display: flex; align-items: center; @@ -162,34 +158,21 @@ margin: 0; } - .footer-primary a, - .footer-primary .footer-separator, - .footer-secondary a, - .footer-secondary .footer-separator { - color: #ffffff; + .footer-links a { text-decoration: none; - font-size: 14px; - } - - .footer-primary .footer-separator { - margin: 0 12px; } - .footer-secondary { - background-color: #e8e8e8; - padding-top: 15px; - } - - .footer-secondary .footer-links { - color: #212121; + .footer-primary .footer-links, + .footer-primary .footer-links a { + color: #ffffff; } - .footer-secondary a, - .footer-secondary .footer-separator { + .footer-secondary .footer-links, + .footer-secondary .footer-links a { color: #212121; } - .footer-secondary .footer-separator { + .footer-separator { margin: 0 12px; } @@ -228,7 +211,7 @@ -

You can access the committee account by .

-

Important: You must have a Login.gov account to sign in to FECfile+. If you don't already have a Login.gov account for this email, select “Create an account” to get started.

-

+

If you are receiving this email in error or have any questions, please contact the FEC Electronic Filing Office toll-free at or locally at .

From b6b398ad3b2649c97cfce30496a6caca40504851 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Fri, 28 Aug 2026 15:54:55 -0400 Subject: [PATCH 24/29] FECFILE-1236: Further refactoring HTML email notification template. --- .../emails/add_member_notification.html | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/django-backend/static/templates/emails/add_member_notification.html b/django-backend/static/templates/emails/add_member_notification.html index cdeafba6fe..f8b1b16202 100644 --- a/django-backend/static/templates/emails/add_member_notification.html +++ b/django-backend/static/templates/emails/add_member_notification.html @@ -124,9 +124,9 @@ flex: 1; } - .email-link { - color: #212121; - text-decoration: underline; + a.email-link { + color: #212121 !important; + text-decoration: underline !important; } .footer-primary, @@ -227,17 +227,23 @@

Added to FECfile+ Committee Account

- You can access the committee account by . + You can access the committee account by + .

- Important: You must have a Login.gov account to sign in to FECfile+. If you don't already have a Login.gov account for this email, select “Create an account” to get started. + Important: You must have a Login.gov account to sign in to FECfile+. + If you don't already have a Login.gov account for this email, select “Create an account” + to get started.

- If you are receiving this email in error or have any questions, please contact the FEC Electronic Filing Office toll-free at or locally at . + If you are receiving this email in error or have any questions, + please contact the FEC Electronic Filing Office + toll-free at + or locally at .

From 8fa2f245731d6b7b14e885d3e8aa75b83920cd4a Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 1 Sep 2026 14:12:10 -0400 Subject: [PATCH 25/29] turn off email on dev and stage --- manifests/manifest-dev.yml | 2 +- manifests/manifest-stage.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/manifest-dev.yml b/manifests/manifest-dev.yml index 5986cb8761..33204b51f4 100644 --- a/manifests/manifest-dev.yml +++ b/manifests/manifest-dev.yml @@ -39,7 +39,7 @@ defaults: &defaults # ---- FEATURE FLAGS ---- FLAG__COMMITTEE_DATA_SOURCE: TEST # Values are PRODUCTION, TEST, MOCKED FLAG__ENABLE_IMPORT: True - FLAG__ENABLE_EMAIL: True + FLAG__ENABLE_EMAIL: False ENABLE_RESTRICTED_COMMANDS: True FEC_FORMAT_VERSION: 8.5 diff --git a/manifests/manifest-stage.yml b/manifests/manifest-stage.yml index c072e9194c..652f14f89d 100644 --- a/manifests/manifest-stage.yml +++ b/manifests/manifest-stage.yml @@ -39,7 +39,7 @@ defaults: &defaults # ---- FEATURE FLAGS ---- FLAG__COMMITTEE_DATA_SOURCE: TEST # Values are TEST and PRODUCTION FLAG__ENABLE_IMPORT: True - FLAG__ENABLE_EMAIL: True + FLAG__ENABLE_EMAIL: False ENABLE_RESTRICTED_COMMANDS: True FEC_FORMAT_VERSION: 8.5 From ba105130cf35a6ba6a4bd2e9d0cb393c11138f41 Mon Sep 17 00:00:00 2001 From: Dan Fowlkes Date: Tue, 1 Sep 2026 15:59:41 -0400 Subject: [PATCH 26/29] FECFILE-1236: Updated HTML email template. --- .../emails/add_member_notification.html | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/django-backend/static/templates/emails/add_member_notification.html b/django-backend/static/templates/emails/add_member_notification.html index f8b1b16202..dac11fa349 100644 --- a/django-backend/static/templates/emails/add_member_notification.html +++ b/django-backend/static/templates/emails/add_member_notification.html @@ -56,7 +56,7 @@ } .email-wrapper { - padding: 20px 10px; + padding: 20px 0px; margin: 0 auto; width: 100%; } @@ -74,8 +74,9 @@ .email-banner img { width: 100%; - max-width: 600px; - height: auto; + max-width: 600px !important; + height: 89px !important; + max-height: 89px !important; display: block; } @@ -97,6 +98,7 @@ .email-title { margin: 0 0 20px; font-family: karla-bold, Arial, Helvetica, sans-serif; + font-weight: bold; font-size: 24px; line-height: 1.3; color: #1a1a1a; @@ -136,12 +138,12 @@ .footer-primary { font-family: karla-bold, Arial, Helvetica, sans-serif; + font-weight: bold; background-color: #112e51; - min-height: 80px; - padding: 0 20px; + height: 80px; display: flex; - align-items: center; - justify-content: center; + align-items: center !important; + justify-content: center !important; } .footer-secondary { @@ -150,12 +152,11 @@ } .footer-links { - display: flex; align-items: center; justify-content: center; flex-wrap: wrap; gap: 0; - margin: 0; + margin: 0 auto; } .footer-links a { @@ -204,7 +205,7 @@