release/sprint-93 -> release/test

.gitignore

@@ -56,7 +56,6 @@ testem.log
 django-backend/.cache/
 django-backend/.local/
 
-
 ### Bower ###
 */bower_components
 *.bower-cache
@@ -89,4 +88,7 @@ django-backend/celerybeat-schedule-wal
 
 #profiling
 static/
-*.prof
+*.prof
+
+# exceptions
+!django-backend/static/

Dockerfile-e2e

@@ -13,4 +13,4 @@ RUN useradd nxgu --no-create-home --home /opt/nxg_fec_e2e && chown -R nxgu:nxgu
 USER nxgu
 
 EXPOSE 8080
-ENTRYPOINT ["/bin/sh", "-c", "python manage.py migrate && python manage.py loaddata fixtures/user-data.json && python manage.py load_mocked_committee_data && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 --threads=8 --reload"]
+ENTRYPOINT ["/bin/sh", "-c", "python manage.py migrate && python manage.py loaddata fixtures/user-data.json && python manage.py load_mocked_committee_data && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w $GUNICORN_WORKERS --threads=$GUNICORN_THREADS --reload"]

bin/entrypoint.sh

@@ -22,4 +22,4 @@ python manage.py load_mocked_committee_data
 
 # Start the Gunicorn server
 echo "Starting Gunicorn server..."
-exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 --threads=8 --reload
+exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w $GUNICORN_WORKERS --threads=$GUNICORN_THREADS --reload

bin/run-api.sh

@@ -3,4 +3,5 @@ cd django-backend
 echo "------ Starting APP ------"
 
 # Run application
-./manage.py collectstatic --noinput --traceback --verbosity 3 && exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 --threads=8
+./manage.py collectstatic --noinput --traceback --verbosity 3 && exec gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w $GUNICORN_WORKERS --threads=$GUNICORN_THREADS
+

django-backend/fecfiler/committee_accounts/tests/test_views.py

@@ -180,6 +180,32 @@ def test_add_membership_requires_correct_parameters(self):
         )
         self.assertEqual(response.status_code, 400)
 
+    @patch("fecfiler.committee_accounts.views.send_email_notification")
+    @patch("fecfiler.committee_accounts.views.settings")
+    def test_add_member_email_template_renders_expected_values(
+        self, mock_settings, mock_send_email
+    ):
+        mock_settings.SPACE = "local"
+
+        viewset = CommitteeMembershipViewSet()
+        viewset.sendAddMemberEmailNotification(
+            committee_id="C12345678",
+            committee_name="Example Committee",
+            email="invitee@fec.gov",
+            full_name="Alex Admin",
+            role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR,
+        )
+
+        self.assertEqual(mock_send_email.call_count, 1)
+        body_text = mock_send_email.call_args.kwargs["body_text"]
+
+        self.assertIn("Alex Admin has added you as a", body_text)
+        self.assertIn("Committee ID: C12345678", body_text)
+        self.assertIn("Committee Name: Example Committee", body_text)
+        self.assertIn("http://localhost:4200/login", body_text)
+        self.assertNotIn("{{", body_text)
+        self.assertNotIn("}}", body_text)
+
     def test_update_membership_forbidden(self):
         user = User.objects.get(id="fb20ffc3-285e-448e-9e56-9ca1fd43e7d3")
         response = self.send_viewset_put_request(

django-backend/fecfiler/committee_accounts/views.py

@@ -18,6 +18,7 @@
     HttpResponseBadRequest,
     HttpResponseServerError,
 )
+from django.template.loader import render_to_string
 from django.core.exceptions import ValidationError
 from drf_spectacular.utils import extend_schema, OpenApiParameter
 from .serializers import CommitteeAccountSerializer, CommitteeMembershipSerializer
@@ -242,16 +243,22 @@ def add_member(self, request):
 
             # if no Exception was returned, send email notification to the user
             if not isinstance(new_member, BaseException):
+                # fall back to email if name is unavailable
+                full_name = request.user.get_full_name() or request.user.email
                 logger.info(
-                    f"User {request.user.first_name} added {email} to committee "
+                    f"User {full_name} added {email} to committee "
                     f"{committee_id} as {role}"
                 )
                 if FLAG__ENABLE_EMAIL:
-                    self.sendAddUserToCommitteeEmail(
+                    committee_data = get_committee_account_data(committee_id)
+                    committee_name = committee_data.get("name", None)
+
+                    self.sendAddMemberEmailNotification(
                         committee_id,
+                        committee_name,
                         email,
-                        request.user.first_name,
-                        role
+                        full_name,
+                        role,
                     )
             else:
                 logger.error(
@@ -362,28 +369,40 @@ def list(self, request, *args, **kwargs):
         serializer = self.get_serializer(queryset, many=True)
         return Response(serializer.data)
 
-    def sendAddUserToCommitteeEmail(self, committee_id, email, first_name, role):
-        subject = f"[FECfile+] Invite to committee {committee_id}"
+    def sendAddMemberEmailNotification(
+            self, committee_id, committee_name, email, full_name, role
+    ):
+        subject = f"{full_name} has added you to a FECfile+ committee account"
 
         # adjust links based on space
-        if settings.SPACE == "prod":
-            envbit = ""
+        if settings.SPACE == "local":
+            fecfile_link = "http://localhost:4200"
         else:
-            envbit = f"{settings.SPACE}."
-
-        body_text = (
-            "ADDED TO FECfile+ COMMITTEE\n"
-            "\n"
-            f"{first_name} has added you as a {role} "
-            f"to {committee_id}.\n"
-            "\n"
-            "You can access the committee account by signing in to FECfile+:\n"
-            f"https://{envbit}fecfile.fec.gov/"
+            if not settings.SPACE or settings.SPACE == "prod":
+                envbit = ""
+            else:
+                envbit = f"{settings.SPACE}."
+            fecfile_link = f"https://{envbit}fecfile.fec.gov"
+
+        email_dict = {
+            "full_name": full_name,
+            "role": role,
+            "committee_id": committee_id,
+            "committee_name": committee_name,
+            "fecfile_link": fecfile_link,
+        }
+        body_text = render_to_string(
+            "emails/add_member_notification.txt",
+            email_dict,
+        )
+        body_html = render_to_string(
+            "emails/add_member_notification.html",
+            email_dict,
         )
 
         try:
             send_email_notification(
-                to_email=email, subject=subject, body_text=body_text
+                to_email=email, subject=subject, body_text=body_text, body_html=body_html
             )
         except Exception as e:
             logger.error(

django-backend/fecfiler/settings/base.py

@@ -7,7 +7,9 @@
 import structlog
 import logging
 import sys
+import re
 
+from copy import deepcopy
 from enum import Enum
 from .env import env
 from fecfiler.shared.utilities import get_float_from_string, get_boolean_from_string
@@ -174,17 +176,19 @@ def custom_silk_filter(request):
 database = dj_database_url.config()
 database.setdefault("OPTIONS", {})
 database["OPTIONS"]["application_name"] = f"{APPLICATION_NAME}_{APPLICATION_INDEX}"
+database["CONN_HEALTH_CHECKS"] = True
+
 # psycopg pool settings:
 # https://www.psycopg.org/psycopg3/docs/api/pool.html#psycopg_pool.ConnectionPool
 # this works for celery workers as well
 # https://docs.celeryq.dev/en/main/django/first-steps-with-django.html#django-connection-pool
 database["OPTIONS"]["pool"] = {
     # min_size: psycopg default is 4
-    "min_size": int(env.get_credential("DB_POOL_MIN_SIZE", "4")),
+    "min_size": int(env.get_credential("DB_POOL_MIN_SIZE", "0")),
     # max_size: default to no overflow
     "max_size": int(env.get_credential("DB_POOL_MAX_SIZE", "4")),
-    # max_idle: psycopg default 10 minutes
-    "max_idle": int(env.get_credential("DB_POOL_MAX_IDLE", "600")),
+    # max_idle: psycopg default is 10m, prune before 350s AWS timeout
+    "max_idle": int(env.get_credential("DB_POOL_MAX_IDLE", "300")),
 }
 
 # Database
@@ -290,33 +294,45 @@ def filter(self, record):
 
 def get_logging_config(log_format=LINE):
     stream_handler = "logging.StreamHandler"
+    json_formatter = {
+        "()": structlog.stdlib.ProcessorFormatter,
+        "processors": [
+            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+            structlog.processors.ExceptionRenderer(
+                structlog.processors.ExceptionDictTransformer(show_locals=False)
+            ),
+            structlog.processors.JSONRenderer(),
+        ],
+    }
+    psycopg_json_formatter = deepcopy(json_formatter)
+    psycopg_json_formatter["processors"] = [process_log_tokens] + psycopg_json_formatter[
+        "processors"
+    ]
+    plain_console_formatter = {
+        "()": structlog.stdlib.ProcessorFormatter,
+        "processors": [
+            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+            structlog.processors.dict_tracebacks,
+            structlog.dev.ConsoleRenderer(
+                colors=True, exception_formatter=structlog.dev.rich_traceback
+            ),
+        ],
+        "foreign_pre_chain": [
+            structlog.contextvars.merge_contextvars,
+        ],
+    }
+    psycopg_plain_console_formatter = deepcopy(plain_console_formatter)
+    psycopg_plain_console_formatter["processors"] = [
+        process_log_tokens
+    ] + psycopg_plain_console_formatter["processors"]
     logging_config = {
         "version": 1,
         "disable_existing_loggers": False,
         "formatters": {
-            "json_formatter": {
-                "()": structlog.stdlib.ProcessorFormatter,
-                "processors": [
-                    structlog.stdlib.ProcessorFormatter.remove_processors_meta,
-                    structlog.processors.ExceptionRenderer(
-                        structlog.processors.ExceptionDictTransformer(show_locals=False)
-                    ),
-                    structlog.processors.JSONRenderer(),
-                ],
-            },
-            "plain_console": {
-                "()": structlog.stdlib.ProcessorFormatter,
-                "processors": [
-                    structlog.stdlib.ProcessorFormatter.remove_processors_meta,
-                    structlog.processors.dict_tracebacks,
-                    structlog.dev.ConsoleRenderer(
-                        colors=True, exception_formatter=structlog.dev.rich_traceback
-                    ),
-                ],
-                "foreign_pre_chain": [
-                    structlog.contextvars.merge_contextvars,
-                ],
-            },
+            "json_formatter": json_formatter,
+            "psycopg_json_formatter": psycopg_json_formatter,
+            "plain_console": plain_console_formatter,
+            "psycopg_plain_console": psycopg_plain_console_formatter,
             "key_value": {
                 "()": structlog.stdlib.ProcessorFormatter,
                 "processors": [
@@ -350,6 +366,11 @@ def get_logging_config(log_format=LINE):
                 "formatter": "plain_console",
                 "stream": sys.stderr,
             },
+            "psycopg_console": {
+                "class": stream_handler,
+                "formatter": "psycopg_plain_console",
+                "stream": sys.stdout,
+            },
             "cloud": {
                 "class": stream_handler,
                 "formatter": "json_formatter",
@@ -362,6 +383,11 @@ def get_logging_config(log_format=LINE):
                 "formatter": "json_formatter",
                 "stream": sys.stderr,
             },
+            "psycopg_cloud": {
+                "class": stream_handler,
+                "formatter": "psycopg_json_formatter",
+                "stream": sys.stdout,
+            },
         },
     }
 
@@ -375,6 +401,11 @@ def get_logging_config(log_format=LINE):
                 "handlers": ["console", "console_error"],
                 "level": "DEBUG",
             },
+            "psycopg": {
+                "handlers": [
+                    "psycopg_console",
+                ],
+            },
         }
         if ENABLE_PL_SQL_LOGGING is True:
             logging_config["loggers"]["django.db.backends"] = {
@@ -391,6 +422,11 @@ def get_logging_config(log_format=LINE):
                 "handlers": ["cloud", "cloud_error"],
                 "level": "INFO",
             },
+            "psycopg": {
+                "handlers": [
+                    "psycopg_cloud",
+                ],
+            },
         }
 
     return logging_config
@@ -402,6 +438,14 @@ def add_migration_logs(logger: logging.Logger, method_name: str, event_dict):
     return event_dict
 
 
+def process_log_tokens(logger, method_name, event_dict):
+    event_dict["event"] = re.sub(r'user ".*?"', 'user "****"', event_dict["event"])
+    event_dict["event"] = re.sub(
+        r'database ".*?"', 'database "****"', event_dict["event"]
+    )
+    return event_dict
+
+
 def get_logging_processors():
     """
     get structlog processors

django-backend/fecfiler/settings/tests/test_logger.py

@@ -0,0 +1,61 @@
+import logging.config
+from django.test import TestCase
+from django.core.exceptions import ValidationError
+import logging
+from fecfiler import settings
+import structlog
+from io import StringIO
+from contextlib import redirect_stdout, redirect_stderr
+
+
+class LoggerTestCase(TestCase):
+
+    # Local
+
+    def test_info_logging_local(self):
+        buffer = StringIO()
+        with redirect_stdout(buffer):
+            logging.config.dictConfig(settings.get_logging_config("LINE"))
+            logger = structlog.get_logger()
+            logger.info("Test info log message")
+            output = buffer.getvalue()
+            self.assertIn("Test info log message", output)
+
+    def test_exception_logging_local(self):
+        buffer = StringIO()
+        with redirect_stderr(buffer):
+            logging.config.dictConfig(settings.get_logging_config("LINE"))
+            logger = structlog.get_logger()
+            try:
+                raise ValidationError("Test exception")
+            except ValidationError as e:
+                logger.exception(e)
+                output = buffer.getvalue()
+            self.assertIn("Test exception", output)
+
+    # Cloud
+
+    def test_info_logging_cloud(self):
+        buffer = StringIO()
+        with redirect_stdout(buffer):
+            logging.config.dictConfig(settings.get_logging_config("NOT_LINE"))
+            logger = structlog.get_logger()
+            logger.info("Test info log message")
+            output = buffer.getvalue()
+            self.assertIn("Test info log message", output)
+
+    def test_exception_logging_cloud(self):
+        buffer = StringIO()
+        with redirect_stderr(buffer):
+            logging.config.dictConfig(settings.get_logging_config("NOT_LINE"))
+            logger = structlog.get_logger()
+            try:
+                raise ValidationError("Test exception1")
+            except ValidationError as e:
+                try:
+                    raise ValidationError("Test exception2") from e
+                except ValidationError as e2:
+                    logger.exception(e2)
+            output = buffer.getvalue()
+            self.assertIn("Test exception1", output)
+            self.assertIn("Test exception2", output)