feat(helm): Add tracing to medcat-trainer (#52)

* feat(helm): Add tracing to medcat-trainer

* feat(medcat-trainer): Add semantic convention resource attributes

Author: alhendrickson

deployment/kubernetes/charts/cogstack-helm-ce/templates/NOTES.txt

@@ -29,8 +29,8 @@ export OPENSEARCH_DASHBOARD_PORT={{ index .Values "opensearch-dashboards" "servi
 kubectl --namespace {{ .Release.Namespace }} port-forward svc/$OPENSEARCH_DASHBOARD_SERVICE 5601:$OPENSEARCH_DASHBOARD_PORT &
 {{- end }}
 
-# 5. Jupyterhub
 {{- if index .Values "cogstack-jupyterhub" "enabled" }}
+# 5. Jupyterhub
 export JUPYTERHUB_PROXY_PUBLIC_SERVICE=proxy-public
 export JUPYTERHUB_PROXY_SERVICE_PORT=$(kubectl get svc $JUPYTERHUB_PROXY_PUBLIC_SERVICE -o jsonpath="{.spec.ports[0].port}" )
 kubectl --namespace {{ .Release.Namespace }} port-forward svc/$JUPYTERHUB_PROXY_PUBLIC_SERVICE 8000:$JUPYTERHUB_PROXY_SERVICE_PORT &

deployment/kubernetes/charts/cogstack-helm-ce/values.yaml

@@ -24,10 +24,9 @@ medcat-service:
     repository: cogstacksystems/medcat-service
     tag: "1.2.0"
 
-
 medcat-trainer:
   image:
-    tag: "latest@sha256:dcbd9f7c480dcb88e072189d233a146317f210a826573fd0c47d469da5360be0"
+    tag: latest@sha256:103215a7540ad614c32866f4cb00ddd91e7aff37cea9abc25dc226c577f9506d
   provisioning:
     enabled: true
     existingConfigMap:

deployment/kubernetes/charts/medcat-trainer-helm/templates/_helpers.tpl

@@ -97,6 +97,30 @@ Return full Solr URL: combines host and port
 http://{{ include "medcat-trainer-helm.solrHost" . }}:{{ include "medcat-trainer-helm.solrPort" . }}
 {{- end }}
 
+{{/*
+Validate tracing.otlp: when otlp.enabled is true, at least one of grpc.enabled or http.enabled must be true.
+*/}}
+{{- define "medcat-trainer-helm.validateTracing" -}}
+{{- if and .Values.tracing .Values.tracing.otlp (eq .Values.tracing.otlp.enabled true) -}}
+{{- if not (or (index .Values.tracing.otlp.grpc "enabled") (index .Values.tracing.otlp.http "enabled")) -}}
+{{- fail "tracing.otlp.enabled is true but neither tracing.otlp.grpc.enabled nor tracing.otlp.http.enabled is true. Enable at least one of tracing.otlp.grpc.enabled or tracing.otlp.http.enabled." -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Convert tracing.resourceAttributes (object) to OTEL CSV format: key1=value1,key2=value2,...
+Values in the map are templated (e.g. "{{ .Release.Name }}") so they are evaluated at render time.
+*/}}
+{{- define "medcat-trainer-helm.tracingResourceAttributesCsv" -}}
+{{- $root := . -}}
+{{- $parts := list -}}
+{{- range $name, $value := .Values.tracing.resourceAttributes -}}
+{{- $parts = append $parts (printf "%s=%s" $name (tpl (toString $value) $root)) -}}
+{{- end -}}
+{{- join "," $parts | quote -}}
+{{- end -}}
+
 {{- define "postgres.service-name" -}}
 {{ include "postgresql.v1.primary.fullname" (dict "Values" .Values.postgresql "Chart" (dict "Name" "postgresql") "Release" .Release) }}
 {{- end -}}

deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-deployment.yaml

@@ -1,3 +1,4 @@
+{{- include "medcat-trainer-helm.validateTracing" . -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -58,6 +59,18 @@ spec:
                 name: {{ include "medcat-trainer-helm.fullname" . }}-env
             - secretRef:
                 name: {{ include "medcat-trainer-helm.fullname" . }}-secret
+          env:
+          {{- range $key, $value := .Values.envValueFrom }}
+          - name: {{ $key | quote }}
+            valueFrom:
+              {{- tpl (toYaml $value) $ | nindent 14 }}
+          {{- end }}
+          {{- if .Values.tracing.otlp.enabled }}
+          - name: K8S_CONTAINER_NAME
+            value: medcat-trainer
+          - name: OTEL_RESOURCE_ATTRIBUTES  
+            value: {{ include "medcat-trainer-helm.tracingResourceAttributesCsv" . }}
+          {{- end }}
           {{- with .Values.livenessProbe }}
           livenessProbe:
             {{- toYaml . | nindent 12 }}

deployment/kubernetes/charts/medcat-trainer-helm/templates/medcat-trainer-env-configmap.yaml

@@ -14,6 +14,28 @@ data:
   DB_PATH: "/home/api/db/db.sqlite3"
   MCT_VERSION: {{ .Values.image.tag | default .Chart.AppVersion }}
   API_URL: http://localhost:{{ .Values.service.apiPort }}/api/
+  REMOTE_MODEL_SERVICE_TYPE: "medcat" # TODO: Remove this for REMOTE_MODEL_SERVICE_TYPE after resolving pr-363 in cogstack-nlp
+
+  {{- if .Values.tracing.otlp.enabled }}
+    # Tracing configuration
+  MCT_ENABLE_TRACING: {{ .Values.tracing.otlp.enabled | quote }}
+  OTEL_TRACES_EXPORTER: "otlp"
+  OTEL_SERVICE_NAME: {{ .Values.tracing.serviceName | quote }}
+  {{- if .Values.tracing.otlp.grpc.enabled }}
+  OTEL_EXPORTER_OTLP_ENDPOINT: {{ .Values.tracing.otlp.grpc.endpoint | quote }}
+  OTEL_EXPORTER_OTLP_PROTOCOL: "grpc"
+  {{- else if .Values.tracing.otlp.http.enabled }}
+  OTEL_EXPORTER_OTLP_ENDPOINT: {{ .Values.tracing.otlp.http.endpoint | quote }}
+  OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf"
+  {{- end }}
+  OTEL_METRICS_EXPORTER: none
+  OTEL_LOGS_EXPORTER: none
+  OTEL_PYTHON_DJANGO_EXCLUDED_URLS: /api/health/*,/metrics
+  OTEL_EXPERIMENTAL_RESOURCE_DETECTORS: {{ .Values.tracing.experimentalResourceDetectors | quote }}
+  OTEL_PYTHON_DISABLED_INSTRUMENTATIONS: {{ .Values.tracing.disabledInstrumentations | quote }}
+
+  {{- end }}
+
   {{- if and .Values.provisioning .Values.provisioning.enabled }}
   LOAD_EXAMPLES: "1"
   PROVISIONING_CONFIG_PATH: "/home/configs/provisioning.yaml"

deployment/kubernetes/charts/medcat-trainer-helm/values.yaml

@@ -39,6 +39,21 @@ env:
   DB_ENGINE: "postgresql"
   DB_PORT: "5432"
 
+# Allow setting env values from field/configmap/secret references. Defaults to include k8s details for observability.
+envValueFrom:
+  K8S_NODE_NAME:
+    fieldRef:
+      fieldPath: spec.nodeName
+  K8S_POD_NAME:
+    fieldRef:
+      fieldPath: metadata.name
+  K8S_POD_UID:
+    fieldRef:
+      fieldPath: metadata.uid
+  K8S_POD_NAMESPACE:
+    fieldRef:
+      fieldPath: metadata.namespace
+
 provisioning:
   # -- Set to true to enable provisioning of projects and models on startup..
   enabled: false
@@ -61,6 +76,35 @@ provisioning:
           description: "Example project for medcat trainer"
           annotationGuidelineLink: "https://docs.google.com/document/d/1xxelBOYbyVzJ7vLlztP2q1Kw9F5Vr1pRwblgrXPS7QM/edit?usp=sharing"
 
+# Enable tracing with opentelemetry for the medcat-trainer backend
+tracing:
+  # The name of the service in the tracing system
+  serviceName: "medcat-trainer"
+  # Resource attributes to add to the traces. Can be templated
+  resourceAttributes:
+    k8s.pod.uid: "$(K8S_POD_UID)"
+    k8s.pod.name: "$(K8S_POD_NAME)"
+    k8s.namespace.name: "$(K8S_POD_NAMESPACE)"
+    k8s.node.name: "$(K8S_NODE_NAME)"
+    service.version: "{{ .Values.image.tag | default .Chart.AppVersion }}"
+    service.instance.id: "$(K8S_POD_NAMESPACE)-$(K8S_POD_NAME)-$(K8S_CONTAINER_NAME)"
+  experimentalResourceDetectors: "containerid,os"
+  # Optionally disable the db instrumentations due to noise.
+  disabledInstrumentations: "psycopg,sqlite3"
+  otlp:
+    # Set to true to enable tracing
+    enabled: false
+    grpc:
+      # Set to true to enable grpc tracing
+      enabled: false
+      # The endpoint to send the traces to
+      endpoint: "http://<unused>:4317"
+    http:
+      # Set to true to enable http tracing over http/protobug
+      enabled: false
+      # The endpoint to send the traces to
+      endpoint: "http://<unused>:4318"
+
 postgresql:
   enabled: true
   # TODO: Support custom DB overrides