SK-313 // write scraped metrics into exported trace file

sk-core/src/trace/mod.rs

@@ -27,7 +27,11 @@ use crate::time::duration_to_ts_from;
 use crate::trace::config::TracerConfig;
 use crate::trace::event::TraceEvent;
 use crate::trace::index::TraceIndex;
-pub use crate::trace::pod_sim_data::PodSimData;
+pub use crate::trace::pod_sim_data::{
+    MetricType,
+    PodMetricsData,
+    PodSimData,
+};
 
 #[derive(Debug, Error)]
 pub enum TraceError {

sk-core/src/trace/pod_sim_data.rs

@@ -23,9 +23,27 @@ pub struct PodMetricsData {
 }
 
 impl PodMetricsData {
+    pub fn insert(&mut self, container: &str, metric_type: MetricType, value: f64) {
+        self.containers
+            .entry(container.into())
+            .or_default()
+            .entry(metric_type)
+            .or_default()
+            .push(value);
+    }
+
     pub fn is_empty(&self) -> bool {
         self.containers.is_empty()
     }
+
+    pub fn merge(&mut self, metrics_data: PodMetricsData) {
+        for (container, data) in metrics_data.containers {
+            let container_entry = self.containers.entry(container).or_default();
+            for (metric_type, mut values) in data {
+                container_entry.entry(metric_type).or_default().append(&mut values)
+            }
+        }
+    }
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
@@ -48,6 +66,10 @@ impl PodSimData {
         }
     }
 
+    pub fn merge_metrics(&mut self, metrics_data: PodMetricsData) {
+        self.metrics.merge(metrics_data);
+    }
+
     pub fn overlaps(&self, start_ts: i64, end_ts: i64) -> bool {
         self.lifecycle.overlaps(start_ts, end_ts)
     }

sk-driver/src/tests/mutation_test.rs

@@ -280,8 +280,6 @@ mod itest {
         mut test_sim: Simulation,
         mut test_pod: corev1::Pod,
         mut adm_req: AdmissionRequest<corev1::Pod>,
-        root_owner_ref: metav1::OwnerReference,
-        depl_owner_ref: metav1::OwnerReference,
         #[case] running_and_has_node_selector: bool,
     ) {
         set_snapshot_suffix!("{running_and_has_node_selector}");
@@ -307,10 +305,9 @@ mod itest {
                 .insert(owner_id, BTreeMap::from([(0, vec![PodSimData::new(PodLifecycleData::Finished(0, 42))])]));
         }
 
-        let owners = vec![root_owner_ref, depl_owner_ref];
         adm_req.object = Some(test_pod.clone());
         let rev = adm_rev(adm_req);
-        let ctx = ctx(test_pod.clone(), owners.clone(), trace);
+        let ctx = ctx(test_pod.clone(), test_pod.owner_references().into(), trace);
 
         let resp = handler(
             rocket::State::from(&ctx),

sk-driver/src/tests/snapshots/sk_driver__tests__mutation_test__itest__mutation_handler_create@false.snap

@@ -22,6 +22,18 @@ Object {
         "name": String("the-pod"),
         "namespace": String("test-namespace"),
         "ownerReferences": Array [
+            Object {
+                "apiVersion": String("simkube.io/v1"),
+                "kind": String(""),
+                "name": String("sk-test-driver-12345-root"),
+                "uid": String(""),
+            },
+            Object {
+                "apiVersion": String("apps/v1"),
+                "kind": String("ReplicaSet"),
+                "name": String("the-replicaset"),
+                "uid": String("asdfasdf"),
+            },
             Object {
                 "apiVersion": String("apps/v1"),
                 "kind": String("Deployment"),

sk-driver/src/tests/snapshots/sk_driver__tests__mutation_test__itest__mutation_handler_create@true.snap

@@ -24,6 +24,18 @@ Object {
         "name": String("the-pod"),
         "namespace": String("test-namespace"),
         "ownerReferences": Array [
+            Object {
+                "apiVersion": String("simkube.io/v1"),
+                "kind": String(""),
+                "name": String("sk-test-driver-12345-root"),
+                "uid": String(""),
+            },
+            Object {
+                "apiVersion": String("apps/v1"),
+                "kind": String("ReplicaSet"),
+                "name": String("the-replicaset"),
+                "uid": String("asdfasdf"),
+            },
             Object {
                 "apiVersion": String("apps/v1"),
                 "kind": String("Deployment"),

sk-tracer/src/manager.rs

@@ -100,9 +100,15 @@ pub(crate) async fn handle_messages(
             },
             Some(request) = pod_rx.recv() => {
                 let mut store = m_store.lock().await;
+
+                // unconditionally compute the pod owners for this pod (if it exists); this
+                // populated the owners cache in the trace store, and makes them available for other
+                // tasks to use.  I don't love that it's coupled in this way but I haven't found a
+                // better solution yet.
+                let owners = store.store_pod_owners(&request.maybe_pod).await;
                 if let Err(err) = store.record_pod_lifecycle(
                     &request.ns_name,
-                    &request.maybe_pod,
+                    owners,
                     request.lifecycle_data.clone(),
                 ).await {
                     error!("could not send pod object update for ({}, {:?}, {:?}): {err}",
@@ -111,7 +117,7 @@ pub(crate) async fn handle_messages(
             },
             Some(samples) = metrics_rx.recv() => {
                 let mut store = m_store.lock().await;
-                if let Err(err) = store.record_pod_metrics(
+                if let Err(err) = store.record_pod_utilization_metrics(
                     samples
                 ).await {
                     error!("could not send metrics data: {err}");

sk-tracer/src/owners_index.rs

@@ -5,11 +5,14 @@ use std::collections::{
 
 use anyhow::*;
 use sk_core::prelude::*;
-use sk_core::trace::PodSimData;
 use sk_core::trace::index::{
     TraceIndex,
     TraceIndexEntry,
 };
+use sk_core::trace::{
+    PodMetricsData,
+    PodSimData,
+};
 use tracing::*;
 
 // The OwnersIndex stores relevant information for all resources tracked by the tracer, as well
@@ -46,10 +49,6 @@ pub(crate) struct OwnersIndex {
 }
 
 impl OwnersIndex {
-    pub fn contains(&self, owner_id: &KubeResourceId) -> bool {
-        self.m.contains_key(owner_id)
-    }
-
     pub fn get_hash(&self, owner_id: &KubeResourceId) -> Option<u64> {
         self.m.get(owner_id)?.hash
     }
@@ -113,15 +112,39 @@ impl OwnersIndex {
         Ok(())
     }
 
+    pub fn store_new_utilization_metrics(
+        &mut self,
+        metrics: HashMap<(KubeResourceId, String), PodMetricsData>,
+    ) -> EmptyResult {
+        for ((owner_id, pod_ns_name), metrics_data) in metrics {
+            let owner_entry = self.m.entry(owner_id.clone()).or_default();
+            let sequence_idx = match self.pod_index.get(&pod_ns_name) {
+                None => {
+                    let idx = owner_entry.pod_sim_data.len() - 1;
+                    self.pod_index.insert(pod_ns_name.clone(), (owner_id.clone(), idx));
+                    idx
+                },
+                Some((_, sequence_idx)) => *sequence_idx,
+            };
+
+            owner_entry
+                .pod_sim_data
+                .get_mut(sequence_idx)
+                .ok_or(anyhow!("no sequence index {sequence_idx} for pod {pod_ns_name}"))?
+                .merge_metrics(metrics_data);
+        }
+
+        Ok(())
+    }
+
     pub fn update_pod_lifecycle(&mut self, pod_ns_name: &str, lifecycle: PodLifecycleData) -> EmptyResult {
         match self.pod_index.get(pod_ns_name) {
             None => bail!("pod {pod_ns_name} not present in index"),
             Some((owner_id, sequence_idx)) => {
-                let owner_entry = self
+                let pod_entry = self
                     .m
                     .get_mut(owner_id)
-                    .ok_or(anyhow!("no owner entry for pod {pod_ns_name}"))?;
-                let pod_entry = owner_entry
+                    .ok_or(anyhow!("no owner entry for pod {pod_ns_name}"))?
                     .pod_sim_data
                     .get_mut(*sequence_idx)
                     .ok_or(anyhow!("no sequence index {sequence_idx} for pod {pod_ns_name}"))?;