fix(otel): enforce unique roots and correlate X-Ray traces

packages/aws-durable-execution-conformance-tests-otel/CONTRIBUTING.md

@@ -69,6 +69,7 @@ The currently supported telemetry assertions are:
 | `minimum_spans` | Minimum number of normalized spans; defaults to `1`. |
 | `minimum_invocations` | Minimum canonical durable invocation span occurrences; identical spans are counted separately. Defaults to `1`. |
 | `require_execution_correlation` | Require the durable execution ARN on the trace; defaults to `true`. |
+| `require_unique_root_per_trace` | Reject any trace ID containing more than one distinct parentless span. Duplicate exports with the same span ID count once. |
 | `require_all_spans` | Require every normalized span to match at least one span assertion. |
 | `span_assertion_scope` | Limit complete span coverage to spans matching this partial selector. |
 | `exact_attribute_prefixes` | Require assertions to enumerate every attribute under the listed prefixes. |
@@ -165,6 +166,13 @@ Keep `ExpectedExecutionHistory` and `ExpectedResult` focused on the execution
 behavior needed to produce the telemetry. Keep `TelemetryAssertions` portable
 across the complete exporter/backend support matrix.
 
+Set `require_unique_root_per_trace: true` for trace-view requirements. A
+backend may return a correlated subset without the upstream root, so the
+assertion permits zero observed roots for a trace ID. It rejects two or more
+distinct parentless spans because that creates a disconnected trace forest and
+makes backend-derived entry point, duration, critical path, and sampling
+behavior ambiguous.
+
 The catalog uses separate requirements when two public plugins intentionally
 produce different trace views. Invocation-view cases assert per-invocation
 operation hierarchy. In execution-view cases, `Invocation` spans remain

packages/aws-durable-execution-conformance-tests-otel/src/aws_durable_execution_conformance_tests_otel/backends/xray.py

@@ -32,6 +32,8 @@
     PollingBackend,
 )
 
+_WORKFLOW_SERVICE_NAME = "Workflow"
+
 
 def _metadata_attributes(metadata: Any) -> dict[str, Any]:
     if not isinstance(metadata, Mapping):
@@ -119,10 +121,13 @@ def _lookup(self, query: TelemetryQuery) -> Trace | None:
             if query.trace_id:
                 trace_ids = [query.trace_id]
             else:
+                # ADOT names a parentless INTERNAL root segment after its span,
+                # so the durable Workflow trace is indexed separately from the
+                # ambient trace that uses the configured service name.
                 summary_request = {
                     "StartTime": query.started_at,
                     "EndTime": query.ended_at,
-                    "FilterExpression": f'service("{query.service_name}")',
+                    "FilterExpression": (f'service("{query.service_name}") OR service("{_WORKFLOW_SERVICE_NAME}")'),
                 }
                 while True:
                     response = self._client.get_trace_summaries(**summary_request)

packages/aws-durable-execution-conformance-tests-otel/src/aws_durable_execution_conformance_tests_otel/validators.py

@@ -54,6 +54,25 @@ def _invocation_count(trace: Trace) -> int:
     )
 
 
+def _unique_root_errors(trace: Trace) -> list[str]:
+    roots_by_trace: dict[str, dict[str, Span]] = {}
+    for span in trace.spans:
+        if span.parent_span_id is None:
+            roots_by_trace.setdefault(span.trace_id, {}).setdefault(span.span_id, span)
+
+    errors: list[str] = []
+    for trace_id, roots_by_id in sorted(roots_by_trace.items()):
+        if len(roots_by_id) <= 1:
+            continue
+        roots = sorted(
+            roots_by_id.values(),
+            key=lambda span: (span.start_time, span.span_id),
+        )
+        root_list = ", ".join(f"{span.name!r} ({span.span_id})" for span in roots)
+        errors.append(f"Trace {trace_id} contains {len(roots)} distinct root spans: {root_list}")
+    return errors
+
+
 def _is_sequence(value: Any) -> bool:
     return isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray))
 
@@ -635,6 +654,12 @@ def validate_trace(
     if len(trace.spans) < minimum_spans:
         errors.append(f"Expected at least {minimum_spans} span(s), found {len(trace.spans)}")
 
+    require_unique_root = assertions.get("require_unique_root_per_trace", False)
+    if not isinstance(require_unique_root, bool):
+        errors.append("require_unique_root_per_trace must be a boolean")
+    elif require_unique_root:
+        errors.extend(_unique_root_errors(trace))
+
     timestamp_tolerance = _timestamp_tolerance(feature_disparities)
     for span in trace.spans:
         if span.start_time - span.end_time > timestamp_tolerance:

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-1.yaml

@@ -20,6 +20,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 4
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-10.yaml

@@ -43,6 +43,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 8
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   minimum_invocations: 2
   span_assertions:
     - select:

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-11.yaml

@@ -22,6 +22,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 6
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   minimum_invocations: 3
   allowed_execution_arns:
     - ${EXECUTION_ARN}

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-12.yaml

@@ -20,6 +20,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 3
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-13.yaml

@@ -32,6 +32,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 4
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-14.yaml

@@ -32,6 +32,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 4
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-15.yaml

@@ -15,6 +15,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 1
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Invocation

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-16.yaml

@@ -20,6 +20,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 4
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-17.yaml

@@ -45,6 +45,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 8
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   minimum_invocations: 2
   span_assertions:
     - select:

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-18.yaml

@@ -22,6 +22,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 6
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   minimum_invocations: 3
   allowed_execution_arns:
     - ${EXECUTION_ARN}

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-19.yaml

@@ -12,6 +12,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 1
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Invocation

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-2.yaml

@@ -26,6 +26,7 @@ TelemetryAssertions:
   minimum_spans: 6
   minimum_invocations: 2
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-3.yaml

@@ -31,6 +31,7 @@ TelemetryAssertions:
   minimum_spans: 6
   minimum_invocations: 2
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-4.yaml

@@ -20,6 +20,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 4
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-5.yaml

@@ -27,6 +27,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 5
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-6.yaml

@@ -39,6 +39,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 9
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-7.yaml

@@ -39,6 +39,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 9
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-8.yaml

@@ -30,6 +30,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 6
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   span_assertions:
     - select:
         name: Workflow

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-execution/otel-execution-9.yaml

@@ -31,6 +31,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 6
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   minimum_invocations: 2
   span_assertions:
     - select:

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-1.yaml

@@ -20,6 +20,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 4
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   require_all_spans: true
   span_assertion_scope:
     attributes:

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-10.yaml

@@ -44,6 +44,7 @@ TelemetryAssertions:
   minimum_spans: 10
   minimum_invocations: 2
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   require_all_spans: true
   span_assertion_scope:
     attributes:
@@ -144,9 +145,6 @@ TelemetryAssertions:
         status: OK
         service_name: ${SERVICE_NAME}
         links:
-          - name: otel-callback
-            attributes:
-              durable.operation.id: ${CALLBACK_CONTEXT}
           - name: Workflow
             attributes:
               durable.execution.arn: ${EXECUTION_ARN}
@@ -203,9 +201,6 @@ TelemetryAssertions:
         status: OK
         service_name: ${SERVICE_NAME}
         links:
-          - name: "${/^(?:otel-callback(?: create callback id|-callback)|CALLBACK)$/}"
-            attributes:
-              durable.operation.id: ${CALLBACK1}
           - name: Workflow
             attributes:
               durable.execution.arn: ${EXECUTION_ARN}

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-11.yaml

@@ -23,6 +23,7 @@ TelemetryAssertions:
   minimum_spans: 7
   minimum_invocations: 3
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   allowed_execution_arns:
     - ${EXECUTION_ARN}
     - ${TARGET_EXECUTION_ARN}
@@ -162,9 +163,6 @@ TelemetryAssertions:
         status: OK
         service_name: ${SERVICE_NAME}
         links:
-          - name: otel-invoke
-            attributes:
-              durable.operation.id: ${INVOKE1}
           - name: Workflow
             attributes:
               durable.execution.arn: ${EXECUTION_ARN}

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-12.yaml

@@ -20,6 +20,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 3
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   require_all_spans: true
   span_assertion_scope:
     attributes:

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-13.yaml

@@ -32,6 +32,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 4
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   require_all_spans: true
   span_assertion_scope:
     attributes:

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-14.yaml

@@ -32,6 +32,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 4
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   require_all_spans: true
   span_assertion_scope:
     attributes:

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-15.yaml

@@ -15,6 +15,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 2
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   require_all_spans: true
   span_assertion_scope:
     attributes:

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-16.yaml

@@ -20,6 +20,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 4
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   require_all_spans: true
   span_assertion_scope:
     attributes:

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-17.yaml

@@ -46,6 +46,7 @@ TelemetryAssertions:
   minimum_spans: 10
   minimum_invocations: 2
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   require_all_spans: true
   span_assertion_scope:
     attributes:
@@ -144,9 +145,6 @@ TelemetryAssertions:
         status: ERROR
         service_name: ${SERVICE_NAME}
         links:
-          - name: otel-failed-callback
-            attributes:
-              durable.operation.id: ${FAILED_CALLBACK_CONTEXT}
           - name: Workflow
             attributes:
               durable.execution.arn: ${EXECUTION_ARN}
@@ -202,9 +200,6 @@ TelemetryAssertions:
         status: ERROR
         service_name: ${SERVICE_NAME}
         links:
-          - name: "${/^(?:otel-failed-callback(?: create callback id|-callback)|CALLBACK)$/}"
-            attributes:
-              durable.operation.id: ${FAILED_CALLBACK}
           - name: Workflow
             attributes:
               durable.execution.arn: ${EXECUTION_ARN}

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-18.yaml

@@ -23,6 +23,7 @@ TelemetryAssertions:
   minimum_spans: 7
   minimum_invocations: 3
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   allowed_execution_arns:
     - ${EXECUTION_ARN}
     - ${TARGET_EXECUTION_ARN}
@@ -164,9 +165,6 @@ TelemetryAssertions:
         status: ERROR
         service_name: ${SERVICE_NAME}
         links:
-          - name: otel-failed-invoke
-            attributes:
-              durable.operation.id: ${FAILED_INVOKE}
           - name: Workflow
             attributes:
               durable.execution.arn: ${EXECUTION_ARN}

packages/aws-durable-execution-conformance-tests-otel/test-requirements/otel-invocation/otel-invocation-19.yaml

@@ -12,6 +12,7 @@ ExpectedResult:
 TelemetryAssertions:
   minimum_spans: 2
   require_execution_correlation: true
+  require_unique_root_per_trace: true
   require_all_spans: true
   span_assertion_scope:
     attributes: