From cf170244a2738d7731fa32765fade20dfb9d10d5 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Thu, 7 May 2026 15:59:39 -0400 Subject: [PATCH 1/4] fix(aws-lambda): force placeholder resource so the Lambda Extension drops dd-tracer-serverless-span The Datadog Lambda Extension filters the placeholder invocation span by comparing `span.resource == "dd-tracer-serverless-span"` (see `bottlecap/src/traces/trace_processor.rs::filter_span_from_lambda_library_or_runtime`). The intent is that the extension drops the placeholder so the inferred `aws.lambda` span (which the end-invocation handshake gives the same `span_id`) is the surviving record under that key in the trace store and remains parented to the inferred apigateway root. In practice the HTTP/JAX-RS instrumentation overwrites the placeholder's resource with the request route (e.g. "POST /") at `HTTP_FRAMEWORK_ROUTE` priority during the invocation, so the extension's filter no longer matches at end-of-invocation. Both records (the placeholder with `parent_id=0`, and the inferred `aws.lambda` with `parent_id=apigateway.span_id`) reach the backend under the same `(trace_id, span_id)` key. The trace store keeps the placeholder, and the rest of the trace is detached from the inferred apigateway root. Fix: in `LambdaHandlerInstrumentation.exit`, force the resource name back to the literal placeholder marker right before `span.finish()`, using `ResourceNamePriorities.MANUAL_INSTRUMENTATION` so the override beats whatever HTTP-framework priority the in-flight instrumentation has written. Verified end-to-end on Quarkus 3.15.4 / Java 21 / Lambda Extension v96: pre-fix the placeholder leaks to the backend with `parent_id=0` and the trace store reports orphans; post-fix zero `dd_tracer_serverless_span` rows reach the backend and the trace store reports a single 11-span chunk with no orphans. Refs: SLES-2837 --- .../v1/lambda/LambdaHandlerInstrumentation.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java index d7407b47b5f..a945fabf647 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java +++ b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java @@ -24,6 +24,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities; import datadog.trace.config.inversion.ConfigHelper; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; @@ -126,6 +127,21 @@ static void exit( String lambdaRequestId = awsContext.getAwsRequestId(); AgentTracer.get().notifyAppSecEnd(span); + // Force the resource name back to the literal placeholder marker right + // before finish so that the Datadog Lambda Extension's filter + // (filter_span_from_lambda_library_or_runtime in + // bottlecap/src/traces/trace_processor.rs, which compares + // span.resource == "dd-tracer-serverless-span") drops the placeholder. + // Other instrumentation (HTTP/JAX-RS) may have overwritten it with the + // route ("POST /") during the invocation, in which case the extension + // would fail to dedup, leading to the placeholder leaking to the backend + // with parent_id=0 and detaching the inferred apigateway root from the + // rest of the trace. + // Use MANUAL_INSTRUMENTATION priority because DDSpanContext.setResourceName + // ignores writes whose priority is below the current resource priority, + // and the HTTP/JAX-RS instrumentation will already have written + // HTTP_FRAMEWORK_ROUTE (3) by this point. + span.setResourceName(INVOCATION_SPAN_NAME, ResourceNamePriorities.MANUAL_INSTRUMENTATION); span.finish(); AgentTracer.get().notifyExtensionEnd(span, result, null != throwable, lambdaRequestId); } finally { From 4adbfdc82461a505b0267f9583acddc963916dcc Mon Sep 17 00:00:00 2001 From: Rithika Narayan Date: Mon, 11 May 2026 15:10:32 -0400 Subject: [PATCH 2/4] use tag interceptor priority to avoid clobbering user-set resource names --- .../aws/v1/lambda/LambdaHandlerInstrumentation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java index a945fabf647..156e922de08 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java +++ b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java @@ -141,7 +141,7 @@ static void exit( // ignores writes whose priority is below the current resource priority, // and the HTTP/JAX-RS instrumentation will already have written // HTTP_FRAMEWORK_ROUTE (3) by this point. - span.setResourceName(INVOCATION_SPAN_NAME, ResourceNamePriorities.MANUAL_INSTRUMENTATION); + span.setResourceName(INVOCATION_SPAN_NAME, ResourceNamePriorities.TAG_INTERCEPTOR); span.finish(); AgentTracer.get().notifyExtensionEnd(span, result, null != throwable, lambdaRequestId); } finally { From 9d828987d8548f87f2b118490bf335d0eec900bc Mon Sep 17 00:00:00 2001 From: Rithika Narayan Date: Mon, 11 May 2026 15:25:36 -0400 Subject: [PATCH 3/4] Add unit test --- ...reamingSimulatesHttpFrameworkResource.java | 26 +++++++++++++++++++ .../LambdaHandlerInstrumentationTest.groovy | 22 ++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/test/groovy/HandlerStreamingSimulatesHttpFrameworkResource.java diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/test/groovy/HandlerStreamingSimulatesHttpFrameworkResource.java b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/test/groovy/HandlerStreamingSimulatesHttpFrameworkResource.java new file mode 100644 index 00000000000..826f9858e25 --- /dev/null +++ b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/test/groovy/HandlerStreamingSimulatesHttpFrameworkResource.java @@ -0,0 +1,26 @@ +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Simulates HTTP server instrumentation updating the local root span resource (e.g. route) while + * the Lambda invocation span is active. + */ +public class HandlerStreamingSimulatesHttpFrameworkResource implements RequestStreamHandler { + + @Override + public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) + throws IOException { + AgentSpan span = AgentTracer.activeSpan(); + if (span != null) { + span.setResourceName("POST /api/simulated", ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE); + } + outputStream.write('O'); + outputStream.write('K'); + } +} diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/test/groovy/LambdaHandlerInstrumentationTest.groovy b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/test/groovy/LambdaHandlerInstrumentationTest.groovy index d5b6a4bbbc1..ed1152ea1aa 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/test/groovy/LambdaHandlerInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/test/groovy/LambdaHandlerInstrumentationTest.groovy @@ -98,6 +98,28 @@ abstract class LambdaHandlerInstrumentationTest extends VersionedNamingTestBase } } + def "serverless invocation span resource reset after simulated HTTP framework overwrite"() { + when: + def input = new ByteArrayInputStream(StandardCharsets.UTF_8.encode("Hello").array()) + def output = new ByteArrayOutputStream() + def ctx = Stub(Context) { + getAwsRequestId() >> requestId + } + new HandlerStreamingSimulatesHttpFrameworkResource().handleRequest(input, output, ctx) + + then: + assertTraces(1) { + trace(1) { + span { + operationName operation() + resourceName operation() + spanType DDSpanTypes.SERVERLESS + errored false + } + } + } + } + def "test streaming handler with error"() { when: def input = new ByteArrayInputStream(StandardCharsets.UTF_8.encode("Hello").array()) From f07b5b0aace6ce8149de10b38e8125adcfb7bec6 Mon Sep 17 00:00:00 2001 From: Rithika Narayan Date: Mon, 11 May 2026 16:24:17 -0400 Subject: [PATCH 4/4] comment --- .../aws/v1/lambda/LambdaHandlerInstrumentation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java index 156e922de08..610b24b2f31 100644 --- a/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java +++ b/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java @@ -137,7 +137,7 @@ static void exit( // would fail to dedup, leading to the placeholder leaking to the backend // with parent_id=0 and detaching the inferred apigateway root from the // rest of the trace. - // Use MANUAL_INSTRUMENTATION priority because DDSpanContext.setResourceName + // Use TAG_INTERCEPTOR priority because DDSpanContext.setResourceName // ignores writes whose priority is below the current resource priority, // and the HTTP/JAX-RS instrumentation will already have written // HTTP_FRAMEWORK_ROUTE (3) by this point.