From 97f99331ec1179cfeafc19c3ba2263b325bb9908 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Thu, 7 May 2026 15:58:35 -0400 Subject: [PATCH 1/2] fix(vertx-web): finish vertx.route-handler via RoutingContext.addEndHandler fallback Vert.x's `Http1xServerResponse.end(Buffer, PromiseInternal)` invokes the registered `endHandler` only when `closed == false` at the moment the response body has been written. In synthetic transports such as quarkus-amazon-lambda-rest's `VirtualClientConnection` (in-memory Netty channel) the writes and the connection close happen synchronously inside `responseComplete()`, so by the time the `!closed` guard runs `closed` is already `true` and `endHandler` is silently skipped. Symptom: `RouteHandlerWrapper` starts a `vertx.route-handler` span for every route in the chain (e.g. Quarkus's AuthenticationHandler) but `EndHandlerWrapper.handle` is never called, so the span is never finished. The span dies in PendingTrace and is not enqueued on the writer. All children parented to that span (`jakarta-rs.request`, `netty.client.request`, downstream `aws.http`/`aws.apigateway` inferred spans) end up orphaned in the trace UI. Fix: also register a finish via `RoutingContext.addEndHandler`, which fires on routing-context completion regardless of underlying connection state and on both success and failure. Both paths funnel through a shared idempotent `finishHandlerSpan` so the second one to fire on real-network transports is a no-op. Verified end-to-end against a Quarkus 3.15.4 / Java 21 Lambda chain (caller -> netty.client.request -> callee) on Datadog Lambda Extension v96. Pre-fix: 5/5 invocations Started, 0/5 Finished. Post-fix: 5/5 Started, 5/5 Finished, single connected trace tree in the UI. Refs: SLES-2837 --- .../vertx_4_0/server/EndHandlerWrapper.java | 10 +------- .../vertx_4_0/server/RouteHandlerWrapper.java | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/EndHandlerWrapper.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/EndHandlerWrapper.java index a71584b11a7..fd4a042edf8 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/EndHandlerWrapper.java +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/EndHandlerWrapper.java @@ -1,9 +1,5 @@ package datadog.trace.instrumentation.vertx_4_0.server; -import static datadog.trace.instrumentation.vertx_4_0.server.RouteHandlerWrapper.HANDLER_SPAN_CONTEXT_KEY; -import static datadog.trace.instrumentation.vertx_4_0.server.VertxDecorator.DECORATE; - -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import io.vertx.core.Handler; import io.vertx.ext.web.RoutingContext; @@ -18,16 +14,12 @@ public class EndHandlerWrapper implements Handler { @Override public void handle(final Void event) { - AgentSpan span = routingContext.get(HANDLER_SPAN_CONTEXT_KEY); try { if (actual != null) { actual.handle(event); } } finally { - if (span != null) { - DECORATE.onResponse(span, routingContext.response()); - span.finish(); - } + RouteHandlerWrapper.finishHandlerSpan(routingContext); } } } diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/RouteHandlerWrapper.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/RouteHandlerWrapper.java index 8706f816e1c..19b344b75f6 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/RouteHandlerWrapper.java +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-4.0/src/main/java/datadog/trace/instrumentation/vertx_4_0/server/RouteHandlerWrapper.java @@ -44,6 +44,16 @@ public void handle(final RoutingContext routingContext) { routingContext.put(HANDLER_SPAN_CONTEXT_KEY, span); routingContext.response().endHandler(new EndHandlerWrapper(routingContext)); + // Fallback finish path: HttpServerResponse.endHandler is silently skipped + // by Vert.x's Http1xServerResponse.end() when the underlying connection + // has already closed (Http1xServerResponse#end gates `endHandler.handle()` + // behind `!closed`). This happens in synthetic transports such as + // quarkus-amazon-lambda-rest's virtual Netty channel, where writes and + // close are synchronous in-memory, leaving the route-handler span unfinished + // and orphaning all jakarta-rs.request / aws.http child spans in the trace. + // RoutingContext#addEndHandler fires on routing-context completion regardless + // of underlying connection state and on both success and failure. + routingContext.addEndHandler(ar -> finishHandlerSpan(routingContext)); DECORATE.afterStart(span); span.setResourceName(DECORATE.className(actual.getClass())); } @@ -60,6 +70,19 @@ public void handle(final RoutingContext routingContext) { } } + // Idempotently finish the route-handler span. Both EndHandlerWrapper (the + // response.endHandler path) and the routingContext.addEndHandler fallback may call + // this; the first one to win clears HANDLER_SPAN_CONTEXT_KEY so the second is a no-op. + static void finishHandlerSpan(final RoutingContext routingContext) { + final AgentSpan span = routingContext.get(HANDLER_SPAN_CONTEXT_KEY); + if (span == null) { + return; + } + routingContext.put(HANDLER_SPAN_CONTEXT_KEY, null); + DECORATE.onResponse(span, routingContext.response()); + span.finish(); + } + private void setRoute(RoutingContext routingContext) { final AgentSpan parentSpan = routingContext.get(PARENT_SPAN_CONTEXT_KEY); if (parentSpan == null) { From ddef3816a9776d16e9a92ae39a555b7328640a06 Mon Sep 17 00:00:00 2001 From: Rithika Narayan Date: Mon, 11 May 2026 14:24:06 -0400 Subject: [PATCH 2/2] Applying changes to vertex-web 3.4 --- .../vertx_3_4/server/CloseHandlerWrapper.java | 25 ++++++++++ .../server/CloseHandlerWrapperAdvice.java | 21 ++++++++ .../vertx_3_4/server/EndHandlerWrapper.java | 10 +--- .../server/ExceptionHandlerWrapper.java | 25 ++++++++++ .../server/ExceptionHandlerWrapperAdvice.java | 23 +++++++++ ...rverResponseEndHandlerInstrumentation.java | 14 ++++++ .../server/RouteHandlerInstrumentation.java | 2 + .../vertx_3_4/server/RouteHandlerWrapper.java | 50 +++++++++++++++++++ 8 files changed, 161 insertions(+), 9 deletions(-) create mode 100644 dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/CloseHandlerWrapper.java create mode 100644 dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/CloseHandlerWrapperAdvice.java create mode 100644 dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/ExceptionHandlerWrapper.java create mode 100644 dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/ExceptionHandlerWrapperAdvice.java diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/CloseHandlerWrapper.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/CloseHandlerWrapper.java new file mode 100644 index 00000000000..340b30317ef --- /dev/null +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/CloseHandlerWrapper.java @@ -0,0 +1,25 @@ +package datadog.trace.instrumentation.vertx_3_4.server; + +import io.vertx.core.Handler; +import io.vertx.ext.web.RoutingContext; + +public class CloseHandlerWrapper implements Handler { + private final RoutingContext routingContext; + + public Handler actual; + + CloseHandlerWrapper(RoutingContext routingContext) { + this.routingContext = routingContext; + } + + @Override + public void handle(final Void event) { + try { + if (actual != null) { + actual.handle(event); + } + } finally { + RouteHandlerWrapper.finishHandlerSpan(routingContext); + } + } +} diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/CloseHandlerWrapperAdvice.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/CloseHandlerWrapperAdvice.java new file mode 100644 index 00000000000..a22e3ca01ef --- /dev/null +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/CloseHandlerWrapperAdvice.java @@ -0,0 +1,21 @@ +package datadog.trace.instrumentation.vertx_3_4.server; + +import io.vertx.core.Handler; +import net.bytebuddy.asm.Advice; + +public class CloseHandlerWrapperAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void wrapHandler( + @Advice.FieldValue(value = "closeHandler", readOnly = false) final Handler closeHandler, + @Advice.Argument(value = 0, readOnly = false) Handler handler) { + if (closeHandler instanceof CloseHandlerWrapper && handler instanceof CloseHandlerWrapper) { + return; + } + if (handler instanceof CloseHandlerWrapper && closeHandler != null) { + ((CloseHandlerWrapper) handler).actual = closeHandler; + } else if (closeHandler instanceof CloseHandlerWrapper) { + ((CloseHandlerWrapper) closeHandler).actual = handler; + handler = closeHandler; + } + } +} diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/EndHandlerWrapper.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/EndHandlerWrapper.java index a8cb1ebb079..f4b30bc4d44 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/EndHandlerWrapper.java +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/EndHandlerWrapper.java @@ -1,9 +1,5 @@ package datadog.trace.instrumentation.vertx_3_4.server; -import static datadog.trace.instrumentation.vertx_3_4.server.RouteHandlerWrapper.HANDLER_SPAN_CONTEXT_KEY; -import static datadog.trace.instrumentation.vertx_3_4.server.VertxDecorator.DECORATE; - -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import io.vertx.core.Handler; import io.vertx.ext.web.RoutingContext; @@ -18,16 +14,12 @@ public class EndHandlerWrapper implements Handler { @Override public void handle(final Void event) { - AgentSpan span = routingContext.get(HANDLER_SPAN_CONTEXT_KEY); try { if (actual != null) { actual.handle(event); } } finally { - if (span != null) { - DECORATE.onResponse(span, routingContext.response()); - span.finish(); - } + RouteHandlerWrapper.finishHandlerSpan(routingContext); } } } diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/ExceptionHandlerWrapper.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/ExceptionHandlerWrapper.java new file mode 100644 index 00000000000..951946badd6 --- /dev/null +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/ExceptionHandlerWrapper.java @@ -0,0 +1,25 @@ +package datadog.trace.instrumentation.vertx_3_4.server; + +import io.vertx.core.Handler; +import io.vertx.ext.web.RoutingContext; + +public class ExceptionHandlerWrapper implements Handler { + private final RoutingContext routingContext; + + public Handler actual; + + ExceptionHandlerWrapper(RoutingContext routingContext) { + this.routingContext = routingContext; + } + + @Override + public void handle(final Throwable event) { + try { + if (actual != null) { + actual.handle(event); + } + } finally { + RouteHandlerWrapper.finishHandlerSpan(routingContext); + } + } +} diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/ExceptionHandlerWrapperAdvice.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/ExceptionHandlerWrapperAdvice.java new file mode 100644 index 00000000000..a45265dd999 --- /dev/null +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/ExceptionHandlerWrapperAdvice.java @@ -0,0 +1,23 @@ +package datadog.trace.instrumentation.vertx_3_4.server; + +import io.vertx.core.Handler; +import net.bytebuddy.asm.Advice; + +public class ExceptionHandlerWrapperAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void wrapHandler( + @Advice.FieldValue(value = "exceptionHandler", readOnly = false) + final Handler exceptionHandler, + @Advice.Argument(value = 0, readOnly = false) Handler handler) { + if (exceptionHandler instanceof ExceptionHandlerWrapper + && handler instanceof ExceptionHandlerWrapper) { + return; + } + if (handler instanceof ExceptionHandlerWrapper && exceptionHandler != null) { + ((ExceptionHandlerWrapper) handler).actual = exceptionHandler; + } else if (exceptionHandler instanceof ExceptionHandlerWrapper) { + ((ExceptionHandlerWrapper) exceptionHandler).actual = handler; + handler = exceptionHandler; + } + } +} diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/HttpServerResponseEndHandlerInstrumentation.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/HttpServerResponseEndHandlerInstrumentation.java index 28aa02442ff..e654aba858f 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/HttpServerResponseEndHandlerInstrumentation.java +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/HttpServerResponseEndHandlerInstrumentation.java @@ -19,7 +19,9 @@ public HttpServerResponseEndHandlerInstrumentation() { @Override public String[] helperClassNames() { return new String[] { + packageName + ".CloseHandlerWrapper", packageName + ".EndHandlerWrapper", + packageName + ".ExceptionHandlerWrapper", packageName + ".RouteHandlerWrapper", packageName + ".VertxDecorator", packageName + ".VertxDecorator$VertxURIDataAdapter", @@ -39,5 +41,17 @@ public void methodAdvice(MethodTransformer transformer) { .and(isPublic()) .and(takesArgument(0, named("io.vertx.core.Handler"))), packageName + ".EndHandlerWrapperAdvice"); + transformer.applyAdvice( + isMethod() + .and(named("closeHandler")) + .and(isPublic()) + .and(takesArgument(0, named("io.vertx.core.Handler"))), + packageName + ".CloseHandlerWrapperAdvice"); + transformer.applyAdvice( + isMethod() + .and(named("exceptionHandler")) + .and(isPublic()) + .and(takesArgument(0, named("io.vertx.core.Handler"))), + packageName + ".ExceptionHandlerWrapperAdvice"); } } diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/RouteHandlerInstrumentation.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/RouteHandlerInstrumentation.java index 6e4c5445d7d..d242ee2e1e5 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/RouteHandlerInstrumentation.java +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/RouteHandlerInstrumentation.java @@ -20,7 +20,9 @@ public RouteHandlerInstrumentation() { @Override public String[] helperClassNames() { return new String[] { + packageName + ".CloseHandlerWrapper", packageName + ".EndHandlerWrapper", + packageName + ".ExceptionHandlerWrapper", packageName + ".RouteHandlerWrapper", packageName + ".VertxDecorator", packageName + ".VertxDecorator$VertxURIDataAdapter", diff --git a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/RouteHandlerWrapper.java b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/RouteHandlerWrapper.java index 37120c2f0cd..979e36d8fe9 100644 --- a/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/RouteHandlerWrapper.java +++ b/dd-java-agent/instrumentation/vertx/vertx-web/vertx-web-3.4/src/main/java/datadog/trace/instrumentation/vertx_3_4/server/RouteHandlerWrapper.java @@ -11,16 +11,32 @@ import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.Tags; +import io.vertx.core.AsyncResult; import io.vertx.core.Handler; import io.vertx.ext.web.RoutingContext; import io.vertx.ext.web.impl.RouteImpl; import io.vertx.ext.web.impl.RouterImpl; +import java.lang.reflect.Method; public class RouteHandlerWrapper implements Handler { static final String PARENT_SPAN_CONTEXT_KEY = AgentSpan.class.getName() + ".parent"; static final String HANDLER_SPAN_CONTEXT_KEY = AgentSpan.class.getName() + ".handler"; static final String ROUTE_CONTEXT_KEY = "dd." + Tags.HTTP_ROUTE; + private static final Object NO_ADD_END_HANDLER = new Object(); + + private static final ClassValue ADD_END_HANDLER_METHOD = + new ClassValue() { + @Override + protected Object computeValue(final Class type) { + try { + return type.getMethod("addEndHandler", Handler.class); + } catch (final NoSuchMethodException e) { + return NO_ADD_END_HANDLER; + } + } + }; + private final Handler actual; private final boolean spanStarter; @@ -48,6 +64,12 @@ public void handle(final RoutingContext routingContext) { routingContext.put(HANDLER_SPAN_CONTEXT_KEY, span); routingContext.response().endHandler(new EndHandlerWrapper(routingContext)); + // Same intent as vertx-web-4.0: finish the route-handler span when the response + // endHandler is skipped (e.g. connection already closed). Prefer + // RoutingContext#addEndHandler + // when present (vertx-web 3.8+); otherwise register close/exception handlers, coordinated + // by HttpServerResponse*WrapperAdvice like EndHandlerWrapperAdvice. + registerSpanCompletionFallback(routingContext); DECORATE.afterStart(span); span.setResourceName(DECORATE.className(actual.getClass())); } @@ -63,6 +85,34 @@ public void handle(final RoutingContext routingContext) { } } + private static void registerSpanCompletionFallback(final RoutingContext routingContext) { + final Object methodOrAbsent = ADD_END_HANDLER_METHOD.get(routingContext.getClass()); + if (methodOrAbsent instanceof Method) { + try { + @SuppressWarnings("unchecked") + final Handler> end = ar -> finishHandlerSpan(routingContext); + ((Method) methodOrAbsent).invoke(routingContext, end); + return; + } catch (final Throwable ignored) { + // Fall through to legacy response hooks (vertx-web 3.4.x without addEndHandler). + } + } + routingContext.response().closeHandler(new CloseHandlerWrapper(routingContext)); + routingContext.response().exceptionHandler(new ExceptionHandlerWrapper(routingContext)); + } + + // Idempotently finish the route-handler span. EndHandlerWrapper, RoutingContext#addEndHandler + // (when used), and CloseHandlerWrapper / ExceptionHandlerWrapper may each call this. + static void finishHandlerSpan(final RoutingContext routingContext) { + final AgentSpan span = routingContext.get(HANDLER_SPAN_CONTEXT_KEY); + if (span == null) { + return; + } + routingContext.put(HANDLER_SPAN_CONTEXT_KEY, null); + DECORATE.onResponse(span, routingContext.response()); + span.finish(); + } + private void setRoute(RoutingContext routingContext) { final AgentSpan parentSpan = routingContext.get(PARENT_SPAN_CONTEXT_KEY); if (parentSpan == null) {