Introduce shared OtelMetricRegistry to track metrics by instrumentation scope

buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleAction.kt

@@ -9,10 +9,14 @@ import java.net.URLClassLoader
 abstract class MuzzleAction : WorkAction<MuzzleWorkParameters> {
     companion object {
         private val lock = Any()
-        private var bootCL: ClassLoader? = null
-        private var toolCL: ClassLoader? = null
+        @Volatile
+        private var lastBootCL: ClassLoader? = null
+        @Volatile
+        private var lastToolCL: ClassLoader? = null
         @Volatile
         private var lastBuildStamp: Long = 0
+        @Volatile
+        private var lastBuildPathCount: Int = 0
 
         fun createClassLoader(cp: FileCollection, parent: ClassLoader = ClassLoader.getSystemClassLoader()): ClassLoader {
             val urls = cp.map { it.toURI().toURL() }.toTypedArray()
@@ -22,12 +26,21 @@ abstract class MuzzleAction : WorkAction<MuzzleWorkParameters> {
 
     override fun execute() {
         val buildStamp = parameters.buildStartedTime.get()
-        if (bootCL == null || toolCL == null || lastBuildStamp < buildStamp) {
+        val buildPathCount = parameters.bootstrapClassPath.count() + parameters.toolingClassPath.count()
+        var bootCL : ClassLoader? = lastBootCL
+        var toolCL : ClassLoader? = lastToolCL
+        // cache boot and tool classloaders for each run; rebuild if either class-path is extended mid-build
+        if (bootCL == null || toolCL == null || lastBuildStamp < buildStamp || lastBuildPathCount < buildPathCount) {
             synchronized(lock) {
-                if (bootCL == null || toolCL == null || lastBuildStamp < buildStamp) {
+                bootCL = lastBootCL
+                toolCL = lastToolCL
+                if (bootCL == null || toolCL == null || lastBuildStamp < buildStamp || lastBuildPathCount < buildPathCount) {
                     bootCL = createClassLoader(parameters.bootstrapClassPath)
                     toolCL = createClassLoader(parameters.toolingClassPath, bootCL!!)
+                    lastBootCL = bootCL
+                    lastToolCL = toolCL
                     lastBuildStamp = buildStamp
+                    lastBuildPathCount = buildPathCount
                 }
             }
         }

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/OtelInstrumentationScope.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/common/OtelInstrumentationScope.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim;
+package datadog.trace.bootstrap.otel.common;
 
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
 import java.util.Objects;

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/OtelInstrument.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/OtelInstrument.java

@@ -1,12 +1,12 @@
-package datadog.opentelemetry.shim.metrics;
+package datadog.trace.bootstrap.otel.metrics;
 
-import datadog.opentelemetry.shim.metrics.data.OtelMetricStorage;
+import datadog.trace.bootstrap.otel.metrics.data.OtelMetricStorage;
 
 /** Ensure all instruments implement the same equivalency. */
-abstract class OtelInstrument {
-  final OtelMetricStorage storage;
+public abstract class OtelInstrument {
+  protected final OtelMetricStorage storage;
 
-  OtelInstrument(OtelMetricStorage storage) {
+  protected OtelInstrument(OtelMetricStorage storage) {
     this.storage = storage;
   }
 

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/OtelInstrumentBuilder.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/OtelInstrumentBuilder.java

@@ -1,8 +1,8 @@
-package datadog.opentelemetry.shim.metrics;
+package datadog.trace.bootstrap.otel.metrics;
 
 import javax.annotation.Nullable;
 
-final class OtelInstrumentBuilder {
+public final class OtelInstrumentBuilder {
   private final String instrumentName;
   private final OtelInstrumentType instrumentType;
   private final boolean longValues;
@@ -17,7 +17,8 @@ final class OtelInstrumentBuilder {
    * @param instrumentType the type of the instrument
    * @return new instrument builder
    */
-  static OtelInstrumentBuilder ofLongs(String instrumentName, OtelInstrumentType instrumentType) {
+  public static OtelInstrumentBuilder ofLongs(
+      String instrumentName, OtelInstrumentType instrumentType) {
     return new OtelInstrumentBuilder(instrumentName, instrumentType, true);
   }
 
@@ -28,7 +29,7 @@ static OtelInstrumentBuilder ofLongs(String instrumentName, OtelInstrumentType i
    * @param instrumentType the type of the instrument
    * @return new instrument builder
    */
-  static OtelInstrumentBuilder ofLongs(
+  public static OtelInstrumentBuilder ofLongs(
       OtelInstrumentBuilder builder, OtelInstrumentType instrumentType) {
     return new OtelInstrumentBuilder(builder.instrumentName, instrumentType, true);
   }
@@ -40,7 +41,8 @@ static OtelInstrumentBuilder ofLongs(
    * @param instrumentType the type of the instrument
    * @return new instrument builder
    */
-  static OtelInstrumentBuilder ofDoubles(String instrumentName, OtelInstrumentType instrumentType) {
+  public static OtelInstrumentBuilder ofDoubles(
+      String instrumentName, OtelInstrumentType instrumentType) {
     return new OtelInstrumentBuilder(instrumentName, instrumentType, false);
   }
 
@@ -51,7 +53,7 @@ static OtelInstrumentBuilder ofDoubles(String instrumentName, OtelInstrumentType
    * @param instrumentType the type of the instrument
    * @return new instrument builder
    */
-  static OtelInstrumentBuilder ofDoubles(
+  public static OtelInstrumentBuilder ofDoubles(
       OtelInstrumentBuilder builder, OtelInstrumentType instrumentType) {
     return new OtelInstrumentBuilder(builder.instrumentName, instrumentType, false);
   }
@@ -63,20 +65,20 @@ private OtelInstrumentBuilder(
     this.longValues = longValues;
   }
 
-  void setDescription(String description) {
+  public void setDescription(String description) {
     this.description = description;
   }
 
-  void setUnit(String unit) {
+  public void setUnit(String unit) {
     this.unit = unit;
   }
 
-  OtelInstrumentDescriptor descriptor() {
+  public OtelInstrumentDescriptor descriptor() {
     return new OtelInstrumentDescriptor(
         instrumentName, instrumentType, longValues, description, unit);
   }
 
-  OtelInstrumentDescriptor observableDescriptor() {
+  public OtelInstrumentDescriptor observableDescriptor() {
     return new OtelInstrumentDescriptor(
         instrumentName, observableType(instrumentType), longValues, description, unit);
   }

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/OtelInstrumentDescriptor.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/OtelInstrumentDescriptor.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics;
+package datadog.trace.bootstrap.otel.metrics;
 
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
 import java.util.Locale;

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/OtelInstrumentType.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/OtelInstrumentType.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics;
+package datadog.trace.bootstrap.otel.metrics;
 
 public enum OtelInstrumentType {
   // same order as io.opentelemetry.sdk.metrics.InstrumentType

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelAggregator.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelAggregator.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics.data;
+package datadog.trace.bootstrap.otel.metrics.data;
 
 /** Common behaviour shared across all aggregators. */
 abstract class OtelAggregator {

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelDoublePoint.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelDoublePoint.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics.data;
+package datadog.trace.bootstrap.otel.metrics.data;
 
 public final class OtelDoublePoint extends OtelPoint {
   public final double value;

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelDoubleSum.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelDoubleSum.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics.data;
+package datadog.trace.bootstrap.otel.metrics.data;
 
 import java.util.concurrent.atomic.DoubleAdder;
 

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelDoubleValue.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelDoubleValue.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics.data;
+package datadog.trace.bootstrap.otel.metrics.data;
 
 final class OtelDoubleValue extends OtelAggregator {
   private volatile double value;

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelHistogramPoint.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelHistogramPoint.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics.data;
+package datadog.trace.bootstrap.otel.metrics.data;
 
 import java.util.List;
 

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelHistogramSketch.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelHistogramSketch.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics.data;
+package datadog.trace.bootstrap.otel.metrics.data;
 
 import datadog.metrics.api.Histogram;
 import java.util.List;

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelLongPoint.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelLongPoint.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics.data;
+package datadog.trace.bootstrap.otel.metrics.data;
 
 public final class OtelLongPoint extends OtelPoint {
   public final long value;

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelLongSum.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelLongSum.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics.data;
+package datadog.trace.bootstrap.otel.metrics.data;
 
 import java.util.concurrent.atomic.LongAdder;
 

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelLongValue.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelLongValue.java

@@ -1,4 +1,4 @@
-package datadog.opentelemetry.shim.metrics.data;
+package datadog.trace.bootstrap.otel.metrics.data;
 
 final class OtelLongValue extends OtelAggregator {
   private volatile long value;

dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelMetricRegistry.java

@@ -0,0 +1,74 @@
+package datadog.trace.bootstrap.otel.metrics.data;
+
+import datadog.trace.bootstrap.otel.common.OtelInstrumentationScope;
+import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor;
+import datadog.trace.bootstrap.otel.metrics.export.OtelMetricsVisitor;
+import datadog.trace.bootstrap.otel.metrics.export.OtelScopedMetricsVisitor;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+
+/** Tracks metric storage and observable callbacks by instrumentation scope. */
+public final class OtelMetricRegistry {
+  public static final OtelMetricRegistry INSTANCE = new OtelMetricRegistry();
+
+  private final Map<OtelInstrumentationScope, Map<OtelInstrumentDescriptor, OtelMetricStorage>>
+      scopedStorage = new ConcurrentHashMap<>();
+
+  private final Map<OtelInstrumentationScope, List<OtelObservable>> scopedObservables =
+      new ConcurrentHashMap<>();
+
+  public OtelMetricStorage registerStorage(
+      OtelInstrumentationScope instrumentationScope,
+      OtelInstrumentDescriptor descriptor,
+      Function<OtelInstrumentDescriptor, OtelMetricStorage> storageFactory) {
+    return scopedStorage
+        .computeIfAbsent(instrumentationScope, unused -> new ConcurrentHashMap<>())
+        .computeIfAbsent(descriptor, storageFactory);
+  }
+
+  public void registerObservable(
+      OtelInstrumentationScope instrumentationScope, OtelObservable observable) {
+    List<OtelObservable> observables =
+        scopedObservables.computeIfAbsent(instrumentationScope, unused -> new ArrayList<>());
+    synchronized (observables) {
+      observables.add(observable);
+    }
+  }
+
+  public boolean unregisterObservable(
+      OtelInstrumentationScope instrumentationScope, OtelObservable observable) {
+    List<OtelObservable> observables = scopedObservables.get(instrumentationScope);
+    if (observables == null) {
+      return false;
+    }
+    synchronized (observables) {
+      return observables.remove(observable);
+    }
+  }
+
+  public void collectMetrics(OtelMetricsVisitor visitor) {
+    scopedStorage.forEach(
+        (scope, storage) ->
+            collectScopedMetrics(scope, storage, visitor.visitScopedMetrics(scope)));
+  }
+
+  private void collectScopedMetrics(
+      OtelInstrumentationScope instrumentationScope,
+      Map<OtelInstrumentDescriptor, OtelMetricStorage> storage,
+      OtelScopedMetricsVisitor visitor) {
+    List<OtelObservable> observables = scopedObservables.get(instrumentationScope);
+    if (observables != null) {
+      // take local snapshot of current observables
+      List<OtelObservable> observablesCopy;
+      synchronized (observables) {
+        observablesCopy = new ArrayList<>(observables);
+      }
+      // must observe measurements outside of lock
+      observablesCopy.forEach(OtelObservable::observeMeasurements);
+    }
+    storage.forEach((descriptor, s) -> s.collectMetric(visitor.visitMetric(descriptor)));
+  }
+}

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/data/OtelMetricStorage.java → dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelMetricStorage.java

@@ -1,11 +1,11 @@
-package datadog.opentelemetry.shim.metrics.data;
+package datadog.trace.bootstrap.otel.metrics.data;
 
-import datadog.opentelemetry.shim.metrics.OtelInstrumentDescriptor;
-import datadog.opentelemetry.shim.metrics.OtelInstrumentType;
-import datadog.opentelemetry.shim.metrics.export.OtelInstrumentVisitor;
 import datadog.trace.api.Config;
 import datadog.trace.api.config.OtlpConfig;
 import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString;
+import datadog.trace.bootstrap.otel.metrics.OtelInstrumentDescriptor;
+import datadog.trace.bootstrap.otel.metrics.OtelInstrumentType;
+import datadog.trace.bootstrap.otel.metrics.export.OtelMetricVisitor;
 import datadog.trace.relocate.api.RatelimitedLogger;
 import io.opentelemetry.api.common.Attributes;
 import java.util.List;
@@ -35,7 +35,7 @@ public final class OtelMetricStorage {
 
   private final OtelInstrumentDescriptor descriptor;
   private final boolean resetOnCollect;
-  private final Function<Attributes, OtelAggregator> aggregatorSupplier;
+  private final Function<Object, OtelAggregator> aggregatorSupplier;
   private volatile Recording currentRecording;
 
   // only used with DELTA temporality
@@ -98,7 +98,7 @@ public OtelInstrumentDescriptor getDescriptor() {
     return descriptor;
   }
 
-  public void recordLong(long value, Attributes attributes) {
+  public void recordLong(long value, Object attributes) {
     if (resetOnCollect) {
       Recording recording = acquireRecordingForWrite();
       try {
@@ -112,7 +112,7 @@ public void recordLong(long value, Attributes attributes) {
     }
   }
 
-  public void recordDouble(double value, Attributes attributes) {
+  public void recordDouble(double value, Object attributes) {
     if (Double.isNaN(value)) {
       LOGGER.debug(
           "Instrument {} has recorded measurement Not-a-Number (NaN) value with attributes {}. Dropping measurement.",
@@ -133,8 +133,7 @@ public void recordDouble(double value, Attributes attributes) {
     }
   }
 
-  private OtelAggregator aggregator(
-      Map<Attributes, OtelAggregator> aggregators, Attributes attributes) {
+  private OtelAggregator aggregator(Map<Object, OtelAggregator> aggregators, Object attributes) {
     Objects.requireNonNull(attributes, "attributes");
     OtelAggregator aggregator = aggregators.get(attributes);
     if (null != aggregator) {
@@ -150,7 +149,7 @@ private OtelAggregator aggregator(
     return aggregators.computeIfAbsent(attributes, aggregatorSupplier);
   }
 
-  public void collect(OtelInstrumentVisitor visitor) {
+  public void collectMetric(OtelMetricVisitor visitor) {
     if (resetOnCollect) {
       doCollectAndReset(visitor);
     } else {
@@ -159,7 +158,7 @@ public void collect(OtelInstrumentVisitor visitor) {
   }
 
   /** Collect data for CUMULATIVE temporality, keeping aggregators for future writes. */
-  private void doCollect(OtelInstrumentVisitor visitor) {
+  private void doCollect(OtelMetricVisitor visitor) {
     // no need to hold writers back if we are not resetting metrics on collect
     currentRecording.aggregators.forEach(
         (attributes, aggregator) -> {
@@ -174,7 +173,7 @@ private void doCollect(OtelInstrumentVisitor visitor) {
    *
    * <p>Each collect request toggles between two groups of aggregators: current / previous.
    */
-  private void doCollectAndReset(OtelInstrumentVisitor visitor) {
+  private void doCollectAndReset(OtelMetricVisitor visitor) {
 
     // capture _current_ recording for collection, its aggregators will be reset at the end
     final Recording recording = currentRecording;
@@ -188,7 +187,7 @@ private void doCollectAndReset(OtelInstrumentVisitor visitor) {
       Thread.yield(); // other threads are still writing to this recording
     }
 
-    Map<Attributes, OtelAggregator> aggregators = recording.aggregators;
+    Map<Object, OtelAggregator> aggregators = recording.aggregators;
 
     // avoid churn: only remove empty aggregators if we're over cardinality
     if (aggregators.size() >= CARDINALITY_LIMIT) {
@@ -233,7 +232,7 @@ private void releaseRecordingAfterWrite(Recording recording) {
   private static final int WRITER = 2;
 
   static final class Recording {
-    final Map<Attributes, OtelAggregator> aggregators;
+    final Map<Object, OtelAggregator> aggregators;
 
     transient volatile int activity;
 

dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelObservable.java

@@ -0,0 +1,5 @@
+package datadog.trace.bootstrap.otel.metrics.data;
+
+public abstract class OtelObservable {
+  protected abstract void observeMeasurements();
+}

dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/data/OtelPoint.java

@@ -0,0 +1,3 @@
+package datadog.trace.bootstrap.otel.metrics.data;
+
+public abstract class OtelPoint {}

dd-java-agent/agent-otel/otel-bootstrap/src/main/java/datadog/trace/bootstrap/otel/metrics/export/OtelMetricVisitor.java

@@ -0,0 +1,9 @@
+package datadog.trace.bootstrap.otel.metrics.export;
+
+import datadog.trace.bootstrap.otel.metrics.data.OtelPoint;
+
+/** A visitor to visit a metric in an instrumentation scope. */
+public interface OtelMetricVisitor {
+  /** Visits a data point in the metric. */
+  void visitPoint(Object attributes, OtelPoint point);
+}