feat(telemetry): add structured status code classification for genai errors

Introduce a `classifyByStatusCode` helper that probes for an HTTP status code via a `StatusCode() int` method before falling back to substring matching. This prevents false positives when error messages incidentally contain strings like "401", "403", or "429" in request IDs, byte counts, or status-line fragments.

Providers that expose HTTP status codes through a structured interface now get classified from the structural signal, while text-only errors continue to use the existing heuristic.

Also add documentation clarifying that `getInstruments` binds to the global MeterProvider on first call via `sync.Once`, which affects test setup requirements.

Author: tdabasinskas

pkg/telemetry/genai/errors.go

@@ -32,6 +32,18 @@ func ClassifyError(err error) string {
 		return "deadline_exceeded"
 	}
 
+	// Prefer a structured status-code probe before falling back to
+	// substring matching. The string heuristic below trips on any
+	// error message that incidentally contains "401", "403", "429" —
+	// request IDs, byte counts, status-line fragments, etc. Providers
+	// that surface HTTP status codes via a `StatusCode() int` method
+	// (or via an OTel-style `HTTPStatusCode() int`) get classified
+	// from the structural signal, while text-only errors fall through
+	// to the heuristic.
+	if t := classifyByStatusCode(err); t != "" {
+		return t
+	}
+
 	msg := strings.ToLower(err.Error())
 	switch {
 	case strings.Contains(msg, "context length") || strings.Contains(msg, "context_length"):
@@ -61,6 +73,27 @@ func ClassifyError(err error) string {
 	return ErrorTypeOther
 }
 
+// classifyByStatusCode returns a low-cardinality `error.type` when err
+// (or anything in its wrap chain) exposes an HTTP status code via a
+// `StatusCode() int` method and the value matches one of the cases
+// ClassifyError handles. Returns "" when no structural signal is
+// available so the caller can fall through to substring heuristics.
+func classifyByStatusCode(err error) string {
+	var sc interface{ StatusCode() int }
+	if !errors.As(err, &sc) {
+		return ""
+	}
+	switch sc.StatusCode() {
+	case 401:
+		return "auth"
+	case 403:
+		return "forbidden"
+	case 429:
+		return "rate_limit"
+	}
+	return ""
+}
+
 // applyExtraAttribute converts a StreamAttributer KeyValue into an OTel
 // attribute and applies it to the span. Unsupported value types are
 // dropped silently — telemetry must never crash request paths.

pkg/telemetry/genai/metrics.go

@@ -44,6 +44,14 @@ var (
 // previously a single early return left every metric permanently
 // disabled, which surprised production debugging when one bucket
 // configuration tripped a registration error.
+//
+// Test note: instruments are bound to the global MeterProvider on first
+// call and frozen for the process lifetime via sync.Once. Replacing the
+// provider with otel.SetMeterProvider after any production code path
+// has already triggered getInstruments will NOT rebind the histograms,
+// so tests that inspect emitted metrics must install their provider
+// before any code under test runs (typically in TestMain or per-test
+// setup before the first instrumented call).
 func getInstruments() *instruments {
 	instOnce.Do(func() {
 		meter := otel.Meter(instrumentationName)

pkg/telemetry/genai/span.go

@@ -353,6 +353,12 @@ func (s *ChatSpan) End() {
 		attribute.String(AttrOperationName, OperationChat),
 		attribute.String(AttrProviderName, s.provider),
 	}
+	// `gen_ai.request.model` is required here by the OTel GenAI
+	// semconv but is unbounded in practice — every dated variant
+	// (e.g. `model-YYYYMMDD`) opens a new metric series. Operators
+	// concerned about backend cardinality should drop or canonicalise
+	// this label at the collector rather than at the agent, so spans
+	// keep full detail while metrics stay bounded.
 	if s.model != "" {
 		commonAttrs = append(commonAttrs, attribute.String(AttrRequestModel, s.model))
 	}