Skip to content

Commit dcc8d51

Browse files
ldetmerNeenu1995
ldetmer
authored and
Neenu1995
committed
chore(bigquery): remove dev gate flag and beta api for tracing telemetry (#12624)
Remove dev gate flag and BetaApi tags from big query open telemetry tracing, as we are feature complete. Note by removing dev gate we will have added additional telemetry may contain sensitive data such as resource names, full URLs, and error messages. Fixes #12100
1 parent f76308d commit dcc8d51

7 files changed

Lines changed: 27 additions & 163 deletions

File tree

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616

1717
package com.google.cloud.bigquery;
1818

19-
import com.google.api.core.BetaApi;
2019
import com.google.api.core.ObsoleteApi;
2120
import com.google.api.gax.retrying.ResultRetryAlgorithm;
2221
import com.google.cloud.ServiceDefaults;
@@ -127,9 +126,21 @@ public Builder setDataFormatOptions(DataFormatOptions dataFormatOptions) {
127126
/**
128127
* Enables OpenTelemetry tracing functionality for this BigQuery instance
129128
*
129+
* <p><div class="warning"> Traces may contain sensitive data such as resource names, full URLs,
130+
* and error messages.
131+
*
132+
* <p>Before configuring subscribers or exporters for traces, review the contents of the spans
133+
* and consult the <a
134+
* href="https://opentelemetry.io/docs/security/handling-sensitive-data/">OpenTelemetry
135+
* documentation</a> to set up filters and formatters to prevent leaking sensitive information,
136+
* depending on your intended use case.
137+
*
138+
* <p>See also: <a
139+
* href="https://opentelemetry.io/docs/concepts/semantic-conventions/">OpenTelemetry Semantic
140+
* Conventions</a> </div>
141+
*
130142
* @param enableOpenTelemetryTracing enables OpenTelemetry tracing if true
131143
*/
132-
@BetaApi
133144
public Builder setEnableOpenTelemetryTracing(boolean enableOpenTelemetryTracing) {
134145
this.enableOpenTelemetryTracing = enableOpenTelemetryTracing;
135146
return this;
@@ -140,7 +151,6 @@ public Builder setEnableOpenTelemetryTracing(boolean enableOpenTelemetryTracing)
140151
*
141152
* @param tracer OpenTelemetry tracer to be used
142153
*/
143-
@BetaApi
144154
public Builder setOpenTelemetryTracer(Tracer tracer) {
145155
this.openTelemetryTracer = tracer;
146156
return this;
@@ -282,7 +292,6 @@ public JobCreationMode getDefaultJobCreationMode() {
282292
*
283293
* @return true if tracing is enabled, false if not
284294
*/
285-
@BetaApi("Span names and attributes are subject to change without notice")
286295
public boolean isOpenTelemetryTracingEnabled() {
287296
return enableOpenTelemetryTracing;
288297
}
@@ -292,7 +301,6 @@ public boolean isOpenTelemetryTracingEnabled() {
292301
*
293302
* @return OpenTelemetry tracer object or {@code null} if not set
294303
*/
295-
@BetaApi("Span names and attributes are subject to change without notice")
296304
public Tracer getOpenTelemetryTracer() {
297305
return openTelemetryTracer;
298306
}

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java‎

Lines changed: 11 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,6 @@ public class HttpBigQueryRpc implements BigQueryRpc {
8888

8989
public static final String DEFAULT_PROJECTION = "full";
9090
private static final String BASE_RESUMABLE_URI = "upload/bigquery/v2/projects/";
91-
static final String HTTP_TRACING_DEV_GATE_PROPERTY =
92-
"com.google.cloud.bigquery.http.tracing.dev.enabled";
9391
static final String RESOURCE_PROJECT_PREFIX = "//bigquery.googleapis.com/projects/";
9492
// see:
9593
// https://cloud.google.com/bigquery/loading-data-post-request#resume-upload
@@ -120,9 +118,7 @@ public HttpBigQueryRpc(BigQueryOptions options) {
120118
this.options = options;
121119
this.urlDomain = new GenericUrl(options.getResolvedApiaryHost("bigquery")).getHost();
122120

123-
if (options.isOpenTelemetryTracingEnabled()
124-
&& options.getOpenTelemetryTracer() != null
125-
&& isHttpTracingEnabled()) {
121+
if (options.isOpenTelemetryTracingEnabled() && options.getOpenTelemetryTracer() != null) {
126122
initializer =
127123
new HttpTracingRequestInitializer(initializer, options.getOpenTelemetryTracer());
128124
}
@@ -2146,14 +2142,11 @@ private Span createRpcTracingSpan(
21462142
.setSpanKind(SpanKind.CLIENT)
21472143
.setAttribute("bq.rpc.service", service)
21482144
.setAttribute("bq.rpc.method", method)
2149-
.setAttribute("bq.rpc.system", "http");
2150-
if (isHttpTracingEnabled()) {
2151-
builder
2152-
.setAttribute(
2153-
BigQueryTelemetryTracer.GCP_RESOURCE_DESTINATION_ID, gcpResourceDestinationId)
2154-
.setAttribute(BigQueryTelemetryTracer.URL_TEMPLATE, urlTemplate)
2155-
.setAttribute(BigQueryTelemetryTracer.URL_DOMAIN, this.urlDomain);
2156-
}
2145+
.setAttribute("bq.rpc.system", "http")
2146+
.setAttribute(
2147+
BigQueryTelemetryTracer.GCP_RESOURCE_DESTINATION_ID, gcpResourceDestinationId)
2148+
.setAttribute(BigQueryTelemetryTracer.URL_TEMPLATE, urlTemplate)
2149+
.setAttribute(BigQueryTelemetryTracer.URL_DOMAIN, this.urlDomain);
21572150

21582151
if (options != null) {
21592152
builder.setAllAttributes(otelAttributesFromOptions(options));
@@ -2175,13 +2168,11 @@ private <T> T executeWithSpan(Span span, SpanOperation<T> operation) throws IOEx
21752168
try (Scope scope = span.makeCurrent()) {
21762169
return operation.execute(span);
21772170
} catch (Exception e) {
2178-
if (isHttpTracingEnabled()) {
2179-
if (e instanceof GoogleJsonResponseException) {
2180-
BigQueryTelemetryTracer.addServerErrorResponseToSpan(
2181-
((GoogleJsonResponseException) e), span);
2182-
} else {
2183-
BigQueryTelemetryTracer.addExceptionToSpan(e, span);
2184-
}
2171+
if (e instanceof GoogleJsonResponseException) {
2172+
BigQueryTelemetryTracer.addServerErrorResponseToSpan(
2173+
((GoogleJsonResponseException) e), span);
2174+
} else {
2175+
BigQueryTelemetryTracer.addExceptionToSpan(e, span);
21852176
}
21862177
throw e;
21872178
} finally {
@@ -2202,13 +2193,4 @@ private static Attributes otelAttributesFromOptions(Map<Option, ?> options) {
22022193
}
22032194
return builder.build();
22042195
}
2205-
2206-
/**
2207-
* Temporary development gate for HttpTracingRequestInitializer rollout: must be explicitly
2208-
* enabled with the system property. tracking ticket for removal:
2209-
* https://github.com/googleapis/google-cloud-java/issues/12100
2210-
*/
2211-
static boolean isHttpTracingEnabled() {
2212-
return Boolean.parseBoolean(System.getProperty(HTTP_TRACING_DEV_GATE_PROPERTY));
2213-
}
22142196
}

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/BigQueryTelemetryTracer.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,12 @@
1818

1919
import com.google.api.client.googleapis.json.GoogleJsonError;
2020
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
21-
import com.google.api.core.BetaApi;
2221
import com.google.api.core.InternalApi;
2322
import io.opentelemetry.api.common.AttributeKey;
2423
import io.opentelemetry.api.trace.Span;
2524
import io.opentelemetry.api.trace.StatusCode;
2625

2726
/** BigQuery Telemetry class that stores generic telemetry attributes and values */
28-
@BetaApi
2927
@InternalApi
3028
public final class BigQueryTelemetryTracer {
3129

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/ErrorTypeUtil.java‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,14 @@
1515
*/
1616
package com.google.cloud.bigquery.telemetry;
1717

18-
import com.google.api.core.BetaApi;
19-
import com.google.common.annotations.VisibleForTesting;
20-
2118
/**
2219
* Utility class for identifying exception types for telemetry tracking. TODO: this class should get
2320
* replaced with gax version when ready work tracked in
2421
* https://github.com/googleapis/google-cloud-java/issues/12105
2522
*/
26-
@BetaApi
27-
@VisibleForTesting
28-
public class ErrorTypeUtil {
23+
class ErrorTypeUtil {
2924

30-
@VisibleForTesting
31-
public enum ErrorType {
25+
enum ErrorType {
3226
CLIENT_TIMEOUT,
3327
CLIENT_CONNECTION_ERROR,
3428
CLIENT_REQUEST_ERROR,

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializer.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
package com.google.cloud.bigquery.telemetry;
1818

1919
import com.google.api.client.http.*;
20-
import com.google.api.core.BetaApi;
2120
import com.google.api.core.InternalApi;
2221
import com.google.cloud.bigquery.BigQueryRetryHelper;
2322
import com.google.common.annotations.VisibleForTesting;
@@ -33,7 +32,6 @@
3332
* HttpRequestInitializer that wraps a delegate initializer, intercepts all HTTP requests, adds
3433
* OpenTelemetry tracing and then invokes delegate interceptor.
3534
*/
36-
@BetaApi
3735
@InternalApi
3836
public class HttpTracingRequestInitializer implements HttpRequestInitializer {
3937

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITOpenTelemetryTest.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
*/
1616
package com.google.cloud.bigquery.it;
1717

18-
import static com.google.cloud.bigquery.telemetry.ErrorTypeUtil.ErrorType;
1918
import static org.junit.jupiter.api.Assertions.assertEquals;
2019
import static org.junit.jupiter.api.Assertions.assertFalse;
2120
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -54,7 +53,6 @@ public class ITOpenTelemetryTest {
5453

5554
@BeforeAll
5655
public static void setUpClass() throws IOException {
57-
System.setProperty("com.google.cloud.bigquery.http.tracing.dev.enabled", "true");
5856
bigqueryHelper = RemoteBigQueryHelper.create();
5957
}
6058

@@ -215,9 +213,7 @@ public void testClientErrorAndRetriesTraced() {
215213
// Error attributes
216214
assertEquals(
217215
"java.net.UnknownHostException", attrs.get(BigQueryTelemetryTracer.EXCEPTION_TYPE));
218-
assertEquals(
219-
ErrorType.CLIENT_CONNECTION_ERROR.toString(),
220-
attrs.get(BigQueryTelemetryTracer.ERROR_TYPE));
216+
assertEquals("CLIENT_CONNECTION_ERROR", attrs.get(BigQueryTelemetryTracer.ERROR_TYPE));
221217
assertEquals(
222218
"UnknownHostException: invalid-host-name-12345.com",
223219
attrs.get(BigQueryTelemetryTracer.STATUS_MESSAGE));

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java‎

Lines changed: 0 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,7 @@
6464
import org.junit.jupiter.api.BeforeEach;
6565
import org.junit.jupiter.api.Nested;
6666
import org.junit.jupiter.api.Test;
67-
import org.junit.jupiter.api.parallel.Execution;
68-
import org.junit.jupiter.api.parallel.ExecutionMode;
6967

70-
// same thread execution temporarily required for using java system properties will get removed in
71-
// issue https://github.com/googleapis/google-cloud-java/issues/12100
72-
@Execution(ExecutionMode.SAME_THREAD)
7368
public class HttpBigQueryRpcTest {
7469

7570
private static final String PROJECT_ID = "test-project";
@@ -225,7 +220,6 @@ class TelemetryEnabled {
225220
@BeforeEach
226221
public void setUp() {
227222
setUpServer();
228-
System.setProperty("com.google.cloud.bigquery.http.tracing.dev.enabled", "true");
229223
rpc = createRpc(true);
230224
}
231225

@@ -1200,119 +1194,13 @@ public void testUrlDomain_OverriddenValue() throws Exception {
12001194
}
12011195
}
12021196

1203-
@Nested
1204-
class TelemetryEnabledDevDisabled {
1205-
private HttpBigQueryRpc rpc;
1206-
1207-
@BeforeEach
1208-
public void setUp() {
1209-
setUpServer();
1210-
System.clearProperty("com.google.cloud.bigquery.http.tracing.dev.enabled");
1211-
rpc = createRpc(true);
1212-
}
1213-
1214-
@Test
1215-
public void testHttpTracingDisabledDoesNotAddAdditionalAttributes() throws Exception {
1216-
setMockResponse(
1217-
"{\"kind\":\"bigquery#dataset\",\"id\":\""
1218-
+ PROJECT_ID
1219-
+ ":"
1220-
+ DATASET_ID
1221-
+ "\",\"datasetReference\":{\"projectId\":\""
1222-
+ PROJECT_ID
1223-
+ "\",\"datasetId\":\""
1224-
+ DATASET_ID
1225-
+ "\"}}");
1226-
1227-
rpc.getDatasetSkipExceptionTranslation(PROJECT_ID, DATASET_ID, new HashMap<>());
1228-
1229-
verifyRequest("GET", "/projects/" + PROJECT_ID + "/datasets/" + DATASET_ID);
1230-
1231-
List<SpanData> spans = spanExporter.getFinishedSpanItems();
1232-
assertThat(spans).isNotEmpty();
1233-
SpanData rpcSpan =
1234-
spans.stream()
1235-
.filter(
1236-
span -> span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.getDataset"))
1237-
.findFirst()
1238-
.orElse(null);
1239-
assertNotNull(rpcSpan);
1240-
verifySpanProductionAttributes(
1241-
"DatasetService",
1242-
"GetDataset",
1243-
Collections.singletonMap("bq.rpc.response.dataset.id", PROJECT_ID + ":" + DATASET_ID),
1244-
rpcSpan);
1245-
1246-
assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.RPC_SYSTEM_NAME));
1247-
assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.GCP_CLIENT_SERVICE));
1248-
assertNull(
1249-
rpcSpan.getAttributes().get(AttributeKey.stringKey("url.template")),
1250-
"url.template attribute should not be set");
1251-
assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.GCP_RESOURCE_DESTINATION_ID));
1252-
assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.URL_DOMAIN));
1253-
assertNull(
1254-
rpcSpan.getAttributes().get(HttpTracingRequestInitializer.HTTP_REQUEST_RESEND_COUNT));
1255-
}
1256-
1257-
@Test
1258-
public void testHttpTracingDisabled_GenericException_DoesNotSetAttributes() throws Exception {
1259-
assertThrows(
1260-
IOException.class,
1261-
() -> {
1262-
rpc.getDatasetSkipExceptionTranslation(PROJECT_ID, DATASET_ID, new HashMap<>());
1263-
});
1264-
1265-
List<io.opentelemetry.sdk.trace.data.SpanData> spans = spanExporter.getFinishedSpanItems();
1266-
assertThat(spans).isNotEmpty();
1267-
io.opentelemetry.sdk.trace.data.SpanData rpcSpan =
1268-
spans.stream()
1269-
.filter(
1270-
span -> span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.getDataset"))
1271-
.findFirst()
1272-
.orElse(null);
1273-
assertNotNull(rpcSpan);
1274-
assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.EXCEPTION_TYPE));
1275-
assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.ERROR_TYPE));
1276-
assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.STATUS_MESSAGE));
1277-
}
1278-
1279-
@Test
1280-
public void testHttpTracingDisabled_GoogleJsonResponseException_DoesNotSetAttributes()
1281-
throws Exception {
1282-
mockResponse.setStatusCode(400);
1283-
mockResponse.setContentType(Json.MEDIA_TYPE);
1284-
mockResponse.setContent(
1285-
"{\"error\":{\"code\":400,\"message\":\"Invalid request\",\"errors\":[{\"message\":\"Invalid request\",\"domain\":\"global\",\"reason\":\"invalid\"}]}}");
1286-
1287-
assertThrows(
1288-
IOException.class,
1289-
() -> {
1290-
rpc.getDatasetSkipExceptionTranslation(PROJECT_ID, DATASET_ID, new HashMap<>());
1291-
});
1292-
1293-
List<io.opentelemetry.sdk.trace.data.SpanData> spans = spanExporter.getFinishedSpanItems();
1294-
assertThat(spans).isNotEmpty();
1295-
io.opentelemetry.sdk.trace.data.SpanData rpcSpan =
1296-
spans.stream()
1297-
.filter(
1298-
span -> span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.getDataset"))
1299-
.findFirst()
1300-
.orElse(null);
1301-
assertNotNull(rpcSpan);
1302-
assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.EXCEPTION_TYPE));
1303-
assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.ERROR_TYPE));
1304-
assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.STATUS_MESSAGE));
1305-
}
1306-
}
1307-
13081197
@Nested
13091198
class TelemetryDisabled {
13101199
private HttpBigQueryRpc rpc;
13111200

13121201
@BeforeEach
13131202
public void setUp() {
13141203
setUpServer();
1315-
System.clearProperty("com.google.cloud.bigquery.http.tracing.dev.enabled");
13161204
rpc = createRpc(false);
13171205
}
13181206

0 commit comments

Comments
 (0)