diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index ce98af380eea..81f8bd95b04d 100644 --- a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -65,6 +65,7 @@ import com.google.cloud.Tuple; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.telemetry.HttpTracingRequestInitializer; import com.google.cloud.http.HttpTransportOptions; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; @@ -85,6 +86,8 @@ public class HttpBigQueryRpc implements BigQueryRpc { public static final String DEFAULT_PROJECTION = "full"; private static final String BASE_RESUMABLE_URI = "upload/bigquery/v2/projects/"; + static final String HTTP_TRACING_DEV_GATE_PROPERTY = + "com.google.cloud.bigquery.http.tracing.dev.enabled"; // see: // https://cloud.google.com/bigquery/loading-data-post-request#resume-upload private static final int HTTP_RESUME_INCOMPLETE = 308; @@ -111,6 +114,14 @@ public HttpBigQueryRpc(BigQueryOptions options) { HttpTransport transport = transportOptions.getHttpTransportFactory().create(); HttpRequestInitializer initializer = transportOptions.getHttpRequestInitializer(options); this.options = options; + + if (options.isOpenTelemetryTracingEnabled() + && options.getOpenTelemetryTracer() != null + && isHttpTracingEnabled()) { + initializer = + new HttpTracingRequestInitializer(initializer, options.getOpenTelemetryTracer()); + } + bigquery = new Bigquery.Builder(transport, new GsonFactory(), initializer) .setRootUrl(options.getResolvedApiaryHost("bigquery")) @@ -1776,4 +1787,13 @@ private static Attributes otelAttributesFromOptions(Map options) { } return builder.build(); } + + /** + * Temporary development gate for HttpTracingRequestInitializer rollout: must be explicitly + * enabled with the system property. tracking ticket for removal: + * https://github.com/googleapis/google-cloud-java/issues/12100 + */ + static boolean isHttpTracingEnabled() { + return Boolean.parseBoolean(System.getProperty(HTTP_TRACING_DEV_GATE_PROPERTY)); + } } diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/BigQueryTelemetryTracer.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/BigQueryTelemetryTracer.java new file mode 100644 index 000000000000..04efbf0c1614 --- /dev/null +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/BigQueryTelemetryTracer.java @@ -0,0 +1,73 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.telemetry; + +import com.google.api.core.BetaApi; +import com.google.api.core.InternalApi; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; + +/** BigQuery Telemetry class that stores generic telemetry attributes and values */ +@BetaApi +@InternalApi +public final class BigQueryTelemetryTracer { + + private BigQueryTelemetryTracer() {} + + public static final String BQ_GCP_CLIENT_SERVICE = "bigquery"; + public static final String BQ_GCP_CLIENT_REPO = "googleapis/java-bigquery"; + public static final String BQ_GCP_CLIENT_ARTIFACT = "google-cloud-bigquery"; + public static final String BQ_GCP_CLIENT_LANGUAGE = "java"; + + // TODO: migrate to use gax attributes keys + // https://github.com/googleapis/google-cloud-java/issues/12099 + // Common GCP Attributes + public static final AttributeKey GCP_CLIENT_SERVICE = + AttributeKey.stringKey("gcp.client.service"); + public static final AttributeKey GCP_CLIENT_VERSION = + AttributeKey.stringKey("gcp.client.version"); + public static final AttributeKey GCP_CLIENT_REPO = + AttributeKey.stringKey("gcp.client.repo"); + public static final AttributeKey GCP_CLIENT_ARTIFACT = + AttributeKey.stringKey("gcp.client.artifact"); + public static final AttributeKey GCP_CLIENT_LANGUAGE = + AttributeKey.stringKey("gcp.client.language"); + public static final AttributeKey GCP_RESOURCE_ID = + AttributeKey.stringKey("gcp.resource.destination.id"); + public static final AttributeKey RPC_SYSTEM_NAME = + AttributeKey.stringKey("rpc.system.name"); + + // Common Error Attributes + public static final AttributeKey ERROR_TYPE = AttributeKey.stringKey("error.type"); + public static final AttributeKey EXCEPTION_TYPE = + AttributeKey.stringKey("exception.type"); + public static final AttributeKey STATUS_MESSAGE = + AttributeKey.stringKey("status.message"); + + // Common Server Attributes + public static final AttributeKey SERVER_ADDRESS = + AttributeKey.stringKey("server.address"); + public static final AttributeKey SERVER_PORT = AttributeKey.longKey("server.port"); + + public static void addCommonAttributeToSpan(Span span) { + span.setAttribute(GCP_CLIENT_SERVICE, BQ_GCP_CLIENT_SERVICE) + .setAttribute(GCP_CLIENT_REPO, BQ_GCP_CLIENT_REPO) + .setAttribute(GCP_CLIENT_ARTIFACT, BQ_GCP_CLIENT_ARTIFACT) + .setAttribute(GCP_CLIENT_LANGUAGE, BQ_GCP_CLIENT_LANGUAGE); + // TODO: add version + } +} diff --git a/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializer.java b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializer.java new file mode 100644 index 000000000000..575e62a28ac2 --- /dev/null +++ b/java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializer.java @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.telemetry; + +import com.google.api.client.http.*; +import com.google.api.core.BetaApi; +import com.google.api.core.InternalApi; +import com.google.common.annotations.VisibleForTesting; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import java.io.IOException; + +/** + * HttpRequestInitializer that wraps a delegate initializer, intercepts all HTTP requests, adds + * OpenTelemetry tracing and then invokes delegate interceptor. + */ +@BetaApi +@InternalApi +public class HttpTracingRequestInitializer implements HttpRequestInitializer { + + // TODO: migrate to use gax attributes keys + // https://github.com/googleapis/google-cloud-java/issues/12099 + // HTTP Specific Telemetry Attributes + public static final AttributeKey HTTP_REQUEST_METHOD = + AttributeKey.stringKey("http.request.method"); + public static final AttributeKey URL_FULL = AttributeKey.stringKey("url.full"); + public static final AttributeKey URL_TEMPLATE = AttributeKey.stringKey("url.template"); + public static final AttributeKey URL_DOMAIN = AttributeKey.stringKey("url.domain"); + public static final AttributeKey HTTP_RESPONSE_STATUS_CODE = + AttributeKey.longKey("http.response.status_code"); + public static final AttributeKey HTTP_REQUEST_RESEND_COUNT = + AttributeKey.longKey("http.request.resend_count"); + public static final AttributeKey HTTP_REQUEST_BODY_SIZE = + AttributeKey.longKey("http.request.body.size"); + public static final AttributeKey HTTP_RESPONSE_BODY_SIZE = + AttributeKey.longKey("http.response.body.size"); + + @VisibleForTesting static final String HTTP_RPC_SYSTEM_NAME = "http"; + + private final HttpRequestInitializer delegate; + private final Tracer tracer; + + public HttpTracingRequestInitializer(HttpRequestInitializer delegate, Tracer tracer) { + this.delegate = delegate; + this.tracer = tracer; + } + + @Override + public void initialize(HttpRequest request) throws IOException { + if (delegate != null) { + delegate.initialize(request); + } + if (tracer == null) { + return; + } + // Get the current active span (created by HttpBigQueryRpc) and add HTTP attributes to it + Span span = Span.current(); + if (!span.getSpanContext().isValid()) { + // No active span to exists, skip instrumentation + return; + } + String host = request.getUrl().getHost(); + int port = request.getUrl().getPort(); + addInitialHttpAttributesToSpan(span, host, port); + } + + /** Add initial HTTP attributes to the existing active span */ + private void addInitialHttpAttributesToSpan(Span span, String host, Integer port) { + BigQueryTelemetryTracer.addCommonAttributeToSpan(span); + span.setAttribute(BigQueryTelemetryTracer.RPC_SYSTEM_NAME, HTTP_RPC_SYSTEM_NAME); + span.setAttribute(BigQueryTelemetryTracer.SERVER_ADDRESS, host); + if (port != null && port > 0) { + span.setAttribute(BigQueryTelemetryTracer.SERVER_PORT, port.longValue()); + } + // TODO add full sanitized url, url domain, request method + } +} diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java index 97b231166410..161d4f030a64 100644 --- a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java @@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -44,6 +45,7 @@ import com.google.api.services.bigquery.model.TableReference; import com.google.cloud.NoCredentials; import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.telemetry.BigQueryTelemetryTracer; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.sdk.OpenTelemetrySdk; @@ -906,6 +908,92 @@ public void testOtelAttributesFromOptionsGetAddedtoSpan() throws Exception { "GetDataset", expectedAttributes); } + + @Test + public void testHttpTracingEnabledAddsAdditionalAttributes() throws Exception { + try { + System.setProperty("com.google.cloud.bigquery.http.tracing.dev.enabled", "true"); + HttpBigQueryRpc customRpc = createRpc(true); + + setMockResponse( + "{\"kind\":\"bigquery#dataset\",\"id\":\"" + + PROJECT_ID + + ":" + + DATASET_ID + + "\",\"datasetReference\":{\"projectId\":\"" + + PROJECT_ID + + "\",\"datasetId\":\"" + + DATASET_ID + + "\"}}"); + + customRpc.getDatasetSkipExceptionTranslation(PROJECT_ID, DATASET_ID, new HashMap<>()); + + verifyRequest("GET", "/projects/" + PROJECT_ID + "/datasets/" + DATASET_ID); + verifySpan( + "com.google.cloud.bigquery.BigQueryRpc.getDataset", + "DatasetService", + "GetDataset", + Collections.singletonMap("bq.rpc.response.dataset.id", PROJECT_ID + ":" + DATASET_ID)); + + List spans = spanExporter.getFinishedSpanItems(); + assertThat(spans).isNotEmpty(); + SpanData rpcSpan = + spans.stream() + .filter( + span -> + span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.getDataset")) + .findFirst() + .orElse(null); + assertNotNull(rpcSpan); + assertEquals("http", rpcSpan.getAttributes().get(BigQueryTelemetryTracer.RPC_SYSTEM_NAME)); + assertNotNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.GCP_CLIENT_SERVICE)); + } finally { + System.clearProperty("com.google.cloud.bigquery.http.tracing.dev.enabled"); + } + } + + @Test + public void testHttpTracingDisabledDoesNotAddAdditionalAttributes() throws Exception { + try { + System.setProperty("com.google.cloud.bigquery.http.tracing.dev.enabled", "false"); + HttpBigQueryRpc customRpc = createRpc(true); + + setMockResponse( + "{\"kind\":\"bigquery#dataset\",\"id\":\"" + + PROJECT_ID + + ":" + + DATASET_ID + + "\",\"datasetReference\":{\"projectId\":\"" + + PROJECT_ID + + "\",\"datasetId\":\"" + + DATASET_ID + + "\"}}"); + + customRpc.getDatasetSkipExceptionTranslation(PROJECT_ID, DATASET_ID, new HashMap<>()); + + verifyRequest("GET", "/projects/" + PROJECT_ID + "/datasets/" + DATASET_ID); + verifySpan( + "com.google.cloud.bigquery.BigQueryRpc.getDataset", + "DatasetService", + "GetDataset", + Collections.singletonMap("bq.rpc.response.dataset.id", PROJECT_ID + ":" + DATASET_ID)); + + List spans = spanExporter.getFinishedSpanItems(); + assertThat(spans).isNotEmpty(); + SpanData rpcSpan = + spans.stream() + .filter( + span -> + span.getName().equals("com.google.cloud.bigquery.BigQueryRpc.getDataset")) + .findFirst() + .orElse(null); + assertNotNull(rpcSpan); + assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.RPC_SYSTEM_NAME)); + assertNull(rpcSpan.getAttributes().get(BigQueryTelemetryTracer.GCP_CLIENT_SERVICE)); + } finally { + System.clearProperty("com.google.cloud.bigquery.http.tracing.dev.enabled"); + } + } } @Nested diff --git a/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializerTest.java b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializerTest.java new file mode 100644 index 000000000000..ea29c20f210f --- /dev/null +++ b/java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/telemetry/HttpTracingRequestInitializerTest.java @@ -0,0 +1,196 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.io.IOException; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class HttpTracingRequestInitializerTest { + + private static final String BASE_URL = + "https://bigquery.googleapis.com:443/bigquery/v2/projects/test/datasets"; + private static final String BIGQUERY_DOMAIN = "bigquery.googleapis.com"; + private static final String CLIENT_ROOT_URL = "https://bigquery.googleapis.com:443"; + private static final String SPAN_NAME = "test-parent-span"; + private InMemorySpanExporter spanExporter; + private Tracer tracer; + private HttpTracingRequestInitializer initializer; + private Span parentSpan; + private Scope spanScope; + + @BeforeEach + public void setUp() { + spanExporter = InMemorySpanExporter.create(); + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + OpenTelemetrySdk openTelemetry = + OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); + tracer = openTelemetry.getTracer("test-tracer"); + initializer = new HttpTracingRequestInitializer(null, tracer); + parentSpan = tracer.spanBuilder(SPAN_NAME).startSpan(); + spanScope = parentSpan.makeCurrent(); + } + + @Test + public void testRequestAttributesAreSetIfSpanExists() throws IOException { + HttpTransport transport = createTransport(); + HttpRequest request = buildGetRequest(transport, initializer, BASE_URL); + + HttpResponse response = request.execute(); + response.disconnect(); + + // End the span before verifying exported spans, so it appears in the exporter + spanScope.close(); + parentSpan.end(); + + List spans = spanExporter.getFinishedSpanItems(); + assertEquals(1, spans.size()); + + SpanData span = spans.get(0); + verifyGeneralSpanData(span); + } + + @Test + public void testExistingParentAttributesAreNotAffectedByRequestAttributes() throws IOException { + parentSpan.setAttribute("parent_attribute", "value"); + HttpTransport transport = createTransport(); + HttpRequest request = buildGetRequest(transport, initializer, BASE_URL); + + HttpResponse response = request.execute(); + response.disconnect(); + + // End the span before verifying exported spans, so it appears in the exporter + spanScope.close(); + parentSpan.end(); + + List spans = spanExporter.getFinishedSpanItems(); + assertEquals(1, spans.size()); + + SpanData span = spans.get(0); + assertEquals("value", span.getAttributes().get(AttributeKey.stringKey("parent_attribute"))); + } + + @Test + public void testNoSpanIsCreatedIfNoActiveSpan() throws IOException { + HttpRequestInitializer delegateInitializer = mock(HttpRequestInitializer.class); + HttpTracingRequestInitializer tracingInitializer = + new HttpTracingRequestInitializer(delegateInitializer, tracer); + + HttpTransport transport = createTransport(); + // close span before building the request so there is no current span during initialization + spanScope.close(); + parentSpan.end(); + + HttpRequest request = buildGetRequest(transport, tracingInitializer, BASE_URL); + + HttpResponse response = request.execute(); + response.disconnect(); + + List spans = spanExporter.getFinishedSpanItems(); + assertEquals(1, spans.size()); + assertNull(spans.get(0).getAttributes().get(BigQueryTelemetryTracer.GCP_CLIENT_ARTIFACT)); + verify(delegateInitializer, times(1)).initialize(any(HttpRequest.class)); + } + + @Test + public void testDelegateInitializerIsCalled() throws IOException { + HttpRequestInitializer delegateInitializer = mock(HttpRequestInitializer.class); + HttpTracingRequestInitializer tracingInitializer = + new HttpTracingRequestInitializer(delegateInitializer, tracer); + + HttpTransport transport = createTransport(); + HttpRequest request = buildGetRequest(transport, tracingInitializer, BASE_URL); + + HttpResponse response = request.execute(); + response.disconnect(); + + verify(delegateInitializer, times(1)).initialize(any(HttpRequest.class)); + } + + private static HttpTransport createTransport() { + return new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() { + return new MockLowLevelHttpResponse(); + } + }; + } + }; + } + + private static HttpRequest buildGetRequest( + HttpTransport transport, HttpRequestInitializer requestInitializer, String url) + throws IOException { + HttpRequestFactory requestFactory = transport.createRequestFactory(requestInitializer); + return requestFactory.buildGetRequest(new GenericUrl(url)); + } + + private void verifyGeneralSpanData(SpanData span) { + assertEquals(SPAN_NAME, span.getName()); + assertEquals(BIGQUERY_DOMAIN, span.getAttributes().get(BigQueryTelemetryTracer.SERVER_ADDRESS)); + assertEquals(443, span.getAttributes().get(BigQueryTelemetryTracer.SERVER_PORT)); + assertEquals( + BigQueryTelemetryTracer.BQ_GCP_CLIENT_SERVICE, + span.getAttributes().get(BigQueryTelemetryTracer.GCP_CLIENT_SERVICE)); + assertEquals( + BigQueryTelemetryTracer.BQ_GCP_CLIENT_REPO, + span.getAttributes().get(BigQueryTelemetryTracer.GCP_CLIENT_REPO)); + assertEquals( + BigQueryTelemetryTracer.BQ_GCP_CLIENT_ARTIFACT, + span.getAttributes().get(BigQueryTelemetryTracer.GCP_CLIENT_ARTIFACT)); + assertEquals( + BigQueryTelemetryTracer.BQ_GCP_CLIENT_LANGUAGE, + span.getAttributes().get(BigQueryTelemetryTracer.GCP_CLIENT_LANGUAGE)); + assertEquals( + HttpTracingRequestInitializer.HTTP_RPC_SYSTEM_NAME, + span.getAttributes().get(BigQueryTelemetryTracer.RPC_SYSTEM_NAME)); + } +}