From c7f72d60ae8810c0d6392f5263ea25a543f8f66c Mon Sep 17 00:00:00 2001 From: ankiaga Date: Tue, 15 Oct 2024 16:56:15 +0530 Subject: [PATCH 1/9] feat: Enabling endToEndTracing support in Connection API --- .../spanner/connection/ConnectionOptions.java | 14 ++++++ .../connection/ConnectionProperties.java | 9 ++++ .../cloud/spanner/connection/SpannerPool.java | 7 +++ .../connection/ConnectionOptionsTest.java | 21 ++++++++ .../spanner/connection/SpannerPoolTest.java | 48 +++++++++++++++++++ 5 files changed, 99 insertions(+) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java index dcc4c663bb3..5c2ae2d1d07 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java @@ -26,6 +26,7 @@ import static com.google.cloud.spanner.connection.ConnectionProperties.DATA_BOOST_ENABLED; import static com.google.cloud.spanner.connection.ConnectionProperties.DIALECT; import static com.google.cloud.spanner.connection.ConnectionProperties.ENABLE_API_TRACING; +import static com.google.cloud.spanner.connection.ConnectionProperties.ENABLE_END_TO_END_TRACING; import static com.google.cloud.spanner.connection.ConnectionProperties.ENABLE_EXTENDED_TRACING; import static com.google.cloud.spanner.connection.ConnectionProperties.ENCODED_CREDENTIALS; import static com.google.cloud.spanner.connection.ConnectionProperties.ENDPOINT; @@ -239,6 +240,7 @@ public String[] getValidValues() { static final boolean DEFAULT_RETURN_COMMIT_STATS = false; static final boolean DEFAULT_LENIENT = false; static final boolean DEFAULT_ROUTE_TO_LEADER = true; + static final boolean DEFAULT_ENABLE_END_TO_END_TRACING = false; static final boolean DEFAULT_DELAY_TRANSACTION_START_UNTIL_FIRST_WRITE = false; static final boolean DEFAULT_KEEP_TRANSACTION_ALIVE = false; static final boolean DEFAULT_TRACK_SESSION_LEAKS = true; @@ -267,6 +269,7 @@ public String[] getValidValues() { /** Name of the 'routeToLeader' connection property. */ public static final String ROUTE_TO_LEADER_PROPERTY_NAME = "routeToLeader"; /** Name of the 'retry aborts internally' connection property. */ + public static final String ENABLE_END_TO_END_TRACING_PROPERTY_NAME = "enableEndToEndTracing"; public static final String RETRY_ABORTS_INTERNALLY_PROPERTY_NAME = "retryAbortsInternally"; /** Name of the property to enable/disable virtual threads for the statement executor. */ public static final String USE_VIRTUAL_THREADS_PROPERTY_NAME = "useVirtualThreads"; @@ -382,6 +385,10 @@ static boolean isEnableTransactionalConnectionStateForPostgreSQL() { ROUTE_TO_LEADER_PROPERTY_NAME, "Should read/write transactions and partitioned DML be routed to leader region (true/false)", DEFAULT_ROUTE_TO_LEADER), + ConnectionProperty.createBooleanProperty( + ENABLE_END_TO_END_TRACING_PROPERTY_NAME, + "Should we enable end to end tracing (true/false)", + DEFAULT_ENABLE_END_TO_END_TRACING), ConnectionProperty.createBooleanProperty( RETRY_ABORTS_INTERNALLY_PROPERTY_NAME, "Should the connection automatically retry Aborted errors (true/false)", @@ -1205,6 +1212,13 @@ public boolean isRouteToLeader() { return getInitialConnectionPropertyValue(ROUTE_TO_LEADER); } + /** + * Whether end-to-end tracing is enabled. + */ + public boolean enableEndToEndTracing() { + return getInitialConnectionPropertyValue(ENABLE_END_TO_END_TRACING); + } + /** * The initial retryAbortsInternally value for connections created by this {@link * ConnectionOptions} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java index c62571c6c03..aa1eff64a8d 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java @@ -39,6 +39,7 @@ import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_DDL_IN_TRANSACTION_MODE; import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_DELAY_TRANSACTION_START_UNTIL_FIRST_WRITE; import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_ENABLE_API_TRACING; +import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_ENABLE_END_TO_END_TRACING; import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_ENABLE_EXTENDED_TRACING; import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_ENDPOINT; import static com.google.cloud.spanner.connection.ConnectionOptions.DEFAULT_KEEP_TRANSACTION_ALIVE; @@ -65,6 +66,7 @@ import static com.google.cloud.spanner.connection.ConnectionOptions.DELAY_TRANSACTION_START_UNTIL_FIRST_WRITE_NAME; import static com.google.cloud.spanner.connection.ConnectionOptions.DIALECT_PROPERTY_NAME; import static com.google.cloud.spanner.connection.ConnectionOptions.ENABLE_API_TRACING_PROPERTY_NAME; +import static com.google.cloud.spanner.connection.ConnectionOptions.ENABLE_END_TO_END_TRACING_PROPERTY_NAME; import static com.google.cloud.spanner.connection.ConnectionOptions.ENABLE_EXTENDED_TRACING_PROPERTY_NAME; import static com.google.cloud.spanner.connection.ConnectionOptions.ENCODED_CREDENTIALS_PROPERTY_NAME; import static com.google.cloud.spanner.connection.ConnectionOptions.ENDPOINT_PROPERTY_NAME; @@ -257,6 +259,13 @@ class ConnectionProperties { DEFAULT_ROUTE_TO_LEADER, BooleanConverter.INSTANCE, Context.STARTUP); + static final ConnectionProperty ENABLE_END_TO_END_TRACING = + create( + ENABLE_END_TO_END_TRACING_PROPERTY_NAME, + "Should we enable end to end tracing (true/false)", + DEFAULT_ENABLE_END_TO_END_TRACING, + BooleanConverter.INSTANCE, + Context.STARTUP); static final ConnectionProperty USE_VIRTUAL_THREADS = create( USE_VIRTUAL_THREADS_PROPERTY_NAME, diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java index 246d340b070..1abf8628fad 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java @@ -156,6 +156,7 @@ static class SpannerPoolKey { private final String userAgent; private final String databaseRole; private final boolean routeToLeader; + private final boolean enableEndToEndTracing; private final boolean useVirtualGrpcTransportThreads; private final OpenTelemetry openTelemetry; private final Boolean enableExtendedTracing; @@ -186,6 +187,7 @@ private SpannerPoolKey(ConnectionOptions options) throws IOException { this.usePlainText = options.isUsePlainText(); this.userAgent = options.getUserAgent(); this.routeToLeader = options.isRouteToLeader(); + this.enableEndToEndTracing = options.enableEndToEndTracing(); this.useVirtualGrpcTransportThreads = options.isUseVirtualGrpcTransportThreads(); this.openTelemetry = options.getOpenTelemetry(); this.enableExtendedTracing = options.isEnableExtendedTracing(); @@ -207,6 +209,7 @@ public boolean equals(Object o) { && Objects.equals(this.usePlainText, other.usePlainText) && Objects.equals(this.userAgent, other.userAgent) && Objects.equals(this.routeToLeader, other.routeToLeader) + && Objects.equals(this.enableEndToEndTracing, other.enableEndToEndTracing) && Objects.equals( this.useVirtualGrpcTransportThreads, other.useVirtualGrpcTransportThreads) && Objects.equals(this.openTelemetry, other.openTelemetry) @@ -226,6 +229,7 @@ public int hashCode() { this.databaseRole, this.userAgent, this.routeToLeader, + this.enableEndToEndTracing, this.useVirtualGrpcTransportThreads, this.openTelemetry, this.enableExtendedTracing, @@ -380,6 +384,9 @@ Spanner createSpanner(SpannerPoolKey key, ConnectionOptions options) { if (!options.isRouteToLeader()) { builder.disableLeaderAwareRouting(); } + if (options.enableEndToEndTracing()) { + builder.setEnableEndToEndTracing(true); + } if (key.usePlainText) { // Credentials may not be sent over a plain text channel. builder.setCredentials(NoCredentials.getInstance()); diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionOptionsTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionOptionsTest.java index 94a44579acf..b33188ff3c8 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionOptionsTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionOptionsTest.java @@ -322,6 +322,27 @@ public void testBuildWithRouteToLeader() { assertTrue(options.isRouteToLeader()); } + @Test + public void testBuildWithEndToEndTracingEnabled() { + final String BASE_URI = + "cloudspanner:/projects/test-project-123/instances/test-instance-123/databases/test-database-123"; + ConnectionOptions.Builder builder = ConnectionOptions.newBuilder(); + builder.setUri(BASE_URI + "?enableEndToEndTracing=true"); + builder.setCredentialsUrl(FILE_TEST_PATH); + ConnectionOptions options = builder.build(); + assertEquals(options.getHost(), DEFAULT_HOST); + assertEquals(options.getProjectId(), TEST_PROJECT); + assertEquals(options.getInstanceId(), TEST_INSTANCE); + assertEquals(options.getDatabaseName(), TEST_DATABASE); + assertTrue(options.enableEndToEndTracing()); + + // Test for default behavior for enableEndToEndTracing property. + builder = ConnectionOptions.newBuilder().setUri(BASE_URI); + builder.setCredentialsUrl(FILE_TEST_PATH); + options = builder.build(); + assertFalse(options.enableEndToEndTracing()); + } + @Test public void testBuildWithAutoConfigEmulatorAndHost() { ConnectionOptions.Builder builder = ConnectionOptions.newBuilder(); diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SpannerPoolTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SpannerPoolTest.java index 19d49139635..b3d405947d1 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SpannerPoolTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SpannerPoolTest.java @@ -563,6 +563,54 @@ public void testEnableApiTracing() { .build())); } + @Test + public void testEnableEndToEndTracing() { + SpannerPoolKey keyWithoutApiTracingConfig = + SpannerPoolKey.of( + ConnectionOptions.newBuilder() + .setUri("cloudspanner:/projects/p/instances/i/databases/d") + .setCredentials(NoCredentials.getInstance()) + .build()); + SpannerPoolKey keyWithApiTracingEnabled = + SpannerPoolKey.of( + ConnectionOptions.newBuilder() + .setUri("cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=true") + .setCredentials(NoCredentials.getInstance()) + .build()); + SpannerPoolKey keyWithApiTracingDisabled = + SpannerPoolKey.of( + ConnectionOptions.newBuilder() + .setUri("cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=false") + .setCredentials(NoCredentials.getInstance()) + .build()); + + assertNotEquals(keyWithoutApiTracingConfig, keyWithApiTracingEnabled); + assertEquals(keyWithoutApiTracingConfig, keyWithApiTracingDisabled); + assertNotEquals(keyWithApiTracingEnabled, keyWithApiTracingDisabled); + + assertEquals( + keyWithApiTracingEnabled, + SpannerPoolKey.of( + ConnectionOptions.newBuilder() + .setUri("cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=true") + .setCredentials(NoCredentials.getInstance()) + .build())); + assertEquals( + keyWithApiTracingDisabled, + SpannerPoolKey.of( + ConnectionOptions.newBuilder() + .setUri("cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=false") + .setCredentials(NoCredentials.getInstance()) + .build())); + assertEquals( + keyWithoutApiTracingConfig, + SpannerPoolKey.of( + ConnectionOptions.newBuilder() + .setUri("cloudspanner:/projects/p/instances/i/databases/d") + .setCredentials(NoCredentials.getInstance()) + .build())); + } + @Test public void testOpenTelemetry() { SpannerPool pool = createSubjectAndMocks(); From 846a00269e460c79702318254e1eef8cfdefc948 Mon Sep 17 00:00:00 2001 From: ankiaga Date: Tue, 15 Oct 2024 17:02:40 +0530 Subject: [PATCH 2/9] fix formatting --- .../cloud/spanner/connection/ConnectionOptions.java | 5 ++--- .../cloud/spanner/connection/SpannerPoolTest.java | 12 ++++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java index 5c2ae2d1d07..8ee76635d48 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java @@ -270,6 +270,7 @@ public String[] getValidValues() { public static final String ROUTE_TO_LEADER_PROPERTY_NAME = "routeToLeader"; /** Name of the 'retry aborts internally' connection property. */ public static final String ENABLE_END_TO_END_TRACING_PROPERTY_NAME = "enableEndToEndTracing"; + public static final String RETRY_ABORTS_INTERNALLY_PROPERTY_NAME = "retryAbortsInternally"; /** Name of the property to enable/disable virtual threads for the statement executor. */ public static final String USE_VIRTUAL_THREADS_PROPERTY_NAME = "useVirtualThreads"; @@ -1212,9 +1213,7 @@ public boolean isRouteToLeader() { return getInitialConnectionPropertyValue(ROUTE_TO_LEADER); } - /** - * Whether end-to-end tracing is enabled. - */ + /** Whether end-to-end tracing is enabled. */ public boolean enableEndToEndTracing() { return getInitialConnectionPropertyValue(ENABLE_END_TO_END_TRACING); } diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SpannerPoolTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SpannerPoolTest.java index b3d405947d1..fea0b8aa6cf 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SpannerPoolTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/SpannerPoolTest.java @@ -574,13 +574,15 @@ public void testEnableEndToEndTracing() { SpannerPoolKey keyWithApiTracingEnabled = SpannerPoolKey.of( ConnectionOptions.newBuilder() - .setUri("cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=true") + .setUri( + "cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=true") .setCredentials(NoCredentials.getInstance()) .build()); SpannerPoolKey keyWithApiTracingDisabled = SpannerPoolKey.of( ConnectionOptions.newBuilder() - .setUri("cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=false") + .setUri( + "cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=false") .setCredentials(NoCredentials.getInstance()) .build()); @@ -592,14 +594,16 @@ public void testEnableEndToEndTracing() { keyWithApiTracingEnabled, SpannerPoolKey.of( ConnectionOptions.newBuilder() - .setUri("cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=true") + .setUri( + "cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=true") .setCredentials(NoCredentials.getInstance()) .build())); assertEquals( keyWithApiTracingDisabled, SpannerPoolKey.of( ConnectionOptions.newBuilder() - .setUri("cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=false") + .setUri( + "cloudspanner:/projects/p/instances/i/databases/d?enableEndToEndTracing=false") .setCredentials(NoCredentials.getInstance()) .build())); assertEquals( From 39d392b5db17e9301b569ac1c86177b079a09ce2 Mon Sep 17 00:00:00 2001 From: ankiaga Date: Mon, 21 Oct 2024 15:38:57 +0530 Subject: [PATCH 3/9] Comments incorporated --- .../spanner/connection/ConnectionOptions.java | 24 +++++++++++-------- .../connection/ConnectionProperties.java | 17 +++++++------ .../cloud/spanner/connection/SpannerPool.java | 6 ++--- .../connection/ConnectionOptionsTest.java | 4 ++-- 4 files changed, 29 insertions(+), 22 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java index 8ee76635d48..6c215e05fc4 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java @@ -240,7 +240,6 @@ public String[] getValidValues() { static final boolean DEFAULT_RETURN_COMMIT_STATS = false; static final boolean DEFAULT_LENIENT = false; static final boolean DEFAULT_ROUTE_TO_LEADER = true; - static final boolean DEFAULT_ENABLE_END_TO_END_TRACING = false; static final boolean DEFAULT_DELAY_TRANSACTION_START_UNTIL_FIRST_WRITE = false; static final boolean DEFAULT_KEEP_TRANSACTION_ALIVE = false; static final boolean DEFAULT_TRACK_SESSION_LEAKS = true; @@ -251,6 +250,7 @@ public String[] getValidValues() { static final int DEFAULT_MAX_PARTITIONED_PARALLELISM = 1; static final Boolean DEFAULT_ENABLE_EXTENDED_TRACING = null; static final Boolean DEFAULT_ENABLE_API_TRACING = null; + static final boolean DEFAULT_ENABLE_END_TO_END_TRACING = false; static final boolean DEFAULT_AUTO_BATCH_DML = false; static final long DEFAULT_AUTO_BATCH_DML_UPDATE_COUNT = 1L; static final boolean DEFAULT_AUTO_BATCH_DML_UPDATE_COUNT_VERIFICATION = true; @@ -269,8 +269,6 @@ public String[] getValidValues() { /** Name of the 'routeToLeader' connection property. */ public static final String ROUTE_TO_LEADER_PROPERTY_NAME = "routeToLeader"; /** Name of the 'retry aborts internally' connection property. */ - public static final String ENABLE_END_TO_END_TRACING_PROPERTY_NAME = "enableEndToEndTracing"; - public static final String RETRY_ABORTS_INTERNALLY_PROPERTY_NAME = "retryAbortsInternally"; /** Name of the property to enable/disable virtual threads for the statement executor. */ public static final String USE_VIRTUAL_THREADS_PROPERTY_NAME = "useVirtualThreads"; @@ -339,6 +337,7 @@ public String[] getValidValues() { public static final String ENABLE_EXTENDED_TRACING_PROPERTY_NAME = "enableExtendedTracing"; public static final String ENABLE_API_TRACING_PROPERTY_NAME = "enableApiTracing"; + public static final String ENABLE_END_TO_END_TRACING_PROPERTY_NAME = "enableEndToEndTracing"; public static final String AUTO_BATCH_DML_PROPERTY_NAME = "auto_batch_dml"; public static final String AUTO_BATCH_DML_UPDATE_COUNT_PROPERTY_NAME = @@ -386,10 +385,6 @@ static boolean isEnableTransactionalConnectionStateForPostgreSQL() { ROUTE_TO_LEADER_PROPERTY_NAME, "Should read/write transactions and partitioned DML be routed to leader region (true/false)", DEFAULT_ROUTE_TO_LEADER), - ConnectionProperty.createBooleanProperty( - ENABLE_END_TO_END_TRACING_PROPERTY_NAME, - "Should we enable end to end tracing (true/false)", - DEFAULT_ENABLE_END_TO_END_TRACING), ConnectionProperty.createBooleanProperty( RETRY_ABORTS_INTERNALLY_PROPERTY_NAME, "Should the connection automatically retry Aborted errors (true/false)", @@ -545,7 +540,14 @@ static boolean isEnableTransactionalConnectionStateForPostgreSQL() { + "to get a detailed view of each RPC that is being executed by your application, " + "or if you want to debug potential latency problems caused by RPCs that are " + "being retried.", - DEFAULT_ENABLE_API_TRACING)))); + DEFAULT_ENABLE_API_TRACING), + ConnectionProperty.createBooleanProperty( + ENABLE_END_TO_END_TRACING_PROPERTY_NAME, + "Enable end-to-end tracing (true/false) to generate traces for both the time " + + "that is spent in the client, as well as time that is spent in the Spanner server. " + + "Server side traces would always go to Google Cloud Trace so to see end to end traces, " + + "client should choose an exporter that exports the traces to Google Cloud Trace.", + DEFAULT_ENABLE_END_TO_END_TRACING)))); private static final Set INTERNAL_PROPERTIES = Collections.unmodifiableSet( @@ -1213,8 +1215,10 @@ public boolean isRouteToLeader() { return getInitialConnectionPropertyValue(ROUTE_TO_LEADER); } - /** Whether end-to-end tracing is enabled. */ - public boolean enableEndToEndTracing() { + /** + * Whether end-to-end tracing is enabled. + */ + public boolean isEndToEndTracingEnabled() { return getInitialConnectionPropertyValue(ENABLE_END_TO_END_TRACING); } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java index aa1eff64a8d..889d5328fee 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java @@ -259,13 +259,6 @@ class ConnectionProperties { DEFAULT_ROUTE_TO_LEADER, BooleanConverter.INSTANCE, Context.STARTUP); - static final ConnectionProperty ENABLE_END_TO_END_TRACING = - create( - ENABLE_END_TO_END_TRACING_PROPERTY_NAME, - "Should we enable end to end tracing (true/false)", - DEFAULT_ENABLE_END_TO_END_TRACING, - BooleanConverter.INSTANCE, - Context.STARTUP); static final ConnectionProperty USE_VIRTUAL_THREADS = create( USE_VIRTUAL_THREADS_PROPERTY_NAME, @@ -301,6 +294,16 @@ class ConnectionProperties { DEFAULT_ENABLE_API_TRACING, BooleanConverter.INSTANCE, Context.STARTUP); + static final ConnectionProperty ENABLE_END_TO_END_TRACING = + create( + ENABLE_END_TO_END_TRACING_PROPERTY_NAME, + "Enable end-to-end tracing (true/false) to generate traces for both the time " + + "that is spent in the client, as well as time that is spent in the Spanner server. " + + "Server side traces would always go to Google Cloud Trace so to see end to end traces, " + + "client should choose an exporter that exports the traces to Google Cloud Trace.", + DEFAULT_ENABLE_END_TO_END_TRACING, + BooleanConverter.INSTANCE, + Context.STARTUP); static final ConnectionProperty MIN_SESSIONS = create( MIN_SESSIONS_PROPERTY_NAME, diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java index 1abf8628fad..22840045ef2 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java @@ -156,11 +156,11 @@ static class SpannerPoolKey { private final String userAgent; private final String databaseRole; private final boolean routeToLeader; - private final boolean enableEndToEndTracing; private final boolean useVirtualGrpcTransportThreads; private final OpenTelemetry openTelemetry; private final Boolean enableExtendedTracing; private final Boolean enableApiTracing; + private final boolean enableEndToEndTracing; @VisibleForTesting static SpannerPoolKey of(ConnectionOptions options) { @@ -187,11 +187,11 @@ private SpannerPoolKey(ConnectionOptions options) throws IOException { this.usePlainText = options.isUsePlainText(); this.userAgent = options.getUserAgent(); this.routeToLeader = options.isRouteToLeader(); - this.enableEndToEndTracing = options.enableEndToEndTracing(); this.useVirtualGrpcTransportThreads = options.isUseVirtualGrpcTransportThreads(); this.openTelemetry = options.getOpenTelemetry(); this.enableExtendedTracing = options.isEnableExtendedTracing(); this.enableApiTracing = options.isEnableApiTracing(); + this.enableEndToEndTracing = options.isEndToEndTracingEnabled(); } @Override @@ -384,7 +384,7 @@ Spanner createSpanner(SpannerPoolKey key, ConnectionOptions options) { if (!options.isRouteToLeader()) { builder.disableLeaderAwareRouting(); } - if (options.enableEndToEndTracing()) { + if (options.isEndToEndTracingEnabled()) { builder.setEnableEndToEndTracing(true); } if (key.usePlainText) { diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionOptionsTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionOptionsTest.java index b33188ff3c8..f826ec08dfc 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionOptionsTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/ConnectionOptionsTest.java @@ -334,13 +334,13 @@ public void testBuildWithEndToEndTracingEnabled() { assertEquals(options.getProjectId(), TEST_PROJECT); assertEquals(options.getInstanceId(), TEST_INSTANCE); assertEquals(options.getDatabaseName(), TEST_DATABASE); - assertTrue(options.enableEndToEndTracing()); + assertTrue(options.isEndToEndTracingEnabled()); // Test for default behavior for enableEndToEndTracing property. builder = ConnectionOptions.newBuilder().setUri(BASE_URI); builder.setCredentialsUrl(FILE_TEST_PATH); options = builder.build(); - assertFalse(options.enableEndToEndTracing()); + assertFalse(options.isEndToEndTracingEnabled()); } @Test From 751c9473d5d03d4ac70b405dd49045231500a9cb Mon Sep 17 00:00:00 2001 From: ankiaga Date: Mon, 21 Oct 2024 15:45:38 +0530 Subject: [PATCH 4/9] formatting fixed --- .../google/cloud/spanner/connection/SpannerPool.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java index 22840045ef2..ed26f76659a 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java @@ -209,12 +209,12 @@ public boolean equals(Object o) { && Objects.equals(this.usePlainText, other.usePlainText) && Objects.equals(this.userAgent, other.userAgent) && Objects.equals(this.routeToLeader, other.routeToLeader) - && Objects.equals(this.enableEndToEndTracing, other.enableEndToEndTracing) && Objects.equals( - this.useVirtualGrpcTransportThreads, other.useVirtualGrpcTransportThreads) + this.useVirtualGrpcTransportThreads, other.useVirtualGrpcTransportThreads) && Objects.equals(this.openTelemetry, other.openTelemetry) && Objects.equals(this.enableExtendedTracing, other.enableExtendedTracing) - && Objects.equals(this.enableApiTracing, other.enableApiTracing); + && Objects.equals(this.enableApiTracing, other.enableApiTracing) + && Objects.equals(this.enableEndToEndTracing, other.enableEndToEndTracing); } @Override @@ -229,11 +229,11 @@ public int hashCode() { this.databaseRole, this.userAgent, this.routeToLeader, - this.enableEndToEndTracing, this.useVirtualGrpcTransportThreads, this.openTelemetry, this.enableExtendedTracing, - this.enableApiTracing); + this.enableApiTracing, + this.enableEndToEndTracing); } } From e4f86e88bcb5ea8a446df2929f3fbe113d10b37a Mon Sep 17 00:00:00 2001 From: ankiaga Date: Mon, 21 Oct 2024 16:57:11 +0530 Subject: [PATCH 5/9] formatting fixed --- .../google/cloud/spanner/connection/ConnectionOptions.java | 4 +--- .../java/com/google/cloud/spanner/connection/SpannerPool.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java index 6c215e05fc4..e4cb67b24e2 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java @@ -1215,9 +1215,7 @@ public boolean isRouteToLeader() { return getInitialConnectionPropertyValue(ROUTE_TO_LEADER); } - /** - * Whether end-to-end tracing is enabled. - */ + /** Whether end-to-end tracing is enabled. */ public boolean isEndToEndTracingEnabled() { return getInitialConnectionPropertyValue(ENABLE_END_TO_END_TRACING); } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java index ed26f76659a..81246e41938 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/SpannerPool.java @@ -210,7 +210,7 @@ public boolean equals(Object o) { && Objects.equals(this.userAgent, other.userAgent) && Objects.equals(this.routeToLeader, other.routeToLeader) && Objects.equals( - this.useVirtualGrpcTransportThreads, other.useVirtualGrpcTransportThreads) + this.useVirtualGrpcTransportThreads, other.useVirtualGrpcTransportThreads) && Objects.equals(this.openTelemetry, other.openTelemetry) && Objects.equals(this.enableExtendedTracing, other.enableExtendedTracing) && Objects.equals(this.enableApiTracing, other.enableApiTracing) From 5de71f8013328db9639d3c78b75d446c005969e1 Mon Sep 17 00:00:00 2001 From: Ankit Agarwal <146331865+ankiaga@users.noreply.github.com> Date: Tue, 22 Oct 2024 08:52:50 +0530 Subject: [PATCH 6/9] Update google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Knut Olav Løite --- .../google/cloud/spanner/connection/ConnectionOptions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java index e4cb67b24e2..66b09c35afd 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionOptions.java @@ -545,8 +545,8 @@ static boolean isEnableTransactionalConnectionStateForPostgreSQL() { ENABLE_END_TO_END_TRACING_PROPERTY_NAME, "Enable end-to-end tracing (true/false) to generate traces for both the time " + "that is spent in the client, as well as time that is spent in the Spanner server. " - + "Server side traces would always go to Google Cloud Trace so to see end to end traces, " - + "client should choose an exporter that exports the traces to Google Cloud Trace.", + + "Server side traces can only go to Google Cloud Trace, so to see end to end traces, " + + "the application should configure an exporter that exports the traces to Google Cloud Trace.", DEFAULT_ENABLE_END_TO_END_TRACING)))); private static final Set INTERNAL_PROPERTIES = From bf3361c15e7e6e6e0d250327200b46f448e13b24 Mon Sep 17 00:00:00 2001 From: Ankit Agarwal <146331865+ankiaga@users.noreply.github.com> Date: Tue, 22 Oct 2024 08:52:56 +0530 Subject: [PATCH 7/9] Update google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Knut Olav Løite --- .../google/cloud/spanner/connection/ConnectionProperties.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java index 889d5328fee..0ca9b7256e2 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/ConnectionProperties.java @@ -299,8 +299,8 @@ class ConnectionProperties { ENABLE_END_TO_END_TRACING_PROPERTY_NAME, "Enable end-to-end tracing (true/false) to generate traces for both the time " + "that is spent in the client, as well as time that is spent in the Spanner server. " - + "Server side traces would always go to Google Cloud Trace so to see end to end traces, " - + "client should choose an exporter that exports the traces to Google Cloud Trace.", + + "Server side traces can only go to Google Cloud Trace, so to see end to end traces, " + + "the application should configure an exporter that exports the traces to Google Cloud Trace.", DEFAULT_ENABLE_END_TO_END_TRACING, BooleanConverter.INSTANCE, Context.STARTUP); From 39e040382a87ac5d8726e4c058ceb1a0e8c5b878 Mon Sep 17 00:00:00 2001 From: ankiaga Date: Tue, 22 Oct 2024 15:50:25 +0530 Subject: [PATCH 8/9] Test changes --- google-cloud-spanner/pom.xml | 32 ++++- .../google/cloud/spanner/it/ITQueryTest.java | 110 +++++++++++++++++- 2 files changed, 137 insertions(+), 5 deletions(-) diff --git a/google-cloud-spanner/pom.xml b/google-cloud-spanner/pom.xml index 9719e081765..8a5a0aec719 100644 --- a/google-cloud-spanner/pom.xml +++ b/google-cloud-spanner/pom.xml @@ -17,8 +17,8 @@ google-cloud-spanner 0.31.1 com.google.cloud.spanner.GceTestEnvConfig - projects/gcloud-devel/instances/spanner-testing-east1 - gcloud-devel + projects/span-cloud-testing/instances/asapha-test + span-cloud-testing projects/gcloud-devel/locations/us-east1/keyRings/cmek-test-key-ring/cryptoKeys/cmek-test-key @@ -445,6 +445,16 @@ test + + io.opentelemetry + opentelemetry-sdk + test + + + io.opentelemetry + opentelemetry-sdk-common + test + io.opentelemetry opentelemetry-sdk-trace @@ -455,6 +465,24 @@ opentelemetry-sdk-testing test + + com.google.cloud.opentelemetry + exporter-trace + 0.31.0 + test + + + com.google.cloud + google-cloud-trace + 2.52.0 + test + + + com.google.api.grpc + proto-google-cloud-trace-v1 + 2.52.0 + test + diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITQueryTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITQueryTest.java index 18044c452b5..4a6cc36db95 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITQueryTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITQueryTest.java @@ -27,6 +27,7 @@ import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; +import com.google.api.gax.core.FixedCredentialsProvider; import com.google.cloud.ByteArray; import com.google.cloud.Date; import com.google.cloud.Timestamp; @@ -37,9 +38,13 @@ import com.google.cloud.spanner.IntegrationTestEnv; import com.google.cloud.spanner.Mutation; import com.google.cloud.spanner.ParallelIntegrationTest; +import com.google.cloud.opentelemetry.trace.TraceConfiguration; +import com.google.cloud.opentelemetry.trace.TraceExporter; import com.google.cloud.spanner.ReadContext.QueryAnalyzeMode; import com.google.cloud.spanner.ResultSet; import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.SpannerOptions; +import com.google.cloud.spanner.SpannerOptionsHelper; import com.google.cloud.spanner.Statement; import com.google.cloud.spanner.Struct; import com.google.cloud.spanner.TimestampBound; @@ -48,14 +53,33 @@ import com.google.cloud.spanner.Value; import com.google.cloud.spanner.connection.ConnectionOptions; import com.google.cloud.spanner.testing.EmulatorSpannerHelper; +import com.google.cloud.trace.v1.TraceServiceClient; +import com.google.cloud.trace.v1.TraceServiceClient.ListTracesPagedResponse; +import com.google.cloud.trace.v1.TraceServiceSettings; import com.google.common.base.Joiner; import com.google.common.collect.Iterables; +import com.google.devtools.cloudtrace.v1.ListTracesRequest; +import com.google.devtools.cloudtrace.v1.Trace; import com.google.spanner.v1.ResultSetStats; +import io.opentelemetry.api.incubator.trace.ExtendedTracer; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Scope; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.io.IOException; import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.ThreadLocalRandom; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -70,9 +94,16 @@ @Category(ParallelIntegrationTest.class) @RunWith(Parameterized.class) public class ITQueryTest { + static { + SpannerOptionsHelper.resetActiveTracingFramework(); + SpannerOptions.enableOpenTelemetryMetrics(); + SpannerOptions.enableOpenTelemetryTraces(); + } + @ClassRule public static IntegrationTestEnv env = new IntegrationTestEnv(); private static DatabaseClient googleStandardSQLClient; private static DatabaseClient postgreSQLClient; + private static OpenTelemetrySdk openTelemetry; private String selectValueQuery; @BeforeClass @@ -87,6 +118,38 @@ public static void setUpDatabase() { } } + @BeforeClass + public static void setupOpenTelemetry() { + assumeFalse("This test requires credentials", EmulatorSpannerHelper.isUsingEmulator()); + + SpannerOptions options = env.getTestHelper().getOptions(); + TraceConfiguration.Builder traceConfigurationBuilder = TraceConfiguration.builder(); + if (options.getCredentials() != null) { + traceConfigurationBuilder.setCredentials(options.getCredentials()); + } + TraceConfiguration traceConfiguration = + traceConfigurationBuilder.setProjectId(options.getProjectId()).build(); + SpanExporter traceExporter = TraceExporter.createWithConfiguration(traceConfiguration); + + String serviceName = + "java-spanner-jdbc-integration-tests-" + ThreadLocalRandom.current().nextInt(); + openTelemetry = + OpenTelemetrySdk.builder() + .setTracerProvider( + SdkTracerProvider.builder() + // Always sample in this test to ensure we know what we get. + .setSampler(Sampler.alwaysOn()) + .setResource(Resource.builder().put("service.name", serviceName).build()) + .addSpanProcessor(BatchSpanProcessor.builder(traceExporter).build()) + .build()) + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .build(); + options.toBuilder().setOpenTelemetry(openTelemetry).setEnableEndToEndTracing(true).build(); + // TODO: Remove when the bug in OpenTelemetry that has SdkTracer implement ExtendedTracer, + // which is only available in the incubator project. + ExtendedTracer ignore = (ExtendedTracer) openTelemetry.getTracer("foo"); + } + @AfterClass public static void teardown() { ConnectionOptions.closeSpanner(); @@ -122,11 +185,46 @@ private DatabaseClient getClient(Dialect dialect) { } @Test - public void simple() { - Struct row = execute(Statement.of("SELECT 1"), Type.int64()); + public void simple() throws InterruptedException, IOException { + String sql = "SELECT 1"; + Struct row = execute(Statement.of(sql), Type.int64()); assertThat(row.getLong(0)).isEqualTo(1); } + private void assertTrace() throws IOException, InterruptedException { + com.google.protobuf.Timestamp timestamp = Timestamp.now().toProto(); + TraceServiceSettings settings = + env.getTestHelper().getOptions().getCredentials() == null + ? TraceServiceSettings.newBuilder().build() + : TraceServiceSettings.newBuilder() + .setCredentialsProvider( + FixedCredentialsProvider.create( + env.getTestHelper().getOptions().getCredentials())) + .build(); + TraceServiceClient client = TraceServiceClient.create(settings); + // It can take a few seconds before the trace is visible. + Thread.sleep(5000L); + boolean foundTrace = false; + for (int attempts = 0; attempts < 2; attempts++) { + ListTracesPagedResponse response = + client.listTraces( + ListTracesRequest.newBuilder() + .setProjectId(env.getTestHelper().getInstanceId().getProject()) + .setFilter("span:arrayOfStruct") + // .setStartTime(timestamp) + .build()); + int size = Iterables.size(response.iterateAll()); + if (size != 0) { + assertEquals(1, size); + foundTrace = true; + break; + } else { + Thread.sleep(5000L); + } + } + assertTrue(foundTrace); + } + @Test public void badQuery() { try { @@ -143,7 +241,10 @@ public void badQuery() { } @Test - public void arrayOfStruct() { + public void arrayOfStruct() throws IOException, InterruptedException { + Tracer tracer = openTelemetry.getTracer(ITQueryTest.class.getName()); + Span span = tracer.spanBuilder("arrayOfStruct").startSpan(); + Scope scope = span.makeCurrent(); assumeFalse("structs are not supported on POSTGRESQL", dialect.dialect == Dialect.POSTGRESQL); Type structType = Type.struct(StructField.of("C1", Type.string()), StructField.of("C2", Type.int64())); @@ -154,6 +255,8 @@ public void arrayOfStruct() { + "FROM (SELECT 'a' AS C1, 1 AS C2 UNION ALL SELECT 'b' AS C1, 2 AS C2) " + "ORDER BY C1 ASC)"), Type.array(structType)); + scope.close(); + span.end(); assertThat(row.isNull(0)).isFalse(); List value = row.getStructList(0); assertThat(value.size()).isEqualTo(2); @@ -178,6 +281,7 @@ public void arrayOfStruct() { Struct.newBuilder().set("C1").to("b").set("C2").to(2).build())) .build(); assertThat(row).isEqualTo(expectedRow); + assertTrace(); } @Test From c2c0e6d238dc89972f46ca6694f2d49603d5413c Mon Sep 17 00:00:00 2001 From: ankiaga Date: Thu, 24 Oct 2024 11:31:11 +0530 Subject: [PATCH 9/9] Test changes --- .../google/cloud/spanner/it/ITQueryTest.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITQueryTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITQueryTest.java index 4a6cc36db95..6f62cc55b77 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITQueryTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITQueryTest.java @@ -61,6 +61,7 @@ import com.google.devtools.cloudtrace.v1.ListTracesRequest; import com.google.devtools.cloudtrace.v1.Trace; import com.google.spanner.v1.ResultSetStats; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.incubator.trace.ExtendedTracer; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; @@ -80,6 +81,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -116,12 +118,12 @@ public static void setUpDatabase() { env.getTestHelper().createTestDatabase(Dialect.POSTGRESQL, Collections.emptyList()); postgreSQLClient = env.getTestHelper().getDatabaseClient(postgreSQLDatabase); } + setupOpenTelemetry(); } - @BeforeClass public static void setupOpenTelemetry() { assumeFalse("This test requires credentials", EmulatorSpannerHelper.isUsingEmulator()); - + // GlobalOpenTelemetry.set(openTelemetry); SpannerOptions options = env.getTestHelper().getOptions(); TraceConfiguration.Builder traceConfigurationBuilder = TraceConfiguration.builder(); if (options.getCredentials() != null) { @@ -144,15 +146,26 @@ public static void setupOpenTelemetry() { .build()) .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) .build(); - options.toBuilder().setOpenTelemetry(openTelemetry).setEnableEndToEndTracing(true).build(); + SpannerOptions.enableOpenTelemetryTraces(); + options.toBuilder() + .setOpenTelemetry(openTelemetry) + .setEnableApiTracing(true) + .setEnableEndToEndTracing(true).build(); // TODO: Remove when the bug in OpenTelemetry that has SdkTracer implement ExtendedTracer, // which is only available in the incubator project. ExtendedTracer ignore = (ExtendedTracer) openTelemetry.getTracer("foo"); } + public static void closeOpenTelemetry() { + if (openTelemetry != null) { + openTelemetry.close(); + } + } + @AfterClass public static void teardown() { ConnectionOptions.closeSpanner(); + closeOpenTelemetry(); } @Before @@ -192,7 +205,7 @@ public void simple() throws InterruptedException, IOException { } private void assertTrace() throws IOException, InterruptedException { - com.google.protobuf.Timestamp timestamp = Timestamp.now().toProto(); + openTelemetry.getSdkTracerProvider().forceFlush().join(10, TimeUnit.SECONDS); TraceServiceSettings settings = env.getTestHelper().getOptions().getCredentials() == null ? TraceServiceSettings.newBuilder().build()