Skip to content

Commit 31936ef

Browse files
BenWhiteheadBenWhitehead
committed
fix: update Grpc Retry Conformance after new additions to testbench
1 parent 9b0253c commit 31936ef

8 files changed

Lines changed: 102 additions & 46 deletions

File tree

‎google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcRetryAlgorithmManager.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import com.google.iam.v1.GetIamPolicyRequest;
2121
import com.google.iam.v1.SetIamPolicyRequest;
2222
import com.google.iam.v1.TestIamPermissionsRequest;
23+
import com.google.protobuf.ByteString;
2324
import com.google.storage.v2.BidiWriteObjectRequest;
2425
import com.google.storage.v2.ComposeObjectRequest;
2526
import com.google.storage.v2.CreateBucketRequest;
@@ -86,11 +87,11 @@ public ResultRetryAlgorithm<?> getFor(CreateNotificationConfigRequest req) {
8687
}
8788

8889
public ResultRetryAlgorithm<?> getFor(DeleteBucketRequest req) {
89-
return retryStrategy.getNonidempotentHandler();
90+
return retryStrategy.getIdempotentHandler();
9091
}
9192

9293
public ResultRetryAlgorithm<?> getFor(DeleteHmacKeyRequest req) {
93-
return retryStrategy.getNonidempotentHandler();
94+
return retryStrategy.getIdempotentHandler();
9495
}
9596

9697
public ResultRetryAlgorithm<?> getFor(DeleteNotificationConfigRequest req) {
@@ -168,8 +169,11 @@ public ResultRetryAlgorithm<?> getFor(RewriteObjectRequest req) {
168169
}
169170

170171
public ResultRetryAlgorithm<?> getFor(SetIamPolicyRequest req) {
171-
// TODO: etag
172-
return retryStrategy.getNonidempotentHandler();
172+
if (req.getPolicy().getEtag().equals(ByteString.empty())) {
173+
return retryStrategy.getNonidempotentHandler();
174+
} else {
175+
return retryStrategy.getIdempotentHandler();
176+
}
173177
}
174178

175179
public ResultRetryAlgorithm<?> getFor(StartResumableWriteRequest req) {

‎google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java‎

Lines changed: 49 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -262,8 +262,32 @@ public Blob create(
262262
@Override
263263
public Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) {
264264
try {
265-
return createFrom(blobInfo, content, options);
266-
} catch (IOException e) {
265+
requireNonNull(blobInfo, "blobInfo must be non null");
266+
InputStream inputStreamParam = firstNonNull(content, new ByteArrayInputStream(ZERO_BYTES));
267+
268+
Opts<ObjectTargetOpt> optsWithDefaults = Opts.unwrap(options).prepend(defaultOpts);
269+
GrpcCallContext grpcCallContext =
270+
optsWithDefaults.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
271+
WriteObjectRequest req = getWriteObjectRequest(blobInfo, optsWithDefaults);
272+
Hasher hasher = Hasher.enabled();
273+
GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
274+
UnbufferedWritableByteChannelSession<WriteObjectResponse> session =
275+
ResumableMedia.gapic()
276+
.write()
277+
.byteChannel(storageClient.writeObjectCallable().withDefaultCallContext(merge))
278+
.setByteStringStrategy(ByteStringStrategy.noCopy())
279+
.setHasher(hasher)
280+
.direct()
281+
.unbuffered()
282+
.setRequest(req)
283+
.build();
284+
285+
try (UnbufferedWritableByteChannel c = session.open()) {
286+
ByteStreams.copy(Channels.newChannel(inputStreamParam), c);
287+
}
288+
ApiFuture<WriteObjectResponse> responseApiFuture = session.getResult();
289+
return this.getBlob(responseApiFuture);
290+
} catch (IOException | ApiException e) {
267291
throw StorageException.coalesce(e);
268292
}
269293
}
@@ -549,17 +573,20 @@ public boolean delete(String bucket, BucketSourceOption... options) {
549573
DeleteBucketRequest.Builder builder =
550574
DeleteBucketRequest.newBuilder().setName(bucketNameCodec.encode(bucket));
551575
DeleteBucketRequest req = opts.deleteBucketsRequest().apply(builder).build();
552-
try {
553-
GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
554-
Retrying.run(
555-
getOptions(),
556-
retryAlgorithmManager.getFor(req),
557-
() -> storageClient.deleteBucketCallable().call(req, merge),
558-
Decoder.identity());
559-
return true;
560-
} catch (StorageException e) {
561-
return false;
562-
}
576+
GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
577+
return Boolean.TRUE.equals(
578+
Retrying.run(
579+
getOptions(),
580+
retryAlgorithmManager.getFor(req),
581+
() -> {
582+
try {
583+
storageClient.deleteBucketCallable().call(req, merge);
584+
return true;
585+
} catch (NotFoundException e) {
586+
return false;
587+
}
588+
},
589+
Decoder.identity()));
563590
}
564591

565592
@Override
@@ -760,11 +787,19 @@ public GrpcBlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options
760787
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
761788
WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts);
762789
Hasher hasher = Hasher.noop();
790+
// in JSON, the starting of the resumable session happens before the invocation of write can
791+
// happen. Emulate the same thing here.
792+
// 1. create the future
793+
ApiFuture<ResumableWrite> startResumableWrite = startResumableWrite(grpcCallContext, req);
794+
// 2. await the result of the future
795+
ResumableWrite resumableWrite = ApiFutureUtils.await(startResumableWrite);
796+
// 3. wrap the result in another future container before constructing the BlobWriteChannel
797+
ApiFuture<ResumableWrite> wrapped = ApiFutures.immediateFuture(resumableWrite);
763798
return new GrpcBlobWriteChannel(
764799
storageClient.writeObjectCallable(),
765800
getOptions(),
766801
retryAlgorithmManager.idempotent(),
767-
() -> startResumableWrite(grpcCallContext, req),
802+
() -> wrapped,
768803
hasher);
769804
}
770805

‎google-cloud-storage/src/test/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import com.google.api.core.ApiFutures;
2121
import com.google.cloud.ReadChannel;
2222
import com.google.cloud.WriteChannel;
23-
import com.google.cloud.storage.BucketInfo.BuilderImpl;
2423
import com.google.common.collect.ImmutableList;
2524
import java.util.Optional;
2625
import java.util.concurrent.ExecutionException;
@@ -40,23 +39,11 @@ public final class PackagePrivateMethodWorkarounds {
4039
private PackagePrivateMethodWorkarounds() {}
4140

4241
public static Bucket bucketCopyWithStorage(Bucket b, Storage s) {
43-
BucketInfo.BuilderImpl builder =
44-
(BuilderImpl)
45-
Conversions.json()
46-
.bucketInfo()
47-
.decode(Conversions.json().bucketInfo().encode(b))
48-
.toBuilder();
49-
return new Bucket(s, builder);
42+
return b.asBucket(s);
5043
}
5144

5245
public static Blob blobCopyWithStorage(Blob b, Storage s) {
53-
BlobInfo.BuilderImpl builder =
54-
(BlobInfo.BuilderImpl)
55-
Conversions.json()
56-
.blobInfo()
57-
.decode(Conversions.json().blobInfo().encode(b))
58-
.toBuilder();
59-
return new Blob(s, builder);
46+
return b.asBlob(s);
6047
}
6148

6249
public static Function<WriteChannel, Optional<BlobInfo>> maybeGetBlobInfoFunction() {

‎google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/CtxFunctions.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,10 @@ static final class Local {
136136
static final class Rpc {
137137
static final CtxFunction createEmptyBlob =
138138
(ctx, c) -> ctx.map(state -> state.with(ctx.getStorage().create(state.getBlobInfo())));
139+
static final CtxFunction bucketIamPolicy =
140+
(ctx, c) ->
141+
ctx.map(
142+
state -> state.with(ctx.getStorage().getIamPolicy(state.getBucket().getName())));
139143
}
140144

141145
static final class ResourceSetup {

‎google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import static com.google.cloud.storage.PackagePrivateMethodWorkarounds.blobCopyWithStorage;
2020
import static com.google.cloud.storage.PackagePrivateMethodWorkarounds.bucketCopyWithStorage;
2121
import static com.google.cloud.storage.conformance.retry.Ctx.ctx;
22+
import static com.google.cloud.storage.conformance.retry.ITRetryConformanceTest.RetryTestCaseResolver.lift;
2223
import static com.google.cloud.storage.conformance.retry.State.empty;
2324
import static com.google.common.truth.Truth.assertThat;
2425
import static java.util.Objects.requireNonNull;
@@ -117,6 +118,7 @@ public void setUp() throws Throwable {
117118
public void tearDown() throws Throwable {
118119
LOGGER.fine("Running teardown...");
119120
if (ctx != null) {
121+
ctx = ctx.leftMap(s -> nonTestStorage);
120122
getReplaceStorageInObjectsFromCtx()
121123
.andThen(mapping.getTearDown())
122124
.apply(ctx, testRetryConformance);
@@ -168,7 +170,11 @@ public ImmutableList<?> parameters() {
168170
.setHost(testBench.getBaseUri().replaceAll("https?://", ""))
169171
.setTestAllowFilter(
170172
RetryTestCaseResolver.includeAll()
171-
.and(RetryTestCaseResolver.lift(trc -> trc.getTransport() == Transport.HTTP)))
173+
.and(
174+
(lift(trc -> trc.getTransport() == Transport.GRPC)
175+
.and(RetryTestCaseResolver.scenarioIdIs(2))
176+
.and((m, trc) -> m == RpcMethod.storage.buckets.setIamPolicy))
177+
.negate()))
172178
.build();
173179

174180
List<RetryTestCase> retryTestCases;

‎google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMapping.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,16 @@
2121
import static org.junit.Assert.fail;
2222

2323
import com.google.cloud.storage.StorageException;
24+
import com.google.cloud.storage.TransportCompatibility.Transport;
2425
import com.google.cloud.storage.conformance.retry.CtxFunctions.ResourceSetup;
2526
import com.google.cloud.storage.conformance.retry.CtxFunctions.ResourceTeardown;
2627
import com.google.cloud.storage.conformance.retry.Functions.CtxFunction;
2728
import com.google.common.base.Preconditions;
2829
import com.google.errorprone.annotations.Immutable;
2930
import java.util.HashSet;
3031
import java.util.function.Predicate;
32+
import java.util.logging.Level;
33+
import java.util.logging.Logger;
3134
import org.junit.AssumptionViolatedException;
3235

3336
/**
@@ -42,6 +45,7 @@
4245
*/
4346
@Immutable
4447
final class RpcMethodMapping {
48+
private static final Logger LOGGER = Logger.getLogger(RpcMethodMapping.class.getName());
4549

4650
private final int mappingId;
4751
private final RpcMethod method;
@@ -90,6 +94,7 @@ public CtxFunction getTest() {
9094
test.apply(ctx, c);
9195
fail("expected failure, but succeeded");
9296
} catch (StorageException e) {
97+
LOGGER.log(Level.CONFIG, "", e);
9398
// We expect an exception to be thrown by mapping and test retry conformance config
9499
// Verify that the exception we received is actually what we expect.
95100
boolean matchExpectedCode = false;
@@ -107,6 +112,14 @@ public CtxFunction getTest() {
107112
if (instructions.contains("return-reset-connection") && code == 0) {
108113
matchExpectedCode = true;
109114
}
115+
// testbench resetting the connection is turned into an UNAVAILABLE in grpc, which we then
116+
// map to 503. Add graceful handling here, since we can't disambiguate between reset
117+
// connection and 503 from the service.
118+
if (c.getTransport() == Transport.GRPC
119+
&& instructions.contains("return-reset-connection")
120+
&& code == 503) {
121+
matchExpectedCode = true;
122+
}
110123

111124
if (matchExpectedCode) {
112125
return ctx;

‎google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import com.google.cloud.storage.Storage.SignUrlOption;
5454
import com.google.cloud.storage.Storage.UriScheme;
5555
import com.google.cloud.storage.StorageRoles;
56+
import com.google.cloud.storage.TransportCompatibility.Transport;
5657
import com.google.cloud.storage.conformance.retry.CtxFunctions.Local;
5758
import com.google.cloud.storage.conformance.retry.CtxFunctions.ResourceSetup;
5859
import com.google.cloud.storage.conformance.retry.CtxFunctions.Rpc;
@@ -1607,7 +1608,9 @@ private static void insert(ArrayList<RpcMethodMapping> a) {
16071608
.build());
16081609
a.add(
16091610
RpcMethodMapping.newBuilder(54, objects.insert)
1610-
.withApplicable(not(TestRetryConformance::isPreconditionsProvided))
1611+
.withApplicable(
1612+
not(TestRetryConformance::isPreconditionsProvided)
1613+
.and(trc -> trc.getTransport() == Transport.HTTP))
16111614
.withSetup(defaultSetup.andThen(Local.blobInfoWithoutGeneration))
16121615
.withTest(
16131616
(ctx, c) ->

‎google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/TestRetryConformance.java‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import com.google.cloud.storage.DataGenerator;
2828
import com.google.cloud.storage.TransportCompatibility.Transport;
2929
import com.google.common.base.Joiner;
30-
import com.google.common.base.Suppliers;
3130
import com.google.common.io.ByteStreams;
3231
import com.google.errorprone.annotations.Immutable;
3332
import java.io.ByteArrayInputStream;
@@ -42,6 +41,7 @@
4241
import java.time.ZoneId;
4342
import java.time.ZoneOffset;
4443
import java.time.format.DateTimeFormatter;
44+
import java.util.List;
4545
import java.util.function.Supplier;
4646
import java.util.stream.Collectors;
4747

@@ -153,13 +153,11 @@ final class TestRetryConformance {
153153
String.format(
154154
"%s_s%03d-%s-m%03d_top1_%s",
155155
BASE_ID, scenarioId, instructionsString.toLowerCase(), mappingId, transportTag);
156+
// define a lazy supplier for bytes.
156157
this.lazyHelloWorldUtf8Bytes =
157-
Suppliers.memoize(
158-
() -> {
159-
// define a lazy supplier for bytes.
160-
return genBytes(method);
161-
});
162-
this.helloWorldFilePath = resolvePathForResource(objectName, method);
158+
() -> genBytes(this.method, this.instruction.getInstructionsList());
159+
this.helloWorldFilePath =
160+
resolvePathForResource(objectName, method, this.instruction.getInstructionsList());
163161
this.serviceAccountCredentials = resolveServiceAccountCredentials();
164162
}
165163

@@ -239,13 +237,14 @@ public String toString() {
239237
return getTestName();
240238
}
241239

242-
private static Supplier<Path> resolvePathForResource(String objectName, Method method) {
240+
private static Supplier<Path> resolvePathForResource(
241+
String objectName, Method method, List<String> instructionList) {
243242
return () -> {
244243
try {
245244
File tempFile = File.createTempFile(objectName, "");
246245
tempFile.deleteOnExit();
247246

248-
byte[] bytes = genBytes(method);
247+
byte[] bytes = genBytes(method, instructionList);
249248
try (ByteArrayInputStream in = new ByteArrayInputStream(bytes);
250249
FileOutputStream out = new FileOutputStream(tempFile)) {
251250
long copy = ByteStreams.copy(in, out);
@@ -276,14 +275,19 @@ public String getTopicName() {
276275
return topicName;
277276
}
278277

279-
private static byte[] genBytes(Method method) {
278+
private static byte[] genBytes(Method method, List<String> instructionsList) {
280279
// Not all tests need data for an object, though some tests - resumable upload - needs
281280
// more than 8MiB.
282281
// We want to avoid allocating 8.1MiB for each test unnecessarily, especially since we
283282
// instantiate all permuted test cases. ~1000 * 8.1MiB ~~ > 8GiB.
284283
switch (method.getName()) {
285284
case "storage.objects.insert":
286-
return DataGenerator.base64Characters().genBytes(_8MiB * 2 + _512KiB);
285+
boolean after8m = instructionsList.stream().anyMatch(s -> s.endsWith("after-8192K"));
286+
if (after8m) {
287+
return DataGenerator.base64Characters().genBytes(_8MiB * 2 + _512KiB);
288+
} else {
289+
return DataGenerator.base64Characters().genBytes(_512KiB);
290+
}
287291
case "storage.objects.get":
288292
return DataGenerator.base64Characters().genBytes(_512KiB);
289293
default:

0 commit comments

Comments
 (0)