You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ec...@apache.org on 2022/03/03 13:40:29 UTC
[beam] branch master updated: [adhoc] Prepare aws2 ClientConfiguration for json serialization and cleanup AWS Module (#16894)
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 06e7c20 [adhoc] Prepare aws2 ClientConfiguration for json serialization and cleanup AWS Module (#16894)
06e7c20 is described below
commit 06e7c201dbdd8fe37d308b0bad2b1684e85e1dc7
Author: Moritz Mack <mm...@talend.com>
AuthorDate: Thu Mar 3 14:38:41 2022 +0100
[adhoc] Prepare aws2 ClientConfiguration for json serialization and cleanup AWS Module (#16894)
* [adhoc] Prepare aws2 ClientConfiguration and related classes for json serialization and cleanup AWS Module
---
sdks/java/io/amazon-web-services2/build.gradle | 1 +
.../sdk/io/aws2/common/ClientConfiguration.java | 95 ++++++++--------------
.../io/aws2/common/HttpClientConfiguration.java | 23 +++++-
.../sdk/io/aws2/common/RetryConfiguration.java | 43 +++++++++-
.../apache/beam/sdk/io/aws2/options/AwsModule.java | 80 ++----------------
.../apache/beam/sdk/io/aws2/s3/SSECustomerKey.java | 6 ++
.../io/aws2/common/ClientConfigurationTest.java | 31 ++++++-
.../aws2/common/HttpClientConfigurationTest.java | 49 +++++++++++
.../sdk/io/aws2/common/RetryConfigurationTest.java | 22 +++++
.../beam/sdk/io/aws2/options/AwsModuleTest.java | 29 -------
.../beam/sdk/io/aws2/s3/SSECustomerKeyTest.java | 17 +++-
11 files changed, 229 insertions(+), 167 deletions(-)
diff --git a/sdks/java/io/amazon-web-services2/build.gradle b/sdks/java/io/amazon-web-services2/build.gradle
index dceb4c4..817b7b4 100644
--- a/sdks/java/io/amazon-web-services2/build.gradle
+++ b/sdks/java/io/amazon-web-services2/build.gradle
@@ -30,6 +30,7 @@ ext.summary = "IO library to read and write Amazon Web Services services from Be
dependencies {
implementation library.java.vendored_guava_26_0_jre
+ implementation library.java.error_prone_annotations
implementation project(path: ":sdks:java:core", configuration: "shadow")
implementation library.java.aws_java_sdk2_apache_client
implementation library.java.aws_java_sdk2_netty_client
diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java
index 4371d75..9ee8eb2 100644
--- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java
+++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/ClientConfiguration.java
@@ -19,19 +19,21 @@ package org.apache.beam.sdk.io.aws2.common;
import static org.apache.beam.sdk.io.aws2.options.AwsSerializableUtils.deserializeAwsCredentialsProvider;
import static org.apache.beam.sdk.io.aws2.options.AwsSerializableUtils.serializeAwsCredentialsProvider;
-import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSetter;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
import com.google.auto.value.AutoValue;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
+import com.google.auto.value.extension.memoized.Memoized;
import java.io.Serializable;
import java.net.URI;
import java.util.function.Consumer;
import javax.annotation.Nullable;
import org.apache.beam.sdk.io.aws2.options.AwsOptions;
import org.checkerframework.dataflow.qual.Pure;
-import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.regions.Region;
@@ -47,18 +49,28 @@ import software.amazon.awssdk.regions.Region;
* uses a backoff strategy with equal jitter for computing the delay before the next retry.
*/
@AutoValue
+@JsonInclude(value = JsonInclude.Include.NON_EMPTY)
+@JsonDeserialize(builder = ClientConfiguration.Builder.class)
public abstract class ClientConfiguration implements Serializable {
/**
* Optional {@link AwsCredentialsProvider}. If set, this overwrites the default in {@link
* AwsOptions#getAwsCredentialsProvider()}.
*/
- public abstract @Nullable @Pure AwsCredentialsProvider credentialsProvider();
+ @JsonProperty
+ @Memoized
+ public @Nullable @Pure AwsCredentialsProvider credentialsProvider() {
+ return credentialsProviderAsJson() != null
+ ? deserializeAwsCredentialsProvider(credentialsProviderAsJson())
+ : null;
+ }
/**
* Optional {@link Region}. If set, this overwrites the default in {@link
* AwsOptions#getAwsRegion()}.
*/
+ @JsonProperty
+ @Memoized
public @Nullable @Pure Region region() {
return regionId() != null ? Region.of(regionId()) : null;
}
@@ -67,20 +79,24 @@ public abstract class ClientConfiguration implements Serializable {
* Optional service endpoint to use AWS compatible services instead, e.g. for testing. If set,
* this overwrites the default in {@link AwsOptions#getEndpoint()}.
*/
+ @JsonProperty
public abstract @Nullable @Pure URI endpoint();
/**
* Optional {@link RetryConfiguration} for AWS clients. If unset, retry behavior will be unchanged
* and use SDK defaults.
*/
+ @JsonProperty
public abstract @Nullable @Pure RetryConfiguration retry();
abstract @Nullable @Pure String regionId();
+ abstract @Nullable @Pure String credentialsProviderAsJson();
+
public abstract Builder toBuilder();
public static Builder builder() {
- return new AutoValue_ClientConfiguration.Builder();
+ return Builder.builder();
}
public static ClientConfiguration create(
@@ -93,12 +109,20 @@ public abstract class ClientConfiguration implements Serializable {
}
@AutoValue.Builder
+ @JsonPOJOBuilder(withPrefix = "")
public abstract static class Builder {
+ @JsonCreator
+ static Builder builder() {
+ return new AutoValue_ClientConfiguration.Builder();
+ }
+
/**
* Optional {@link AwsCredentialsProvider}. If set, this overwrites the default in {@link
* AwsOptions#getAwsCredentialsProvider()}.
*/
- public abstract Builder credentialsProvider(AwsCredentialsProvider credentialsProvider);
+ public Builder credentialsProvider(AwsCredentialsProvider credentialsProvider) {
+ return credentialsProviderAsJson(serializeAwsCredentialsProvider(credentialsProvider));
+ }
/**
* Optional {@link Region}. If set, this overwrites the default in {@link
@@ -118,6 +142,7 @@ public abstract class ClientConfiguration implements Serializable {
* Optional {@link RetryConfiguration} for AWS clients. If unset, retry behavior will be
* unchanged and use SDK defaults.
*/
+ @JsonSetter
public abstract Builder retry(RetryConfiguration retry);
/**
@@ -132,58 +157,8 @@ public abstract class ClientConfiguration implements Serializable {
abstract Builder regionId(String region);
- abstract AwsCredentialsProvider credentialsProvider();
-
- abstract ClientConfiguration autoBuild();
+ abstract Builder credentialsProviderAsJson(String credentialsProvider);
- public ClientConfiguration build() {
- if (credentialsProvider() != null) {
- credentialsProvider(new SerializableAwsCredentialsProvider(credentialsProvider()));
- }
- return autoBuild();
- }
- }
-
- /** Internal serializable {@link AwsCredentialsProvider}. */
- private static class SerializableAwsCredentialsProvider
- implements AwsCredentialsProvider, Serializable {
- private transient AwsCredentialsProvider provider;
- private String serializedProvider;
-
- SerializableAwsCredentialsProvider(AwsCredentialsProvider provider) {
- this.provider = checkNotNull(provider, "AwsCredentialsProvider cannot be null");
- this.serializedProvider = serializeAwsCredentialsProvider(provider);
- }
-
- private void writeObject(ObjectOutputStream out) throws IOException {
- out.writeUTF(serializedProvider);
- }
-
- private void readObject(ObjectInputStream in) throws IOException {
- serializedProvider = in.readUTF();
- provider = deserializeAwsCredentialsProvider(serializedProvider);
- }
-
- @Override
- public AwsCredentials resolveCredentials() {
- return provider.resolveCredentials();
- }
-
- @Override
- public boolean equals(@Nullable Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- SerializableAwsCredentialsProvider that = (SerializableAwsCredentialsProvider) o;
- return serializedProvider.equals(that.serializedProvider);
- }
-
- @Override
- public int hashCode() {
- return serializedProvider.hashCode();
- }
+ public abstract ClientConfiguration build();
}
}
diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfiguration.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfiguration.java
index 3b9cb1d..65b882a 100644
--- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfiguration.java
+++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfiguration.java
@@ -17,6 +17,11 @@
*/
package org.apache.beam.sdk.io.aws2.common;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
import com.google.auto.value.AutoValue;
import java.io.Serializable;
import javax.annotation.Nullable;
@@ -32,11 +37,14 @@ import org.checkerframework.dataflow.qual.Pure;
* HTTP Configuration</a>
*/
@AutoValue
+@JsonInclude(value = JsonInclude.Include.NON_EMPTY)
+@JsonDeserialize(builder = HttpClientConfiguration.Builder.class)
public abstract class HttpClientConfiguration implements Serializable {
/**
* Milliseconds to wait when acquiring a connection from the pool before giving up and timing out.
*/
+ @JsonProperty
public abstract @Nullable @Pure Integer connectionAcquisitionTimeout();
/**
@@ -45,12 +53,14 @@ public abstract class HttpClientConfiguration implements Serializable {
* <p>This will never close a connection that is currently in use, so long-lived connections may
* remain open longer than this time.
*/
+ @JsonProperty
public abstract @Nullable @Pure Integer connectionMaxIdleTime();
/**
* Milliseconds to wait when initially establishing a connection before giving up and timing out.
* A duration of 0 means infinity, and is not recommended.
*/
+ @JsonProperty
public abstract @Nullable @Pure Integer connectionTimeout();
/**
@@ -60,12 +70,14 @@ public abstract class HttpClientConfiguration implements Serializable {
* <p>This will never close a connection that is currently in use, so long-lived connections may
* remain open longer than this time.
*/
+ @JsonProperty
public abstract @Nullable @Pure Integer connectionTimeToLive();
/**
* Milliseconds to wait for data to be transferred over an established, open connection before the
* connection is timed out. A duration of 0 means infinity, and is not recommended.
*/
+ @JsonProperty
public abstract @Nullable @Pure Integer socketTimeout();
/**
@@ -75,6 +87,7 @@ public abstract class HttpClientConfiguration implements Serializable {
* <p>Note: Read timeout is only supported for async clients and ignored otherwise. Use {@link
* #socketTimeout()} instead.
*/
+ @JsonProperty
public abstract @Nullable @Pure Integer readTimeout();
/**
@@ -84,6 +97,7 @@ public abstract class HttpClientConfiguration implements Serializable {
* <p>Note: Write timeout is only supported for async clients and ignored otherwise. Use {@link
* #socketTimeout()} instead.
*/
+ @JsonProperty
public abstract @Nullable @Pure Integer writeTimeout();
/**
@@ -94,14 +108,21 @@ public abstract class HttpClientConfiguration implements Serializable {
* concurrent requests. When using HTTP/2 the number of connections that will be used depends on
* the max streams allowed per connection.
*/
+ @JsonProperty
public abstract @Nullable @Pure Integer maxConnections();
public static Builder builder() {
- return new AutoValue_HttpClientConfiguration.Builder();
+ return Builder.builder();
}
@AutoValue.Builder
+ @JsonPOJOBuilder(withPrefix = "")
public abstract static class Builder {
+ @JsonCreator
+ static Builder builder() {
+ return new AutoValue_HttpClientConfiguration.Builder();
+ }
+
/**
* Milliseconds to wait when acquiring a connection from the pool before giving up and timing
* out.
diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java
index 1c22601..6ca3429 100644
--- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java
+++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/common/RetryConfiguration.java
@@ -20,6 +20,13 @@ package org.apache.beam.sdk.io.aws2.common;
import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;
import static org.joda.time.Duration.ZERO;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.util.StdConverter;
import com.google.auto.value.AutoValue;
import java.io.Serializable;
import javax.annotation.Nullable;
@@ -36,31 +43,51 @@ import software.amazon.awssdk.core.retry.backoff.EqualJitterBackoffStrategy;
* SdkDefaultRetrySetting} for further details.
*/
@AutoValue
+@JsonInclude(value = JsonInclude.Include.NON_EMPTY)
+@JsonDeserialize(builder = RetryConfiguration.Builder.class)
public abstract class RetryConfiguration implements Serializable {
private static final java.time.Duration BASE_BACKOFF = java.time.Duration.ofMillis(100);
private static final java.time.Duration THROTTLED_BASE_BACKOFF = java.time.Duration.ofSeconds(1);
private static final java.time.Duration MAX_BACKOFF = java.time.Duration.ofSeconds(20);
+ @JsonProperty
public abstract @Pure int numRetries();
+ @JsonProperty
+ @JsonSerialize(converter = DurationToMillis.class)
public abstract @Nullable @Pure Duration baseBackoff();
+ @JsonProperty
+ @JsonSerialize(converter = DurationToMillis.class)
public abstract @Nullable @Pure Duration throttledBaseBackoff();
+ @JsonProperty
+ @JsonSerialize(converter = DurationToMillis.class)
public abstract @Nullable @Pure Duration maxBackoff();
+ public abstract RetryConfiguration.Builder toBuilder();
+
public static Builder builder() {
- return new AutoValue_RetryConfiguration.Builder();
+ return Builder.builder();
}
@AutoValue.Builder
+ @JsonPOJOBuilder(withPrefix = "")
public abstract static class Builder {
+ @JsonCreator
+ static Builder builder() {
+ return new AutoValue_RetryConfiguration.Builder();
+ }
+
public abstract Builder numRetries(int numRetries);
+ @JsonDeserialize(converter = MillisToDuration.class)
public abstract Builder baseBackoff(Duration baseBackoff);
+ @JsonDeserialize(converter = MillisToDuration.class)
public abstract Builder throttledBaseBackoff(Duration baseBackoff);
+ @JsonDeserialize(converter = MillisToDuration.class)
public abstract Builder maxBackoff(Duration maxBackoff);
abstract RetryConfiguration autoBuild();
@@ -115,4 +142,18 @@ public abstract class RetryConfiguration implements Serializable {
private @Nullable static java.time.Duration toJava(@Nullable Duration duration) {
return duration == null ? null : java.time.Duration.ofMillis(duration.getMillis());
}
+
+ static class DurationToMillis extends StdConverter<Duration, Long> {
+ @Override
+ public Long convert(Duration duration) {
+ return duration.getMillis();
+ }
+ }
+
+ static class MillisToDuration extends StdConverter<Long, Duration> {
+ @Override
+ public Duration convert(Long millis) {
+ return Duration.millis(millis);
+ }
+ }
}
diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java
index ff54a18..0f8b138 100644
--- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java
+++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/options/AwsModule.java
@@ -20,6 +20,7 @@ package org.apache.beam.sdk.io.aws2.options;
import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkNotNull;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.core.JsonGenerator;
@@ -42,14 +43,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.util.NameTransformer;
import com.google.auto.service.AutoService;
import java.io.IOException;
-import java.net.URI;
-import java.util.Map;
import java.util.function.Supplier;
import org.apache.beam.repackaged.core.org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.annotations.Experimental.Kind;
-import org.apache.beam.sdk.io.aws2.common.HttpClientConfiguration;
-import org.apache.beam.sdk.io.aws2.s3.SSECustomerKey;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
import org.checkerframework.checker.nullness.qual.NonNull;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
@@ -67,7 +64,6 @@ import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.sts.StsClient;
import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
-import software.amazon.awssdk.utils.AttributeMap;
/**
* A Jackson {@link Module} that registers a {@link JsonSerializer} and {@link JsonDeserializer} for
@@ -88,15 +84,11 @@ public class AwsModule extends SimpleModule {
public void setupModule(SetupContext cxt) {
cxt.setMixInAnnotations(AwsCredentialsProvider.class, AwsCredentialsProviderMixin.class);
cxt.setMixInAnnotations(ProxyConfiguration.class, ProxyConfigurationMixin.class);
- cxt.setMixInAnnotations(HttpClientConfiguration.class, HttpClientConfigurationMixin.class);
cxt.setMixInAnnotations(
- HttpClientConfiguration.Builder.class, HttpClientConfigurationMixin.Builder.class);
- cxt.setMixInAnnotations(SSECustomerKey.class, SSECustomerKeyMixin.class);
- cxt.setMixInAnnotations(SSECustomerKey.Builder.class, SSECustomerKeyMixin.Builder.class);
+ ProxyConfiguration.Builder.class, ProxyConfigurationMixin.Builder.class);
cxt.setMixInAnnotations(Region.class, RegionMixin.class);
- addValueInstantiator(HttpClientConfiguration.Builder.class, HttpClientConfiguration::builder);
-
+ addValueInstantiator(ProxyConfiguration.Builder.class, ProxyConfiguration::builder);
super.setupModule(cxt);
}
@@ -266,71 +258,11 @@ public class AwsModule extends SimpleModule {
}
/** A mixin to add Jackson annotations to {@link ProxyConfiguration}. */
- @JsonDeserialize(using = ProxyConfigurationDeserializer.class)
- @JsonSerialize(using = ProxyConfigurationSerializer.class)
- private static class ProxyConfigurationMixin {}
-
- private static class ProxyConfigurationDeserializer extends JsonDeserializer<ProxyConfiguration> {
- @Override
- public ProxyConfiguration deserialize(JsonParser jsonParser, DeserializationContext context)
- throws IOException {
- Map<String, String> asMap =
- checkNotNull(
- jsonParser.readValueAs(new TypeReference<Map<String, String>>() {}),
- "Serialized ProxyConfiguration is null");
-
- ProxyConfiguration.Builder builder = ProxyConfiguration.builder();
- final String endpoint = asMap.get("endpoint");
- if (endpoint != null) {
- builder.endpoint(URI.create(endpoint));
- }
- final String username = asMap.get("username");
- if (username != null) {
- builder.username(username);
- }
- final String password = asMap.get("password");
- if (password != null) {
- builder.password(password);
- }
- // defaults to FALSE / disabled
- Boolean useSystemPropertyValues = Boolean.valueOf(asMap.get("useSystemPropertyValues"));
- return builder.useSystemPropertyValues(useSystemPropertyValues).build();
- }
- }
-
- private static class ProxyConfigurationSerializer extends JsonSerializer<ProxyConfiguration> {
- @Override
- public void serialize(
- ProxyConfiguration proxyConfiguration,
- JsonGenerator jsonGenerator,
- SerializerProvider serializer)
- throws IOException {
- // proxyConfiguration.endpoint() is private so we have to build it manually.
- final String endpoint =
- proxyConfiguration.scheme()
- + "://"
- + proxyConfiguration.host()
- + ":"
- + proxyConfiguration.port();
- jsonGenerator.writeStartObject();
- jsonGenerator.writeStringField("endpoint", endpoint);
- jsonGenerator.writeStringField("username", proxyConfiguration.username());
- jsonGenerator.writeStringField("password", proxyConfiguration.password());
- jsonGenerator.writeEndObject();
- }
- }
-
- /** A mixin to add Jackson annotations to {@link AttributeMap}. */
- @JsonDeserialize(builder = HttpClientConfiguration.Builder.class)
+ @JsonDeserialize(builder = ProxyConfiguration.Builder.class)
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
+ @JsonIgnoreProperties(value = {"host", "port", "scheme"})
@JsonInclude(value = JsonInclude.Include.NON_EMPTY)
- private static class HttpClientConfigurationMixin {
- @JsonPOJOBuilder(withPrefix = "")
- static class Builder {}
- }
-
- @JsonDeserialize(builder = SSECustomerKey.Builder.class)
- private static class SSECustomerKeyMixin {
+ private static class ProxyConfigurationMixin {
@JsonPOJOBuilder(withPrefix = "")
static class Builder {}
}
diff --git a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java
index 64ee4aa..8e0ea42 100644
--- a/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java
+++ b/sdks/java/io/amazon-web-services2/src/main/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKey.java
@@ -19,12 +19,17 @@ package org.apache.beam.sdk.io.aws2.s3;
import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import org.apache.commons.codec.digest.DigestUtils;
import org.checkerframework.checker.nullness.qual.Nullable;
/** Customer provided key for use with Amazon S3 server-side encryption. */
+@JsonInclude(value = JsonInclude.Include.NON_EMPTY)
+@JsonDeserialize(builder = SSECustomerKey.Builder.class)
public class SSECustomerKey {
private final @Nullable String key;
@@ -63,6 +68,7 @@ public class SSECustomerKey {
return new Builder();
}
+ @JsonPOJOBuilder(withPrefix = "")
public static class Builder {
private @Nullable String key;
diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/ClientConfigurationTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/ClientConfigurationTest.java
index 93615f5..0c2c91b 100644
--- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/ClientConfigurationTest.java
+++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/ClientConfigurationTest.java
@@ -22,16 +22,18 @@ import static org.apache.beam.sdk.util.SerializableUtils.serializeToByteArray;
import static org.assertj.core.api.Assertions.assertThat;
import java.net.URI;
+import org.apache.beam.sdk.io.aws2.options.SerializationTestUtil;
import org.junit.Test;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
public class ClientConfigurationTest {
@Test
- public void testSerialization() {
+ public void testJavaSerialization() {
AwsCredentialsProvider credentials =
StaticCredentialsProvider.create(AwsBasicCredentials.create("key", "secret"));
@@ -50,4 +52,31 @@ public class ClientConfigurationTest {
assertThat(deserializedConfig).isEqualTo(config);
}
+
+ @Test
+ public void testJsonSerialization() {
+ ClientConfiguration config = ClientConfiguration.builder().build();
+ assertThat(jsonSerializeDeserialize(config)).isEqualTo(config);
+
+ config = config.toBuilder().region(Region.US_WEST_1).build();
+ assertThat(jsonSerializeDeserialize(config)).isEqualTo(config);
+
+ config = config.toBuilder().credentialsProvider(DefaultCredentialsProvider.create()).build();
+ assertThat(jsonSerializeDeserialize(config)).isEqualTo(config);
+
+ AwsBasicCredentials credentials = AwsBasicCredentials.create("key", "secret");
+ StaticCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(credentials);
+ config = config.toBuilder().credentialsProvider(credentialsProvider).build();
+ assertThat(jsonSerializeDeserialize(config)).isEqualTo(config);
+
+ config = config.toBuilder().endpoint(URI.create("https://localhost:8080")).build();
+ assertThat(jsonSerializeDeserialize(config)).isEqualTo(config);
+
+ config = config.toBuilder().retry(r -> r.numRetries(10)).build();
+ assertThat(jsonSerializeDeserialize(config)).isEqualTo(config);
+ }
+
+ private ClientConfiguration jsonSerializeDeserialize(ClientConfiguration obj) {
+ return SerializationTestUtil.serializeDeserialize(ClientConfiguration.class, obj);
+ }
}
diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfigurationTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfigurationTest.java
new file mode 100644
index 0000000..b3147ac
--- /dev/null
+++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/HttpClientConfigurationTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.aws2.common;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.beam.sdk.io.aws2.options.SerializationTestUtil;
+import org.junit.Test;
+
+public class HttpClientConfigurationTest {
+ @Test
+ public void testJsonSerialization() {
+ HttpClientConfiguration expected = HttpClientConfiguration.builder().build();
+ assertThat(serializeAndDeserialize(expected)).isEqualTo(expected);
+
+ expected =
+ HttpClientConfiguration.builder()
+ .connectionAcquisitionTimeout(100)
+ .connectionMaxIdleTime(200)
+ .connectionTimeout(300)
+ .connectionTimeToLive(400)
+ .socketTimeout(500)
+ .readTimeout(600)
+ .writeTimeout(700)
+ .maxConnections(10)
+ .build();
+
+ assertThat(serializeAndDeserialize(expected)).isEqualTo(expected);
+ }
+
+ private HttpClientConfiguration serializeAndDeserialize(HttpClientConfiguration obj) {
+ return SerializationTestUtil.serializeDeserialize(HttpClientConfiguration.class, obj);
+ }
+}
diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/RetryConfigurationTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/RetryConfigurationTest.java
index 7ec86cb..dca607d 100644
--- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/RetryConfigurationTest.java
+++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/common/RetryConfigurationTest.java
@@ -17,9 +17,12 @@
*/
package org.apache.beam.sdk.io.aws2.common;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.joda.time.Duration.ZERO;
+import org.apache.beam.sdk.io.aws2.options.SerializationTestUtil;
+import org.joda.time.Duration;
import org.junit.Test;
public class RetryConfigurationTest {
@@ -47,4 +50,23 @@ public class RetryConfigurationTest {
assertThatThrownBy(() -> RetryConfiguration.builder().numRetries(1).maxBackoff(ZERO).build())
.hasMessage("maxBackoff must be greater than 0");
}
+
+ @Test
+ public void testJsonSerialization() {
+ RetryConfiguration config = RetryConfiguration.builder().numRetries(10).build();
+ assertThat(jsonSerializeDeserialize(config)).isEqualTo(config);
+
+ config = config.toBuilder().maxBackoff(Duration.millis(1000)).build();
+ assertThat(jsonSerializeDeserialize(config)).isEqualTo(config);
+
+ config = config.toBuilder().baseBackoff(Duration.millis(200)).build();
+ assertThat(jsonSerializeDeserialize(config)).isEqualTo(config);
+
+ config = config.toBuilder().throttledBaseBackoff(Duration.millis(100)).build();
+ assertThat(jsonSerializeDeserialize(config)).isEqualTo(config);
+ }
+
+ private RetryConfiguration jsonSerializeDeserialize(RetryConfiguration obj) {
+ return SerializationTestUtil.serializeDeserialize(RetryConfiguration.class, obj);
+ }
}
diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java
index 2876b90..b294c3f 100644
--- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java
+++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/options/AwsModuleTest.java
@@ -33,8 +33,6 @@ import java.net.URI;
import java.util.List;
import java.util.Properties;
import java.util.function.Supplier;
-import org.apache.beam.sdk.io.aws2.common.HttpClientConfiguration;
-import org.apache.beam.sdk.io.aws2.s3.SSECustomerKey;
import org.apache.beam.sdk.util.ThrowingSupplier;
import org.apache.beam.sdk.util.common.ReflectHelpers;
import org.hamcrest.MatcherAssert;
@@ -151,33 +149,6 @@ public class AwsModuleTest {
assertEquals("password", deserializedProxyConfiguration.password());
}
- @Test
- public void testHttpClientConfigurationSerializationDeserialization() throws Exception {
- HttpClientConfiguration expected =
- HttpClientConfiguration.builder()
- .connectionAcquisitionTimeout(100)
- .connectionMaxIdleTime(200)
- .connectionTimeout(300)
- .connectionTimeToLive(400)
- .socketTimeout(500)
- .readTimeout(600)
- .writeTimeout(700)
- .maxConnections(10)
- .build();
-
- assertThat(serializeAndDeserialize(expected)).isEqualTo(expected);
- }
-
- @Test
- public void testSSECustomerKeySerializationDeserialization() throws Exception {
- // default key created by S3Options.SSECustomerKeyFactory
- SSECustomerKey emptyKey = SSECustomerKey.builder().build();
- assertThat(serializeAndDeserialize(emptyKey)).isEqualToComparingFieldByField(emptyKey);
-
- SSECustomerKey key = SSECustomerKey.builder().key("key").algorithm("algo").md5("md5").build();
- assertThat(serializeAndDeserialize(key)).isEqualToComparingFieldByField(key);
- }
-
private <T> T withSystemPropertyOverrides(Properties overrides, ThrowingSupplier<T> fun)
throws Exception {
Properties systemProps = System.getProperties();
diff --git a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKeyTest.java b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKeyTest.java
index ac39552..943a3fd 100644
--- a/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKeyTest.java
+++ b/sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/s3/SSECustomerKeyTest.java
@@ -17,9 +17,11 @@
*/
package org.apache.beam.sdk.io.aws2.s3;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
+import org.apache.beam.sdk.io.aws2.options.SerializationTestUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -30,7 +32,6 @@ import org.junit.runners.JUnit4;
"nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
})
public class SSECustomerKeyTest {
-
@Test
public void testBuild() {
assertThrows(
@@ -54,4 +55,18 @@ public class SSECustomerKeyTest {
assertEquals(algorithm, sseCustomerKey.getAlgorithm());
assertEquals(encodedMD5, sseCustomerKey.getMD5());
}
+
+ @Test
+ public void testJsonSerializeDeserialize() {
+ // default key created by S3Options.SSECustomerKeyFactory
+ SSECustomerKey emptyKey = SSECustomerKey.builder().build();
+ assertThat(jsonSerializeDeserialize(emptyKey)).isEqualToComparingFieldByField(emptyKey);
+
+ SSECustomerKey key = SSECustomerKey.builder().key("key").algorithm("algo").md5("md5").build();
+ assertThat(jsonSerializeDeserialize(key)).isEqualToComparingFieldByField(key);
+ }
+
+ private SSECustomerKey jsonSerializeDeserialize(SSECustomerKey key) {
+ return SerializationTestUtil.serializeDeserialize(SSECustomerKey.class, key);
+ }
}