You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2022/01/26 15:13:26 UTC
[GitHub] [flink] vahmed-hamdy commented on a change in pull request #18518: [FLINK-24229][connectors/dynamodb] Added DynamoDB connector
vahmed-hamdy commented on a change in pull request #18518:
URL: https://github.com/apache/flink/pull/18518#discussion_r792693997
##########
File path: flink-connectors/flink-connector-dynamodb/src/main/java/org/apache/flink/streaming/connectors/dynamodb/sink/key/PrimaryKey.java
##########
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.dynamodb.sink.key;
+
+import org.apache.flink.streaming.connectors.dynamodb.config.DynamoDbTablesConfig;
+import org.apache.flink.streaming.connectors.dynamodb.sink.InvalidRequestException;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
+import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
+
+import javax.annotation.Nullable;
+
+import java.util.Map;
+import java.util.UUID;
+
+/** Represents DynamoDB primary key. */
+public class PrimaryKey {
+
+ private final String partitionKeyValue;
+ @Nullable private final String sortKeyValue;
+
+ private PrimaryKey(String partitionKeyValue) {
+ this.partitionKeyValue = partitionKeyValue;
+ this.sortKeyValue = null;
+ }
+
+ private PrimaryKey(String partitionKeyValue, @Nullable String sortKeyValue) {
+ this.partitionKeyValue = partitionKeyValue;
+ this.sortKeyValue = sortKeyValue;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ PrimaryKey that = (PrimaryKey) o;
+
+ return new EqualsBuilder()
+ .append(partitionKeyValue, that.partitionKeyValue)
+ .append(sortKeyValue, that.sortKeyValue)
+ .isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 37)
+ .append(partitionKeyValue)
+ .append(sortKeyValue)
+ .toHashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "PrimaryKey{"
+ + "partitionKeyValue='"
+ + partitionKeyValue
+ + '\''
+ + ", sortKeyValue='"
+ + sortKeyValue
+ + '\''
+ + '}';
+ }
+
+ public static PrimaryKey build(DynamoDbTablesConfig.TableConfig config, WriteRequest request) {
+ if (config != null) {
Review comment:
Are we expecting a nullable parameter?
might be worth adding a `@Nullable` annotation
##########
File path: flink-connectors/flink-connector-dynamodb/src/main/java/org/apache/flink/streaming/connectors/dynamodb/sink/DynamoDbSinkException.java
##########
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.dynamodb.sink;
+
+/** Exception is thrown when DynamoDb sink failed to write data. */
+public class DynamoDbSinkException extends RuntimeException {
Review comment:
Missing compatibility annotation
##########
File path: flink-connectors/flink-connector-dynamodb/src/main/java/org/apache/flink/streaming/connectors/dynamodb/sink/TableRequestsContainer.java
##########
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.dynamodb.sink;
+
+import org.apache.flink.streaming.connectors.dynamodb.config.DynamoDbTablesConfig;
+import org.apache.flink.streaming.connectors.dynamodb.sink.key.PrimaryKey;
+
+import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableList;
+
+import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Container to accumulate requests in batches per table. De-duplicates batch request entities as
+ * per PrimaryKey definition. DynamoDB Batch API rejects the whole batch request if the request
+ * contains at least two items with identical hash and range keys (which essentially is two put
+ * operations).
+ */
+class TableRequestsContainer {
+
+ private final DynamoDbTablesConfig tablesConfig;
+ private final LinkedHashMap<String, Map<PrimaryKey, WriteRequest>> container;
Review comment:
nit: Do we need `container` to be a `LinkedHashMap`?
##########
File path: flink-connectors/flink-connector-dynamodb/src/test/java/org/apache/flink/streaming/connectors/dynamodb/sink/TableRequestsContainerTest.java
##########
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.dynamodb.sink;
+
+import org.apache.flink.streaming.connectors.dynamodb.config.DynamoDbTablesConfig;
+
+import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableMap;
+
+import org.junit.Test;
+import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
+import software.amazon.awssdk.services.dynamodb.model.PutRequest;
+import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
+
+import static org.junit.Assert.assertEquals;
Review comment:
Can we use `assertj` instead to match the [guidelines](https://flink.apache.org/contributing/code-style-and-quality-common.html#testing)?
##########
File path: flink-connectors/flink-connector-dynamodb/src/test/java/org/apache/flink/streaming/connectors/dynamodb/sink/DynamoDbSinkBuilderTest.java
##########
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.dynamodb.sink;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+/** Tests for {@link DynamoDbSinkBuilder}. */
+public class DynamoDbSinkBuilderTest {
Review comment:
In my opinion, I prefer having at least 1 positive test (i.e build a sink and verify it).
##########
File path: flink-connectors/flink-connector-dynamodb/src/main/resources/META-INF/NOTICE
##########
@@ -0,0 +1,257 @@
+flink-connector-dynamodb
+Copyright 2014-2021 The Apache Software Foundation
Review comment:
should this be 2022?
##########
File path: flink-connectors/flink-connector-dynamodb/src/main/java/org/apache/flink/streaming/connectors/dynamodb/sink/key/PrimaryKey.java
##########
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.dynamodb.sink.key;
+
+import org.apache.flink.streaming.connectors.dynamodb.config.DynamoDbTablesConfig;
+import org.apache.flink.streaming.connectors.dynamodb.sink.InvalidRequestException;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
+import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
+
+import javax.annotation.Nullable;
+
+import java.util.Map;
+import java.util.UUID;
+
+/** Represents DynamoDB primary key. */
+public class PrimaryKey {
+
+ private final String partitionKeyValue;
+ @Nullable private final String sortKeyValue;
+
+ private PrimaryKey(String partitionKeyValue) {
+ this.partitionKeyValue = partitionKeyValue;
+ this.sortKeyValue = null;
+ }
+
+ private PrimaryKey(String partitionKeyValue, @Nullable String sortKeyValue) {
+ this.partitionKeyValue = partitionKeyValue;
+ this.sortKeyValue = sortKeyValue;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ PrimaryKey that = (PrimaryKey) o;
+
+ return new EqualsBuilder()
+ .append(partitionKeyValue, that.partitionKeyValue)
+ .append(sortKeyValue, that.sortKeyValue)
+ .isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17, 37)
+ .append(partitionKeyValue)
+ .append(sortKeyValue)
+ .toHashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "PrimaryKey{"
+ + "partitionKeyValue='"
+ + partitionKeyValue
+ + '\''
+ + ", sortKeyValue='"
+ + sortKeyValue
+ + '\''
+ + '}';
+ }
+
+ public static PrimaryKey build(DynamoDbTablesConfig.TableConfig config, WriteRequest request) {
+ if (config != null) {
+ Map<String, AttributeValue> requestItems = getRequestItems(request);
+
+ AttributeValue partitionKeyAttributeValue =
+ requestItems.get(config.getPartitionKeyName());
+ AttributeValue sortKeyAttributeValue = requestItems.get(config.getSortKeyName());
+
+ if (config.getPartitionKeyName() != null && partitionKeyAttributeValue == null) {
Review comment:
Can `config.getPartitionKeyName()` ever be null?
##########
File path: flink-connectors/flink-connector-dynamodb/src/main/java/org/apache/flink/streaming/connectors/dynamodb/sink/DynamoDbWriteRequest.java
##########
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.dynamodb.sink;
+
+import software.amazon.awssdk.services.dynamodb.model.WriteRequest;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Represents a single DynamoDb {@link WriteRequest}. Contains the name of the DynamoDb table name
+ * to write to as well as the {@link WriteRequest}
+ */
+public class DynamoDbWriteRequest implements Serializable {
Review comment:
Missing compatibility annotation, might be worth having a look across the whole PR
##########
File path: flink-connectors/flink-connector-dynamodb/src/test/java/org/apache/flink/streaming/connectors/dynamodb/sink/key/PrimaryKeyTest.java
##########
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.dynamodb.sink.key;
+
+import org.apache.flink.streaming.connectors.dynamodb.config.DynamoDbTablesConfig;
+import org.apache.flink.streaming.connectors.dynamodb.sink.InvalidRequestException;
+
+import org.apache.flink.shaded.guava30.com.google.common.collect.ImmutableMap;
+
+import org.junit.Assert;
Review comment:
Same as above
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org