[GitHub] [beam] aromanenko-dev commented on a change in pull request #14244: [BEAM-11907] SQS source with full functionality

[GitBox <gi...@apache.org>]

aromanenko-dev commented on a change in pull request #14244:
URL: https://github.com/apache/beam/pull/14244#discussion_r629317202



##########
File path: sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsUnboundedReader.java
##########
@@ -17,51 +17,385 @@
  */
 package org.apache.beam.sdk.io.aws.sqs;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toMap;
+import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState;
+
+import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.model.BatchResultErrorEntry;
+import com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchRequestEntry;
+import com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchResult;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchResult;
+import com.amazonaws.services.sqs.model.GetQueueAttributesRequest;
 import com.amazonaws.services.sqs.model.Message;
+import com.amazonaws.services.sqs.model.MessageAttributeValue;
 import com.amazonaws.services.sqs.model.MessageSystemAttributeName;
+import com.amazonaws.services.sqs.model.QueueAttributeName;
 import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
 import com.amazonaws.services.sqs.model.ReceiveMessageResult;
-import java.io.Serializable;
-import java.nio.charset.StandardCharsets;
+import java.io.IOException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Objects;
 import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.BucketingFunction;
+import org.apache.beam.sdk.util.MovingFunction;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.EvictingQueue;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists;
+import org.joda.time.Duration;
 import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @SuppressWarnings({
   "nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
 })
-class SqsUnboundedReader extends UnboundedSource.UnboundedReader<Message> implements Serializable {
+class SqsUnboundedReader extends UnboundedSource.UnboundedReader<Message> {
+  private static final Logger LOG = LoggerFactory.getLogger(SqsUnboundedReader.class);
 
+  /** Maximum number of messages to pull from SQS per request. */
   public static final int MAX_NUMBER_OF_MESSAGES = 10;
+
+  /** Maximum times to retry batch SQS operations upon partial success. */
+  private static final int BATCH_OPERATION_MAX_RETIRES = 5;

Review comment:
       How the values of this and other constants below were estimated?

##########
File path: sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sqs/SqsIOTest.java
##########
@@ -46,25 +52,151 @@
 /** Tests on {@link SqsIO}. */
 @RunWith(JUnit4.class)
 public class SqsIOTest {
+  private static final String DATA = "testData";
 
   @Rule public TestPipeline pipeline = TestPipeline.create();
 
   @Rule public EmbeddedSqsServer embeddedSqsRestServer = new EmbeddedSqsServer();
 
-  @Test
-  public void testRead() {
+  private SqsUnboundedSource source;
+
+  private void setupOneMessage() {
     final AmazonSQS client = embeddedSqsRestServer.getClient();
     final String queueUrl = embeddedSqsRestServer.getQueueUrl();
+    client.sendMessage(queueUrl, DATA);
+    source =
+        new SqsUnboundedSource(
+            SqsIO.read().withQueueUrl(queueUrl).withMaxNumRecords(1),
+            new SqsConfiguration(pipeline.getOptions().as(AwsOptions.class)));
+  }
+
+  private void setupMessages(List<String> messages) {
+    final AmazonSQS client = embeddedSqsRestServer.getClient();
+    final String queueUrl = embeddedSqsRestServer.getQueueUrl();
+    for (String message : messages) {
+      client.sendMessage(queueUrl, message);
+    }
+    source =
+        new SqsUnboundedSource(
+            SqsIO.read().withQueueUrl(queueUrl).withMaxNumRecords(1),
+            new SqsConfiguration(pipeline.getOptions().as(AwsOptions.class)));
+  }
+
+  @Test
+  public void checkpointCoderIsSane() {

Review comment:
       Please, add "test" prefix for all test methods




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org