[GitHub] [beam] dennisylyung commented on a change in pull request #14244: [BEAM-11907] SQS source with full functionality

[GitBox <gi...@apache.org>]

dennisylyung commented on a change in pull request #14244:
URL: https://github.com/apache/beam/pull/14244#discussion_r654247286



##########
File path: sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sqs/SqsIOTest.java
##########
@@ -46,25 +52,151 @@
 /** Tests on {@link SqsIO}. */
 @RunWith(JUnit4.class)
 public class SqsIOTest {
+  private static final String DATA = "testData";
 
   @Rule public TestPipeline pipeline = TestPipeline.create();
 
   @Rule public EmbeddedSqsServer embeddedSqsRestServer = new EmbeddedSqsServer();
 
-  @Test
-  public void testRead() {
+  private SqsUnboundedSource source;
+
+  private void setupOneMessage() {
     final AmazonSQS client = embeddedSqsRestServer.getClient();
     final String queueUrl = embeddedSqsRestServer.getQueueUrl();
+    client.sendMessage(queueUrl, DATA);
+    source =
+        new SqsUnboundedSource(
+            SqsIO.read().withQueueUrl(queueUrl).withMaxNumRecords(1),
+            new SqsConfiguration(pipeline.getOptions().as(AwsOptions.class)));
+  }
+
+  private void setupMessages(List<String> messages) {
+    final AmazonSQS client = embeddedSqsRestServer.getClient();
+    final String queueUrl = embeddedSqsRestServer.getQueueUrl();
+    for (String message : messages) {
+      client.sendMessage(queueUrl, message);
+    }
+    source =
+        new SqsUnboundedSource(
+            SqsIO.read().withQueueUrl(queueUrl).withMaxNumRecords(1),
+            new SqsConfiguration(pipeline.getOptions().as(AwsOptions.class)));
+  }
+
+  @Test
+  public void testCheckpointCoderIsSane() {
+    setupOneMessage();
+    CoderProperties.coderSerializable(source.getCheckpointMarkCoder());
+    // Since we only serialize/deserialize the 'notYetReadIds', and we don't want to make
+    // equals on checkpoints ignore those fields, we'll test serialization and deserialization
+    // of checkpoints in multipleReaders below.
+  }
 
-    final PCollection<Message> output =
-        pipeline.apply(SqsIO.read().withQueueUrl(queueUrl).withMaxNumRecords(100));
+  @Test
+  public void testReadOneMessage() throws IOException {

Review comment:
       changed in the new commit

##########
File path: sdks/java/io/amazon-web-services2/src/test/java/org/apache/beam/sdk/io/aws2/sqs/SqsIOTest.java
##########
@@ -17,59 +17,200 @@
  */
 package org.apache.beam.sdk.io.aws2.sqs;
 
+import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
-import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.testing.CoderProperties;
 import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.values.PCollection;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.apache.beam.sdk.util.CoderUtils;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import software.amazon.awssdk.services.sqs.SqsClient;
+import software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityRequest;
 import software.amazon.awssdk.services.sqs.model.Message;
 import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest;
 import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse;
 import software.amazon.awssdk.services.sqs.model.SendMessageRequest;
 
 /** Tests on {@link SqsIO}. */
 @RunWith(JUnit4.class)
+@SuppressWarnings({
+  "nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
+})
 public class SqsIOTest {
+  private static final String DATA = "testData";
 
   @Rule public TestPipeline pipeline = TestPipeline.create();
 
-  @Test
-  public void testRead() {
-    final SqsClient client = EmbeddedSqsServer.getClient();
-    final String queueUrl = EmbeddedSqsServer.getQueueUrl();
+  @Rule public EmbeddedSqsServer embeddedSqsRestServer = new EmbeddedSqsServer();
+
+  private SqsUnboundedSource source;
 
-    final PCollection<SqsMessage> output =
-        pipeline.apply(
+  private void setupOneMessage() {
+    final SqsClient client = embeddedSqsRestServer.getClient();
+    final String queueUrl = embeddedSqsRestServer.getQueueUrl();
+    client.sendMessage(SendMessageRequest.builder().queueUrl(queueUrl).messageBody(DATA).build());
+    source =
+        new SqsUnboundedSource(
             SqsIO.read()
+                .withQueueUrl(queueUrl)
                 .withSqsClientProvider(SqsClientProviderMock.of(client))
+                .withMaxNumRecords(1));
+  }
+
+  private void setupMessages(List<String> messages) {
+    final SqsClient client = embeddedSqsRestServer.getClient();
+    final String queueUrl = embeddedSqsRestServer.getQueueUrl();
+    for (String message : messages) {
+      client.sendMessage(
+          SendMessageRequest.builder().queueUrl(queueUrl).messageBody(message).build());
+    }
+    source =
+        new SqsUnboundedSource(
+            SqsIO.read()
                 .withQueueUrl(queueUrl)
-                .withMaxNumRecords(100));
+                .withSqsClientProvider(SqsClientProviderMock.of(client))
+                .withMaxNumRecords(messages.size()));
+  }
 
-    PAssert.thatSingleton(output.apply(Count.globally())).isEqualTo(100L);
+  @Test
+  public void testCheckpointCoderIsSane() {
+    setupOneMessage();
+    CoderProperties.coderSerializable(source.getCheckpointMarkCoder());
+    // Since we only serialize/deserialize the 'notYetReadIds', and we don't want to make
+    // equals on checkpoints ignore those fields, we'll test serialization and deserialization
+    // of checkpoints in multipleReaders below.
+  }
 
+  @Test
+  public void testReadOneMessage() throws IOException {

Review comment:
       changed in the new commit




-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org