[kudu] 01/06: [java] Deflake TestKuduScanner.testDiffScan

[gr...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit 072260b6f91989128cb3284b17cefc506dec38a4
Author: Will Berkeley <wd...@gmail.com>
AuthorDate: Sun Jun 2 21:41:09 2019 -0700

    [java] Deflake TestKuduScanner.testDiffScan
    
    I noticed a funny failure in TestKuduScanner.testDiffScan:
    
    21:55:41.681 [DEBUG - New I/O worker #13] (AsyncKuduScanner.java:556) Can not open scanner
    org.apache.kudu.client.NonRecoverableException: snapshot scan start timestamp is earlier than the ancient history mark. Consider increasing the value of the configuration parameter --tablet_history_max_age_sec. Snapshot timestamp: P: 0 usec, L: 0 Ancient History Mark: P: 1558907741677535 usec, L: 0 Physical time difference: -1558907741.678s
    
    The client logs the request it sent, which indeed has a
    'snap_start_timestamp' of 0. This timestamp is supposed to come from
    incrementing the propagated timestamp received from the client after
    some random inserts and mutations. Fortunately, this test has nice
    logging: it logs the operations it did and the timestamp:
    
    21:55:40.874 [INFO - Time-limited test] (TestKuduScanner.java:250) Before: {}
    21:55:40.874 [INFO - Time-limited test] (TestKuduScanner.java:255) startHT: 0
    
    'generateMutationOperations' could generate 0 mutations, causing this
    test to fail. This requires the random number generator to generate 0 3
    times in a row when sampling uniformly from {0, 1, 2, 3, 4}, so the odds
    of failure due to this defect in the test were only 1/125. I fixed this
    by adjusting the number of initial inserts generated so it is always at
    least 1.
    
    I ran the test 1250 times with the fix and saw zero failures. Without
    the fix, I saw 9/1250 failures.
    
    Change-Id: Iadb46e5dae71724aa8ff88d04c40ef4eaf1ddd2a
    Reviewed-on: http://gerrit.cloudera.org:8080/13495
    Tested-by: Will Berkeley <wd...@gmail.com>
    Reviewed-by: Alexey Serbin <as...@cloudera.com>
---
 .../org/apache/kudu/client/TestKuduScanner.java    | 34 +++++++++++++---------
 .../java/org/apache/kudu/test/RandomUtils.java     | 12 ++++++++
 2 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduScanner.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduScanner.java
index fcc744f..911f98e 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduScanner.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduScanner.java
@@ -242,10 +242,13 @@ public class TestKuduScanner {
 
     KuduTable table = client.createTable(tableName, schema, getBasicCreateTableOptions());
 
-    // Generate some rows before the start time.
+    // Generate some rows before the start time. Ensure there's at least one insert.
     int beforeBounds = 5;
-    List<Operation> beforeOps = generateMutationOperations(table, random.nextInt(beforeBounds),
-        random.nextInt(beforeBounds), random.nextInt(beforeBounds));
+    int numInserts = RandomUtils.nextIntInRange(random, 1, beforeBounds);
+    int numUpdates = random.nextInt(beforeBounds);
+    int numDeletes = random.nextInt(beforeBounds);
+    List<Operation> beforeOps =
+        generateMutationOperations(table, numInserts, numUpdates, numDeletes);
     Map<Integer, ChangeType> before = applyOperations(beforeOps);
     LOG.info("Before: {}", before);
 
@@ -254,13 +257,13 @@ public class TestKuduScanner {
     long startHT = client.getLastPropagatedTimestamp() + 1;
     LOG.info("startHT: {}", startHT);
 
-    // Generate row mutations.
+    // Generate row mutations. The mutations performed here are what should be seen by the diff scan.
     int mutationBounds = 10;
-    int numInserts = random.nextInt(mutationBounds);
-    int numUpdates = random.nextInt(mutationBounds);
-    int numDeletes = random.nextInt(mutationBounds);
+    int expectedNumInserts = random.nextInt(mutationBounds);
+    int expectedNumUpdates = random.nextInt(mutationBounds);
+    int expectedNumDeletes = random.nextInt(mutationBounds);
     List<Operation> operations =
-        generateMutationOperations(table, numInserts, numUpdates, numDeletes);
+        generateMutationOperations(table, expectedNumInserts, expectedNumUpdates, expectedNumDeletes);
     Map<Integer, ChangeType> mutations = applyOperations(operations);
     LOG.info("Mutations: {}", mutations);
 
@@ -269,10 +272,13 @@ public class TestKuduScanner {
     long endHT = client.getLastPropagatedTimestamp() + 1;
     LOG.info("endHT: {}", endHT);
 
-    // Generate Some Rows after the end time.
+    // Generate some rows after the end time.
     int afterBounds = 5;
-    List<Operation> afterOps = generateMutationOperations(table, random.nextInt(afterBounds),
-        random.nextInt(afterBounds), random.nextInt(afterBounds));
+    numInserts = random.nextInt(afterBounds);
+    numUpdates = random.nextInt(afterBounds);
+    numDeletes = random.nextInt(afterBounds);
+    List<Operation> afterOps =
+        generateMutationOperations(table, numInserts, numUpdates, numDeletes);
     Map<Integer, ChangeType> after = applyOperations(afterOps);
     LOG.info("After: {}", after);
 
@@ -328,9 +334,9 @@ public class TestKuduScanner {
         resultExtra++;
       }
     }
-    assertEquals(numInserts, resultNumInserts);
-    assertEquals(numUpdates, resultNumUpdates);
-    assertEquals(numDeletes, resultNumDeletes);
+    assertEquals(expectedNumInserts, resultNumInserts);
+    assertEquals(expectedNumUpdates, resultNumUpdates);
+    assertEquals(expectedNumDeletes, resultNumDeletes);
     assertEquals(0, resultExtra);
   }
 
diff --git a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/RandomUtils.java b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/RandomUtils.java
index 0328e15..a6490df 100644
--- a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/RandomUtils.java
+++ b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/RandomUtils.java
@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.kudu.test;
 
+import com.google.common.base.Preconditions;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.yetus.audience.InterfaceStability;
 import org.slf4j.Logger;
@@ -46,4 +47,15 @@ public class RandomUtils {
     LOG.info("Using random seed: {}", seed);
     return new Random(seed);
   }
+
+  /*
+   * Return the next pseudorandom integer generated by 'random' in the range [start, end).
+   * 'start' must be strictly less than 'end'.
+   */
+  public static int nextIntInRange(Random random, int start, int end) {
+    Preconditions.checkArgument(
+        start < end,
+        String.format("start must be strictly less than end (%d < %d)", start, end));
+    return start + random.nextInt(end - start);
+  }
 }