You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2018/02/04 00:04:26 UTC

[incubator-pulsar] branch master updated: Introduce retries for flaky tests (#1166)

This is an automated email from the ASF dual-hosted git repository.

mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 3de54fd  Introduce retries for flaky tests (#1166)
3de54fd is described below

commit 3de54fd5db51d6cf03fb31ccea8266e75a3d4100
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Sat Feb 3 16:04:25 2018 -0800

    Introduce retries for flaky tests (#1166)
    
    * Introduce retries for flaky tests
    
    * Avoid conflicts in allocating ports across different processes
    
    * Removed duplicated builtools module after merge
    
    * Also do retries on pulsar-discovery-service
---
 buildtools/pom.xml                                 |  7 ++
 .../apache/pulsar/tests/AnnotationListener.java    | 35 ++++++++
 .../org/apache/pulsar/tests/RetryAnalyzer.java     | 36 +++++++++
 managed-ledger/pom.xml                             | 19 +++++
 .../org/apache/bookkeeper/test/PortManager.java    | 93 ++++++++++++++++++----
 pulsar-broker/pom.xml                              | 19 +++++
 .../pulsar/broker/service/ReplicatorTest.java      |  9 ++-
 pulsar-discovery-service/pom.xml                   | 22 +++++
 8 files changed, 222 insertions(+), 18 deletions(-)

diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index f4e4172..fc75fe7 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -33,4 +33,11 @@
   <packaging>jar</packaging>
   <name>Pulsar Build Tools</name>
 
+  <dependencies>
+    <dependency>
+      <groupId>org.testng</groupId>
+      <artifactId>testng</artifactId>
+    </dependency>
+  </dependencies>
+
 </project>
diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java
new file mode 100644
index 0000000..0682105
--- /dev/null
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+
+import org.testng.IAnnotationTransformer;
+import org.testng.annotations.ITestAnnotation;
+
+@SuppressWarnings("rawtypes")
+public class AnnotationListener implements IAnnotationTransformer {
+
+    @Override
+    public void transform(ITestAnnotation annotation, Class testClass, Constructor testConstructor, Method testMethod) {
+        annotation.setRetryAnalyzer(RetryAnalyzer.class);
+    }
+
+}
diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/RetryAnalyzer.java b/buildtools/src/main/java/org/apache/pulsar/tests/RetryAnalyzer.java
new file mode 100644
index 0000000..c4caa04
--- /dev/null
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/RetryAnalyzer.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests;
+
+import org.testng.IRetryAnalyzer;
+import org.testng.ITestResult;
+
+public class RetryAnalyzer implements IRetryAnalyzer {
+
+    private int count = 0;
+
+    // Only try again once
+    private static final int MAX_RETRIES = 1;
+
+    @Override
+    public boolean retry(ITestResult result) {
+        return count++ < MAX_RETRIES;
+    }
+
+}
diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml
index 7314eb9..34a0792 100644
--- a/managed-ledger/pom.xml
+++ b/managed-ledger/pom.xml
@@ -98,12 +98,31 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>buildtools</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <properties>
+            <property>
+              <name>listener</name>
+              <value>org.apache.pulsar.tests.AnnotationListener</value>
+            </property>
+          </properties>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
         <executions>
           <execution>
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/PortManager.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/PortManager.java
index e34f5f5..a18cc04 100644
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/PortManager.java
+++ b/managed-ledger/src/test/java/org/apache/bookkeeper/test/PortManager.java
@@ -18,37 +18,96 @@
  */
 package org.apache.bookkeeper.test;
 
-import java.net.ServerSocket;
+import java.io.FileReader;
+import java.io.FileWriter;
 import java.io.IOException;
+import java.net.ServerSocket;
+import java.nio.CharBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 
 /**
  * Port manager allows a base port to be specified on the commandline. Tests will then use ports, counting up from this
  * base port. This allows multiple instances of the bookkeeper tests to run at once.
  */
 public class PortManager {
-    private static int nextPort = getBasePort();
 
+    private static final String lockFilename = System.getProperty("test.lockFilename",
+            "/tmp/pulsar-test-port-manager.lock");
+    private static final int basePort = Integer.valueOf(System.getProperty("test.basePort", "15000"));
+
+    private static final int maxPort = 32000;
+
+    /**
+     * Return a TCP port that is currently unused.
+     *
+     * Keeps track of assigned ports and avoid race condition between different processes
+     */
     public synchronized static int nextFreePort() {
-        while (true) {
-            ServerSocket ss = null;
+        Path path = Paths.get(lockFilename);
+
+        try {
+            FileChannel fileChannel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+            FileLock lock = fileChannel.lock();
+
             try {
-                int port = nextPort++;
-                ss = new ServerSocket(port);
-                ss.setReuseAddress(true);
-                return port;
-            } catch (IOException ioe) {
-            } finally {
-                if (ss != null) {
-                    try {
-                        ss.close();
-                    } catch (IOException ioe) {
-                    }
+
+                FileReader reader = new FileReader(lockFilename);
+                CharBuffer buffer = CharBuffer.allocate(16);
+                int len = reader.read(buffer);
+                buffer.flip();
+
+                int lastUsedPort = basePort;
+                if (len > 0) {
+                    String lastUsedPortStr = buffer.toString();
+                    lastUsedPort = Integer.parseInt(lastUsedPortStr);
                 }
+
+                int freePort = probeFreePort(lastUsedPort + 1);
+
+                FileWriter writer = new FileWriter(lockFilename);
+                writer.write(Integer.toString(freePort));
+
+                reader.close();
+                writer.close();
+
+                return freePort;
+
+            } finally {
+                lock.release();
+                fileChannel.close();
             }
+        } catch (IOException e) {
+            throw new RuntimeException(e);
         }
     }
 
-    private static int getBasePort() {
-        return Integer.valueOf(System.getProperty("test.basePort", "15000"));
+    private static final int MAX_PORT_CONFLICTS = 10;
+
+    private synchronized static int probeFreePort(int port) {
+        int exceptionCount = 0;
+        while (true) {
+            if (port == maxPort) {
+                // Rollover the port probe
+                port = basePort;
+            }
+
+            try (ServerSocket ss = new ServerSocket(port)) {
+                ss.close();
+                // Give it some time to truly close the connection
+                Thread.sleep(100);
+                return port;
+
+            } catch (Exception e) {
+                port++;
+                exceptionCount++;
+                if (exceptionCount > MAX_PORT_CONFLICTS) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
     }
 }
diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index 2f8811d..9a703e0 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -224,11 +224,30 @@
       <groupId>com.ea.agentloader</groupId>
       <artifactId>ea-agent-loader</artifactId>
     </dependency>
+
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>buildtools</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>
     <plugins>
       <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <properties>
+            <property>
+              <name>listener</name>
+              <value>org.apache.pulsar.tests.AnnotationListener</value>
+            </property>
+          </properties>
+        </configuration>
+      </plugin>
+      <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>aspectj-maven-plugin</artifactId>
         <version>1.10</version>
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
index 8397727..f1394d1 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
@@ -86,7 +87,13 @@ public class ReplicatorTest extends ReplicatorTestBase {
     @Override
     @AfterClass(timeOut = 30000)
     void shutdown() throws Exception {
-        super.shutdown();
+        ForkJoinPool.commonPool().execute(() -> {
+            try {
+                super.shutdown();
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        });
     }
 
     @Test(enabled = true, timeOut = 30000)
diff --git a/pulsar-discovery-service/pom.xml b/pulsar-discovery-service/pom.xml
index 3c89757..c6566e7 100644
--- a/pulsar-discovery-service/pom.xml
+++ b/pulsar-discovery-service/pom.xml
@@ -135,6 +135,28 @@
       <type>test-jar</type>
     </dependency>
 
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>buildtools</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <properties>
+            <property>
+              <name>listener</name>
+              <value>org.apache.pulsar.tests.AnnotationListener</value>
+            </property>
+          </properties>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
 </project>

-- 
To stop receiving notification emails like this one, please contact
mmerli@apache.org.