You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tomee.apache.org by db...@apache.org on 2012/03/04 00:48:02 UTC

svn commit: r1296726 - in /openejb/trunk/openejb: container/openejb-core/src/main/java/org/apache/openejb/config/ itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/ itests/failover/ itests/failover/src/main/java/org/apache/openejb...

Author: dblevins
Date: Sat Mar  3 23:48:01 2012
New Revision: 1296726

URL: http://svn.apache.org/viewvc?rev=1296726&view=rev
Log:
OPENEJB-1794 - Multipoint Automatic Reconnect
OPENEJB-1793 - Multipoint.reconnectDelay configures how log to wait between attempts to rejoin the multipoint network
OPENEJB-1789 - Multipoint.discoveryHost allows for "bind" of 0.0.0.0

Added:
    openejb/trunk/openejb/itests/failover/src/main/java/org/apache/openejb/itest/failover/Repository.java
    openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/DedicatedRootServerTest.java
      - copied, changed from r1295442, openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java
    openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/DiscoveryHostTest.java
      - copied, changed from r1295442, openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java
    openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java
    openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/ReconnectDelayTest.java
      - copied, changed from r1295442, openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java
    openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/SpeedTest.java
Modified:
    openejb/trunk/openejb/container/openejb-core/src/main/java/org/apache/openejb/config/Deploy.java
    openejb/trunk/openejb/itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/Calculator.java
    openejb/trunk/openejb/itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/CalculatorBean.java
    openejb/trunk/openejb/itests/failover/pom.xml
    openejb/trunk/openejb/itests/failover/src/main/java/org/apache/openejb/server/control/StandaloneServer.java
    openejb/trunk/openejb/server/openejb-multicast/src/main/java/org/apache/openejb/server/discovery/MultipointDiscoveryAgent.java
    openejb/trunk/openejb/server/openejb-multicast/src/main/java/org/apache/openejb/server/discovery/MultipointServer.java

Modified: openejb/trunk/openejb/container/openejb-core/src/main/java/org/apache/openejb/config/Deploy.java
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/container/openejb-core/src/main/java/org/apache/openejb/config/Deploy.java?rev=1296726&r1=1296725&r2=1296726&view=diff
==============================================================================
--- openejb/trunk/openejb/container/openejb-core/src/main/java/org/apache/openejb/config/Deploy.java (original)
+++ openejb/trunk/openejb/container/openejb-core/src/main/java/org/apache/openejb/config/Deploy.java Sat Mar  3 23:48:01 2012
@@ -64,7 +64,7 @@ public class Deploy {
     private static final int BUF_SIZE = 8192;
 
 
-    public static void main(String[] args) throws SystemExitException {
+    public static void main(String... args) throws SystemExitException {
 
         CommandLineParser parser = new PosixParser();
 

Modified: openejb/trunk/openejb/itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/Calculator.java
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/Calculator.java?rev=1296726&r1=1296725&r2=1296726&view=diff
==============================================================================
--- openejb/trunk/openejb/itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/Calculator.java (original)
+++ openejb/trunk/openejb/itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/Calculator.java Sat Mar  3 23:48:01 2012
@@ -23,4 +23,5 @@ public interface Calculator {
 
     int sum(int a, int b);
 
+    String name();
 }

Modified: openejb/trunk/openejb/itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/CalculatorBean.java
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/CalculatorBean.java?rev=1296726&r1=1296725&r2=1296726&view=diff
==============================================================================
--- openejb/trunk/openejb/itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/CalculatorBean.java (original)
+++ openejb/trunk/openejb/itests/failover-ejb/src/main/java/org/apache/openejb/itest/failover/ejb/CalculatorBean.java Sat Mar  3 23:48:01 2012
@@ -25,4 +25,8 @@ public class CalculatorBean implements C
     public int sum(int a, int b) {
         return a + b;
     }
+
+    public String name() {
+        return System.getProperty("name");
+    }
 }

Modified: openejb/trunk/openejb/itests/failover/pom.xml
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/itests/failover/pom.xml?rev=1296726&r1=1296725&r2=1296726&view=diff
==============================================================================
--- openejb/trunk/openejb/itests/failover/pom.xml (original)
+++ openejb/trunk/openejb/itests/failover/pom.xml Sat Mar  3 23:48:01 2012
@@ -27,6 +27,19 @@
   <artifactId>failover</artifactId>
   <packaging>jar</packaging>
   <name>OpenEJB :: iTests Beans</name>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <systemPropertyVariables>
+            <version>${project.version}</version>
+          </systemPropertyVariables>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
   <dependencies>
     <dependency>
       <groupId>org.apache.openejb</groupId>

Added: openejb/trunk/openejb/itests/failover/src/main/java/org/apache/openejb/itest/failover/Repository.java
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/itests/failover/src/main/java/org/apache/openejb/itest/failover/Repository.java?rev=1296726&view=auto
==============================================================================
--- openejb/trunk/openejb/itests/failover/src/main/java/org/apache/openejb/itest/failover/Repository.java (added)
+++ openejb/trunk/openejb/itests/failover/src/main/java/org/apache/openejb/itest/failover/Repository.java Sat Mar  3 23:48:01 2012
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.openejb.itest.failover;
+
+import org.apache.openejb.loader.Files;
+import org.apache.openejb.util.Join;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @version $Rev$ $Date$
+ */
+public class Repository {
+    private final File repository;
+
+    public Repository() {
+        this(defaultRepository());
+    }
+
+    public Repository(File repository) {
+        Files.exists(repository);
+        Files.dir(repository);
+
+        this.repository = repository;
+    }
+
+    /**
+     * Checks the system property 'repo'
+     * If not found, defaults to '${user.home}/.m2/repository'
+     * @return
+     */
+    private static File defaultRepository() {
+        final File home = new File(System.getProperty("user.home"));
+        final String property = System.getProperty("repo", Files.path(home, ".m2", "repository").getAbsolutePath());
+        return new File(property);
+    }
+
+    public Artifact getArtifact(String groupId, String artifactId, String type) {
+        return new Artifact(groupId, artifactId, type);
+    }
+
+    public class Artifact {
+        private final String groupId;
+        private final String artifactId;
+        private final String type;
+
+        public Artifact(String groupId, String artifactId, String type) {
+            this.groupId = groupId;
+            this.artifactId = artifactId;
+            this.type = type;
+        }
+
+        public File get(String version) {
+            List<String> path = new ArrayList<String>();
+
+            // GroupID path
+            Collections.addAll(path, groupId.split("\\."));
+            path.add(artifactId);
+            path.add(version);
+            path.add(artifactId + "-" + version + "." + type);
+
+            File file = new File(repository, Join.join(File.separator, path));
+            Files.exists(file);
+            Files.file(file);
+            Files.readable(file);
+
+            return file;
+        }
+
+        public File get() {
+            return get(guessVersion());
+        }
+
+        private String guessVersion() {
+            String[] keys = {artifactId + ".version", groupId + ".version", "version"};
+            for (String key : keys) {
+                final String value = System.getProperty(key);
+                if (value != null) {
+                    return value;
+                }
+            }
+
+            String message = String.format("Cannot find version for %s. Checked the following system properties: %s", this, Join.join(", ", keys));
+            throw new IllegalStateException(message);
+        }
+
+        @Override
+        public String toString() {
+            return "Artifact{" +
+                    "groupId='" + groupId + '\'' +
+                    ", artifactId='" + artifactId + '\'' +
+                    ", type='" + type + '\'' +
+                    '}';
+        }
+    }
+}

Modified: openejb/trunk/openejb/itests/failover/src/main/java/org/apache/openejb/server/control/StandaloneServer.java
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/itests/failover/src/main/java/org/apache/openejb/server/control/StandaloneServer.java?rev=1296726&r1=1296725&r2=1296726&view=diff
==============================================================================
--- openejb/trunk/openejb/itests/failover/src/main/java/org/apache/openejb/server/control/StandaloneServer.java (original)
+++ openejb/trunk/openejb/itests/failover/src/main/java/org/apache/openejb/server/control/StandaloneServer.java Sat Mar  3 23:48:01 2012
@@ -57,6 +57,10 @@ public class StandaloneServer {
     private OutputStream out = System.out;
     private Options options = new Options(properties);
 
+    public StandaloneServer(File home) {
+        this(home, home);
+    }
+
     public StandaloneServer(File home, File base) {
         this.home = home;
         this.base = base;
@@ -80,9 +84,6 @@ public class StandaloneServer {
 
     public class ServerService {
 
-        private String bind;
-        private int threads;
-
         private final String name;
 
         public ServerService(String name) {
@@ -101,10 +102,18 @@ public class StandaloneServer {
             return options.get(name + ".disabled", true);
         }
 
+        public boolean isEnabled() {
+            return !isDisabled();
+        }
+
         public void setDisabled(boolean b) {
             properties.put(name + ".disabled", b + "");
         }
 
+        public void setEnabled(boolean b) {
+            setDisabled(!b);
+        }
+
         public String getBind() {
             return options.get(name + ".bind", "");
         }
@@ -121,9 +130,37 @@ public class StandaloneServer {
             properties.put(name + ".threads", threads + "");
         }
 
-        public void set(String name, String value) {
+        public ServerService set(String name, String value) {
             properties.put(this.name + "." + name, value);
+            return this;
+        }
+
+        public ServerService threads(int threads) {
+            setThreads(threads);
+            return this;
+        }
+
+        public ServerService port(int port) {
+            setPort(port);
+            return this;
+        }
+
+        public ServerService enable() {
+            setEnabled(true);
+            return this;
         }
+
+        public ServerService disable() {
+            setDisabled(true);
+            return this;
+        }
+
+        public ServerService bind(String host) {
+            setBind(host);
+            return this;
+        }
+
+
     }
 
     public File getHome() {
@@ -186,6 +223,10 @@ public class StandaloneServer {
         return properties;
     }
 
+    public Object setProperty(String key, String value) {
+        return getProperties().setProperty(key, value);
+    }
+
     public void start() {
         start(0, TimeUnit.MILLISECONDS);
     }
@@ -249,7 +290,6 @@ public class StandaloneServer {
     }
 
     private void waitForExit() {
-        kill.remove(this);
         try {
             process.waitFor();
         } catch (InterruptedException e) {
@@ -285,14 +325,14 @@ public class StandaloneServer {
     }
 
     public void deploy(String path) {
-        final int code = command("deploy", path);
+        final int code = command("deploy", getServerUrl(), path);
         if (code != 0) {
             throw new DeployException(home, code, path);
         }
     }
 
     public void undeploy(String path) {
-        final int code = command("undeploy", path);
+        final int code = command("undeploy", getServerUrl(), path);
         if (code != 0) {
             throw new UndeployException(home, code, path);
         }
@@ -308,6 +348,7 @@ public class StandaloneServer {
 
 
     public void killOnExit() {
+        if (kill.contains(this)) return;
         kill.add(this);
     }
 
@@ -405,4 +446,15 @@ public class StandaloneServer {
         }
     }
 
+    private String getServerUrl() {
+        final ServerService ejbd = getServerService("ejbd");
+
+        int port = ejbd.getPort();
+        if (port == 0) port = 4201;
+
+        String host = ejbd.getBind();
+        if (host == null || host.length() == 0) host = "localhost";
+
+        return String.format("--server-url=ejbd://%s:%s", host, port);
+    }
 }

Copied: openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/DedicatedRootServerTest.java (from r1295442, openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java)
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/DedicatedRootServerTest.java?p2=openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/DedicatedRootServerTest.java&p1=openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java&r1=1295442&r2=1296726&rev=1296726&view=diff
==============================================================================
--- openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java (original)
+++ openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/DedicatedRootServerTest.java Sat Mar  3 23:48:01 2012
@@ -22,52 +22,77 @@ import org.apache.openejb.loader.Files;
 import org.apache.openejb.loader.IO;
 import org.apache.openejb.loader.Zips;
 import org.apache.openejb.server.control.StandaloneServer;
-import org.apache.openejb.util.Join;
+import org.apache.openejb.util.NetworkUtil;
 import org.junit.Assert;
 import org.junit.Test;
-import org.junit.Ignore;
 
 import javax.ejb.EJBException;
 import javax.naming.Context;
 import javax.naming.InitialContext;
+import javax.naming.NamingException;
 import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.NetworkInterface;
-import java.net.ServerSocket;
-import java.net.SocketAddress;
-import java.nio.channels.ServerSocketChannel;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
-public class FailoverTest {
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * In some situations it can be desirable to have
+ * one dedicated multipoint root server which does
+ * no other function other than to serve as a central
+ * hub for making multipoint introductions.
+ *
+ * This dedicate root server will not serve applications
+ * and not be added to the list of servers that can
+ * service EJB requests.
+ *
+ */
+public class DedicatedRootServerTest {
 
     @Test
-    public void testNothing() throws Exception {
-    }
+    public void test() throws Exception {
 
-    @Test @Ignore
-    public void testFailover() throws Exception {
+        // To run in an IDE, uncomment and update this line
+        //System.setProperty("version", "4.0.0-beta-3-SNAPSHOT");
 
-        final File zip = new File("/Users/dblevins/.m2/repository/org/apache/openejb/openejb-standalone/4.0.0-beta-3-SNAPSHOT/openejb-standalone-4.0.0-beta-3-SNAPSHOT.zip");
-        final File app = new File("/Users/dblevins/.m2/repository/org/apache/openejb/itests/failover-ejb/4.0.0-beta-3-SNAPSHOT/failover-ejb-4.0.0-beta-3-SNAPSHOT.jar");
+        final Repository repository = new Repository();
+        final File zip = repository.getArtifact("org.apache.openejb", "openejb-standalone", "zip").get();
+        final File app = repository.getArtifact("org.apache.openejb.itests", "failover-ejb", "jar").get();
 
         final File dir = Files.tmpdir();
 
-        final String[] serverNames = {"red", "green", "blue"};
-//        final String[] serverNames = {"red", "green", "blue", "yellow", "orange", "purple"};
-//        final String[] serverNames = {"red", "green"};
+        final StandaloneServer root;
+        {
+            StandaloneServer root1;
+            final String name = "root";
+            final File home = new File(dir, name);
+
+            Files.mkdir(home);
+            Zips.unzip(zip, home, true);
+
+            root1 = new StandaloneServer(home, home);
+            root1.killOnExit();
+            root1.ignoreOut();
+            root1.setProperty("name", name);
+            root1.setProperty("openejb.extract.configuration", "false");
+
+            final StandaloneServer.ServerService multipoint = root1.getServerService("multipoint");
+            multipoint.setBind("localhost");
+            multipoint.setPort(getAvailablePort());
+            multipoint.setDisabled(false);
+            multipoint.set("discoveryName", name);
+            root = root1;
 
-        final List<StandaloneServer> servers = new ArrayList<StandaloneServer>();
+            root.start();
+        }
 
-        final List<String> initialServers = new ArrayList<String>();
 
-        for (String name : serverNames) {
+        final Map<String, StandaloneServer> servers = new HashMap<String, StandaloneServer>();
+        for (String name : new String[]{"red", "green", "blue"}) {
             final File home = new File(dir, name);
             Files.mkdir(home);
             Zips.unzip(zip, home, true);
@@ -75,88 +100,98 @@ public class FailoverTest {
             final StandaloneServer server = new StandaloneServer(home, home);
             server.killOnExit();
             server.ignoreOut();
-            server.getProperties().put("openejb.extract.configuration", "false");
+            server.setProperty("name", name);
+            server.setProperty("openejb.extract.configuration", "false");
 
             IO.copy(app, Files.path(home, "apps", "itest.jar"));
             IO.copy(IO.read("<openejb><Deployments dir=\"apps/\"/></openejb>"), Files.path(home, "conf", "openejb.xml"));
-            /*
-            server      = org.apache.openejb.server.ejbd.EjbServer
-            bind        = 127.0.0.1
-            port        = 4201
-            disabled    = false
-            threads     = 200
-            backlog     = 200
-            discovery   = ejb:ejbd://{bind}:{port}
-             */
+
             final StandaloneServer.ServerService ejbd = server.getServerService("ejbd");
             ejbd.setDisabled(false);
             ejbd.setPort(getAvailablePort());
             ejbd.setThreads(5);
-            ejbd.set("discovery", "ejb:ejbd://{bind}:{port}/" + name);
 
-            /*
-            server      = org.apache.openejb.server.discovery.MultipointDiscoveryAgent
-            bind        = 127.0.0.1
-            port        = 4212
-            disabled    = true
-
-            initialServers         =
-            group                  = default
-            heart_rate             = 500
-            loopback_mode          = false
-            max_missed_heartbeats  = 10
-             */
             final StandaloneServer.ServerService multipoint = server.getServerService("multipoint");
             multipoint.setPort(getAvailablePort());
             multipoint.setDisabled(false);
+            multipoint.set("discoveryName", name);
+            multipoint.set("initialServers", "localhost:"+root.getServerService("multipoint").getPort());
 
-            initialServers.add("localhost:" + multipoint.getPort());
-
-            servers.add(server);
-        }
-
-        servers.get(0).setOut(System.out);
-
-        for (StandaloneServer server : servers) {
-            final StandaloneServer.ServerService multipoint = server.getServerService("multipoint");
-            multipoint.set("initialServers", Join.join(",", initialServers));
-        }
-
-        for (StandaloneServer server : servers) {
+            servers.put(name, server);
             server.start(1, TimeUnit.MINUTES);
-        }
 
-        Collections.reverse(servers);
+            invoke(name, server);
+        }
 
         System.setProperty("openejb.client.requestretry", "true");
 
         final Properties environment = new Properties();
         environment.put(Context.INITIAL_CONTEXT_FACTORY, RemoteInitialContextFactory.class.getName());
-        environment.put(Context.PROVIDER_URL, "failover:ejbd://localhost:" + servers.get(0).getServerService("ejbd").getPort());
+        environment.put(Context.PROVIDER_URL, "ejbd://localhost:" + servers.values().iterator().next().getServerService("ejbd").getPort());
 
         final InitialContext context = new InitialContext(environment);
         final Calculator bean = (Calculator) context.lookup("CalculatorBeanRemote");
 
-        for (StandaloneServer server : servers) {
-            System.out.println(String.format("Average invocation time %s microseconds", invoke(bean, 10000)));
-            server.kill();
+
+        String previous = null;
+        for (StandaloneServer ignored : servers.values()) {
+
+            // What server are we talking to now?
+            final String name = bean.name();
+
+            // The root should not be serving apps
+            assertFalse("root".equals(name));
+
+            // Should not be the same server we were talking with previously (we killed that server)
+            if (previous != null) assertFalse(name.equals(previous));
+            previous = name;
+
+            // Should be the same server for the next N calls
+            invoke(bean, 1000, name);
+
+            // Now let's kill that server
+            servers.get(name).kill();
         }
 
         System.out.println("All servers destroyed");
 
         try {
-            System.out.println(String.format("Average invocation time %s microseconds", invoke(bean, 10000)));
-            Assert.fail("Server should be destroyed");
+            final String name = bean.name();
+            Assert.fail("Server should be destroyed: " + name);
         } catch (EJBException e) {
             // good
         }
 
-        for (StandaloneServer server : servers) {
-            server.start(1, TimeUnit.MINUTES);
-            System.out.println(String.format("Average invocation time %s microseconds", invoke(bean, 10000)));
+        // Let's start a server again and invocations should now succeed
+        final Iterator<StandaloneServer> iterator = servers.values().iterator();
+        iterator.next();
+        iterator.next().start(1, TimeUnit.MINUTES);
+
+        assertEquals(5, bean.sum(2, 3));
+    }
+
+    private void invoke(String name, StandaloneServer server) throws NamingException {
+        final Properties environment = new Properties();
+        environment.put(Context.INITIAL_CONTEXT_FACTORY, RemoteInitialContextFactory.class.getName());
+        environment.put(Context.PROVIDER_URL, "ejbd://localhost:" + server.getServerService("ejbd").getPort());
+
+        final InitialContext context = new InitialContext(environment);
+        final Calculator bean = (Calculator) context.lookup("CalculatorBeanRemote");
+        assertEquals(name, bean.name());
+    }
+
+    private long invoke(Calculator bean, int max, String expectedName) {
+
+        long total = 0;
+
+        for (int i = 0; i < max; i++) {
+            final long start = System.nanoTime();
+            String name = bean.name();
+            Assert.assertEquals(expectedName, name);
+            total += System.nanoTime() - start;
         }
 
-        System.out.println("DONE");
+        return TimeUnit.NANOSECONDS.toMicros(total / max);
     }
 
     private long invoke(Calculator bean, int max) {
@@ -173,14 +208,6 @@ public class FailoverTest {
     }
 
     private int getAvailablePort() {
-        try {
-            final ServerSocket serverSocket = new ServerSocket(0);
-            final int port = serverSocket.getLocalPort();
-            serverSocket.close();
-
-            return port;
-        } catch (IOException e) {
-            throw new IllegalStateException("Unable to acquire a available port");
-        }
+        return NetworkUtil.getNextAvailablePort();
     }
 }

Copied: openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/DiscoveryHostTest.java (from r1295442, openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java)
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/DiscoveryHostTest.java?p2=openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/DiscoveryHostTest.java&p1=openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java&r1=1295442&r2=1296726&rev=1296726&view=diff
==============================================================================
--- openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java (original)
+++ openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/DiscoveryHostTest.java Sat Mar  3 23:48:01 2012
@@ -23,6 +23,8 @@ import org.apache.openejb.loader.IO;
 import org.apache.openejb.loader.Zips;
 import org.apache.openejb.server.control.StandaloneServer;
 import org.apache.openejb.util.Join;
+import org.apache.openejb.util.NetworkUtil;
+import org.apache.xbean.finder.ResourceFinder;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.Ignore;
@@ -37,6 +39,7 @@ import java.net.InetSocketAddress;
 import java.net.NetworkInterface;
 import java.net.ServerSocket;
 import java.net.SocketAddress;
+import java.net.URL;
 import java.nio.channels.ServerSocketChannel;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -45,29 +48,25 @@ import java.util.List;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
-public class FailoverTest {
+public class DiscoveryHostTest {
 
     @Test
-    public void testNothing() throws Exception {
-    }
+    public void test() throws Exception {
 
-    @Test @Ignore
-    public void testFailover() throws Exception {
+        // To run in an IDE, uncomment and update this line
+        //System.setProperty("version", "4.0.0-beta-3-SNAPSHOT");
 
-        final File zip = new File("/Users/dblevins/.m2/repository/org/apache/openejb/openejb-standalone/4.0.0-beta-3-SNAPSHOT/openejb-standalone-4.0.0-beta-3-SNAPSHOT.zip");
-        final File app = new File("/Users/dblevins/.m2/repository/org/apache/openejb/itests/failover-ejb/4.0.0-beta-3-SNAPSHOT/failover-ejb-4.0.0-beta-3-SNAPSHOT.jar");
+        final Repository repository = new Repository();
+        final File zip = repository.getArtifact("org.apache.openejb", "openejb-standalone", "zip").get();
+        final File app = repository.getArtifact("org.apache.openejb.itests", "failover-ejb", "jar").get();
 
         final File dir = Files.tmpdir();
 
-        final String[] serverNames = {"red", "green", "blue"};
-//        final String[] serverNames = {"red", "green", "blue", "yellow", "orange", "purple"};
-//        final String[] serverNames = {"red", "green"};
-
         final List<StandaloneServer> servers = new ArrayList<StandaloneServer>();
 
         final List<String> initialServers = new ArrayList<String>();
 
-        for (String name : serverNames) {
+        for (String name : new String[]{"red", "green", "blue"}) {
             final File home = new File(dir, name);
             Files.mkdir(home);
             Zips.unzip(zip, home, true);
@@ -75,40 +74,25 @@ public class FailoverTest {
             final StandaloneServer server = new StandaloneServer(home, home);
             server.killOnExit();
             server.ignoreOut();
+            server.getProperties().put("name", name);
             server.getProperties().put("openejb.extract.configuration", "false");
 
             IO.copy(app, Files.path(home, "apps", "itest.jar"));
             IO.copy(IO.read("<openejb><Deployments dir=\"apps/\"/></openejb>"), Files.path(home, "conf", "openejb.xml"));
-            /*
-            server      = org.apache.openejb.server.ejbd.EjbServer
-            bind        = 127.0.0.1
-            port        = 4201
-            disabled    = false
-            threads     = 200
-            backlog     = 200
-            discovery   = ejb:ejbd://{bind}:{port}
-             */
+
             final StandaloneServer.ServerService ejbd = server.getServerService("ejbd");
             ejbd.setDisabled(false);
+            ejbd.setBind("0.0.0.0");
             ejbd.setPort(getAvailablePort());
             ejbd.setThreads(5);
-            ejbd.set("discovery", "ejb:ejbd://{bind}:{port}/" + name);
+            ejbd.set("discoveryHost", "localhost");
+            ejbd.set("discovery", "ejb:ejbd://{discoveryHost}:{port}/" + name);
 
-            /*
-            server      = org.apache.openejb.server.discovery.MultipointDiscoveryAgent
-            bind        = 127.0.0.1
-            port        = 4212
-            disabled    = true
-
-            initialServers         =
-            group                  = default
-            heart_rate             = 500
-            loopback_mode          = false
-            max_missed_heartbeats  = 10
-             */
             final StandaloneServer.ServerService multipoint = server.getServerService("multipoint");
+            multipoint.setBind("0.0.0.0");
             multipoint.setPort(getAvailablePort());
             multipoint.setDisabled(false);
+            multipoint.set("discoveryHost", "localhost");
 
             initialServers.add("localhost:" + multipoint.getPort());
 
@@ -159,6 +143,21 @@ public class FailoverTest {
         System.out.println("DONE");
     }
 
+    private long invoke(Calculator bean, int max, String expectedName) {
+
+        long total = 0;
+
+        for (int i = 0; i < max; i++) {
+            final long start = System.nanoTime();
+            String name = bean.name();
+            System.out.println(name);
+            Assert.assertEquals(expectedName, name);
+            total += System.nanoTime() - start;
+        }
+
+        return TimeUnit.NANOSECONDS.toMicros(total / max);
+    }
+
     private long invoke(Calculator bean, int max) {
 
         long total = 0;
@@ -173,14 +172,6 @@ public class FailoverTest {
     }
 
     private int getAvailablePort() {
-        try {
-            final ServerSocket serverSocket = new ServerSocket(0);
-            final int port = serverSocket.getLocalPort();
-            serverSocket.close();
-
-            return port;
-        } catch (IOException e) {
-            throw new IllegalStateException("Unable to acquire a available port");
-        }
+        return NetworkUtil.getNextAvailablePort();
     }
 }

Added: openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java?rev=1296726&view=auto
==============================================================================
--- openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java (added)
+++ openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java Sat Mar  3 23:48:01 2012
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.openejb.itest.failover;
+
+import org.apache.openejb.itest.failover.ejb.Calculator;
+import org.apache.openejb.loader.Files;
+import org.apache.openejb.loader.Zips;
+import org.apache.openejb.server.control.StandaloneServer;
+import org.apache.openejb.util.NetworkUtil;
+import org.junit.Assert;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @version $Rev$ $Date$
+ */
+public class FailoverTest {
+    private long invoke(Calculator bean, int max, String expectedName) {
+
+        long total = 0;
+
+        for (int i = 0; i < max; i++) {
+            final long start = System.nanoTime();
+            String name = bean.name();
+            System.out.println(name);
+            Assert.assertEquals(expectedName, name);
+            total += System.nanoTime() - start;
+        }
+
+        return TimeUnit.NANOSECONDS.toMicros(total / max);
+    }
+
+    protected long invoke(Calculator bean, int max) {
+
+        long total = 0;
+
+        for (int i = 0; i < max; i++) {
+            final long start = System.nanoTime();
+            Assert.assertEquals(3, bean.sum(1, 2));
+            total += System.nanoTime() - start;
+        }
+
+        return TimeUnit.NANOSECONDS.toMicros(total / max);
+    }
+
+    public StandaloneServer createMultipointServer(File zip, File dir, String name) throws IOException {
+        final File home = new File(dir, name);
+
+        Files.mkdir(home);
+        Zips.unzip(zip, home, true);
+
+        StandaloneServer server = new StandaloneServer(home, home);
+        server.killOnExit();
+        server.ignoreOut();
+        server.setProperty("name", name);
+        server.setProperty("TestName", this.getClass().getName());
+        server.setProperty("openejb.extract.configuration", "false");
+
+        final StandaloneServer.ServerService multipoint = server.getServerService("multipoint");
+        multipoint.setBind("localhost");
+        multipoint.setPort(NetworkUtil.getNextAvailablePort());
+        multipoint.setEnabled(true);
+        multipoint.set("discoveryName", name);
+        return server;
+    }
+
+    public int getAvailablePort() {
+        return NetworkUtil.getNextAvailablePort();
+    }
+}

Copied: openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/ReconnectDelayTest.java (from r1295442, openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java)
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/ReconnectDelayTest.java?p2=openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/ReconnectDelayTest.java&p1=openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java&r1=1295442&r2=1296726&rev=1296726&view=diff
==============================================================================
--- openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/FailoverTest.java (original)
+++ openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/ReconnectDelayTest.java Sat Mar  3 23:48:01 2012
@@ -22,52 +22,51 @@ import org.apache.openejb.loader.Files;
 import org.apache.openejb.loader.IO;
 import org.apache.openejb.loader.Zips;
 import org.apache.openejb.server.control.StandaloneServer;
-import org.apache.openejb.util.Join;
+import org.apache.openejb.util.Duration;
+import org.apache.openejb.util.NetworkUtil;
 import org.junit.Assert;
 import org.junit.Test;
-import org.junit.Ignore;
 
-import javax.ejb.EJBException;
 import javax.naming.Context;
 import javax.naming.InitialContext;
 import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.NetworkInterface;
-import java.net.ServerSocket;
-import java.net.SocketAddress;
-import java.nio.channels.ServerSocketChannel;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
-public class FailoverTest {
+/**
+ * This test verifies the situation where none of the
+ * servers listed in the 'initialServers' list can be contacted.
+ *
+ * A server that is unable to connect to any of its peers should
+ * continue trying to connect to servers on the 'initialServers'
+ * list.  How long to wait between attempts is dictated by the
+ * 'reconnectDelay' setting, which is 30 seconds by default.
+ */
+public class ReconnectDelayTest {
+
+    private static final String MULTIPOINT = "multipoint";
 
     @Test
-    public void testNothing() throws Exception {
-    }
+    public void test() throws Exception {
 
-    @Test @Ignore
-    public void testFailover() throws Exception {
+        // To run in an IDE, uncomment and update this line
+        //System.setProperty("version", "4.0.0-beta-3-SNAPSHOT");
 
-        final File zip = new File("/Users/dblevins/.m2/repository/org/apache/openejb/openejb-standalone/4.0.0-beta-3-SNAPSHOT/openejb-standalone-4.0.0-beta-3-SNAPSHOT.zip");
-        final File app = new File("/Users/dblevins/.m2/repository/org/apache/openejb/itests/failover-ejb/4.0.0-beta-3-SNAPSHOT/failover-ejb-4.0.0-beta-3-SNAPSHOT.jar");
+        Duration reconnectDelay = new Duration("1 second");
 
-        final File dir = Files.tmpdir();
+        final Repository repository = new Repository();
+        final File zip = repository.getArtifact("org.apache.openejb", "openejb-standalone", "zip").get();
+        final File app = repository.getArtifact("org.apache.openejb.itests", "failover-ejb", "jar").get();
 
-        final String[] serverNames = {"red", "green", "blue"};
-//        final String[] serverNames = {"red", "green", "blue", "yellow", "orange", "purple"};
-//        final String[] serverNames = {"red", "green"};
+        final File dir = Files.tmpdir();
 
-        final List<StandaloneServer> servers = new ArrayList<StandaloneServer>();
+        System.setProperty("openejb.client.requestretry", "true");
 
-        final List<String> initialServers = new ArrayList<String>();
+        final Map<String, StandaloneServer> servers = new HashMap<String, StandaloneServer>();
 
-        for (String name : serverNames) {
+        for (String name : new String[]{"red", "green", "blue"}) {
             final File home = new File(dir, name);
             Files.mkdir(home);
             Zips.unzip(zip, home, true);
@@ -75,97 +74,118 @@ public class FailoverTest {
             final StandaloneServer server = new StandaloneServer(home, home);
             server.killOnExit();
             server.ignoreOut();
+            server.getProperties().put("name", name);
             server.getProperties().put("openejb.extract.configuration", "false");
 
             IO.copy(app, Files.path(home, "apps", "itest.jar"));
             IO.copy(IO.read("<openejb><Deployments dir=\"apps/\"/></openejb>"), Files.path(home, "conf", "openejb.xml"));
-            /*
-            server      = org.apache.openejb.server.ejbd.EjbServer
-            bind        = 127.0.0.1
-            port        = 4201
-            disabled    = false
-            threads     = 200
-            backlog     = 200
-            discovery   = ejb:ejbd://{bind}:{port}
-             */
+
             final StandaloneServer.ServerService ejbd = server.getServerService("ejbd");
             ejbd.setDisabled(false);
+            ejbd.setBind("0.0.0.0");
             ejbd.setPort(getAvailablePort());
             ejbd.setThreads(5);
-            ejbd.set("discovery", "ejb:ejbd://{bind}:{port}/" + name);
+            ejbd.set("discoveryHost", "localhost");
+            ejbd.set("discovery", "ejb:ejbd://{discoveryHost}:{port}/" + name);
 
-            /*
-            server      = org.apache.openejb.server.discovery.MultipointDiscoveryAgent
-            bind        = 127.0.0.1
-            port        = 4212
-            disabled    = true
-
-            initialServers         =
-            group                  = default
-            heart_rate             = 500
-            loopback_mode          = false
-            max_missed_heartbeats  = 10
-             */
-            final StandaloneServer.ServerService multipoint = server.getServerService("multipoint");
+            final StandaloneServer.ServerService multipoint = server.getServerService(MULTIPOINT);
+            multipoint.setBind("0.0.0.0");
             multipoint.setPort(getAvailablePort());
             multipoint.setDisabled(false);
+            multipoint.set("discoveryHost", "localhost");
+            multipoint.set("discoveryName", name);
+            multipoint.set("reconnectDelay", reconnectDelay.toString());
 
-            initialServers.add("localhost:" + multipoint.getPort());
-
-            servers.add(server);
+            servers.put(name, server);
         }
 
-        servers.get(0).setOut(System.out);
+        final StandaloneServer red = servers.get("red");
 
-        for (StandaloneServer server : servers) {
-            final StandaloneServer.ServerService multipoint = server.getServerService("multipoint");
-            multipoint.set("initialServers", Join.join(",", initialServers));
+        // Set all the initialServers to point to RED
+        for (Map.Entry<String, StandaloneServer> entry : servers.entrySet()) {
+            final StandaloneServer server = entry.getValue();
+            final StandaloneServer.ServerService multipoint = server.getServerService(MULTIPOINT);
+            multipoint.set("initialServers", "localhost:" + red.getServerService(MULTIPOINT).getPort());
         }
 
-        for (StandaloneServer server : servers) {
-            server.start(1, TimeUnit.MINUTES);
+        // Start all the servers except RED
+        for (Map.Entry<String, StandaloneServer> entry : servers.entrySet()) {
+            if (entry.getKey().equals("red")) continue;
+            entry.getValue().start(1, TimeUnit.MINUTES);
         }
 
-        Collections.reverse(servers);
+        // Verify Failover is not yet functional
 
-        System.setProperty("openejb.client.requestretry", "true");
+        {
+            // RED was never started so BLUE never found any peers
 
-        final Properties environment = new Properties();
-        environment.put(Context.INITIAL_CONTEXT_FACTORY, RemoteInitialContextFactory.class.getName());
-        environment.put(Context.PROVIDER_URL, "failover:ejbd://localhost:" + servers.get(0).getServerService("ejbd").getPort());
+            // Lets invoke BLUE then shut it down and verify we have
+            // no other peers to invoke
+            final StandaloneServer blue = servers.get("blue");
+            final Properties environment = new Properties();
+            environment.put(Context.INITIAL_CONTEXT_FACTORY, RemoteInitialContextFactory.class.getName());
+            environment.put(Context.PROVIDER_URL, "ejbd://localhost:" + blue.getServerService("ejbd").getPort());
 
-        final InitialContext context = new InitialContext(environment);
-        final Calculator bean = (Calculator) context.lookup("CalculatorBeanRemote");
+            final InitialContext context = new InitialContext(environment);
+            final Calculator bean = (Calculator) context.lookup("CalculatorBeanRemote");
 
-        for (StandaloneServer server : servers) {
-            System.out.println(String.format("Average invocation time %s microseconds", invoke(bean, 10000)));
-            server.kill();
-        }
+            // Invoke BLUE a few times
+            invoke(bean, 10, "blue");
 
-        System.out.println("All servers destroyed");
+            // Kill BLUE
+            blue.kill();
 
-        try {
-            System.out.println(String.format("Average invocation time %s microseconds", invoke(bean, 10000)));
-            Assert.fail("Server should be destroyed");
-        } catch (EJBException e) {
-            // good
+            // Invocations should now fail (and not failover)
+            try {
+                bean.name();
+                Assert.fail("Server should be down and failover not hooked up");
+            } catch (Exception e) {
+                // pass
+            }
         }
 
-        for (StandaloneServer server : servers) {
-            server.start(1, TimeUnit.MINUTES);
-            System.out.println(String.format("Average invocation time %s microseconds", invoke(bean, 10000)));
-        }
+        // Now we start RED
+        red.start(1, TimeUnit.MINUTES);
 
-        System.out.println("DONE");
+        // Wait for the reconnectDelay so GREEN can find RED
+        Thread.sleep((long) (reconnectDelay.getTime(TimeUnit.MILLISECONDS) * 1.5));
+
+        // Verify Failover is now functional
+
+        {
+            // RED was never started so GREEN never found any peers
+
+            // Lets invoke GREEN then shut it down and verify we have
+            // no other peers to invoke
+            final StandaloneServer green = servers.get("green");
+            final Properties environment = new Properties();
+            environment.put(Context.INITIAL_CONTEXT_FACTORY, RemoteInitialContextFactory.class.getName());
+            environment.put(Context.PROVIDER_URL, "ejbd://localhost:" + green.getServerService("ejbd").getPort());
+
+            final InitialContext context = new InitialContext(environment);
+            final Calculator bean = (Calculator) context.lookup("CalculatorBeanRemote");
+
+
+            // Invoke GREEN a few times
+            invoke(bean, 10, "green");
+
+            // Kill GREEN
+            green.kill();
+
+            // Invocations should now failover to RED
+            invoke(bean, 10, "red");
+        }
     }
 
-    private long invoke(Calculator bean, int max) {
+    private long invoke(Calculator bean, int max, String expectedName) {
 
         long total = 0;
 
         for (int i = 0; i < max; i++) {
             final long start = System.nanoTime();
-            Assert.assertEquals(3, bean.sum(1, 2));
+            String name = bean.name();
+            System.out.println(name);
+            Assert.assertEquals(expectedName, name);
             total += System.nanoTime() - start;
         }
 
@@ -173,14 +193,6 @@ public class FailoverTest {
     }
 
     private int getAvailablePort() {
-        try {
-            final ServerSocket serverSocket = new ServerSocket(0);
-            final int port = serverSocket.getLocalPort();
-            serverSocket.close();
-
-            return port;
-        } catch (IOException e) {
-            throw new IllegalStateException("Unable to acquire a available port");
-        }
+        return NetworkUtil.getNextAvailablePort();
     }
 }

Added: openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/SpeedTest.java
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/SpeedTest.java?rev=1296726&view=auto
==============================================================================
--- openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/SpeedTest.java (added)
+++ openejb/trunk/openejb/itests/failover/src/test/java/org/apache/openejb/itest/failover/SpeedTest.java Sat Mar  3 23:48:01 2012
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.openejb.itest.failover;
+
+import org.apache.openejb.client.RemoteInitialContextFactory;
+import org.apache.openejb.itest.failover.ejb.Calculator;
+import org.apache.openejb.loader.Files;
+import org.apache.openejb.loader.IO;
+import org.apache.openejb.loader.Zips;
+import org.apache.openejb.server.control.StandaloneServer;
+import org.apache.openejb.util.Join;
+import org.apache.openejb.util.NetworkUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.ejb.EJBException;
+import javax.naming.Context;
+import javax.naming.InitialContext;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+public class SpeedTest extends FailoverTest {
+
+    @Test
+    public void test() throws Exception {
+
+        // To run in an IDE, uncomment and update this line
+        //System.setProperty("version", "4.0.0-beta-3-SNAPSHOT");
+
+        final Repository repository = new Repository();
+        final File zip = repository.getArtifact("org.apache.openejb", "openejb-standalone", "zip").get();
+        final File app = repository.getArtifact("org.apache.openejb.itests", "failover-ejb", "jar").get();
+
+        final File dir = Files.tmpdir();
+
+        final StandaloneServer root = createMultipointServer(zip, dir, "root");
+        root.setOut(System.out);
+        root.start(1, TimeUnit.MINUTES);
+
+        final List<StandaloneServer> servers = new ArrayList<StandaloneServer>();
+
+        for (String name : new String[]{"red", "green", "blue", "yellow", "orange"}) {
+
+            final File home = new File(dir, name);
+            Files.mkdir(home);
+            Zips.unzip(zip, home, true);
+
+            final StandaloneServer server = new StandaloneServer(home, home);
+            server.killOnExit();
+            server.ignoreOut();
+            server.setProperty("name", name);
+            server.setProperty("openejb.extract.configuration", "false");
+
+            IO.copy(app, Files.path(home, "apps", "itest.jar"));
+            IO.copy(IO.read("<openejb><Deployments dir=\"apps/\"/></openejb>"), Files.path(home, "conf", "openejb.xml"));
+
+            final StandaloneServer.ServerService ejbd = server.getServerService("ejbd");
+            ejbd.setDisabled(false);
+            ejbd.setPort(getAvailablePort());
+            ejbd.setThreads(5);
+
+            final StandaloneServer.ServerService multipoint = server.getServerService("multipoint");
+            multipoint.setPort(getAvailablePort());
+            multipoint.setDisabled(false);
+            multipoint.set("discoveryName", name);
+            multipoint.set("initialServers", "localhost:"+root.getServerService("multipoint").getPort());
+
+            server.start(1, TimeUnit.MINUTES);
+
+            servers.add(server);
+        }
+
+        Collections.reverse(servers);
+
+        System.setProperty("openejb.client.requestretry", "true");
+
+        final Properties environment = new Properties();
+        environment.put(Context.INITIAL_CONTEXT_FACTORY, RemoteInitialContextFactory.class.getName());
+        environment.put(Context.PROVIDER_URL, "failover:ejbd://localhost:" + servers.get(0).getServerService("ejbd").getPort());
+
+        final InitialContext context = new InitialContext(environment);
+        final Calculator bean = (Calculator) context.lookup("CalculatorBeanRemote");
+
+        // Hotspotting doesn't kick in with just a few invocations
+        // Later we'll be doing thousands of invocations and that will
+        // bring the invoke time down significantly
+
+        // With this test we're looking for issues with failed servers
+        // causing delays even though they should have been removed from
+        // the clients invocation list
+
+        final long slowest = invoke(bean, 5);
+        System.out.printf("Base invocation speed: %s microseconds", slowest);
+        System.out.println();
+
+        for (StandaloneServer server : servers) {
+            final long speed = invoke(bean, 10000);
+            Assert.assertTrue(String.format("Average invocation time %s microseconds higher than pre-hotspot time of %s microseconds", speed, slowest), speed < slowest);
+            server.kill();
+        }
+
+        System.out.println("All servers destroyed");
+
+        try {
+            System.out.println(String.format("Average invocation time %s microseconds", invoke(bean, 10000)));
+            Assert.fail("Server should be destroyed");
+        } catch (EJBException e) {
+            // good
+        }
+
+        final StandaloneServer server = servers.get(0);
+        server.start(1, TimeUnit.MINUTES);
+
+        final long speed = invoke(bean, 10000);
+        Assert.assertTrue(String.format("Average invocation time %s microseconds higher than pre-hotspot time of %s microseconds", speed, slowest), speed < slowest);
+
+    }
+}

Modified: openejb/trunk/openejb/server/openejb-multicast/src/main/java/org/apache/openejb/server/discovery/MultipointDiscoveryAgent.java
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/server/openejb-multicast/src/main/java/org/apache/openejb/server/discovery/MultipointDiscoveryAgent.java?rev=1296726&r1=1296725&r2=1296726&view=diff
==============================================================================
--- openejb/trunk/openejb/server/openejb-multicast/src/main/java/org/apache/openejb/server/discovery/MultipointDiscoveryAgent.java (original)
+++ openejb/trunk/openejb/server/openejb-multicast/src/main/java/org/apache/openejb/server/discovery/MultipointDiscoveryAgent.java Sat Mar  3 23:48:01 2012
@@ -21,6 +21,7 @@ import org.apache.openejb.server.ServerS
 import org.apache.openejb.server.ServiceException;
 import org.apache.openejb.server.DiscoveryAgent;
 import org.apache.openejb.server.DiscoveryListener;
+import org.apache.openejb.util.Duration;
 import org.apache.openejb.util.LogCategory;
 import org.apache.openejb.util.Logger;
 import org.apache.openejb.loader.Options;
@@ -31,8 +32,9 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.Socket;
 import java.net.URI;
-import java.util.Map;
+import java.util.LinkedHashSet;
 import java.util.Properties;
+import java.util.Set;
 import java.util.StringTokenizer;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -57,6 +59,8 @@ public class MultipointDiscoveryAgent im
     private boolean debug = true;
     private String name;
     private String discoveryHost;
+    private Set<URI> roots;
+    private Duration reconnectDelay;
 
     public MultipointDiscoveryAgent() {
     }
@@ -68,7 +72,7 @@ public class MultipointDiscoveryAgent im
 
     public void init(Properties props) {
 
-        Options options = new Options(props);
+        final Options options = new Options(props);
         options.setLogger(new OptionsLog(log));
 
         host = props.getProperty("bind", host);
@@ -77,7 +81,18 @@ public class MultipointDiscoveryAgent im
         heartRate = options.get("heart_rate", heartRate);
         discoveryHost = options.get("discoveryHost", host);
         name = options.get("discoveryName", MultipointServer.randomColor());
+        reconnectDelay = options.get("reconnectDelay", new Duration("30 seconds"));
 
+        final Set<URI> uris = new LinkedHashSet<URI>();
+
+        // Connect the initial set of peer servers
+        StringTokenizer st = new StringTokenizer(initialServers, ",");
+        while (st.hasMoreTokens()) {
+            URI uri = URI.create("conn://" + st.nextToken().trim());
+            uris.add(uri);
+        }
+
+        roots = uris;
 
         Tracker.Builder builder = new Tracker.Builder();
         builder.setHeartRate(heartRate);
@@ -136,15 +151,9 @@ public class MultipointDiscoveryAgent im
         try {
             if (running.compareAndSet(false, true)) {
 
-                multipointServer = new MultipointServer(host, discoveryHost, port, tracker, name, debug).start();
+                multipointServer = new MultipointServer(host, discoveryHost, port, tracker, name, debug, roots, reconnectDelay).start();
 
                 this.port = multipointServer.getPort();
-                
-                // Connect the initial set of peer servers
-                StringTokenizer st = new StringTokenizer(initialServers, ",");
-                while (st.hasMoreTokens()) {
-                    multipointServer.connect(URI.create("conn://"+st.nextToken().trim()));
-                }
 
             }
         } catch (Exception e) {

Modified: openejb/trunk/openejb/server/openejb-multicast/src/main/java/org/apache/openejb/server/discovery/MultipointServer.java
URL: http://svn.apache.org/viewvc/openejb/trunk/openejb/server/openejb-multicast/src/main/java/org/apache/openejb/server/discovery/MultipointServer.java?rev=1296726&r1=1296725&r2=1296726&view=diff
==============================================================================
--- openejb/trunk/openejb/server/openejb-multicast/src/main/java/org/apache/openejb/server/discovery/MultipointServer.java (original)
+++ openejb/trunk/openejb/server/openejb-multicast/src/main/java/org/apache/openejb/server/discovery/MultipointServer.java Sat Mar  3 23:48:01 2012
@@ -16,6 +16,7 @@
  */
 package org.apache.openejb.server.discovery;
 
+import org.apache.openejb.util.Duration;
 import org.apache.openejb.util.Join;
 import org.apache.openejb.util.LogCategory;
 import org.apache.openejb.util.Logger;
@@ -37,9 +38,11 @@ import java.nio.channels.SocketChannel;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -59,50 +62,62 @@ public class MultipointServer {
     private final int port;
     private final Selector selector;
     private final URI me;
+    private final Set<URI> roots = new LinkedHashSet<URI>();
 
     /**
      * Only used for toString to make debugging easier
      */
     private final String name;
 
-
     private final Tracker tracker;
 
     private final LinkedList<URI> connect = new LinkedList<URI>();
     private final Map<URI, Session> connections = new HashMap<URI, Session>();
     private boolean debug = true;
 
-    public MultipointServer(int port, Tracker tracker) throws IOException {
-        this("localhost", port, tracker);
-    }
+    private long joined = 0;
+    private long reconnectDelay;
 
-    public MultipointServer(String host, int port, Tracker tracker) throws IOException {
-        this(host, port, tracker, randomColor());
+    public MultipointServer(int port, Tracker tracker) throws IOException {
+        this("localhost", "localhost", port, tracker, randomColor(), true, Collections.EMPTY_SET, new Duration(30, TimeUnit.SECONDS));
     }
 
-    public MultipointServer(String host, int port, Tracker tracker, String name) throws IOException {
-        this(host, port, tracker, name, true);
-    }
+    public MultipointServer(String bindHost, String broadcastHost, int port, Tracker tracker, String name, boolean debug, Set<URI> roots, Duration reconnectDelay) throws IOException {
+        if (tracker == null) throw new NullPointerException("tracker cannot be null");
+        if (bindHost == null) throw new NullPointerException("host cannot be null");
 
-    public MultipointServer(String host, int port, Tracker tracker, String name, boolean debug) throws IOException {
-        this(host, host, port, tracker, name, debug);
-    }
+        if (broadcastHost == null) broadcastHost = bindHost;
+        if (reconnectDelay == null) reconnectDelay = new Duration(30, TimeUnit.SECONDS);
 
-    public MultipointServer(String bindHost, String broadcastHost, int port, Tracker tracker, String name, boolean debug) throws IOException {
-        if (tracker == null) throw new NullPointerException("tracker cannot be null");
         this.tracker = tracker;
         this.name = name;
         this.debug = debug;
-        String format = String.format("MultipointServer(bindHost=%s, discoveryHost=%s, port=%s, name=%s, debug=%s)", bindHost, broadcastHost, port, name, debug);
+        if (roots != null) {
+            this.roots.addAll(roots);
+        }
+
+        this.reconnectDelay = reconnectDelay.getTime(TimeUnit.NANOSECONDS);
+
+        final String format = String.format("MultipointServer(bindHost=%s, discoveryHost=%s, port=%s, name=%s, debug=%s, roots=%s, reconnectDelay='%s')",
+                bindHost,
+                broadcastHost,
+                port,
+                name,
+                debug,
+                this.roots.size(),
+                reconnectDelay.toString());
+
         log.debug(format);
-        ServerSocketChannel serverChannel = ServerSocketChannel.open();
 
-        ServerSocket serverSocket = serverChannel.socket();
-        InetSocketAddress address = new InetSocketAddress(bindHost, port);
-        serverSocket.bind(address);
+        final InetSocketAddress address = new InetSocketAddress(bindHost, port);
+
+        final ServerSocketChannel serverChannel = ServerSocketChannel.open();
         serverChannel.configureBlocking(false);
 
+        final ServerSocket serverSocket = serverChannel.socket();
+        serverSocket.bind(address);
         this.port = serverSocket.getLocalPort();
+
         if (name != null) {
             me = URI.create("conn://" + broadcastHost + ":" + this.port + "/" + name);
         } else {
@@ -120,6 +135,23 @@ public class MultipointServer {
         return port;
     }
 
+    /**
+     * Attempt to connect back to the network if
+     *  - We aren't already connected
+     *  - We aren't already attempting to connect
+     *  - It has been a while since we last tried (reconnectDelay)
+     */
+    private void rejoin() {
+        if (connections.size() > 0) return;
+        if (connect.size() > 0) return;
+        if (System.nanoTime() - joined <= reconnectDelay) return;
+
+        for (URI root : roots) {
+            connect(root);
+        }
+
+        this.joined = System.nanoTime();
+    }
     public MultipointServer start() {
         if (running.compareAndSet(false, true)) {
             Thread thread = new Thread(new Runnable() {
@@ -380,8 +412,14 @@ public class MultipointServer {
             // Here is where we actually will expire missing services
             tracker.checkServices();
 
+            // Fill 'connections' list if we are fully disconnected
+            rejoin();
+
+            // Connect to anyone in the 'connections' list
             initiateConnections();
 
+            // Adjust selector timeout so we execute in even increments
+            // This keeps the heartbeat and rejoin regular
             selectorTimeout = adjustedSelectorTimeout(start);
         }
     }
@@ -715,7 +753,7 @@ public class MultipointServer {
         connect(URI.create("conn://localhost:" + port));
     }
 
-    public void connect(URI uri) throws Exception {
+    public void connect(URI uri) {
         if (me.equals(uri)) return;
 
         synchronized (connect) {