You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/07/16 15:51:07 UTC

[12/18] incubator-ignite git commit: ignite-890: fixed

ignite-890: fixed


Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/baeb924e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/baeb924e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/baeb924e

Branch: refs/heads/master
Commit: baeb924e05d928df067493bf63c492c5ffda6f21
Parents: da5d7e5
Author: Denis Magda <dm...@gridgain.com>
Authored: Tue Jul 14 15:46:15 2015 +0300
Committer: Denis Magda <dm...@gridgain.com>
Committed: Tue Jul 14 15:46:15 2015 +0300

----------------------------------------------------------------------
 .../discovery/GridDiscoveryManager.java         |   4 +-
 .../communication/tcp/TcpCommunicationSpi.java  |  22 +--
 .../ignite/spi/discovery/tcp/ClientImpl.java    |  19 +--
 .../ignite/spi/discovery/tcp/ServerImpl.java    |  62 ++------
 .../spi/discovery/tcp/TcpDiscoverySpi.java      |   8 --
 .../tcp/internal/TcpDiscoveryNode.java          |   6 -
 .../messages/TcpDiscoveryNodeFailedMessage.java |   1 -
 .../tcp/TcpDiscoveryMultiThreadedTest.java      | 142 ++++++++++++-------
 .../testframework/junits/GridAbstractTest.java  |  13 +-
 9 files changed, 114 insertions(+), 163 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/baeb924e/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 80f7f98..eae07ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -1589,7 +1589,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
             new Runnable() {
                 @Override public void run() {
                     ctx.markSegmented();
-                    System.out.println("Stopping grid on segmentation: " + ctx.gridName());
+
                     G.stop(ctx.gridName(), true);
                 }
             }
@@ -1896,7 +1896,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
                     segmented = true;
 
                     if (!isLocDaemon)
-                        U.warn(log, "Local node SEGMENTED: " + node + ", remote_nodes = " + getSpi().getRemoteNodes());
+                        U.warn(log, "Local node SEGMENTED: " + node);
                     else if (log.isDebugEnabled())
                         log.debug("Local node SEGMENTED: " + node);
 

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/baeb924e/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 423f4ef..df7d098 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -1899,8 +1899,6 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
                 }
             }
             catch (IgniteCheckedException | RuntimeException | Error e) {
-                //tryFailClient(node, e);
-
                 if (log.isDebugEnabled())
                     log.debug(
                         "Caught exception (will close client) [err=" + e.getMessage() + ", client=" + client + ']');
@@ -2141,8 +2139,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
         if (client == null) {
             assert errs != null;
 
-//            if (!tryFailClient(node, errs) && X.hasCause(errs, ConnectException.class))
-              if (X.hasCause(errs, ConnectException.class))
+            if (X.hasCause(errs, ConnectException.class))
                 LT.warn(log, null, "Failed to connect to a remote node " +
                     "(make sure that destination node is alive and " +
                     "operating system firewall is disabled on local and remote hosts) " +
@@ -2203,7 +2200,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
                     UUID rmtNodeId0 = U.bytesToUuid(buf.array(), 1);
 
                     if (!rmtNodeId.equals(rmtNodeId0))
-                        throw new HandshakeFailureException("Remote node ID is not as expected [expected=" + rmtNodeId +
+                        throw new IgniteCheckedException("Remote node ID is not as expected [expected=" + rmtNodeId +
                             ", rcvd=" + rmtNodeId0 + ']');
                     else if (log.isDebugEnabled())
                         log.debug("Received remote node ID: " + rmtNodeId0);
@@ -2245,7 +2242,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
                             int read = ch.read(buf);
 
                             if (read == -1)
-                                throw new HandshakeFailureException("Failed to read remote node recovery handshake " +
+                                throw new IgniteCheckedException("Failed to read remote node recovery handshake " +
                                     "(connection closed).");
 
                             i += read;
@@ -2424,19 +2421,6 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter
         }
     }
 
-    /** Internal exception class for handshake failure handling. */
-    private static class HandshakeFailureException extends IgniteCheckedException {
-        /** */
-        private static final long serialVersionUID = 0L;
-
-        /**
-         * @param msg Message.
-         */
-        HandshakeFailureException(String msg) {
-            super(msg);
-        }
-    }
-
     /**
      * This worker takes responsibility to shut the server down when stopping,
      * No other thread shall stop passed server.

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/baeb924e/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 467acc6..838b89a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -129,8 +129,7 @@ class ClientImpl extends TcpDiscoveryImpl {
 
         b.append("Stats: ").append(spi.stats).append(U.nl());
 
-        System.out.println(b.toString());
-//        U.quietAndInfo(log, b.toString());
+        U.quietAndInfo(log, b.toString());
     }
 
     /** {@inheritDoc} */
@@ -432,10 +431,8 @@ class ClientImpl extends TcpDiscoveryImpl {
             }
 
             if (addrs.isEmpty()) {
-                if (timeout > 0 && (U.currentTimeMillis() - startTime) > timeout) {
-                    System.out.println("Client reconnect timeout: " + getLocalNodeId());
+                if (timeout > 0 && (U.currentTimeMillis() - startTime) > timeout)
                     return null;
-                }
 
                 U.warn(log, "Failed to connect to any address from IP finder (will retry to join topology " +
                     "in 2000ms): " + addrs0);
@@ -796,7 +793,7 @@ class ClientImpl extends TcpDiscoveryImpl {
                 catch (IOException e) {
                     msgWorker.addMessage(new SocketClosedMessage(sock));
 
-                    //if (log.isDebugEnabled())
+                    if (log.isDebugEnabled())
                         U.error(log, "Connection failed [sock=" + sock + ", locNodeId=" + getLocalNodeId() + ']', e);
                 }
                 finally {
@@ -1134,24 +1131,19 @@ class ClientImpl extends TcpDiscoveryImpl {
                         assert spi.getSpiContext().isStopping();
 
                         if (currSock != null) {
-                            System.out.println("Sending node left msg: " + getLocalNodeId());
-
                             TcpDiscoveryAbstractMessage leftMsg = new TcpDiscoveryNodeLeftMessage(getLocalNodeId());
 
                             leftMsg.client(true);
 
                             sockWriter.sendMessage(leftMsg);
                         }
-                        else {
-                            System.out.println("No connection on leave: " + getLocalNodeId());
+                        else
                             leaveLatch.countDown();
-                        }
                     }
                     else if (msg instanceof SocketClosedMessage) {
                         if (((SocketClosedMessage)msg).sock == currSock) {
                             currSock = null;
 
-                            System.out.println("Socket closed. Join latch: " + joinLatch.getCount() + ". Node: " + getLocalNodeId());
                             boolean join = joinLatch.getCount() > 0;
 
                             if (spi.getSpiContext().isStopping() || segmented) {
@@ -1166,7 +1158,6 @@ class ClientImpl extends TcpDiscoveryImpl {
                             else {
                                 assert reconnector == null;
 
-                                System.out.println("Starting reconnector: " + getLocalNodeId());
                                 final Reconnector reconnector = new Reconnector(join);
                                 this.reconnector = reconnector;
                                 reconnector.start();
@@ -1180,8 +1171,6 @@ class ClientImpl extends TcpDiscoveryImpl {
                             reconnector.cancel();
                             reconnector.join();
 
-                            System.out.println("RECONNECT FAILED: sending segmentation error: " + locNode);
-
                             notifyDiscovery(EVT_NODE_SEGMENTED, topVer, locNode, allVisibleNodes());
                         }
                     }

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/baeb924e/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index cda026f..bca2d98 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1335,7 +1335,7 @@ class ServerImpl extends TcpDiscoveryImpl {
             return;
         }
 
-        //assert log.isInfoEnabled();
+        assert log.isInfoEnabled();
 
         synchronized (mux) {
             StringBuilder b = new StringBuilder(U.nl());
@@ -1388,8 +1388,7 @@ class ServerImpl extends TcpDiscoveryImpl {
 
             b.append("Stats: ").append(spi.stats).append(U.nl());
 
-            System.out.println(b.toString());
-            //U.quietAndInfo(log, b.toString());
+            U.quietAndInfo(log, b.toString());
         }
     }
 
@@ -1830,8 +1829,6 @@ class ServerImpl extends TcpDiscoveryImpl {
         }
     }
 
-    private static volatile boolean dumping;
-
     /**
      * Message worker thread for messages processing.
      */
@@ -2228,8 +2225,7 @@ class ServerImpl extends TcpDiscoveryImpl {
                                         ", next=" + next.id() +
                                         ", res=" + res + ']');
 
-                                if (debugMode && !(msg instanceof TcpDiscoveryDiscardMessage) && !(msg instanceof TcpDiscoveryHeartbeatMessage)
-                                    && !(msg instanceof  TcpDiscoveryCustomEventMessage))
+                                if (debugMode)
                                     debugLog("Message has been sent to next node [msg=" + msg +
                                         ", next=" + next.id() +
                                         ", res=" + res + ']');
@@ -2817,48 +2813,28 @@ class ServerImpl extends TcpDiscoveryImpl {
                         if (log.isDebugEnabled())
                             log.debug("Accept client reconnect, restored pending messages " +
                                 "[locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
-
-                        if (debugMode)
-                            debugLog("Accept client reconnect, restored pending messages " +
-                                         "[locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
-
                     }
                     else {
                         if (log.isDebugEnabled())
                             log.debug("Failing reconnecting client node because failed to restore pending " +
                                 "messages [locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
 
-                        if (debugMode)
-                            debugLog("Failing reconnecting client node because failed to restore pending " +
-                                         "messages [locNodeId=" + locNodeId + ", clientNodeId=" + nodeId + ']');
-
                         processNodeFailedMessage(new TcpDiscoveryNodeFailedMessage(locNodeId,
                             node.id(), node.internalOrder()));
                     }
                 }
             }
-            else {
-                if (log.isDebugEnabled())
-                    log.debug("Reconnecting client node is already failed [nodeId=" + nodeId + ']');
-
-                if (debugMode)
-                    debugLog("Reconnecting client node is already failed [nodeId=" + nodeId + ']');
-            }
+            else if (log.isDebugEnabled())
+                log.debug("Reconnecting client node is already failed [nodeId=" + nodeId + ']');
 
             if (isLocNodeRouter) {
                 ClientMessageWorker wrk = clientMsgWorkers.get(nodeId);
 
                 if (wrk != null)
                     wrk.addMessage(msg);
-                else {
-                    if (log.isDebugEnabled())
-                        log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
-                                      locNodeId + ", clientNodeId=" + nodeId + ']');
-
-                    if (debugMode)
-                        debugLog("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
-                                      locNodeId + ", clientNodeId=" + nodeId + ']');
-                }
+                else if (log.isDebugEnabled())
+                    log.debug("Failed to reconnect client node (disconnected during the process) [locNodeId=" +
+                        locNodeId + ", clientNodeId=" + nodeId + ']');
             }
             else {
                 if (ring.hasRemoteNodes())
@@ -3252,9 +3228,6 @@ class ServerImpl extends TcpDiscoveryImpl {
                         if (log.isDebugEnabled())
                             log.debug("Starting local node stop procedure.");
 
-                        if (debugMode)
-                            debugLog("Starting local node stop procedure.");
-
                         spiState = STOPPING;
 
                         mux.notifyAll();
@@ -3262,9 +3235,6 @@ class ServerImpl extends TcpDiscoveryImpl {
                 }
 
                 if (msg.verified() || !ring.hasRemoteNodes() || msg.senderNodeId() != null) {
-                    if (debugMode)
-                        debugLog("Do stop local node: [msg=" + msg + ", hasRemote=" + ring.hasRemoteNodes() + ']');
-
                     if (spi.ipFinder.isShared() && !ring.hasRemoteNodes()) {
                         try {
                             spi.ipFinder.unregisterAddresses(locNode.socketAddresses());
@@ -3765,22 +3735,11 @@ class ServerImpl extends TcpDiscoveryImpl {
                             if (clientNodeIds.contains(clientNode.id()))
                                 clientNode.aliveCheck(spi.maxMissedClientHbs);
                             else {
-                                if (!isLocalNodeCoordinator())
-                                    continue;
-
-                                if (clientNode.aliveCheck() == 0)
-                                    // Node can just became coordinator without receiving any client heartbeat.
-                                    clientNode.aliveCheck(spi.maxMissedClientHbs);
-
                                 int aliveCheck = clientNode.decrementAliveCheck();
 
-                                if (aliveCheck == 0) {
-                                    // Make aliveCheck negative
-                                    clientNode.decrementAliveCheck();
-
+                                if (aliveCheck <= 0 && isLocalNodeCoordinator() && !failedNodes.contains(clientNode))
                                     processNodeFailedMessage(new TcpDiscoveryNodeFailedMessage(locNodeId,
                                         clientNode.id(), clientNode.internalOrder()));
-                                }
                             }
                         }
                     }
@@ -4354,8 +4313,7 @@ class ServerImpl extends TcpDiscoveryImpl {
 
                         spi.stats.onMessageReceived(msg);
 
-                        if (debugMode && !(msg instanceof TcpDiscoveryDiscardMessage) && !(msg instanceof TcpDiscoveryHeartbeatMessage)
-                            && !(msg instanceof  TcpDiscoveryCustomEventMessage) && recordable(msg))
+                        if (debugMode && recordable(msg))
                             debugLog("Message has been received: " + msg);
 
                         if (msg instanceof TcpDiscoveryJoinRequestMessage) {

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/baeb924e/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index b84e6c8..7663fe6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -327,8 +327,6 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, T
     /** */
     private boolean forceSrvMode;
 
-    public static volatile ArrayList<TcpDiscoverySpi> allSpis = new ArrayList<>();
-
     /** {@inheritDoc} */
     @Override public String getSpiState() {
         return impl.getSpiState();
@@ -1563,12 +1561,6 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, T
             impl = new ServerImpl(this);
         }
 
-//        impl.setDebugMode(true);
-//
-//        synchronized (allSpis) {
-//            allSpis.add(this);
-//        }
-
         assertParameter(ipFinder != null, "ipFinder != null");
         assertParameter(hbFreq > 0, "heartbeatFreq > 0");
         assertParameter(netTimeout > 0, "networkTimeout > 0");

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/baeb924e/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
index 46844ba..22f56c3 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
@@ -427,12 +427,6 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Cluste
         this.aliveCheck = aliveCheck;
     }
 
-    public int aliveCheck() {
-        assert isClient();
-
-        return aliveCheck;
-    }
-
     /**
      * @return Client router node ID.
      */

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/baeb924e/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryNodeFailedMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryNodeFailedMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryNodeFailedMessage.java
index cd85bfd..93ecdaa 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryNodeFailedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/messages/TcpDiscoveryNodeFailedMessage.java
@@ -51,7 +51,6 @@ public class TcpDiscoveryNodeFailedMessage extends TcpDiscoveryAbstractMessage {
     public TcpDiscoveryNodeFailedMessage(UUID creatorNodeId, UUID failedNodeId, long order) {
         super(creatorNodeId);
 
-        U.dumpStack("Failed = " + failedNodeId + ", creator=" + creatorNodeId);
         assert failedNodeId != null;
         assert order > 0;
 

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/baeb924e/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryMultiThreadedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryMultiThreadedTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryMultiThreadedTest.java
index 4e5c68e..fa70828 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryMultiThreadedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryMultiThreadedTest.java
@@ -40,7 +40,7 @@ public class TcpDiscoveryMultiThreadedTest extends GridCommonAbstractTest {
     private static final int GRID_CNT = 5;
 
     /** */
-    private static final int CLIENT_GRID_CNT = 1;
+    private static final int CLIENT_GRID_CNT = 5;
 
     /** */
     private static final ThreadLocal<Boolean> clientFlagPerThread = new ThreadLocal<>();
@@ -95,78 +95,60 @@ public class TcpDiscoveryMultiThreadedTest extends GridCommonAbstractTest {
 
     /** {@inheritDoc} */
     @Override protected long getTestTimeout() {
-        return 2 * 60 * 1000;
+        return 3 * 60 * 1000;
     }
 
     /**
      * @throws Exception If any error occurs.
      */
-    public void testMultiThreaded() throws Exception {
-        //fail("https://issues.apache.org/jira/browse/IGNITE-1100");
+    public void testMultiThreadedClientsRestart() throws Exception {
+        clientFlagGlobal = false;
 
-        execute();
-    }
+        info("Test timeout: " + (getTestTimeout() / (60 * 1000)) + " min.");
 
-    /**
-     * @throws Exception If any error occurs.
-     */
-    public void testTopologyVersion() throws Exception {
         startGridsMultiThreaded(GRID_CNT);
 
-        long prev = 0;
-
-        for (Ignite g : G.allGrids()) {
-            IgniteKernal kernal = (IgniteKernal)g;
-
-            long ver = kernal.context().discovery().topologyVersion();
-
-            info("Top ver: " + ver);
-
-            if (prev == 0)
-                prev = ver;
-        }
-
-        info("Test finished.");
-    }
-
-    /**
-     * @throws Exception If any error occurs.
-     */
-    public void testMultipleStartOnCoordinatorStop() throws Exception{
-        startGrids(GRID_CNT);
-
-        final CyclicBarrier barrier = new CyclicBarrier(GRID_CNT + 4);
+        clientFlagGlobal = true;
 
-        final AtomicInteger startIdx = new AtomicInteger(GRID_CNT);
+        startGridsMultiThreaded(GRID_CNT, CLIENT_GRID_CNT);
 
-        IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                barrier.await();
+        final AtomicBoolean done = new AtomicBoolean();
 
-                Ignite ignite = startGrid(startIdx.getAndIncrement());
+        final AtomicInteger clientIdx = new AtomicInteger(GRID_CNT);
 
-                assertFalse(ignite.configuration().isClientMode());
+        IgniteInternalFuture<?> fut1 = multithreadedAsync(
+            new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    clientFlagPerThread.set(true);
 
-                log.info("Started node: " + ignite.name());
+                    int idx = clientIdx.getAndIncrement();
 
-                return null;
-            }
-        }, GRID_CNT + 3, "start-thread");
+                    while (!done.get()) {
+                        stopGrid(idx);
+                        startGrid(idx);
+                    }
 
-        barrier.await();
+                    return null;
+                }
+            },
+            CLIENT_GRID_CNT
+        );
 
-        U.sleep(ThreadLocalRandom.current().nextInt(10, 100));
+        Thread.sleep(getTestTimeout() - 60 * 1000);
 
-        for (int i = 0; i < GRID_CNT; i++)
-            stopGrid(i);
+        done.set(true);
 
-        fut.get();
+        fut1.get();
     }
 
     /**
-     * @throws Exception If failed.
+     * @throws Exception If any error occurs.
      */
-    private void execute() throws Exception {
+    public void testMultiThreadedClientsServersRestart() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-1123");
+        
+        clientFlagGlobal = false;
+
         info("Test timeout: " + (getTestTimeout() / (60 * 1000)) + " min.");
 
         startGridsMultiThreaded(GRID_CNT);
@@ -229,4 +211,64 @@ public class TcpDiscoveryMultiThreadedTest extends GridCommonAbstractTest {
         fut1.get();
         fut2.get();
     }
+
+    /**
+     * @throws Exception If any error occurs.
+     */
+    public void testTopologyVersion() throws Exception {
+        clientFlagGlobal = false;
+
+        startGridsMultiThreaded(GRID_CNT);
+
+        long prev = 0;
+
+        for (Ignite g : G.allGrids()) {
+            IgniteKernal kernal = (IgniteKernal)g;
+
+            long ver = kernal.context().discovery().topologyVersion();
+
+            info("Top ver: " + ver);
+
+            if (prev == 0)
+                prev = ver;
+        }
+
+        info("Test finished.");
+    }
+
+    /**
+     * @throws Exception If any error occurs.
+     */
+    public void testMultipleStartOnCoordinatorStop() throws Exception{
+        clientFlagGlobal = false;
+
+        startGrids(GRID_CNT);
+
+        final CyclicBarrier barrier = new CyclicBarrier(GRID_CNT + 4);
+
+        final AtomicInteger startIdx = new AtomicInteger(GRID_CNT);
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                barrier.await();
+
+                Ignite ignite = startGrid(startIdx.getAndIncrement());
+
+                assertFalse(ignite.configuration().isClientMode());
+
+                log.info("Started node: " + ignite.name());
+
+                return null;
+            }
+        }, GRID_CNT + 3, "start-thread");
+
+        barrier.await();
+
+        U.sleep(ThreadLocalRandom.current().nextInt(10, 100));
+
+        for (int i = 0; i < GRID_CNT; i++)
+            stopGrid(i);
+
+        fut.get();
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/baeb924e/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index f7ec124..06a1523 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -1022,33 +1022,26 @@ public abstract class GridAbstractTest extends TestCase {
      * @param cancel Cancel flag.
      */
     @SuppressWarnings("deprecation")
-    protected boolean stopGrid(int idx, boolean cancel) {
+    protected void stopGrid(int idx, boolean cancel) {
         String gridName = getTestGridName(idx);
 
-        boolean res;
-
         try {
             Ignite ignite = G.ignite(gridName);
 
             assert ignite != null : "Ignite returned null grid for name: " + gridName;
 
-            info(">>> Stopping grid [name=" + ignite.name() + ", id=" + ignite.cluster().localNode().id() +
-                ", isClient=" + ignite.configuration().isClientMode() + ']');
+            info(">>> Stopping grid [name=" + ignite.name() + ", id=" + ignite.cluster().localNode().id() + ']');
 
-            res = G.stop(gridName, cancel);
+            G.stop(gridName, cancel);
         }
         catch (IllegalStateException ignored) {
             // Ignore error if grid already stopped.
-            res = true;
         }
         catch (Throwable e) {
             error("Failed to stop grid [gridName=" + gridName + ", cancel=" + cancel + ']', e);
 
             stopGridErr = true;
-            res = false;
         }
-
-        return res;
     }
 
     /**