You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2018/07/25 17:57:48 UTC
[10/13] ignite git commit: IGNITE-9040 StopNodeFailureHandler is not
able to stop node correctly on node segmentation
IGNITE-9040 StopNodeFailureHandler is not able to stop node correctly on node segmentation
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/469aaba5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/469aaba5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/469aaba5
Branch: refs/heads/ignite-8297
Commit: 469aaba59c0539507972f4725642b2f2f81c08a0
Parents: 49d7a24
Author: Sergey Chugunov <se...@gmail.com>
Authored: Wed Jul 25 16:26:12 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Wed Jul 25 16:26:41 2018 +0300
----------------------------------------------------------------------
.../ignite/internal/GridKernalContext.java | 7 +
.../ignite/internal/GridKernalContextImpl.java | 14 +-
.../org/apache/ignite/internal/IgnitionEx.java | 13 +-
.../wal/reader/StandaloneGridKernalContext.java | 5 +
.../zk/internal/ZookeeperDiscoverySpiTest.java | 139 ++++++++++++++++++-
5 files changed, 164 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/469aaba5/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 505c3d6..051978c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -445,6 +445,13 @@ public interface GridKernalContext extends Iterable<GridComponent> {
public boolean invalid();
/**
+ * Checks whether this node detected its segmentation from the rest of the grid.
+ *
+ * @return {@code True} if this node has segmented, {@code false} otherwise.
+ */
+ public boolean segmented();
+
+ /**
* Gets failure processor.
*/
public FailureProcessor failure();
http://git-wip-us.apache.org/repos/asf/ignite/blob/469aaba5/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 2f4ecbc..2be64e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -37,6 +37,7 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
import org.apache.ignite.internal.managers.collision.GridCollisionManager;
import org.apache.ignite.internal.managers.communication.GridIoManager;
@@ -1125,7 +1126,18 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
@Override public boolean invalid() {
FailureProcessor failureProc = failure();
- return failureProc != null && failureProc.failureContext() != null;
+ return failureProc != null
+ && failureProc.failureContext() != null
+ && failureProc.failureContext().type() != FailureType.SEGMENTATION;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean segmented() {
+ FailureProcessor failureProc = failure();
+
+ return failureProc != null
+ && failureProc.failureContext() != null
+ && failureProc.failureContext().type() == FailureType.SEGMENTATION;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/469aaba5/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index f90fe82..aff19c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -2604,15 +2604,12 @@ public class IgnitionEx {
throw e;
}
finally {
- if (!grid0.context().invalid())
+ if (grid0.context().segmented())
+ state = STOPPED_ON_SEGMENTATION;
+ else if (grid0.context().invalid())
+ state = STOPPED_ON_FAILURE;
+ else
state = STOPPED;
- else {
- FailureContext failure = grid0.context().failure().failureContext();
-
- state = failure.type() == FailureType.SEGMENTATION ?
- STOPPED_ON_SEGMENTATION :
- STOPPED_ON_FAILURE;
- }
grid = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/469aaba5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index 795d460..b9ab76a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -472,6 +472,11 @@ public class StandaloneGridKernalContext implements GridKernalContext {
}
/** {@inheritDoc} */
+ @Override public boolean segmented() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public FailureProcessor failure() {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/469aaba5/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
----------------------------------------------------------------------
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index 4ccafc3..077c4da 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -59,6 +59,8 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteState;
+import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cluster.ClusterNode;
@@ -127,6 +129,7 @@ import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiMBean;
import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiTestSuite2;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZKUtil;
import org.apache.zookeeper.ZkTestClientCnxnSocketNIO;
@@ -147,6 +150,9 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTAN
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2;
import static org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
import static org.apache.zookeeper.ZooKeeper.ZOOKEEPER_CLIENT_CNXN_SOCKET;
/**
@@ -182,6 +188,12 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
private boolean testSockNio;
/** */
+ private CacheAtomicityMode atomicityMode;
+
+ /** */
+ private int backups = -1;
+
+ /** */
private boolean testCommSpi;
/** */
@@ -278,11 +290,7 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
cfg.setDiscoverySpi(zkSpi);
- CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
-
- ccfg.setWriteSynchronizationMode(FULL_SYNC);
-
- cfg.setCacheConfiguration(ccfg);
+ cfg.setCacheConfiguration(getCacheConfiguration());
Boolean clientMode = clientThreadLoc.get();
@@ -366,6 +374,21 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
return cfg;
}
+ /** */
+ private CacheConfiguration getCacheConfiguration() {
+ CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ if (atomicityMode != null)
+ ccfg.setAtomicityMode(atomicityMode);
+
+ if (backups > 0)
+ ccfg.setBackups(backups);
+
+ return ccfg;
+ }
+
/**
* @param clientMode Client mode flag for started nodes.
*/
@@ -1075,6 +1098,112 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
}
/**
+ * Verifies correct handling of SEGMENTATION event with STOP segmentation policy: node is stopped successfully,
+ * all its threads are shut down.
+ *
+ * @throws Exception If failed.
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/IGNITE-9040">IGNITE-9040</a> ticket for more context of the test.
+ */
+ public void testStopNodeOnSegmentaion() throws Exception {
+ try {
+ System.setProperty("IGNITE_WAL_LOG_TX_RECORDS", "true");
+
+ sesTimeout = 2000;
+ testSockNio = true;
+ persistence = true;
+ atomicityMode = CacheAtomicityMode.TRANSACTIONAL;
+ backups = 2;
+
+ final Ignite node0 = startGrid(0);
+
+ sesTimeout = 10_000;
+ testSockNio = false;
+
+ startGrid(1);
+
+ node0.cluster().active(true);
+
+ clientMode(true);
+
+ final IgniteEx client = startGrid(2);
+
+ //first transaction
+ client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 0, 0);
+ client.cache(DEFAULT_CACHE_NAME).put(0, 0);
+
+ //second transaction to create a deadlock with the first one
+ // and guarantee transaction futures will be presented on segmented node
+ // (erroneous write to WAL on segmented node stop happens
+ // on completing transaction with NodeStoppingException)
+ GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ Transaction tx2 = client.transactions().txStart(OPTIMISTIC, READ_COMMITTED, 0, 0);
+ client.cache(DEFAULT_CACHE_NAME).put(0, 0);
+ tx2.commit();
+ }
+ });
+
+ //next block simulates Ignite node segmentation by closing socket of ZooKeeper client
+ {
+ final CountDownLatch l = new CountDownLatch(1);
+
+ node0.events().localListen(new IgnitePredicate<Event>() {
+ @Override public boolean apply(Event evt) {
+ l.countDown();
+
+ return false;
+ }
+ }, EventType.EVT_NODE_SEGMENTED);
+
+ ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+ c0.closeSocket(true);
+
+ for (int i = 0; i < 10; i++) {
+ Thread.sleep(1_000);
+
+ if (l.getCount() == 0)
+ break;
+ }
+
+ info("Allow connect");
+
+ c0.allowConnect();
+
+ assertTrue(l.await(10, TimeUnit.SECONDS));
+ }
+
+ waitForNodeStop(node0.name());
+
+ checkStoppedNodeThreads(node0.name());
+ }
+ finally {
+ System.clearProperty("IGNITE_WAL_LOG_TX_RECORDS");
+ }
+ }
+
+ /** */
+ private void checkStoppedNodeThreads(String nodeName) {
+ Set<Thread> threads = Thread.getAllStackTraces().keySet();
+
+ for (Thread t : threads) {
+ if (t.getName().contains(nodeName))
+ throw new AssertionError("Thread from stopped node has been found: " + t.getName());
+ }
+ }
+
+ /** */
+ private void waitForNodeStop(String name) throws Exception {
+ while (true) {
+ if (IgnitionEx.state(name).equals(IgniteState.STARTED))
+ Thread.sleep(2000);
+ else
+ break;
+ }
+ }
+
+ /**
* @throws Exception If failed.
*/
public void testSegmentation1() throws Exception {