You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/03/28 16:06:25 UTC

ignite git commit: IGNITE-6890 Critical failures handling API (IEP-14)

Repository: ignite
Updated Branches:
  refs/heads/master 8bc4886b4 -> 15022e021


IGNITE-6890 Critical failures handling API (IEP-14)

Signed-off-by: Andrey Gura <ag...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/15022e02
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/15022e02
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/15022e02

Branch: refs/heads/master
Commit: 15022e0219010979a3ea09eabce0393b40a6584e
Parents: 8bc4886
Author: Dmitriy Sorokin <d....@gmail.com>
Authored: Wed Mar 28 16:03:27 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Wed Mar 28 18:34:23 2018 +0300

----------------------------------------------------------------------
 .../java/org/apache/ignite/IgniteState.java     |   7 +-
 .../configuration/IgniteConfiguration.java      |  26 +++++
 .../apache/ignite/failure/FailureContext.java   |  65 +++++++++++
 .../apache/ignite/failure/FailureHandler.java   |  39 +++++++
 .../org/apache/ignite/failure/FailureType.java  |  32 ++++++
 .../ignite/failure/NoOpFailureHandler.java      |  30 +++++
 .../failure/RestartProcessFailureHandler.java   |  45 ++++++++
 .../ignite/failure/StopNodeFailureHandler.java  |  43 +++++++
 .../failure/StopNodeOrHaltFailureHandler.java   | 102 +++++++++++++++++
 .../org/apache/ignite/failure/package-info.java |  22 ++++
 .../ignite/internal/GridKernalContext.java      |  27 ++---
 .../ignite/internal/GridKernalContextImpl.java  |  27 ++---
 .../apache/ignite/internal/IgniteKernal.java    |   3 +
 .../org/apache/ignite/internal/IgnitionEx.java  |  15 ++-
 .../apache/ignite/internal/NodeInvalidator.java |  53 ---------
 .../discovery/GridDiscoveryManager.java         |  46 ++------
 .../GridCachePartitionExchangeManager.java      |  22 +++-
 .../GridCacheDatabaseSharedManager.java         |   5 +-
 .../persistence/GridCacheOffheapManager.java    |   5 +-
 .../persistence/file/FilePageStoreManager.java  |  22 ++--
 .../wal/FileWriteAheadLogManager.java           |  20 ++--
 .../wal/FsyncModeFileWriteAheadLogManager.java  |  17 +--
 .../wal/reader/StandaloneGridKernalContext.java |  18 +--
 .../processors/failure/FailureProcessor.java    | 113 +++++++++++++++++++
 .../processors/failure/package-info.java        |  22 ++++
 .../startup/cmdline/CommandLineStartup.java     |   5 +-
 26 files changed, 656 insertions(+), 175 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/IgniteState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteState.java b/modules/core/src/main/java/org/apache/ignite/IgniteState.java
index e3c04c9..a3704fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteState.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteState.java
@@ -43,7 +43,12 @@ public enum IgniteState {
      * set to {@link SegmentationPolicy#STOP} or {@link SegmentationPolicy#RESTART_JVM}
      * and node is stopped from internals of Ignite after segment becomes invalid.
      */
-    STOPPED_ON_SEGMENTATION;
+    STOPPED_ON_SEGMENTATION,
+
+    /**
+     * Grid factory stopped due to a critical failure.
+     */
+    STOPPED_ON_FAILURE;
 
     /** Enumerated values. */
     private static final IgniteState[] VALS = values();

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index aa2fb00..add3880 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -42,6 +42,7 @@ import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeTask;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.failure.FailureHandler;
 import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
 import org.apache.ignite.internal.processors.odbc.ClientListenerProcessor;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -489,6 +490,9 @@ public class IgniteConfiguration {
     /** User authentication enabled. */
     private boolean authEnabled;
 
+    /** Failure handler. */
+    private FailureHandler failureHnd;
+
     /**
      * Creates valid grid configuration with all default values.
      */
@@ -547,6 +551,7 @@ public class IgniteConfiguration {
         hadoopCfg = cfg.getHadoopConfiguration();
         igfsCfg = cfg.getFileSystemConfiguration();
         igfsPoolSize = cfg.getIgfsThreadPoolSize();
+        failureHnd = cfg.getFailureHandler();
         igniteHome = cfg.getIgniteHome();
         igniteInstanceName = cfg.getIgniteInstanceName();
         igniteWorkDir = cfg.getWorkDirectory();
@@ -2923,6 +2928,27 @@ public class IgniteConfiguration {
     }
 
     /**
+     * Gets failure handler.
+     *
+     * @return Failure handler.
+     */
+    public FailureHandler getFailureHandler() {
+        return failureHnd;
+    }
+
+    /**
+     * Sets failure handler.
+     *
+     * @param failureHnd Failure handler.
+     * @return {@code This} for chaining.
+     */
+    public IgniteConfiguration setFailureHandler(FailureHandler failureHnd) {
+        this.failureHnd = failureHnd;
+
+        return this;
+    }
+
+    /**
      * Gets client connector configuration.
      *
      * @return Client connector configuration.

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/FailureContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/FailureContext.java b/modules/core/src/main/java/org/apache/ignite/failure/FailureContext.java
new file mode 100644
index 0000000..066e151
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/FailureContext.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Failure context contains information about failure type and exception if applicable.
+ * This information could be used for appropriate handling of the failure.
+ */
+public class FailureContext {
+    /** Type. */
+    private final FailureType type;
+
+    /** Error. */
+    private final Throwable err;
+
+    /**
+     * Creates instance of {@link FailureContext}.
+     *
+     * @param type Failure type.
+     * @param err Exception.
+     */
+    public FailureContext(FailureType type, Throwable err) {
+        A.notNull(type, "type");
+
+        this.type = type;
+        this.err = err;
+    }
+
+    /**
+     * @return Failure type.
+     */
+    public FailureType type() {
+        return type;
+    }
+
+    /**
+     * @return Exception or {@code null}.
+     */
+    public Throwable error() {
+        return err;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(FailureContext.class, this);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/FailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/FailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/FailureHandler.java
new file mode 100644
index 0000000..8717b16
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/FailureHandler.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
+
+/**
+ * Provides facility to handle failures by custom user implementations,
+ * which can be configured by {@link IgniteConfiguration#setFailureHandler(FailureHandler)} method.
+ */
+public interface FailureHandler {
+    /**
+     * Handles failure occurred on {@code ignite} instance.
+     * Failure details is contained in {@code failureCtx}.
+     * Returns {@code true} if kernal context must be invalidated by {@link FailureProcessor} after calling this method.
+     *
+     * @param ignite Ignite instance.
+     * @param failureCtx Failure context.
+     * @return Whether kernal context must be invalidated or not.
+     */
+    public boolean onFailure(Ignite ignite, FailureContext failureCtx);
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java b/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java
new file mode 100644
index 0000000..d933420
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+/**
+ * Types of failures.
+ */
+public enum FailureType {
+    /** Node segmentation. */
+    SEGMENTATION,
+
+    /** System worker termination. */
+    SYSTEM_WORKER_TERMINATION,
+
+    /** Critical error - error which leads to the system's inoperability. */
+    CRITICAL_ERROR
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java
new file mode 100644
index 0000000..2ec645e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+
+/**
+ * Just ignores any failure. It's useful for tests and debugging.
+ */
+public class NoOpFailureHandler implements FailureHandler {
+    /** {@inheritDoc} */
+    @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java
new file mode 100644
index 0000000..4f4432f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * This handler could be used only with ignite.(sh|bat) script.
+ * Process will be terminated using {@link Ignition#restart(boolean)} call.
+ */
+public class RestartProcessFailureHandler implements FailureHandler {
+    /** {@inheritDoc} */
+    @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+        new Thread(
+            new Runnable() {
+                @Override public void run() {
+                    U.error(ignite.log(), "Restarting JVM on Ignite failure: [failureCtx=" + failureCtx + ']');
+
+                    G.restart(true);
+                }
+            },
+            "node-restarter"
+        ).start();
+
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java
new file mode 100644
index 0000000..f05cc76
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Handler will stop node in case of critical error using {@code IgnitionEx.stop(nodeName, true, true)} call.
+ */
+public class StopNodeFailureHandler implements FailureHandler {
+    /** {@inheritDoc} */
+    @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+        new Thread(
+            new Runnable() {
+                @Override public void run() {
+                    U.error(ignite.log(), "Stopping local node on Ignite failure: [failureCtx=" + failureCtx + ']');
+
+                    IgnitionEx.stop(ignite.name(), true, true);
+                }
+            },
+            "node-stopper"
+        ).start();
+
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
new file mode 100644
index 0000000..4f74406
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Handler will try to stop node if {@code tryStop} value is {@code true}.
+ * If node can't be stopped during provided {@code timeout} or {@code tryStop} value is {@code false}
+ * then JVM process will be terminated forcibly using {@code Runtime.getRuntime().halt()}.
+ */
+public class StopNodeOrHaltFailureHandler implements FailureHandler {
+    /** Try stop. */
+    private final boolean tryStop;
+
+    /** Timeout. */
+    private final long timeout;
+
+    /**
+     * Default constructor.
+     */
+    public StopNodeOrHaltFailureHandler() {
+        this(false, 0);
+    }
+
+    /**
+     * @param tryStop Try stop.
+     * @param timeout Stop node timeout.
+     */
+    public StopNodeOrHaltFailureHandler(boolean tryStop, long timeout) {
+        this.tryStop = tryStop;
+        this.timeout = timeout;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+        IgniteLogger log = ignite.log();
+
+        if (tryStop) {
+            final CountDownLatch latch = new CountDownLatch(1);
+
+            new Thread(
+                new Runnable() {
+                    @Override public void run() {
+                        U.error(log, "Stopping local node on Ignite failure: [failureCtx=" + failureCtx + ']');
+
+                        IgnitionEx.stop(ignite.name(), true, true);
+
+                        latch.countDown();
+                    }
+                },
+                "node-stopper"
+            ).start();
+
+            new Thread(
+                new Runnable() {
+                    @Override public void run() {
+                        try {
+                            if (!latch.await(timeout, TimeUnit.MILLISECONDS)) {
+                                U.error(log, "Stopping local node timeout, JVM will be halted.");
+
+                                Runtime.getRuntime().halt(Ignition.KILL_EXIT_CODE);
+                            }
+                        }
+                        catch (InterruptedException e) {
+                            // No-op.
+                        }
+                    }
+                },
+                "jvm-halt-on-stop-timeout"
+            ).start();
+        }
+        else {
+            U.error(log, "JVM will be halted immediately on ignite failure: [failureCtx=" + failureCtx + ']');
+
+            Runtime.getRuntime().halt(Ignition.KILL_EXIT_CODE);
+        }
+
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/package-info.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/package-info.java b/modules/core/src/main/java/org/apache/ignite/failure/package-info.java
new file mode 100644
index 0000000..2128ddb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains <b>Failure handling public API</b> with standard FailureHandler implementations.
+ */
+package org.apache.ignite.failure;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index ce53b47..0b40054 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -43,6 +43,7 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
 import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.processors.hadoop.HadoopHelper;
 import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.igfs.IgfsHelper;
@@ -429,16 +430,16 @@ public interface GridKernalContext extends Iterable<GridComponent> {
     public DataStructuresProcessor dataStructures();
 
     /**
-     * Sets segmented flag to {@code true} when node is stopped due to segmentation issues.
+     * Checks whether this node is invalid due to a critical error or not.
+     *
+     * @return {@code True} if this node is invalid, {@code false} otherwise.
      */
-    public void markSegmented();
+    public boolean invalid();
 
     /**
-     * Gets segmented flag.
-     *
-     * @return {@code True} if network is currently segmented, {@code false} otherwise.
+     * Gets failure processor.
      */
-    public boolean segmented();
+    public FailureProcessor failure();
 
     /**
      * Print grid kernal memory stats (sizes of internal structures, etc.).
@@ -663,18 +664,4 @@ public interface GridKernalContext extends Iterable<GridComponent> {
      * @return subscription processor to manage internal-only (strict node-local) subscriptions between components.
      */
     public GridInternalSubscriptionProcessor internalSubscriptionProcessor();
-
-    /**
-     * TODO: Should be replaced with proper implementation in https://issues.apache.org/jira/browse/IGNITE-6891
-     *
-     * @return {@code true} if node was invalidated, false in other case.
-     */
-    public boolean invalidated();
-
-    /**
-     * Invalidates node.
-     *
-     * TODO: Should be replaced with proper implementation in https://issues.apache.org/jira/browse/IGNITE-6891
-     */
-    public void invalidate();
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 430b671..3408334 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -59,6 +59,7 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
 import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.processors.hadoop.HadoopHelper;
 import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.igfs.IgfsHelper;
@@ -392,8 +393,8 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     /** */
     private GridInternalSubscriptionProcessor internalSubscriptionProc;
 
-    /** Node invalidation flag. */
-    private volatile boolean invalidated;
+    /** Failure processor. */
+    private FailureProcessor failureProc;
 
     /**
      * No-arg constructor is required by externalization.
@@ -542,6 +543,8 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
          * ==========
          */
 
+        else if (comp instanceof FailureProcessor)
+            failureProc = (FailureProcessor)comp;
         else if (comp instanceof GridTaskProcessor)
             taskProc = (GridTaskProcessor)comp;
         else if (comp instanceof GridJobProcessor)
@@ -878,16 +881,6 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     }
 
     /** {@inheritDoc} */
-    @Override public void markSegmented() {
-        segFlag = true;
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean segmented() {
-        return segFlag;
-    }
-
-    /** {@inheritDoc} */
     @Override public GridPerformanceSuggestions performance() {
         return perf;
     }
@@ -1116,13 +1109,15 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     }
 
     /** {@inheritDoc} */
-    @Override public boolean invalidated() {
-        return invalidated;
+    @Override public boolean invalid() {
+        FailureProcessor failureProc = failure();
+
+        return failureProc != null && failureProc.failureContext() != null;
     }
 
     /** {@inheritDoc} */
-    @Override public void invalidate() {
-        invalidated = true;
+    @Override public FailureProcessor failure() {
+        return failureProc;
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 843b6e1..8bc46fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -104,6 +104,7 @@ import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.binary.BinaryUtils;
 import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
 import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.managers.GridManager;
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
 import org.apache.ignite.internal.managers.collision.GridCollisionManager;
@@ -906,6 +907,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
 
             startProcessor(new IgnitePluginProcessor(ctx, cfg, plugins));
 
+            startProcessor(new FailureProcessor(ctx));
+
             startProcessor(new PoolProcessor(ctx));
 
             // Closure processor should be started before all others

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index f5cb798..8073faa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -35,6 +35,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executors;
@@ -70,6 +71,8 @@ import org.apache.ignite.configuration.MemoryConfiguration;
 import org.apache.ignite.configuration.MemoryPolicyConfiguration;
 import org.apache.ignite.configuration.PersistentStoreConfiguration;
 import org.apache.ignite.configuration.TransactionConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
@@ -113,10 +116,10 @@ import org.apache.ignite.thread.IgniteStripedThreadPoolExecutor;
 import org.apache.ignite.thread.IgniteThread;
 import org.apache.ignite.thread.IgniteThreadPoolExecutor;
 import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
 
 import static org.apache.ignite.IgniteState.STARTED;
 import static org.apache.ignite.IgniteState.STOPPED;
+import static org.apache.ignite.IgniteState.STOPPED_ON_FAILURE;
 import static org.apache.ignite.IgniteState.STOPPED_ON_SEGMENTATION;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_CLIENT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONFIG_URL;
@@ -2554,7 +2557,15 @@ public class IgnitionEx {
                     throw e;
             }
             finally {
-                state = grid0.context().segmented() ? STOPPED_ON_SEGMENTATION : STOPPED;
+                if (!grid0.context().invalid())
+                    state = STOPPED;
+                else {
+                    FailureContext failure = grid0.context().failure().failureContext();
+
+                    state = failure.type() == FailureType.SEGMENTATION ?
+                        STOPPED_ON_SEGMENTATION :
+                        STOPPED_ON_FAILURE;
+                }
 
                 grid = null;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java b/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java
deleted file mode 100644
index b19ec08..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal;
-
-import org.apache.ignite.IgniteLogger;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Temporary functionality to invalidate and stop the node.
- * TODO: Should be replaced on proper implementation in https://issues.apache.org/jira/browse/IGNITE-6891
- */
-public class NodeInvalidator {
-    public static NodeInvalidator INSTANCE = new NodeInvalidator();
-
-    private static final long STOP_TIMEOUT_MS = 60 * 1000;
-
-    private NodeInvalidator() {
-        // Empty
-    }
-
-    public void invalidate(@NotNull GridKernalContext ctx, @NotNull Throwable error) {
-        if (ctx.invalidated())
-            return;
-
-        ctx.invalidate();
-
-        final String gridName = ctx.igniteInstanceName();
-        final IgniteLogger logger = ctx.log(getClass());
-
-        logger.error("Critical error with " + gridName + " is happened. " +
-                "All further operations will be failed and local node will be stopped.", error);
-
-        new Thread("node-stopper") {
-            @Override public void run() {
-                IgnitionEx.stop(gridName, true, true, STOP_TIMEOUT_MS);
-            }
-        }.start();
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 707b1ba..2e814d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -58,6 +58,10 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.failure.RestartProcessFailureHandler;
+import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.GridComponent;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
@@ -75,7 +79,6 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.GridCacheUtils;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
@@ -93,7 +96,6 @@ import org.apache.ignite.internal.util.lang.GridTuple6;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.P1;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -2377,32 +2379,6 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
         cacheNodes.add(rich);
     }
 
-    /** Stops local node. */
-    private void stopNode() {
-        new Thread(
-            new Runnable() {
-                @Override public void run() {
-                    ctx.markSegmented();
-
-                    G.stop(ctx.igniteInstanceName(), true);
-                }
-            }
-        ).start();
-    }
-
-    /** Restarts JVM. */
-    private void restartJvm() {
-        new Thread(
-            new Runnable() {
-                @Override public void run() {
-                    ctx.markSegmented();
-
-                    G.restart(true);
-                }
-            }
-        ).start();
-    }
-
     /** Worker for network segment checks. */
     private class SegmentCheckWorker extends GridWorker {
         /** */
@@ -2497,6 +2473,12 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
         private final BlockingQueue<GridTuple6<Integer, AffinityTopologyVersion, ClusterNode,
             DiscoCache, Collection<ClusterNode>, DiscoveryCustomMessage>> evts = new LinkedBlockingQueue<>();
 
+        /** Restart process handler. */
+        private final RestartProcessFailureHandler restartProcHnd = new RestartProcessFailureHandler();
+
+        /** Stop node handler. */
+        private final StopNodeFailureHandler stopNodeHnd = new StopNodeFailureHandler();
+
         /** Node segmented event fired flag. */
         private boolean nodeSegFired;
 
@@ -2780,16 +2762,12 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
 
             switch (segPlc) {
                 case RESTART_JVM:
-                    U.warn(log, "Restarting JVM according to configured segmentation policy.");
-
-                    restartJvm();
+                    ctx.failure().process(new FailureContext(FailureType.SEGMENTATION, null), restartProcHnd);
 
                     break;
 
                 case STOP:
-                    U.warn(log, "Stopping local node according to configured segmentation policy.");
-
-                    stopNode();
+                    ctx.failure().process(new FailureContext(FailureType.SEGMENTATION, null), stopNodeHnd);
 
                     break;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 4e93822..5e57299 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -32,6 +32,7 @@ import java.util.Map;
 import java.util.NavigableMap;
 import java.util.TreeMap;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.LinkedBlockingDeque;
@@ -47,6 +48,7 @@ import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteDiagnosticAware;
 import org.apache.ignite.internal.IgniteDiagnosticPrepareContext;
@@ -112,7 +114,6 @@ import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_IO_DUMP_ON_TIMEOUT;
@@ -122,6 +123,7 @@ import static org.apache.ignite.IgniteSystemProperties.getLong;
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE;
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
@@ -2245,6 +2247,24 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
 
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+            Throwable err = null;
+
+            try {
+                body0();
+            }
+            catch (Throwable e) {
+                err = e;
+            }
+            finally {
+                if (!stop)
+                    cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+            }
+        }
+
+        /**
+         *
+         */
+        private void body0() throws InterruptedException, IgniteInterruptedCheckedException {
             long timeout = cctx.gridConfig().getNetworkTimeout();
 
             long cnt = 0;

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 410e9b9..73802c1 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -75,10 +75,11 @@ import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.NodeInvalidator;
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
@@ -2876,7 +2877,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                             chp.progress.cpFinishFut.onDone(e);
 
                             // In case of writing error node should be invalidated and stopped.
-                            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+                            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
                             return;
                         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 1a35b24..766333f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -27,7 +27,8 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.NodeInvalidator;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -1185,7 +1186,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
                     U.error(log, "Unhandled exception during page store initialization. All further operations will " +
                         "be failed and local node will be stopped.", ex);
 
-                    NodeInvalidator.INSTANCE.invalidate(ctx.kernalContext(), ex);
+                    ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
 
                     throw ex;
                 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 9549269..6313eac 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -38,8 +38,9 @@ import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.NodeInvalidator;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
@@ -49,9 +50,8 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
-import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
-import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.util.IgniteUtils;
@@ -202,7 +202,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
             }
         }
         catch (PersistentStorageIOException e) {
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
         }
@@ -322,7 +322,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
             store.read(pageId, pageBuf, keepCrc);
         }
         catch (PersistentStorageIOException e) {
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
         }
@@ -343,7 +343,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
             store.readHeader(buf);
         }
         catch (PersistentStorageIOException e) {
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
         }
@@ -379,7 +379,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
             store.write(pageId, pageBuf, tag, calculateCrc);
         }
         catch (PersistentStorageIOException e) {
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
         }
@@ -458,7 +458,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
             return new CacheStoreHolder(idxStore, partStores);
         }
         catch (PersistentStorageIOException e) {
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
         }
@@ -550,7 +550,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
             getStore(grpId, partId).sync();
         }
         catch (PersistentStorageIOException e) {
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
         }
@@ -562,7 +562,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
             getStore(grpId, partId).ensure();
         }
         catch (PersistentStorageIOException e) {
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
         }
@@ -580,7 +580,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
             return PageIdUtils.pageId(partId, flags, (int)pageIdx);
         }
         catch (PersistentStorageIOException e) {
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 306fb2c..2fff481 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -65,10 +65,11 @@ import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.events.WalSegmentArchivedEvent;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.NodeInvalidator;
 import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.pagemem.wal.StorageException;
@@ -681,7 +682,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
         catch (IgniteCheckedException e) {
             U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e);
 
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
         }
     }
 
@@ -1233,7 +1234,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
         catch (IOException e) {
             StorageException se = new StorageException("Unable to initialize WAL segment", e);
 
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, se));
 
             throw se;
         }
@@ -1382,7 +1383,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
      * @throws StorageException If node is no longer valid and we missed a WAL operation.
      */
     private void checkNode() throws StorageException {
-        if (cctx.kernalContext().invalidated())
+        if (cctx.kernalContext().invalid())
             throw new StorageException("Failed to perform WAL operation (environment was invalidated by a " +
                     "previous error)");
     }
@@ -1884,7 +1885,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                 catch (IgniteCheckedException | IOException e) {
                     U.error(log, "Unexpected error during WAL compression", e);
 
-                    NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+                    cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
                 }
                 catch (InterruptedException ignore) {
                     Thread.currentThread().interrupt();
@@ -2036,7 +2037,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
                 catch (IOException e) {
                     U.error(log, "Unexpected error during WAL decompression", e);
 
-                    NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+                    cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
                 }
             }
         }
@@ -2814,7 +2815,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
             lock.lock();
 
             try {
-                assert cctx.kernalContext().invalidated() || written == lastFsyncPos || mode != WALMode.FSYNC :
+                assert cctx.kernalContext().invalid() ||
+                    written == lastFsyncPos || mode != WALMode.FSYNC :
                     "fsync [written=" + written + ", lastFsync=" + lastFsyncPos + ", idx=" + idx + ']';
 
                 fileIO = null;
@@ -3281,7 +3283,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
             Throwable err = walWriter.err;
 
             if (err != null)
-                NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), err);
+                cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, err));
 
             if (expPos == UNCONDITIONAL_FLUSH)
                 expPos = (currentHandle().buf.tail());
@@ -3370,7 +3372,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
             catch (IOException e) {
                 StorageException se = new StorageException("Unable to write", e);
 
-                NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+                cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, se));
 
                 throw se;
             }

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 6934f4f..59196bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -61,10 +61,10 @@ import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.events.WalSegmentArchivedEvent;
+import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.NodeInvalidator;
 import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.pagemem.wal.StorageException;
@@ -109,6 +109,7 @@ import static java.nio.file.StandardOpenOption.CREATE;
 import static java.nio.file.StandardOpenOption.READ;
 import static java.nio.file.StandardOpenOption.WRITE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
 
 /**
  * File WAL manager.
@@ -599,7 +600,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
         catch (IgniteCheckedException e) {
             U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e);
 
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+            cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
         }
     }
 
@@ -1056,7 +1057,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
         catch (IOException e) {
             StorageException se = new StorageException("Unable to initialize WAL segment", e);
 
-            NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+            cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
 
             throw se;
         }
@@ -1196,7 +1197,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
      * @throws StorageException If node is no longer valid and we missed a WAL operation.
      */
     private void checkNode() throws StorageException {
-        if (cctx.kernalContext().invalidated())
+        if (cctx.kernalContext().invalid())
             throw new StorageException("Failed to perform WAL operation (environment was invalidated by a " +
                     "previous error)");
     }
@@ -1721,7 +1722,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
                 catch (IgniteCheckedException | IOException e) {
                     U.error(log, "Unexpected error during WAL compression", e);
 
-                    NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
                 }
                 catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
@@ -1845,7 +1846,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
                 catch (IOException e) {
                     U.error(log, "Unexpected error during WAL decompression", e);
 
-                    NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
                 }
             }
         }
@@ -2563,7 +2564,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
             catch (Throwable e) {
                 StorageException se = new StorageException("Unable to write", new IOException(e));
 
-                NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
 
                 // All workers waiting for a next segment must be woken up and stopped
                 signalNextAvailable();
@@ -2877,7 +2878,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
                 catch (IOException e) {
                     StorageException se = new StorageException("Unable to write", e);
 
-                    NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
 
                     throw se;
                 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index fdcc053..429a5ce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -58,6 +58,7 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
 import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.processors.hadoop.HadoopHelper;
 import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.igfs.IgfsHelper;
@@ -458,12 +459,13 @@ public class StandaloneGridKernalContext implements GridKernalContext {
     }
 
     /** {@inheritDoc} */
-    @Override public void markSegmented() {
+    @Override public boolean invalid() {
+        return false;
     }
 
     /** {@inheritDoc} */
-    @Override public boolean segmented() {
-        return false;
+    @Override public FailureProcessor failure() {
+        return null;
     }
 
     /** {@inheritDoc} */
@@ -631,16 +633,6 @@ public class StandaloneGridKernalContext implements GridKernalContext {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean invalidated() {
-        return false;
-    }
-
-    /** {@inheritDoc} */
-    @Override public void invalidate() {
-
-    }
-
-    /** {@inheritDoc} */
     @NotNull @Override public Iterator<GridComponent> iterator() {
         return null;
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
new file mode 100644
index 0000000..615fb9f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * General failure processing API
+ */
+public class FailureProcessor extends GridProcessorAdapter {
+    /** Ignite. */
+    private final Ignite ignite;
+
+    /** Handler. */
+    private volatile FailureHandler hnd;
+
+    /** Failure context. */
+    private volatile FailureContext failureCtx;
+
+    /**
+     * @param ctx Context.
+     */
+    public FailureProcessor(GridKernalContext ctx) {
+        super(ctx);
+
+        this.ignite = ctx.grid();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void start() throws IgniteCheckedException {
+        FailureHandler hnd = ctx.config().getFailureHandler();
+
+        if (hnd == null)
+            hnd = getDefaultFailureHandler();
+
+        assert hnd != null;
+
+        this.hnd = hnd;
+    }
+
+    /**
+     * This method is used to initialize local failure handler if {@link IgniteConfiguration} don't contain configured one.
+     *
+     * @return Default {@link FailureHandler} implementation.
+     */
+    protected FailureHandler getDefaultFailureHandler() {
+        return new StopNodeOrHaltFailureHandler();
+    }
+
+    /**
+     * @return Failure context.
+     */
+    public FailureContext failureContext() {
+        return failureCtx;
+    }
+
+    /**
+     * Processes failure accordingly to configured {@link FailureHandler}.
+     *
+     * @param failureCtx Failure context.
+     */
+    public void process(FailureContext failureCtx) {
+        process(failureCtx, hnd);
+    }
+
+    /**
+     * Processes failure accordingly to given failure handler.
+     *
+     * @param failureCtx Failure context.
+     * @param hnd Failure handler.
+     */
+    public synchronized void process(FailureContext failureCtx, FailureHandler hnd) {
+        assert failureCtx != null;
+        assert hnd != null;
+
+        if (this.failureCtx != null) // Node already terminating, no reason to process more errors.
+            return;
+
+        U.error(ignite.log(), "Critical failure. Will be handled accordingly to configured handler [hnd=" +
+            hnd.getClass() + ", failureCtx=" + failureCtx + ']', failureCtx.error());
+
+        boolean invalidated = hnd.onFailure(ignite, failureCtx);
+
+        if (invalidated) {
+            this.failureCtx = failureCtx;
+
+            log.error("Ignite node is in invalid state due to a critical failure.");
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/package-info.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/package-info.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/package-info.java
new file mode 100644
index 0000000..1c1ad13
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Failure processor.
+ */
+package org.apache.ignite.internal.processors.failure;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineStartup.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineStartup.java b/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineStartup.java
index 9b05071..821af8a 100644
--- a/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineStartup.java
+++ b/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineStartup.java
@@ -43,8 +43,7 @@ import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
-import static org.apache.ignite.IgniteState.STOPPED;
-import static org.apache.ignite.IgniteState.STOPPED_ON_SEGMENTATION;
+import static org.apache.ignite.IgniteState.STARTED;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PROG_NAME;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_RESTART_CODE;
 import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR;
@@ -326,7 +325,7 @@ public final class CommandLineStartup {
                 if (!F.eq(igniteInstanceName, name))
                     return;
 
-                if (state == STOPPED || state == STOPPED_ON_SEGMENTATION)
+                if (state != STARTED)
                     latch.countDown();
             }
         });