You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/03/28 16:06:25 UTC
ignite git commit: IGNITE-6890 Critical failures handling API (IEP-14)
Repository: ignite
Updated Branches:
refs/heads/master 8bc4886b4 -> 15022e021
IGNITE-6890 Critical failures handling API (IEP-14)
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/15022e02
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/15022e02
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/15022e02
Branch: refs/heads/master
Commit: 15022e0219010979a3ea09eabce0393b40a6584e
Parents: 8bc4886
Author: Dmitriy Sorokin <d....@gmail.com>
Authored: Wed Mar 28 16:03:27 2018 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Wed Mar 28 18:34:23 2018 +0300
----------------------------------------------------------------------
.../java/org/apache/ignite/IgniteState.java | 7 +-
.../configuration/IgniteConfiguration.java | 26 +++++
.../apache/ignite/failure/FailureContext.java | 65 +++++++++++
.../apache/ignite/failure/FailureHandler.java | 39 +++++++
.../org/apache/ignite/failure/FailureType.java | 32 ++++++
.../ignite/failure/NoOpFailureHandler.java | 30 +++++
.../failure/RestartProcessFailureHandler.java | 45 ++++++++
.../ignite/failure/StopNodeFailureHandler.java | 43 +++++++
.../failure/StopNodeOrHaltFailureHandler.java | 102 +++++++++++++++++
.../org/apache/ignite/failure/package-info.java | 22 ++++
.../ignite/internal/GridKernalContext.java | 27 ++---
.../ignite/internal/GridKernalContextImpl.java | 27 ++---
.../apache/ignite/internal/IgniteKernal.java | 3 +
.../org/apache/ignite/internal/IgnitionEx.java | 15 ++-
.../apache/ignite/internal/NodeInvalidator.java | 53 ---------
.../discovery/GridDiscoveryManager.java | 46 ++------
.../GridCachePartitionExchangeManager.java | 22 +++-
.../GridCacheDatabaseSharedManager.java | 5 +-
.../persistence/GridCacheOffheapManager.java | 5 +-
.../persistence/file/FilePageStoreManager.java | 22 ++--
.../wal/FileWriteAheadLogManager.java | 20 ++--
.../wal/FsyncModeFileWriteAheadLogManager.java | 17 +--
.../wal/reader/StandaloneGridKernalContext.java | 18 +--
.../processors/failure/FailureProcessor.java | 113 +++++++++++++++++++
.../processors/failure/package-info.java | 22 ++++
.../startup/cmdline/CommandLineStartup.java | 5 +-
26 files changed, 656 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/IgniteState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteState.java b/modules/core/src/main/java/org/apache/ignite/IgniteState.java
index e3c04c9..a3704fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteState.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteState.java
@@ -43,7 +43,12 @@ public enum IgniteState {
* set to {@link SegmentationPolicy#STOP} or {@link SegmentationPolicy#RESTART_JVM}
* and node is stopped from internals of Ignite after segment becomes invalid.
*/
- STOPPED_ON_SEGMENTATION;
+ STOPPED_ON_SEGMENTATION,
+
+ /**
+ * Grid factory stopped due to a critical failure.
+ */
+ STOPPED_ON_FAILURE;
/** Enumerated values. */
private static final IgniteState[] VALS = values();
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index aa2fb00..add3880 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -42,6 +42,7 @@ import org.apache.ignite.compute.ComputeJob;
import org.apache.ignite.compute.ComputeTask;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
+import org.apache.ignite.failure.FailureHandler;
import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.processors.odbc.ClientListenerProcessor;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -489,6 +490,9 @@ public class IgniteConfiguration {
/** User authentication enabled. */
private boolean authEnabled;
+ /** Failure handler. */
+ private FailureHandler failureHnd;
+
/**
* Creates valid grid configuration with all default values.
*/
@@ -547,6 +551,7 @@ public class IgniteConfiguration {
hadoopCfg = cfg.getHadoopConfiguration();
igfsCfg = cfg.getFileSystemConfiguration();
igfsPoolSize = cfg.getIgfsThreadPoolSize();
+ failureHnd = cfg.getFailureHandler();
igniteHome = cfg.getIgniteHome();
igniteInstanceName = cfg.getIgniteInstanceName();
igniteWorkDir = cfg.getWorkDirectory();
@@ -2923,6 +2928,27 @@ public class IgniteConfiguration {
}
/**
+ * Gets failure handler.
+ *
+ * @return Failure handler.
+ */
+ public FailureHandler getFailureHandler() {
+ return failureHnd;
+ }
+
+ /**
+ * Sets failure handler.
+ *
+ * @param failureHnd Failure handler.
+ * @return {@code This} for chaining.
+ */
+ public IgniteConfiguration setFailureHandler(FailureHandler failureHnd) {
+ this.failureHnd = failureHnd;
+
+ return this;
+ }
+
+ /**
* Gets client connector configuration.
*
* @return Client connector configuration.
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/FailureContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/FailureContext.java b/modules/core/src/main/java/org/apache/ignite/failure/FailureContext.java
new file mode 100644
index 0000000..066e151
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/FailureContext.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Failure context contains information about failure type and exception if applicable.
+ * This information could be used for appropriate handling of the failure.
+ */
+public class FailureContext {
+ /** Type. */
+ private final FailureType type;
+
+ /** Error. */
+ private final Throwable err;
+
+ /**
+ * Creates instance of {@link FailureContext}.
+ *
+ * @param type Failure type.
+ * @param err Exception.
+ */
+ public FailureContext(FailureType type, Throwable err) {
+ A.notNull(type, "type");
+
+ this.type = type;
+ this.err = err;
+ }
+
+ /**
+ * @return Failure type.
+ */
+ public FailureType type() {
+ return type;
+ }
+
+ /**
+ * @return Exception or {@code null}.
+ */
+ public Throwable error() {
+ return err;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(FailureContext.class, this);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/FailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/FailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/FailureHandler.java
new file mode 100644
index 0000000..8717b16
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/FailureHandler.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
+
+/**
+ * Provides facility to handle failures by custom user implementations,
+ * which can be configured by {@link IgniteConfiguration#setFailureHandler(FailureHandler)} method.
+ */
+public interface FailureHandler {
+ /**
+ * Handles failure occurred on {@code ignite} instance.
+ * Failure details is contained in {@code failureCtx}.
+ * Returns {@code true} if kernal context must be invalidated by {@link FailureProcessor} after calling this method.
+ *
+ * @param ignite Ignite instance.
+ * @param failureCtx Failure context.
+ * @return Whether kernal context must be invalidated or not.
+ */
+ public boolean onFailure(Ignite ignite, FailureContext failureCtx);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java b/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java
new file mode 100644
index 0000000..d933420
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/FailureType.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+/**
+ * Types of failures.
+ */
+public enum FailureType {
+ /** Node segmentation. */
+ SEGMENTATION,
+
+ /** System worker termination. */
+ SYSTEM_WORKER_TERMINATION,
+
+ /** Critical error - error which leads to the system's inoperability. */
+ CRITICAL_ERROR
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java
new file mode 100644
index 0000000..2ec645e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+
+/**
+ * Just ignores any failure. It's useful for tests and debugging.
+ */
+public class NoOpFailureHandler implements FailureHandler {
+ /** {@inheritDoc} */
+ @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java
new file mode 100644
index 0000000..4f4432f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * This handler could be used only with ignite.(sh|bat) script.
+ * Process will be terminated using {@link Ignition#restart(boolean)} call.
+ */
+public class RestartProcessFailureHandler implements FailureHandler {
+ /** {@inheritDoc} */
+ @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+ new Thread(
+ new Runnable() {
+ @Override public void run() {
+ U.error(ignite.log(), "Restarting JVM on Ignite failure: [failureCtx=" + failureCtx + ']');
+
+ G.restart(true);
+ }
+ },
+ "node-restarter"
+ ).start();
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java
new file mode 100644
index 0000000..f05cc76
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Handler will stop node in case of critical error using {@code IgnitionEx.stop(nodeName, true, true)} call.
+ */
+public class StopNodeFailureHandler implements FailureHandler {
+ /** {@inheritDoc} */
+ @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+ new Thread(
+ new Runnable() {
+ @Override public void run() {
+ U.error(ignite.log(), "Stopping local node on Ignite failure: [failureCtx=" + failureCtx + ']');
+
+ IgnitionEx.stop(ignite.name(), true, true);
+ }
+ },
+ "node-stopper"
+ ).start();
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
new file mode 100644
index 0000000..4f74406
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Handler will try to stop node if {@code tryStop} value is {@code true}.
+ * If node can't be stopped during provided {@code timeout} or {@code tryStop} value is {@code false}
+ * then JVM process will be terminated forcibly using {@code Runtime.getRuntime().halt()}.
+ */
+public class StopNodeOrHaltFailureHandler implements FailureHandler {
+ /** Try stop. */
+ private final boolean tryStop;
+
+ /** Timeout. */
+ private final long timeout;
+
+ /**
+ * Default constructor.
+ */
+ public StopNodeOrHaltFailureHandler() {
+ this(false, 0);
+ }
+
+ /**
+ * @param tryStop Try stop.
+ * @param timeout Stop node timeout.
+ */
+ public StopNodeOrHaltFailureHandler(boolean tryStop, long timeout) {
+ this.tryStop = tryStop;
+ this.timeout = timeout;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+ IgniteLogger log = ignite.log();
+
+ if (tryStop) {
+ final CountDownLatch latch = new CountDownLatch(1);
+
+ new Thread(
+ new Runnable() {
+ @Override public void run() {
+ U.error(log, "Stopping local node on Ignite failure: [failureCtx=" + failureCtx + ']');
+
+ IgnitionEx.stop(ignite.name(), true, true);
+
+ latch.countDown();
+ }
+ },
+ "node-stopper"
+ ).start();
+
+ new Thread(
+ new Runnable() {
+ @Override public void run() {
+ try {
+ if (!latch.await(timeout, TimeUnit.MILLISECONDS)) {
+ U.error(log, "Stopping local node timeout, JVM will be halted.");
+
+ Runtime.getRuntime().halt(Ignition.KILL_EXIT_CODE);
+ }
+ }
+ catch (InterruptedException e) {
+ // No-op.
+ }
+ }
+ },
+ "jvm-halt-on-stop-timeout"
+ ).start();
+ }
+ else {
+ U.error(log, "JVM will be halted immediately on ignite failure: [failureCtx=" + failureCtx + ']');
+
+ Runtime.getRuntime().halt(Ignition.KILL_EXIT_CODE);
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/failure/package-info.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/package-info.java b/modules/core/src/main/java/org/apache/ignite/failure/package-info.java
new file mode 100644
index 0000000..2128ddb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/failure/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains <b>Failure handling public API</b> with standard FailureHandler implementations.
+ */
+package org.apache.ignite.failure;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index ce53b47..0b40054 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -43,6 +43,7 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
import org.apache.ignite.internal.processors.hadoop.HadoopHelper;
import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
import org.apache.ignite.internal.processors.igfs.IgfsHelper;
@@ -429,16 +430,16 @@ public interface GridKernalContext extends Iterable<GridComponent> {
public DataStructuresProcessor dataStructures();
/**
- * Sets segmented flag to {@code true} when node is stopped due to segmentation issues.
+ * Checks whether this node is invalid due to a critical error or not.
+ *
+ * @return {@code True} if this node is invalid, {@code false} otherwise.
*/
- public void markSegmented();
+ public boolean invalid();
/**
- * Gets segmented flag.
- *
- * @return {@code True} if network is currently segmented, {@code false} otherwise.
+ * Gets failure processor.
*/
- public boolean segmented();
+ public FailureProcessor failure();
/**
* Print grid kernal memory stats (sizes of internal structures, etc.).
@@ -663,18 +664,4 @@ public interface GridKernalContext extends Iterable<GridComponent> {
* @return subscription processor to manage internal-only (strict node-local) subscriptions between components.
*/
public GridInternalSubscriptionProcessor internalSubscriptionProcessor();
-
- /**
- * TODO: Should be replaced with proper implementation in https://issues.apache.org/jira/browse/IGNITE-6891
- *
- * @return {@code true} if node was invalidated, false in other case.
- */
- public boolean invalidated();
-
- /**
- * Invalidates node.
- *
- * TODO: Should be replaced with proper implementation in https://issues.apache.org/jira/browse/IGNITE-6891
- */
- public void invalidate();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 430b671..3408334 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -59,6 +59,7 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
import org.apache.ignite.internal.processors.hadoop.HadoopHelper;
import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
import org.apache.ignite.internal.processors.igfs.IgfsHelper;
@@ -392,8 +393,8 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
/** */
private GridInternalSubscriptionProcessor internalSubscriptionProc;
- /** Node invalidation flag. */
- private volatile boolean invalidated;
+ /** Failure processor. */
+ private FailureProcessor failureProc;
/**
* No-arg constructor is required by externalization.
@@ -542,6 +543,8 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
* ==========
*/
+ else if (comp instanceof FailureProcessor)
+ failureProc = (FailureProcessor)comp;
else if (comp instanceof GridTaskProcessor)
taskProc = (GridTaskProcessor)comp;
else if (comp instanceof GridJobProcessor)
@@ -878,16 +881,6 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
}
/** {@inheritDoc} */
- @Override public void markSegmented() {
- segFlag = true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean segmented() {
- return segFlag;
- }
-
- /** {@inheritDoc} */
@Override public GridPerformanceSuggestions performance() {
return perf;
}
@@ -1116,13 +1109,15 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
}
/** {@inheritDoc} */
- @Override public boolean invalidated() {
- return invalidated;
+ @Override public boolean invalid() {
+ FailureProcessor failureProc = failure();
+
+ return failureProc != null && failureProc.failureContext() != null;
}
/** {@inheritDoc} */
- @Override public void invalidate() {
- invalidated = true;
+ @Override public FailureProcessor failure() {
+ return failureProc;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 843b6e1..8bc46fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -104,6 +104,7 @@ import org.apache.ignite.internal.binary.BinaryMarshaller;
import org.apache.ignite.internal.binary.BinaryUtils;
import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
import org.apache.ignite.internal.managers.GridManager;
import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
import org.apache.ignite.internal.managers.collision.GridCollisionManager;
@@ -906,6 +907,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
startProcessor(new IgnitePluginProcessor(ctx, cfg, plugins));
+ startProcessor(new FailureProcessor(ctx));
+
startProcessor(new PoolProcessor(ctx));
// Closure processor should be started before all others
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index f5cb798..8073faa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -35,6 +35,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
@@ -70,6 +71,8 @@ import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.configuration.MemoryPolicyConfiguration;
import org.apache.ignite.configuration.PersistentStoreConfiguration;
import org.apache.ignite.configuration.TransactionConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.binary.BinaryMarshaller;
import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
@@ -113,10 +116,10 @@ import org.apache.ignite.thread.IgniteStripedThreadPoolExecutor;
import org.apache.ignite.thread.IgniteThread;
import org.apache.ignite.thread.IgniteThreadPoolExecutor;
import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
import static org.apache.ignite.IgniteState.STARTED;
import static org.apache.ignite.IgniteState.STOPPED;
+import static org.apache.ignite.IgniteState.STOPPED_ON_FAILURE;
import static org.apache.ignite.IgniteState.STOPPED_ON_SEGMENTATION;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_CLIENT;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONFIG_URL;
@@ -2554,7 +2557,15 @@ public class IgnitionEx {
throw e;
}
finally {
- state = grid0.context().segmented() ? STOPPED_ON_SEGMENTATION : STOPPED;
+ if (!grid0.context().invalid())
+ state = STOPPED;
+ else {
+ FailureContext failure = grid0.context().failure().failureContext();
+
+ state = failure.type() == FailureType.SEGMENTATION ?
+ STOPPED_ON_SEGMENTATION :
+ STOPPED_ON_FAILURE;
+ }
grid = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java b/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java
deleted file mode 100644
index b19ec08..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/NodeInvalidator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal;
-
-import org.apache.ignite.IgniteLogger;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Temporary functionality to invalidate and stop the node.
- * TODO: Should be replaced on proper implementation in https://issues.apache.org/jira/browse/IGNITE-6891
- */
-public class NodeInvalidator {
- public static NodeInvalidator INSTANCE = new NodeInvalidator();
-
- private static final long STOP_TIMEOUT_MS = 60 * 1000;
-
- private NodeInvalidator() {
- // Empty
- }
-
- public void invalidate(@NotNull GridKernalContext ctx, @NotNull Throwable error) {
- if (ctx.invalidated())
- return;
-
- ctx.invalidate();
-
- final String gridName = ctx.igniteInstanceName();
- final IgniteLogger logger = ctx.log(getClass());
-
- logger.error("Critical error with " + gridName + " is happened. " +
- "All further operations will be failed and local node will be stopped.", error);
-
- new Thread("node-stopper") {
- @Override public void run() {
- IgnitionEx.stop(gridName, true, true, STOP_TIMEOUT_MS);
- }
- }.start();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 707b1ba..2e814d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -58,6 +58,10 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.failure.RestartProcessFailureHandler;
+import org.apache.ignite.failure.StopNodeFailureHandler;
import org.apache.ignite.internal.GridComponent;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
@@ -75,7 +79,6 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest;
import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.GridCacheUtils;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cluster.BaselineTopology;
import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
@@ -93,7 +96,6 @@ import org.apache.ignite.internal.util.lang.GridTuple6;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.P1;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -2377,32 +2379,6 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
cacheNodes.add(rich);
}
- /** Stops local node. */
- private void stopNode() {
- new Thread(
- new Runnable() {
- @Override public void run() {
- ctx.markSegmented();
-
- G.stop(ctx.igniteInstanceName(), true);
- }
- }
- ).start();
- }
-
- /** Restarts JVM. */
- private void restartJvm() {
- new Thread(
- new Runnable() {
- @Override public void run() {
- ctx.markSegmented();
-
- G.restart(true);
- }
- }
- ).start();
- }
-
/** Worker for network segment checks. */
private class SegmentCheckWorker extends GridWorker {
/** */
@@ -2497,6 +2473,12 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
private final BlockingQueue<GridTuple6<Integer, AffinityTopologyVersion, ClusterNode,
DiscoCache, Collection<ClusterNode>, DiscoveryCustomMessage>> evts = new LinkedBlockingQueue<>();
+ /** Restart process handler. */
+ private final RestartProcessFailureHandler restartProcHnd = new RestartProcessFailureHandler();
+
+ /** Stop node handler. */
+ private final StopNodeFailureHandler stopNodeHnd = new StopNodeFailureHandler();
+
/** Node segmented event fired flag. */
private boolean nodeSegFired;
@@ -2780,16 +2762,12 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
switch (segPlc) {
case RESTART_JVM:
- U.warn(log, "Restarting JVM according to configured segmentation policy.");
-
- restartJvm();
+ ctx.failure().process(new FailureContext(FailureType.SEGMENTATION, null), restartProcHnd);
break;
case STOP:
- U.warn(log, "Stopping local node according to configured segmentation policy.");
-
- stopNode();
+ ctx.failure().process(new FailureContext(FailureType.SEGMENTATION, null), stopNodeHnd);
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 4e93822..5e57299 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -32,6 +32,7 @@ import java.util.Map;
import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.LinkedBlockingDeque;
@@ -47,6 +48,7 @@ import org.apache.ignite.cache.affinity.AffinityFunction;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
import org.apache.ignite.internal.IgniteDiagnosticAware;
import org.apache.ignite.internal.IgniteDiagnosticPrepareContext;
@@ -112,7 +114,6 @@ import org.apache.ignite.lang.IgniteProductVersion;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.thread.IgniteThread;
import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_IO_DUMP_ON_TIMEOUT;
@@ -122,6 +123,7 @@ import static org.apache.ignite.IgniteSystemProperties.getLong;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE;
import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
@@ -2245,6 +2247,24 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
/** {@inheritDoc} */
@Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+ Throwable err = null;
+
+ try {
+ body0();
+ }
+ catch (Throwable e) {
+ err = e;
+ }
+ finally {
+ if (!stop)
+ cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+ }
+ }
+
+ /**
+ *
+ */
+ private void body0() throws InterruptedException, IgniteInterruptedCheckedException {
long timeout = cctx.gridConfig().getNetworkTimeout();
long cnt = 0;
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 410e9b9..73802c1 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -75,10 +75,11 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.EventType;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.NodeInvalidator;
import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
@@ -2876,7 +2877,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
chp.progress.cpFinishFut.onDone(e);
// In case of writing error node should be invalidated and stopped.
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
return;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 1a35b24..766333f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -27,7 +27,8 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.NodeInvalidator;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -1185,7 +1186,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
U.error(log, "Unhandled exception during page store initialization. All further operations will " +
"be failed and local node will be stopped.", ex);
- NodeInvalidator.INSTANCE.invalidate(ctx.kernalContext(), ex);
+ ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
throw ex;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 9549269..6313eac 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -38,8 +38,9 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.NodeInvalidator;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
@@ -49,9 +50,8 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
import org.apache.ignite.internal.processors.cache.StoredCacheData;
-import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
-import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
import org.apache.ignite.internal.util.IgniteUtils;
@@ -202,7 +202,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
}
}
catch (PersistentStorageIOException e) {
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
throw e;
}
@@ -322,7 +322,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
store.read(pageId, pageBuf, keepCrc);
}
catch (PersistentStorageIOException e) {
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
throw e;
}
@@ -343,7 +343,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
store.readHeader(buf);
}
catch (PersistentStorageIOException e) {
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
throw e;
}
@@ -379,7 +379,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
store.write(pageId, pageBuf, tag, calculateCrc);
}
catch (PersistentStorageIOException e) {
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
throw e;
}
@@ -458,7 +458,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
return new CacheStoreHolder(idxStore, partStores);
}
catch (PersistentStorageIOException e) {
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
throw e;
}
@@ -550,7 +550,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
getStore(grpId, partId).sync();
}
catch (PersistentStorageIOException e) {
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
throw e;
}
@@ -562,7 +562,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
getStore(grpId, partId).ensure();
}
catch (PersistentStorageIOException e) {
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
throw e;
}
@@ -580,7 +580,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
return PageIdUtils.pageId(partId, flags, (int)pageIdx);
}
catch (PersistentStorageIOException e) {
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
throw e;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 306fb2c..2fff481 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -65,10 +65,11 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.events.EventType;
import org.apache.ignite.events.WalSegmentArchivedEvent;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.NodeInvalidator;
import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.pagemem.wal.StorageException;
@@ -681,7 +682,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IgniteCheckedException e) {
U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e);
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
}
}
@@ -1233,7 +1234,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IOException e) {
StorageException se = new StorageException("Unable to initialize WAL segment", e);
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, se));
throw se;
}
@@ -1382,7 +1383,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
* @throws StorageException If node is no longer valid and we missed a WAL operation.
*/
private void checkNode() throws StorageException {
- if (cctx.kernalContext().invalidated())
+ if (cctx.kernalContext().invalid())
throw new StorageException("Failed to perform WAL operation (environment was invalidated by a " +
"previous error)");
}
@@ -1884,7 +1885,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IgniteCheckedException | IOException e) {
U.error(log, "Unexpected error during WAL compression", e);
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
}
catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
@@ -2036,7 +2037,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IOException e) {
U.error(log, "Unexpected error during WAL decompression", e);
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
}
}
}
@@ -2814,7 +2815,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
lock.lock();
try {
- assert cctx.kernalContext().invalidated() || written == lastFsyncPos || mode != WALMode.FSYNC :
+ assert cctx.kernalContext().invalid() ||
+ written == lastFsyncPos || mode != WALMode.FSYNC :
"fsync [written=" + written + ", lastFsync=" + lastFsyncPos + ", idx=" + idx + ']';
fileIO = null;
@@ -3281,7 +3283,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
Throwable err = walWriter.err;
if (err != null)
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), err);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, err));
if (expPos == UNCONDITIONAL_FLUSH)
expPos = (currentHandle().buf.tail());
@@ -3370,7 +3372,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
catch (IOException e) {
StorageException se = new StorageException("Unable to write", e);
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+ cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, se));
throw se;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 6934f4f..59196bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -61,10 +61,10 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.events.EventType;
import org.apache.ignite.events.WalSegmentArchivedEvent;
+import org.apache.ignite.failure.FailureContext;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.NodeInvalidator;
import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.pagemem.wal.StorageException;
@@ -109,6 +109,7 @@ import static java.nio.file.StandardOpenOption.CREATE;
import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
/**
* File WAL manager.
@@ -599,7 +600,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
catch (IgniteCheckedException e) {
U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e);
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
}
}
@@ -1056,7 +1057,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
catch (IOException e) {
StorageException se = new StorageException("Unable to initialize WAL segment", e);
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
throw se;
}
@@ -1196,7 +1197,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
* @throws StorageException If node is no longer valid and we missed a WAL operation.
*/
private void checkNode() throws StorageException {
- if (cctx.kernalContext().invalidated())
+ if (cctx.kernalContext().invalid())
throw new StorageException("Failed to perform WAL operation (environment was invalidated by a " +
"previous error)");
}
@@ -1721,7 +1722,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
catch (IgniteCheckedException | IOException e) {
U.error(log, "Unexpected error during WAL compression", e);
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
@@ -1845,7 +1846,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
catch (IOException e) {
U.error(log, "Unexpected error during WAL decompression", e);
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), e);
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
}
}
}
@@ -2563,7 +2564,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
catch (Throwable e) {
StorageException se = new StorageException("Unable to write", new IOException(e));
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
// All workers waiting for a next segment must be woken up and stopped
signalNextAvailable();
@@ -2877,7 +2878,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
catch (IOException e) {
StorageException se = new StorageException("Unable to write", e);
- NodeInvalidator.INSTANCE.invalidate(cctx.kernalContext(), se);
+ cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
throw se;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index fdcc053..429a5ce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -58,6 +58,7 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
import org.apache.ignite.internal.processors.hadoop.HadoopHelper;
import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
import org.apache.ignite.internal.processors.igfs.IgfsHelper;
@@ -458,12 +459,13 @@ public class StandaloneGridKernalContext implements GridKernalContext {
}
/** {@inheritDoc} */
- @Override public void markSegmented() {
+ @Override public boolean invalid() {
+ return false;
}
/** {@inheritDoc} */
- @Override public boolean segmented() {
- return false;
+ @Override public FailureProcessor failure() {
+ return null;
}
/** {@inheritDoc} */
@@ -631,16 +633,6 @@ public class StandaloneGridKernalContext implements GridKernalContext {
}
/** {@inheritDoc} */
- @Override public boolean invalidated() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public void invalidate() {
-
- }
-
- /** {@inheritDoc} */
@NotNull @Override public Iterator<GridComponent> iterator() {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
new file mode 100644
index 0000000..615fb9f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * General failure processing API
+ */
+public class FailureProcessor extends GridProcessorAdapter {
+ /** Ignite. */
+ private final Ignite ignite;
+
+ /** Handler. */
+ private volatile FailureHandler hnd;
+
+ /** Failure context. */
+ private volatile FailureContext failureCtx;
+
+ /**
+ * @param ctx Context.
+ */
+ public FailureProcessor(GridKernalContext ctx) {
+ super(ctx);
+
+ this.ignite = ctx.grid();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteCheckedException {
+ FailureHandler hnd = ctx.config().getFailureHandler();
+
+ if (hnd == null)
+ hnd = getDefaultFailureHandler();
+
+ assert hnd != null;
+
+ this.hnd = hnd;
+ }
+
+ /**
+ * This method is used to initialize local failure handler if {@link IgniteConfiguration} don't contain configured one.
+ *
+ * @return Default {@link FailureHandler} implementation.
+ */
+ protected FailureHandler getDefaultFailureHandler() {
+ return new StopNodeOrHaltFailureHandler();
+ }
+
+ /**
+ * @return Failure context.
+ */
+ public FailureContext failureContext() {
+ return failureCtx;
+ }
+
+ /**
+ * Processes failure accordingly to configured {@link FailureHandler}.
+ *
+ * @param failureCtx Failure context.
+ */
+ public void process(FailureContext failureCtx) {
+ process(failureCtx, hnd);
+ }
+
+ /**
+ * Processes failure accordingly to given failure handler.
+ *
+ * @param failureCtx Failure context.
+ * @param hnd Failure handler.
+ */
+ public synchronized void process(FailureContext failureCtx, FailureHandler hnd) {
+ assert failureCtx != null;
+ assert hnd != null;
+
+ if (this.failureCtx != null) // Node already terminating, no reason to process more errors.
+ return;
+
+ U.error(ignite.log(), "Critical failure. Will be handled accordingly to configured handler [hnd=" +
+ hnd.getClass() + ", failureCtx=" + failureCtx + ']', failureCtx.error());
+
+ boolean invalidated = hnd.onFailure(ignite, failureCtx);
+
+ if (invalidated) {
+ this.failureCtx = failureCtx;
+
+ log.error("Ignite node is in invalid state due to a critical failure.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/package-info.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/package-info.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/package-info.java
new file mode 100644
index 0000000..1c1ad13
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Failure processor.
+ */
+package org.apache.ignite.internal.processors.failure;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/15022e02/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineStartup.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineStartup.java b/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineStartup.java
index 9b05071..821af8a 100644
--- a/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineStartup.java
+++ b/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineStartup.java
@@ -43,8 +43,7 @@ import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
-import static org.apache.ignite.IgniteState.STOPPED;
-import static org.apache.ignite.IgniteState.STOPPED_ON_SEGMENTATION;
+import static org.apache.ignite.IgniteState.STARTED;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PROG_NAME;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_RESTART_CODE;
import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR;
@@ -326,7 +325,7 @@ public final class CommandLineStartup {
if (!F.eq(igniteInstanceName, name))
return;
- if (state == STOPPED || state == STOPPED_ON_SEGMENTATION)
+ if (state != STARTED)
latch.countDown();
}
});