You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2015/08/09 09:42:25 UTC
[01/10] cassandra git commit: Ensure commit log stop policy is
enforced at startup
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 98a08ebcf -> 3b7934f1a
refs/heads/cassandra-2.2 6fd41ab15 -> 087153394
refs/heads/cassandra-3.0 7ea2ce124 -> e2ad7d569
refs/heads/trunk 6292aaa72 -> 98cc1ee09
Ensure commit log stop policy is enforced at startup
patch by paulo; reviewed by benedict for CASSANDRA-8515
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3b7934f1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3b7934f1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3b7934f1
Branch: refs/heads/cassandra-2.1
Commit: 3b7934f1aa20d2210866afd9b88472e9cb1aed8d
Parents: 98a08eb
Author: Paulo Motta <pa...@gmail.com>
Authored: Mon Jul 13 19:35:50 2015 -0300
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Sun Aug 9 09:34:05 2015 +0200
----------------------------------------------------------------------
.../cassandra/service/CassandraDaemon.java | 7 ++
.../cassandra/utils/JVMStabilityInspector.java | 9 +-
.../org/apache/cassandra/db/CommitLogTest.java | 110 +++++++++++++++++++
.../apache/cassandra/utils/KillerForTests.java | 11 +-
4 files changed, 134 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index 2c141a6..d078203 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -34,6 +34,7 @@ import javax.management.remote.JMXConnectorServer;
import javax.management.remote.JMXServiceURL;
import javax.management.remote.rmi.RMIConnectorServer;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
import org.slf4j.Logger;
@@ -425,6 +426,12 @@ public class CassandraDaemon
int nativePort = DatabaseDescriptor.getNativeTransportPort();
nativeServer = new org.apache.cassandra.transport.Server(nativeAddr, nativePort);
+ completeSetup();
+ }
+
+ @VisibleForTesting
+ public void completeSetup()
+ {
setupCompleted = true;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
index c0ab84f..de396bb 100644
--- a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
+++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
@@ -28,7 +28,9 @@ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.thrift.Cassandra;
/**
* Responsible for deciding whether to kill the JVM if it gets in an "unstable" state (think OOM).
@@ -38,6 +40,7 @@ public final class JVMStabilityInspector
private static final Logger logger = LoggerFactory.getLogger(JVMStabilityInspector.class);
private static Killer killer = new Killer();
+
private JVMStabilityInspector() {}
/**
@@ -66,7 +69,11 @@ public final class JVMStabilityInspector
public static void inspectCommitLogThrowable(Throwable t)
{
- if (DatabaseDescriptor.getCommitFailurePolicy() == Config.CommitFailurePolicy.die)
+ if (!StorageService.instance.isSetupCompleted())
+ {
+ logger.error("Exiting due to error while processing commit log during initialization.", t);
+ killer.killCurrentJVM(t, true);
+ } else if (DatabaseDescriptor.getCommitFailurePolicy() == Config.CommitFailurePolicy.die)
killer.killCurrentJVM(t);
else
inspectThrowable(t);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/CommitLogTest.java b/test/unit/org/apache/cassandra/db/CommitLogTest.java
index 9a8a1dc..1c3daab 100644
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.commitlog.CommitLogSegment;
import org.apache.cassandra.db.composites.CellName;
@@ -43,6 +44,7 @@ import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -279,6 +281,10 @@ public class CommitLogTest extends SchemaLoader
@Test
public void testCommitFailurePolicy_stop() throws ConfigurationException
{
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
// Need storage service active so stop policy can shutdown gossip
StorageService.instance.initServer();
Assert.assertTrue(Gossiper.instance.isEnabled());
@@ -299,6 +305,10 @@ public class CommitLogTest extends SchemaLoader
@Test
public void testCommitFailurePolicy_die()
{
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
KillerForTests killerForTests = new KillerForTests();
JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
@@ -307,11 +317,111 @@ public class CommitLogTest extends SchemaLoader
DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);
CommitLog.handleCommitError("Testing die policy", new Throwable());
Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_mustDieIfNotStartedUp()
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ //even though policy is ignore, JVM must die because Daemon has not finished initializing
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager();
+
+ //busy wait since commitlogsegmentmanager spawns another thread
+ int retries = 0;
+ while (!killerForTests.wasKilled() && retries++ < 5)
+ Thread.sleep(10);
+
+ //since failure was before CassandraDaemon startup, the JVM must be killed
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureAfterInitialization_mustRespectFailurePolicy() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager();
+
+ //wait commit log segment manager thread to execute
+ Thread.sleep(50);
+
+ //error policy is set to IGNORE, so JVM must not be killed if error ocurs after startup
+ Assert.assertFalse(killerForTests.wasKilled());
}
finally
{
DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/test/unit/org/apache/cassandra/utils/KillerForTests.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/KillerForTests.java b/test/unit/org/apache/cassandra/utils/KillerForTests.java
index d488f97..abc7952 100644
--- a/test/unit/org/apache/cassandra/utils/KillerForTests.java
+++ b/test/unit/org/apache/cassandra/utils/KillerForTests.java
@@ -24,11 +24,13 @@ package org.apache.cassandra.utils;
public class KillerForTests extends JVMStabilityInspector.Killer
{
private boolean killed = false;
+ private boolean quiet = false;
@Override
- protected void killCurrentJVM(Throwable t)
+ protected void killCurrentJVM(Throwable t, boolean quiet)
{
- killed = true;
+ this.killed = true;
+ this.quiet = quiet;
}
public boolean wasKilled()
@@ -36,6 +38,11 @@ public class KillerForTests extends JVMStabilityInspector.Killer
return killed;
}
+ public boolean wasKilledQuietly()
+ {
+ return quiet;
+ }
+
public void reset()
{
killed = false;
[09/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by be...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Conflicts:
test/unit/org/apache/cassandra/db/CommitLogTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e2ad7d56
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e2ad7d56
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e2ad7d56
Branch: refs/heads/trunk
Commit: e2ad7d5695d4044a021f14933762f105390c9beb
Parents: 7ea2ce1 0871533
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Sun Aug 9 09:41:38 2015 +0200
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Sun Aug 9 09:41:38 2015 +0200
----------------------------------------------------------------------
.../cassandra/service/CassandraDaemon.java | 8 +
.../cassandra/utils/JVMStabilityInspector.java | 9 +-
.../org/apache/cassandra/db/CommitLogTest.java | 161 ++++++++++++++++++-
.../apache/cassandra/utils/KillerForTests.java | 11 +-
4 files changed, 184 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2ad7d56/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/CassandraDaemon.java
index fddf593,2020201..3b123c3
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@@ -40,8 -40,10 +40,10 @@@ import com.addthis.metrics3.reporter.co
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistryListener;
import com.codahale.metrics.SharedMetricRegistries;
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.cassandra.metrics.DefaultNameFactory;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2ad7d56/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2ad7d56/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/CommitLogTest.java
index 21bdd9b,536f0cb..512a3d2
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@@ -47,20 -42,31 +48,29 @@@ import org.junit.BeforeClass
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
- import org.apache.cassandra.db.commitlog.CommitLogSegment;
+ import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.ReplayPosition;
+ import org.apache.cassandra.db.commitlog.CommitLogSegment;
import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.composites.CellName;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
+ import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.io.util.ByteBufferDataInput;
import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.vint.VIntCoding;
+ import org.apache.cassandra.service.CassandraDaemon;
+ import org.apache.cassandra.service.StorageService;
+ import org.apache.cassandra.utils.ByteBufferUtil;
+ import org.apache.cassandra.utils.FBUtilities;
+ import org.apache.cassandra.utils.JVMStabilityInspector;
+ import org.apache.cassandra.utils.KillerForTests;
public class CommitLogTest
{
@@@ -307,40 -319,179 +317,187 @@@
}
@Test
- public void testTruncateWithoutSnapshot() throws IOException
+ public void testCommitFailurePolicy_stop() throws ConfigurationException
+ {
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ // Need storage service active so stop policy can shutdown gossip
+ StorageService.instance.initServer();
+ Assert.assertTrue(Gossiper.instance.isEnabled());
+
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.stop);
+ CommitLog.handleCommitError("Test stop error", new Throwable());
+ Assert.assertFalse(Gossiper.instance.isEnabled());
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_die()
+ {
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_mustDieIfNotStartedUp()
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ //even though policy is ignore, JVM must die because Daemon has not finished initializing
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager(CommitLog.instance);
+
+ //busy wait since commitlogsegmentmanager spawns another thread
+ int retries = 0;
+ while (!killerForTests.wasKilled() && retries++ < 5)
+ Thread.sleep(10);
+
+ //since failure was before CassandraDaemon startup, the JVM must be killed
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureAfterInitialization_mustRespectFailurePolicy() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager(CommitLog.instance);
+
+ //wait commit log segment manager thread to execute
+ Thread.sleep(50);
+
+ //error policy is set to IGNORE, so JVM must not be killed if error ocurs after startup
+ Assert.assertFalse(killerForTests.wasKilled());
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testTruncateWithoutSnapshot() throws ExecutionException, InterruptedException, IOException
{
- CommitLog.instance.resetUnsafe(true);
- boolean prev = DatabaseDescriptor.isAutoSnapshot();
- DatabaseDescriptor.setAutoSnapshot(false);
- ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
- ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard2");
-
- final Mutation rm1 = new Mutation(KEYSPACE1, bytes("k"));
- rm1.add("Standard1", Util.cellname("c1"), ByteBuffer.allocate(100), 0);
- rm1.apply();
- cfs1.truncateBlocking();
- DatabaseDescriptor.setAutoSnapshot(prev);
- final Mutation rm2 = new Mutation(KEYSPACE1, bytes("k"));
- rm2.add("Standard2", Util.cellname("c1"), ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4), 0);
-
- for (int i = 0 ; i < 5 ; i++)
- CommitLog.instance.add(rm2);
-
- Assert.assertEquals(2, CommitLog.instance.activeSegments());
- ReplayPosition position = CommitLog.instance.getContext();
- for (Keyspace ks : Keyspace.system())
- for (ColumnFamilyStore syscfs : ks.getColumnFamilyStores())
- CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, position);
- CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, position);
- Assert.assertEquals(1, CommitLog.instance.activeSegments());
+ boolean originalState = DatabaseDescriptor.isAutoSnapshot();
+ try
+ {
+ CommitLog.instance.resetUnsafe(true);
+ boolean prev = DatabaseDescriptor.isAutoSnapshot();
+ DatabaseDescriptor.setAutoSnapshot(false);
+ ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+ ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD2);
+
+ new RowUpdateBuilder(cfs1.metadata, 0, "k").clustering("bytes").add("val", ByteBuffer.allocate(100)).build().applyUnsafe();
+ cfs1.truncateBlocking();
+ DatabaseDescriptor.setAutoSnapshot(prev);
+ Mutation m2 = new RowUpdateBuilder(cfs2.metadata, 0, "k")
+ .clustering("bytes")
+ .add("val", ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4))
+ .build();
+
+ for (int i = 0 ; i < 5 ; i++)
+ CommitLog.instance.add(m2);
+
+ assertEquals(2, CommitLog.instance.activeSegments());
+ ReplayPosition position = CommitLog.instance.getContext();
+ for (Keyspace ks : Keyspace.system())
+ for (ColumnFamilyStore syscfs : ks.getColumnFamilyStores())
+ CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, position);
+ CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, position);
+ assertEquals(1, CommitLog.instance.activeSegments());
+ }
+ finally
+ {
+ DatabaseDescriptor.setAutoSnapshot(originalState);
+ }
}
@Test
[02/10] cassandra git commit: Ensure commit log stop policy is
enforced at startup
Posted by be...@apache.org.
Ensure commit log stop policy is enforced at startup
patch by paulo; reviewed by benedict for CASSANDRA-8515
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3b7934f1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3b7934f1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3b7934f1
Branch: refs/heads/cassandra-2.2
Commit: 3b7934f1aa20d2210866afd9b88472e9cb1aed8d
Parents: 98a08eb
Author: Paulo Motta <pa...@gmail.com>
Authored: Mon Jul 13 19:35:50 2015 -0300
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Sun Aug 9 09:34:05 2015 +0200
----------------------------------------------------------------------
.../cassandra/service/CassandraDaemon.java | 7 ++
.../cassandra/utils/JVMStabilityInspector.java | 9 +-
.../org/apache/cassandra/db/CommitLogTest.java | 110 +++++++++++++++++++
.../apache/cassandra/utils/KillerForTests.java | 11 +-
4 files changed, 134 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index 2c141a6..d078203 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -34,6 +34,7 @@ import javax.management.remote.JMXConnectorServer;
import javax.management.remote.JMXServiceURL;
import javax.management.remote.rmi.RMIConnectorServer;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
import org.slf4j.Logger;
@@ -425,6 +426,12 @@ public class CassandraDaemon
int nativePort = DatabaseDescriptor.getNativeTransportPort();
nativeServer = new org.apache.cassandra.transport.Server(nativeAddr, nativePort);
+ completeSetup();
+ }
+
+ @VisibleForTesting
+ public void completeSetup()
+ {
setupCompleted = true;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
index c0ab84f..de396bb 100644
--- a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
+++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
@@ -28,7 +28,9 @@ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.thrift.Cassandra;
/**
* Responsible for deciding whether to kill the JVM if it gets in an "unstable" state (think OOM).
@@ -38,6 +40,7 @@ public final class JVMStabilityInspector
private static final Logger logger = LoggerFactory.getLogger(JVMStabilityInspector.class);
private static Killer killer = new Killer();
+
private JVMStabilityInspector() {}
/**
@@ -66,7 +69,11 @@ public final class JVMStabilityInspector
public static void inspectCommitLogThrowable(Throwable t)
{
- if (DatabaseDescriptor.getCommitFailurePolicy() == Config.CommitFailurePolicy.die)
+ if (!StorageService.instance.isSetupCompleted())
+ {
+ logger.error("Exiting due to error while processing commit log during initialization.", t);
+ killer.killCurrentJVM(t, true);
+ } else if (DatabaseDescriptor.getCommitFailurePolicy() == Config.CommitFailurePolicy.die)
killer.killCurrentJVM(t);
else
inspectThrowable(t);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/CommitLogTest.java b/test/unit/org/apache/cassandra/db/CommitLogTest.java
index 9a8a1dc..1c3daab 100644
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.commitlog.CommitLogSegment;
import org.apache.cassandra.db.composites.CellName;
@@ -43,6 +44,7 @@ import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -279,6 +281,10 @@ public class CommitLogTest extends SchemaLoader
@Test
public void testCommitFailurePolicy_stop() throws ConfigurationException
{
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
// Need storage service active so stop policy can shutdown gossip
StorageService.instance.initServer();
Assert.assertTrue(Gossiper.instance.isEnabled());
@@ -299,6 +305,10 @@ public class CommitLogTest extends SchemaLoader
@Test
public void testCommitFailurePolicy_die()
{
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
KillerForTests killerForTests = new KillerForTests();
JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
@@ -307,11 +317,111 @@ public class CommitLogTest extends SchemaLoader
DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);
CommitLog.handleCommitError("Testing die policy", new Throwable());
Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_mustDieIfNotStartedUp()
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ //even though policy is ignore, JVM must die because Daemon has not finished initializing
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager();
+
+ //busy wait since commitlogsegmentmanager spawns another thread
+ int retries = 0;
+ while (!killerForTests.wasKilled() && retries++ < 5)
+ Thread.sleep(10);
+
+ //since failure was before CassandraDaemon startup, the JVM must be killed
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureAfterInitialization_mustRespectFailurePolicy() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager();
+
+ //wait commit log segment manager thread to execute
+ Thread.sleep(50);
+
+ //error policy is set to IGNORE, so JVM must not be killed if error ocurs after startup
+ Assert.assertFalse(killerForTests.wasKilled());
}
finally
{
DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/test/unit/org/apache/cassandra/utils/KillerForTests.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/KillerForTests.java b/test/unit/org/apache/cassandra/utils/KillerForTests.java
index d488f97..abc7952 100644
--- a/test/unit/org/apache/cassandra/utils/KillerForTests.java
+++ b/test/unit/org/apache/cassandra/utils/KillerForTests.java
@@ -24,11 +24,13 @@ package org.apache.cassandra.utils;
public class KillerForTests extends JVMStabilityInspector.Killer
{
private boolean killed = false;
+ private boolean quiet = false;
@Override
- protected void killCurrentJVM(Throwable t)
+ protected void killCurrentJVM(Throwable t, boolean quiet)
{
- killed = true;
+ this.killed = true;
+ this.quiet = quiet;
}
public boolean wasKilled()
@@ -36,6 +38,11 @@ public class KillerForTests extends JVMStabilityInspector.Killer
return killed;
}
+ public boolean wasKilledQuietly()
+ {
+ return quiet;
+ }
+
public void reset()
{
killed = false;
[07/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by be...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Conflicts:
src/java/org/apache/cassandra/service/CassandraDaemon.java
test/unit/org/apache/cassandra/db/CommitLogTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/08715339
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/08715339
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/08715339
Branch: refs/heads/trunk
Commit: 0871533945b19bb5ccf96ab7a28233cadd537a59
Parents: 6fd41ab 3b7934f
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Sun Aug 9 09:40:09 2015 +0200
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Sun Aug 9 09:40:09 2015 +0200
----------------------------------------------------------------------
.../cassandra/service/CassandraDaemon.java | 8 +
.../cassandra/utils/JVMStabilityInspector.java | 9 +-
.../org/apache/cassandra/db/CommitLogTest.java | 159 ++++++++++++++++++-
.../apache/cassandra/utils/KillerForTests.java | 11 +-
4 files changed, 182 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/08715339/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/CassandraDaemon.java
index 10aa4b2,d078203..2020201
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@@ -37,11 -34,9 +37,13 @@@ import javax.management.remote.JMXConne
import javax.management.remote.JMXServiceURL;
import javax.management.remote.rmi.RMIConnectorServer;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistryListener;
+import com.codahale.metrics.SharedMetricRegistries;
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.metrics.DefaultNameFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/08715339/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/CommitLogTest.java
index c53d371,1c3daab..536f0cb
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@@ -19,74 -19,42 +19,82 @@@
package org.apache.cassandra.db;
-import java.io.*;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
import java.util.UUID;
+ import java.util.concurrent.ExecutionException;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
+import com.google.common.collect.ImmutableMap;
+
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
-
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.KSMetaData;
+import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
- import org.apache.cassandra.db.commitlog.CommitLogSegment;
+ import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.ReplayPosition;
+ import org.apache.cassandra.db.commitlog.CommitLogSegment;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
+ import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.util.ByteBufferDataInput;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.net.MessagingService;
+ import org.apache.cassandra.service.CassandraDaemon;
+ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+ import org.apache.cassandra.utils.JVMStabilityInspector;
+ import org.apache.cassandra.utils.KillerForTests;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-
-public class CommitLogTest extends SchemaLoader
+public class CommitLogTest
{
+ private static final String KEYSPACE1 = "CommitLogTest";
+ private static final String KEYSPACE2 = "CommitLogTestNonDurable";
+ private static final String CF1 = "Standard1";
+ private static final String CF2 = "Standard2";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF2));
+ SchemaLoader.createKeyspace(KEYSPACE2,
+ false,
+ true,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF2));
+ System.setProperty("cassandra.commitlog.stop_on_errors", "true");
+ CompactionManager.instance.disableAutoCompaction();
+ }
+
@Test
public void testRecoveryWithEmptyLog() throws Exception
{
@@@ -311,15 -279,162 +319,162 @@@
}
@Test
- public void testTruncateWithoutSnapshot() throws IOException
+ public void testCommitFailurePolicy_stop() throws ConfigurationException
+ {
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ // Need storage service active so stop policy can shutdown gossip
+ StorageService.instance.initServer();
+ Assert.assertTrue(Gossiper.instance.isEnabled());
+
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.stop);
+ CommitLog.handleCommitError("Test stop error", new Throwable());
+ Assert.assertFalse(Gossiper.instance.isEnabled());
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_die()
+ {
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_mustDieIfNotStartedUp()
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ //even though policy is ignore, JVM must die because Daemon has not finished initializing
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
- new CommitLogSegmentManager();
++ new CommitLogSegmentManager(CommitLog.instance);
+
+ //busy wait since commitlogsegmentmanager spawns another thread
+ int retries = 0;
+ while (!killerForTests.wasKilled() && retries++ < 5)
+ Thread.sleep(10);
+
+ //since failure was before CassandraDaemon startup, the JVM must be killed
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureAfterInitialization_mustRespectFailurePolicy() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
- new CommitLogSegmentManager();
++ new CommitLogSegmentManager(CommitLog.instance);
+
+ //wait commit log segment manager thread to execute
+ Thread.sleep(50);
+
+ //error policy is set to IGNORE, so JVM must not be killed if error ocurs after startup
+ Assert.assertFalse(killerForTests.wasKilled());
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
- public void testTruncateWithoutSnapshot() throws ExecutionException, InterruptedException
++ public void testTruncateWithoutSnapshot() throws ExecutionException, InterruptedException, IOException
{
- CommitLog.instance.resetUnsafe();
+ CommitLog.instance.resetUnsafe(true);
boolean prev = DatabaseDescriptor.isAutoSnapshot();
DatabaseDescriptor.setAutoSnapshot(false);
- ColumnFamilyStore cfs1 = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
- ColumnFamilyStore cfs2 = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard2");
+ ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
+ ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard2");
- final Mutation rm1 = new Mutation("Keyspace1", bytes("k"));
+ final Mutation rm1 = new Mutation(KEYSPACE1, bytes("k"));
rm1.add("Standard1", Util.cellname("c1"), ByteBuffer.allocate(100), 0);
rm1.apply();
cfs1.truncateBlocking();
[03/10] cassandra git commit: Ensure commit log stop policy is
enforced at startup
Posted by be...@apache.org.
Ensure commit log stop policy is enforced at startup
patch by paulo; reviewed by benedict for CASSANDRA-8515
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3b7934f1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3b7934f1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3b7934f1
Branch: refs/heads/cassandra-3.0
Commit: 3b7934f1aa20d2210866afd9b88472e9cb1aed8d
Parents: 98a08eb
Author: Paulo Motta <pa...@gmail.com>
Authored: Mon Jul 13 19:35:50 2015 -0300
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Sun Aug 9 09:34:05 2015 +0200
----------------------------------------------------------------------
.../cassandra/service/CassandraDaemon.java | 7 ++
.../cassandra/utils/JVMStabilityInspector.java | 9 +-
.../org/apache/cassandra/db/CommitLogTest.java | 110 +++++++++++++++++++
.../apache/cassandra/utils/KillerForTests.java | 11 +-
4 files changed, 134 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index 2c141a6..d078203 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -34,6 +34,7 @@ import javax.management.remote.JMXConnectorServer;
import javax.management.remote.JMXServiceURL;
import javax.management.remote.rmi.RMIConnectorServer;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
import org.slf4j.Logger;
@@ -425,6 +426,12 @@ public class CassandraDaemon
int nativePort = DatabaseDescriptor.getNativeTransportPort();
nativeServer = new org.apache.cassandra.transport.Server(nativeAddr, nativePort);
+ completeSetup();
+ }
+
+ @VisibleForTesting
+ public void completeSetup()
+ {
setupCompleted = true;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
index c0ab84f..de396bb 100644
--- a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
+++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
@@ -28,7 +28,9 @@ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.thrift.Cassandra;
/**
* Responsible for deciding whether to kill the JVM if it gets in an "unstable" state (think OOM).
@@ -38,6 +40,7 @@ public final class JVMStabilityInspector
private static final Logger logger = LoggerFactory.getLogger(JVMStabilityInspector.class);
private static Killer killer = new Killer();
+
private JVMStabilityInspector() {}
/**
@@ -66,7 +69,11 @@ public final class JVMStabilityInspector
public static void inspectCommitLogThrowable(Throwable t)
{
- if (DatabaseDescriptor.getCommitFailurePolicy() == Config.CommitFailurePolicy.die)
+ if (!StorageService.instance.isSetupCompleted())
+ {
+ logger.error("Exiting due to error while processing commit log during initialization.", t);
+ killer.killCurrentJVM(t, true);
+ } else if (DatabaseDescriptor.getCommitFailurePolicy() == Config.CommitFailurePolicy.die)
killer.killCurrentJVM(t);
else
inspectThrowable(t);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/CommitLogTest.java b/test/unit/org/apache/cassandra/db/CommitLogTest.java
index 9a8a1dc..1c3daab 100644
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.commitlog.CommitLogSegment;
import org.apache.cassandra.db.composites.CellName;
@@ -43,6 +44,7 @@ import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -279,6 +281,10 @@ public class CommitLogTest extends SchemaLoader
@Test
public void testCommitFailurePolicy_stop() throws ConfigurationException
{
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
// Need storage service active so stop policy can shutdown gossip
StorageService.instance.initServer();
Assert.assertTrue(Gossiper.instance.isEnabled());
@@ -299,6 +305,10 @@ public class CommitLogTest extends SchemaLoader
@Test
public void testCommitFailurePolicy_die()
{
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
KillerForTests killerForTests = new KillerForTests();
JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
@@ -307,11 +317,111 @@ public class CommitLogTest extends SchemaLoader
DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);
CommitLog.handleCommitError("Testing die policy", new Throwable());
Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_mustDieIfNotStartedUp()
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ //even though policy is ignore, JVM must die because Daemon has not finished initializing
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager();
+
+ //busy wait since commitlogsegmentmanager spawns another thread
+ int retries = 0;
+ while (!killerForTests.wasKilled() && retries++ < 5)
+ Thread.sleep(10);
+
+ //since failure was before CassandraDaemon startup, the JVM must be killed
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureAfterInitialization_mustRespectFailurePolicy() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager();
+
+ //wait commit log segment manager thread to execute
+ Thread.sleep(50);
+
+ //error policy is set to IGNORE, so JVM must not be killed if error ocurs after startup
+ Assert.assertFalse(killerForTests.wasKilled());
}
finally
{
DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/test/unit/org/apache/cassandra/utils/KillerForTests.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/KillerForTests.java b/test/unit/org/apache/cassandra/utils/KillerForTests.java
index d488f97..abc7952 100644
--- a/test/unit/org/apache/cassandra/utils/KillerForTests.java
+++ b/test/unit/org/apache/cassandra/utils/KillerForTests.java
@@ -24,11 +24,13 @@ package org.apache.cassandra.utils;
public class KillerForTests extends JVMStabilityInspector.Killer
{
private boolean killed = false;
+ private boolean quiet = false;
@Override
- protected void killCurrentJVM(Throwable t)
+ protected void killCurrentJVM(Throwable t, boolean quiet)
{
- killed = true;
+ this.killed = true;
+ this.quiet = quiet;
}
public boolean wasKilled()
@@ -36,6 +38,11 @@ public class KillerForTests extends JVMStabilityInspector.Killer
return killed;
}
+ public boolean wasKilledQuietly()
+ {
+ return quiet;
+ }
+
public void reset()
{
killed = false;
[05/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by be...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Conflicts:
src/java/org/apache/cassandra/service/CassandraDaemon.java
test/unit/org/apache/cassandra/db/CommitLogTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/08715339
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/08715339
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/08715339
Branch: refs/heads/cassandra-3.0
Commit: 0871533945b19bb5ccf96ab7a28233cadd537a59
Parents: 6fd41ab 3b7934f
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Sun Aug 9 09:40:09 2015 +0200
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Sun Aug 9 09:40:09 2015 +0200
----------------------------------------------------------------------
.../cassandra/service/CassandraDaemon.java | 8 +
.../cassandra/utils/JVMStabilityInspector.java | 9 +-
.../org/apache/cassandra/db/CommitLogTest.java | 159 ++++++++++++++++++-
.../apache/cassandra/utils/KillerForTests.java | 11 +-
4 files changed, 182 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/08715339/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/CassandraDaemon.java
index 10aa4b2,d078203..2020201
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@@ -37,11 -34,9 +37,13 @@@ import javax.management.remote.JMXConne
import javax.management.remote.JMXServiceURL;
import javax.management.remote.rmi.RMIConnectorServer;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistryListener;
+import com.codahale.metrics.SharedMetricRegistries;
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.metrics.DefaultNameFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/08715339/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/CommitLogTest.java
index c53d371,1c3daab..536f0cb
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@@ -19,74 -19,42 +19,82 @@@
package org.apache.cassandra.db;
-import java.io.*;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
import java.util.UUID;
+ import java.util.concurrent.ExecutionException;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
+import com.google.common.collect.ImmutableMap;
+
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
-
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.KSMetaData;
+import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
- import org.apache.cassandra.db.commitlog.CommitLogSegment;
+ import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.ReplayPosition;
+ import org.apache.cassandra.db.commitlog.CommitLogSegment;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
+ import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.util.ByteBufferDataInput;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.net.MessagingService;
+ import org.apache.cassandra.service.CassandraDaemon;
+ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+ import org.apache.cassandra.utils.JVMStabilityInspector;
+ import org.apache.cassandra.utils.KillerForTests;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-
-public class CommitLogTest extends SchemaLoader
+public class CommitLogTest
{
+ private static final String KEYSPACE1 = "CommitLogTest";
+ private static final String KEYSPACE2 = "CommitLogTestNonDurable";
+ private static final String CF1 = "Standard1";
+ private static final String CF2 = "Standard2";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF2));
+ SchemaLoader.createKeyspace(KEYSPACE2,
+ false,
+ true,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF2));
+ System.setProperty("cassandra.commitlog.stop_on_errors", "true");
+ CompactionManager.instance.disableAutoCompaction();
+ }
+
@Test
public void testRecoveryWithEmptyLog() throws Exception
{
@@@ -311,15 -279,162 +319,162 @@@
}
@Test
- public void testTruncateWithoutSnapshot() throws IOException
+ public void testCommitFailurePolicy_stop() throws ConfigurationException
+ {
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ // Need storage service active so stop policy can shutdown gossip
+ StorageService.instance.initServer();
+ Assert.assertTrue(Gossiper.instance.isEnabled());
+
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.stop);
+ CommitLog.handleCommitError("Test stop error", new Throwable());
+ Assert.assertFalse(Gossiper.instance.isEnabled());
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_die()
+ {
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_mustDieIfNotStartedUp()
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ //even though policy is ignore, JVM must die because Daemon has not finished initializing
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
- new CommitLogSegmentManager();
++ new CommitLogSegmentManager(CommitLog.instance);
+
+ //busy wait since commitlogsegmentmanager spawns another thread
+ int retries = 0;
+ while (!killerForTests.wasKilled() && retries++ < 5)
+ Thread.sleep(10);
+
+ //since failure was before CassandraDaemon startup, the JVM must be killed
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureAfterInitialization_mustRespectFailurePolicy() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
- new CommitLogSegmentManager();
++ new CommitLogSegmentManager(CommitLog.instance);
+
+ //wait commit log segment manager thread to execute
+ Thread.sleep(50);
+
+ //error policy is set to IGNORE, so JVM must not be killed if error ocurs after startup
+ Assert.assertFalse(killerForTests.wasKilled());
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
- public void testTruncateWithoutSnapshot() throws ExecutionException, InterruptedException
++ public void testTruncateWithoutSnapshot() throws ExecutionException, InterruptedException, IOException
{
- CommitLog.instance.resetUnsafe();
+ CommitLog.instance.resetUnsafe(true);
boolean prev = DatabaseDescriptor.isAutoSnapshot();
DatabaseDescriptor.setAutoSnapshot(false);
- ColumnFamilyStore cfs1 = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
- ColumnFamilyStore cfs2 = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard2");
+ ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
+ ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard2");
- final Mutation rm1 = new Mutation("Keyspace1", bytes("k"));
+ final Mutation rm1 = new Mutation(KEYSPACE1, bytes("k"));
rm1.add("Standard1", Util.cellname("c1"), ByteBuffer.allocate(100), 0);
rm1.apply();
cfs1.truncateBlocking();
[10/10] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Posted by be...@apache.org.
Merge branch 'cassandra-3.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/98cc1ee0
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/98cc1ee0
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/98cc1ee0
Branch: refs/heads/trunk
Commit: 98cc1ee0994060ca136bc0db968fa4fdf3d772bd
Parents: 6292aaa e2ad7d5
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Sun Aug 9 09:41:44 2015 +0200
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Sun Aug 9 09:41:44 2015 +0200
----------------------------------------------------------------------
.../cassandra/service/CassandraDaemon.java | 8 +
.../cassandra/utils/JVMStabilityInspector.java | 9 +-
.../org/apache/cassandra/db/CommitLogTest.java | 161 ++++++++++++++++++-
.../apache/cassandra/utils/KillerForTests.java | 11 +-
4 files changed, 184 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[06/10] cassandra git commit: Merge branch 'cassandra-2.1' into
cassandra-2.2
Posted by be...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2
Conflicts:
src/java/org/apache/cassandra/service/CassandraDaemon.java
test/unit/org/apache/cassandra/db/CommitLogTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/08715339
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/08715339
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/08715339
Branch: refs/heads/cassandra-2.2
Commit: 0871533945b19bb5ccf96ab7a28233cadd537a59
Parents: 6fd41ab 3b7934f
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Sun Aug 9 09:40:09 2015 +0200
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Sun Aug 9 09:40:09 2015 +0200
----------------------------------------------------------------------
.../cassandra/service/CassandraDaemon.java | 8 +
.../cassandra/utils/JVMStabilityInspector.java | 9 +-
.../org/apache/cassandra/db/CommitLogTest.java | 159 ++++++++++++++++++-
.../apache/cassandra/utils/KillerForTests.java | 11 +-
4 files changed, 182 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/08715339/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/CassandraDaemon.java
index 10aa4b2,d078203..2020201
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@@ -37,11 -34,9 +37,13 @@@ import javax.management.remote.JMXConne
import javax.management.remote.JMXServiceURL;
import javax.management.remote.rmi.RMIConnectorServer;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistryListener;
+import com.codahale.metrics.SharedMetricRegistries;
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.metrics.DefaultNameFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/08715339/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/CommitLogTest.java
index c53d371,1c3daab..536f0cb
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@@ -19,74 -19,42 +19,82 @@@
package org.apache.cassandra.db;
-import java.io.*;
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
import java.util.UUID;
+ import java.util.concurrent.ExecutionException;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
+import com.google.common.collect.ImmutableMap;
+
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
-
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.KSMetaData;
+import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
- import org.apache.cassandra.db.commitlog.CommitLogSegment;
+ import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.ReplayPosition;
+ import org.apache.cassandra.db.commitlog.CommitLogSegment;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
+ import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.util.ByteBufferDataInput;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.net.MessagingService;
+ import org.apache.cassandra.service.CassandraDaemon;
+ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+ import org.apache.cassandra.utils.JVMStabilityInspector;
+ import org.apache.cassandra.utils.KillerForTests;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
-
-public class CommitLogTest extends SchemaLoader
+public class CommitLogTest
{
+ private static final String KEYSPACE1 = "CommitLogTest";
+ private static final String KEYSPACE2 = "CommitLogTestNonDurable";
+ private static final String CF1 = "Standard1";
+ private static final String CF2 = "Standard2";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF2));
+ SchemaLoader.createKeyspace(KEYSPACE2,
+ false,
+ true,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF2));
+ System.setProperty("cassandra.commitlog.stop_on_errors", "true");
+ CompactionManager.instance.disableAutoCompaction();
+ }
+
@Test
public void testRecoveryWithEmptyLog() throws Exception
{
@@@ -311,15 -279,162 +319,162 @@@
}
@Test
- public void testTruncateWithoutSnapshot() throws IOException
+ public void testCommitFailurePolicy_stop() throws ConfigurationException
+ {
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ // Need storage service active so stop policy can shutdown gossip
+ StorageService.instance.initServer();
+ Assert.assertTrue(Gossiper.instance.isEnabled());
+
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.stop);
+ CommitLog.handleCommitError("Test stop error", new Throwable());
+ Assert.assertFalse(Gossiper.instance.isEnabled());
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_die()
+ {
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_mustDieIfNotStartedUp()
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ //even though policy is ignore, JVM must die because Daemon has not finished initializing
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
- new CommitLogSegmentManager();
++ new CommitLogSegmentManager(CommitLog.instance);
+
+ //busy wait since commitlogsegmentmanager spawns another thread
+ int retries = 0;
+ while (!killerForTests.wasKilled() && retries++ < 5)
+ Thread.sleep(10);
+
+ //since failure was before CassandraDaemon startup, the JVM must be killed
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureAfterInitialization_mustRespectFailurePolicy() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
- new CommitLogSegmentManager();
++ new CommitLogSegmentManager(CommitLog.instance);
+
+ //wait commit log segment manager thread to execute
+ Thread.sleep(50);
+
+ //error policy is set to IGNORE, so JVM must not be killed if error ocurs after startup
+ Assert.assertFalse(killerForTests.wasKilled());
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
- public void testTruncateWithoutSnapshot() throws ExecutionException, InterruptedException
++ public void testTruncateWithoutSnapshot() throws ExecutionException, InterruptedException, IOException
{
- CommitLog.instance.resetUnsafe();
+ CommitLog.instance.resetUnsafe(true);
boolean prev = DatabaseDescriptor.isAutoSnapshot();
DatabaseDescriptor.setAutoSnapshot(false);
- ColumnFamilyStore cfs1 = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
- ColumnFamilyStore cfs2 = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard2");
+ ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
+ ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard2");
- final Mutation rm1 = new Mutation("Keyspace1", bytes("k"));
+ final Mutation rm1 = new Mutation(KEYSPACE1, bytes("k"));
rm1.add("Standard1", Util.cellname("c1"), ByteBuffer.allocate(100), 0);
rm1.apply();
cfs1.truncateBlocking();
[08/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by be...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Conflicts:
test/unit/org/apache/cassandra/db/CommitLogTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e2ad7d56
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e2ad7d56
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e2ad7d56
Branch: refs/heads/cassandra-3.0
Commit: e2ad7d5695d4044a021f14933762f105390c9beb
Parents: 7ea2ce1 0871533
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Sun Aug 9 09:41:38 2015 +0200
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Sun Aug 9 09:41:38 2015 +0200
----------------------------------------------------------------------
.../cassandra/service/CassandraDaemon.java | 8 +
.../cassandra/utils/JVMStabilityInspector.java | 9 +-
.../org/apache/cassandra/db/CommitLogTest.java | 161 ++++++++++++++++++-
.../apache/cassandra/utils/KillerForTests.java | 11 +-
4 files changed, 184 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2ad7d56/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/CassandraDaemon.java
index fddf593,2020201..3b123c3
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@@ -40,8 -40,10 +40,10 @@@ import com.addthis.metrics3.reporter.co
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistryListener;
import com.codahale.metrics.SharedMetricRegistries;
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
-import org.apache.cassandra.metrics.DefaultNameFactory;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2ad7d56/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2ad7d56/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/CommitLogTest.java
index 21bdd9b,536f0cb..512a3d2
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@@ -47,20 -42,31 +48,29 @@@ import org.junit.BeforeClass
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.ParameterizedClass;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
- import org.apache.cassandra.db.commitlog.CommitLogSegment;
+ import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.ReplayPosition;
+ import org.apache.cassandra.db.commitlog.CommitLogSegment;
import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.composites.CellName;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
+ import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.io.util.ByteBufferDataInput;
import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.vint.VIntCoding;
+ import org.apache.cassandra.service.CassandraDaemon;
+ import org.apache.cassandra.service.StorageService;
+ import org.apache.cassandra.utils.ByteBufferUtil;
+ import org.apache.cassandra.utils.FBUtilities;
+ import org.apache.cassandra.utils.JVMStabilityInspector;
+ import org.apache.cassandra.utils.KillerForTests;
public class CommitLogTest
{
@@@ -307,40 -319,179 +317,187 @@@
}
@Test
- public void testTruncateWithoutSnapshot() throws IOException
+ public void testCommitFailurePolicy_stop() throws ConfigurationException
+ {
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ // Need storage service active so stop policy can shutdown gossip
+ StorageService.instance.initServer();
+ Assert.assertTrue(Gossiper.instance.isEnabled());
+
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.stop);
+ CommitLog.handleCommitError("Test stop error", new Throwable());
+ Assert.assertFalse(Gossiper.instance.isEnabled());
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_die()
+ {
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_mustDieIfNotStartedUp()
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ //even though policy is ignore, JVM must die because Daemon has not finished initializing
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager(CommitLog.instance);
+
+ //busy wait since commitlogsegmentmanager spawns another thread
+ int retries = 0;
+ while (!killerForTests.wasKilled() && retries++ < 5)
+ Thread.sleep(10);
+
+ //since failure was before CassandraDaemon startup, the JVM must be killed
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureAfterInitialization_mustRespectFailurePolicy() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager(CommitLog.instance);
+
+ //wait commit log segment manager thread to execute
+ Thread.sleep(50);
+
+ //error policy is set to IGNORE, so JVM must not be killed if error ocurs after startup
+ Assert.assertFalse(killerForTests.wasKilled());
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testTruncateWithoutSnapshot() throws ExecutionException, InterruptedException, IOException
{
- CommitLog.instance.resetUnsafe(true);
- boolean prev = DatabaseDescriptor.isAutoSnapshot();
- DatabaseDescriptor.setAutoSnapshot(false);
- ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
- ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard2");
-
- final Mutation rm1 = new Mutation(KEYSPACE1, bytes("k"));
- rm1.add("Standard1", Util.cellname("c1"), ByteBuffer.allocate(100), 0);
- rm1.apply();
- cfs1.truncateBlocking();
- DatabaseDescriptor.setAutoSnapshot(prev);
- final Mutation rm2 = new Mutation(KEYSPACE1, bytes("k"));
- rm2.add("Standard2", Util.cellname("c1"), ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4), 0);
-
- for (int i = 0 ; i < 5 ; i++)
- CommitLog.instance.add(rm2);
-
- Assert.assertEquals(2, CommitLog.instance.activeSegments());
- ReplayPosition position = CommitLog.instance.getContext();
- for (Keyspace ks : Keyspace.system())
- for (ColumnFamilyStore syscfs : ks.getColumnFamilyStores())
- CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, position);
- CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, position);
- Assert.assertEquals(1, CommitLog.instance.activeSegments());
+ boolean originalState = DatabaseDescriptor.isAutoSnapshot();
+ try
+ {
+ CommitLog.instance.resetUnsafe(true);
+ boolean prev = DatabaseDescriptor.isAutoSnapshot();
+ DatabaseDescriptor.setAutoSnapshot(false);
+ ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+ ColumnFamilyStore cfs2 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD2);
+
+ new RowUpdateBuilder(cfs1.metadata, 0, "k").clustering("bytes").add("val", ByteBuffer.allocate(100)).build().applyUnsafe();
+ cfs1.truncateBlocking();
+ DatabaseDescriptor.setAutoSnapshot(prev);
+ Mutation m2 = new RowUpdateBuilder(cfs2.metadata, 0, "k")
+ .clustering("bytes")
+ .add("val", ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4))
+ .build();
+
+ for (int i = 0 ; i < 5 ; i++)
+ CommitLog.instance.add(m2);
+
+ assertEquals(2, CommitLog.instance.activeSegments());
+ ReplayPosition position = CommitLog.instance.getContext();
+ for (Keyspace ks : Keyspace.system())
+ for (ColumnFamilyStore syscfs : ks.getColumnFamilyStores())
+ CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, position);
+ CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, position);
+ assertEquals(1, CommitLog.instance.activeSegments());
+ }
+ finally
+ {
+ DatabaseDescriptor.setAutoSnapshot(originalState);
+ }
}
@Test
[04/10] cassandra git commit: Ensure commit log stop policy is
enforced at startup
Posted by be...@apache.org.
Ensure commit log stop policy is enforced at startup
patch by paulo; reviewed by benedict for CASSANDRA-8515
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3b7934f1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3b7934f1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3b7934f1
Branch: refs/heads/trunk
Commit: 3b7934f1aa20d2210866afd9b88472e9cb1aed8d
Parents: 98a08eb
Author: Paulo Motta <pa...@gmail.com>
Authored: Mon Jul 13 19:35:50 2015 -0300
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Sun Aug 9 09:34:05 2015 +0200
----------------------------------------------------------------------
.../cassandra/service/CassandraDaemon.java | 7 ++
.../cassandra/utils/JVMStabilityInspector.java | 9 +-
.../org/apache/cassandra/db/CommitLogTest.java | 110 +++++++++++++++++++
.../apache/cassandra/utils/KillerForTests.java | 11 +-
4 files changed, 134 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index 2c141a6..d078203 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -34,6 +34,7 @@ import javax.management.remote.JMXConnectorServer;
import javax.management.remote.JMXServiceURL;
import javax.management.remote.rmi.RMIConnectorServer;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Uninterruptibles;
import org.slf4j.Logger;
@@ -425,6 +426,12 @@ public class CassandraDaemon
int nativePort = DatabaseDescriptor.getNativeTransportPort();
nativeServer = new org.apache.cassandra.transport.Server(nativeAddr, nativePort);
+ completeSetup();
+ }
+
+ @VisibleForTesting
+ public void completeSetup()
+ {
setupCompleted = true;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
index c0ab84f..de396bb 100644
--- a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
+++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
@@ -28,7 +28,9 @@ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.thrift.Cassandra;
/**
* Responsible for deciding whether to kill the JVM if it gets in an "unstable" state (think OOM).
@@ -38,6 +40,7 @@ public final class JVMStabilityInspector
private static final Logger logger = LoggerFactory.getLogger(JVMStabilityInspector.class);
private static Killer killer = new Killer();
+
private JVMStabilityInspector() {}
/**
@@ -66,7 +69,11 @@ public final class JVMStabilityInspector
public static void inspectCommitLogThrowable(Throwable t)
{
- if (DatabaseDescriptor.getCommitFailurePolicy() == Config.CommitFailurePolicy.die)
+ if (!StorageService.instance.isSetupCompleted())
+ {
+ logger.error("Exiting due to error while processing commit log during initialization.", t);
+ killer.killCurrentJVM(t, true);
+ } else if (DatabaseDescriptor.getCommitFailurePolicy() == Config.CommitFailurePolicy.die)
killer.killCurrentJVM(t);
else
inspectThrowable(t);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/CommitLogTest.java b/test/unit/org/apache/cassandra/db/CommitLogTest.java
index 9a8a1dc..1c3daab 100644
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.commitlog.CommitLogSegment;
import org.apache.cassandra.db.composites.CellName;
@@ -43,6 +44,7 @@ import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -279,6 +281,10 @@ public class CommitLogTest extends SchemaLoader
@Test
public void testCommitFailurePolicy_stop() throws ConfigurationException
{
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
// Need storage service active so stop policy can shutdown gossip
StorageService.instance.initServer();
Assert.assertTrue(Gossiper.instance.isEnabled());
@@ -299,6 +305,10 @@ public class CommitLogTest extends SchemaLoader
@Test
public void testCommitFailurePolicy_die()
{
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
KillerForTests killerForTests = new KillerForTests();
JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
@@ -307,11 +317,111 @@ public class CommitLogTest extends SchemaLoader
DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.die);
CommitLog.handleCommitError("Testing die policy", new Throwable());
Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitFailurePolicy_mustDieIfNotStartedUp()
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ //even though policy is ignore, JVM must die because Daemon has not finished initializing
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+ CommitLog.handleCommitError("Testing die policy", new Throwable());
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureBeforeInitialization_mustKillJVM() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager();
+
+ //busy wait since commitlogsegmentmanager spawns another thread
+ int retries = 0;
+ while (!killerForTests.wasKilled() && retries++ < 5)
+ Thread.sleep(10);
+
+ //since failure was before CassandraDaemon startup, the JVM must be killed
+ Assert.assertTrue(killerForTests.wasKilled());
+ Assert.assertTrue(killerForTests.wasKilledQuietly()); //killed quietly due to startup failure
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
+ JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
+ }
+ }
+
+ @Test
+ public void testCommitLogFailureAfterInitialization_mustRespectFailurePolicy() throws Exception
+ {
+ //startup was not completed successfuly (since method completeSetup() was not called)
+ CassandraDaemon daemon = new CassandraDaemon();
+ daemon.completeSetup(); //startup must be completed, otherwise commit log failure must kill JVM regardless of failure policy
+ StorageService.instance.registerDaemon(daemon);
+
+ //let's make the commit log directory non-writable
+ File commitLogDir = new File(DatabaseDescriptor.getCommitLogLocation());
+ commitLogDir.setWritable(false);
+
+ KillerForTests killerForTests = new KillerForTests();
+ JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+ Config.CommitFailurePolicy oldPolicy = DatabaseDescriptor.getCommitFailurePolicy();
+ try
+ {
+ DatabaseDescriptor.setCommitFailurePolicy(Config.CommitFailurePolicy.ignore);
+
+ //now let's create a commit log segment manager and wait for it to fail
+ new CommitLogSegmentManager();
+
+ //wait commit log segment manager thread to execute
+ Thread.sleep(50);
+
+ //error policy is set to IGNORE, so JVM must not be killed if error ocurs after startup
+ Assert.assertFalse(killerForTests.wasKilled());
}
finally
{
DatabaseDescriptor.setCommitFailurePolicy(oldPolicy);
JVMStabilityInspector.replaceKiller(originalKiller);
+ commitLogDir.setWritable(true);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3b7934f1/test/unit/org/apache/cassandra/utils/KillerForTests.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/KillerForTests.java b/test/unit/org/apache/cassandra/utils/KillerForTests.java
index d488f97..abc7952 100644
--- a/test/unit/org/apache/cassandra/utils/KillerForTests.java
+++ b/test/unit/org/apache/cassandra/utils/KillerForTests.java
@@ -24,11 +24,13 @@ package org.apache.cassandra.utils;
public class KillerForTests extends JVMStabilityInspector.Killer
{
private boolean killed = false;
+ private boolean quiet = false;
@Override
- protected void killCurrentJVM(Throwable t)
+ protected void killCurrentJVM(Throwable t, boolean quiet)
{
- killed = true;
+ this.killed = true;
+ this.quiet = quiet;
}
public boolean wasKilled()
@@ -36,6 +38,11 @@ public class KillerForTests extends JVMStabilityInspector.Killer
return killed;
}
+ public boolean wasKilledQuietly()
+ {
+ return quiet;
+ }
+
public void reset()
{
killed = false;