You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jm...@apache.org on 2016/04/01 17:55:38 UTC

[01/15] cassandra git commit: Fix out-of-space error treatment in memtable flushing

Repository: cassandra
Updated Branches:
  refs/heads/cassandra-2.1 105fbb35d -> f3b3c410a
  refs/heads/cassandra-2.2 106750c78 -> caaa9fc82
  refs/heads/cassandra-3.0 be3808140 -> bd4cab24b
  refs/heads/cassandra-3.5 4111e4fdf -> af9b9cd5a
  refs/heads/trunk bdae4520c -> 6d1254ac2


Fix out-of-space error treatment in memtable flushing

Patch by Branimir Lambov; reviewed by Joshua McKenzie for
CASSANDRA-11448


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f3b3c410
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f3b3c410
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f3b3c410

Branch: refs/heads/cassandra-2.1
Commit: f3b3c410a0d84a4348cf05954b38df6b087762a7
Parents: 105fbb3
Author: Branimir Lambov <br...@datastax.com>
Authored: Wed Mar 30 16:29:55 2016 +0300
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:45:18 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af2518c..50bc894 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.14
+ * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
  * Backport CASSANDRA-10859 (CASSANDRA-11415)
  * COPY FROM fails when importing blob (CASSANDRA-11375)
  * Backport CASSANDRA-10679 (CASSANDRA-9598)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 3d66d3a..edc3564 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -967,6 +967,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         final OpOrder.Barrier writeBarrier;
         final CountDownLatch latch = new CountDownLatch(1);
         final ReplayPosition lastReplayPosition;
+        volatile FSWriteError flushFailure = null;
 
         private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
         {
@@ -1009,12 +1010,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             // must check lastReplayPosition != null because Flush may find that all memtables are clean
             // and so not set a lastReplayPosition
-            if (lastReplayPosition != null)
+            // If a flush errored out but the error was ignored, make sure we don't discard the commit log.
+            if (lastReplayPosition != null && flushFailure == null)
             {
                 CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
             }
 
             metric.pendingFlushes.dec();
+
+            if (flushFailure != null)
+                throw flushFailure;
         }
     }
 
@@ -1114,11 +1119,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             metric.memtableSwitchCount.inc();
 
-            for (Memtable memtable : memtables)
+            try
+            {
+                for (Memtable memtable : memtables)
+                {
+                    // flush the memtable
+                    MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
+                    reclaim(memtable);
+                }
+            }
+            catch (FSWriteError e)
             {
-                // flush the memtable
-                MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
-                reclaim(memtable);
+                JVMStabilityInspector.inspectThrowable(e);
+                // If we weren't killed, try to continue work but do not allow CommitLog to be discarded.
+                postFlush.flushFailure = e;
             }
 
             // signal the post-flush we've done our work

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 810c336..35aa447 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -310,7 +310,7 @@ public class Directories
             if (tooBig)
                 return null;
             else
-                throw new IOError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"));
+                throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
 
         // shortcut for single data directory systems
         if (candidates.size() == 1)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 0ea54ab..6838de6 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@ -35,6 +35,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.*;
 import org.slf4j.Logger;
@@ -554,7 +555,8 @@ public class CommitLogSegmentManager
     /**
      * @return a read-only collection of the active commit log segments
      */
-    Collection<CommitLogSegment> getActiveSegments()
+    @VisibleForTesting
+    public Collection<CommitLogSegment> getActiveSegments()
     {
         return Collections.unmodifiableCollection(activeSegments);
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
index 925efd6..1a15d6f 100644
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
@@ -17,7 +17,10 @@
  */
 package org.apache.cassandra.io.util;
 
+import java.io.IOException;
+
 import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.utils.WrappedRunnable;
 
 public abstract class DiskAwareRunnable extends WrappedRunnable
@@ -26,7 +29,7 @@ public abstract class DiskAwareRunnable extends WrappedRunnable
     {
         Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
         if (directory == null)
-            throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
+            throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
 
         return directory;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
new file mode 100644
index 0000000..8304aff
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import static junit.framework.Assert.fail;
+
+import java.io.IOError;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config.DiskFailurePolicy;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.BlacklistedDirectories;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories.DataDirectory;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.commitlog.CommitLogSegment;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.KillerForTests;
+
+/**
+ * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
+ */
+public class OutOfSpaceTest extends CQLTester
+{
+    @Test
+    public void testFlushUnwriteableDie() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        KillerForTests killerForTests = new KillerForTests();
+        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.die);
+            flushAndExpectError();
+            Assert.assertTrue(killerForTests.wasKilled());
+            Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+            JVMStabilityInspector.replaceKiller(originalKiller);
+        }
+    }
+
+    @Test
+    public void testFlushUnwriteableStop() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.stop);
+            flushAndExpectError();
+            Assert.assertFalse(Gossiper.instance.isEnabled());
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+    }
+
+    @Test
+    public void testFlushUnwriteableIgnore() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
+            flushAndExpectError();
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+
+        // Next flush should succeed.
+        makeTable();
+        flush();
+    }
+
+    public void makeTable() throws Throwable
+    {
+        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+
+        // insert exactly the amount of tombstones that shouldn't trigger an exception
+        for (int i = 0; i < 10; i++)
+            execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+    }
+
+    public void markDirectoriesUnwriteable()
+    {
+        ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+        try
+        {
+            for ( ; ; )
+            {
+                DataDirectory dir = cfs.directories.getWriteableLocation(1);
+                BlacklistedDirectories.maybeMarkUnwritable(cfs.directories.getLocationForDisk(dir));
+            }
+        }
+        catch (IOError e)
+        {
+            // Expected -- marked all directories as unwritable
+        }
+    }
+
+    public void flushAndExpectError() throws InterruptedException, ExecutionException
+    {
+        try
+        {
+            Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).forceFlush().get();
+            fail("FSWriteError expected.");
+        }
+        catch (ExecutionException e)
+        {
+            // Correct path.
+            Assert.assertTrue(e.getCause() instanceof FSWriteError);
+        }
+
+        // Make sure commit log wasn't discarded.
+        UUID cfid = currentTableMetadata().cfId;
+        for (CommitLogSegment segment : CommitLog.instance.allocator.getActiveSegments())
+            if (segment.getDirtyCFIDs().contains(cfid))
+                return;
+        fail("Expected commit log to remain dirty for the affected table.");
+    }
+}


[06/15] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2

Posted by jm...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/caaa9fc8
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/caaa9fc8
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/caaa9fc8

Branch: refs/heads/trunk
Commit: caaa9fc82cd51b7321da4dad0715ebc561d180dd
Parents: 106750c f3b3c41
Author: Josh McKenzie <jo...@datastax.com>
Authored: Fri Apr 1 11:46:15 2016 -0400
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:46:15 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 3e51935,50bc894..78ea961
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,45 -1,7 +1,46 @@@
 -2.1.14
 +2.2.6
 + * Add cassandra-stress keystore option (CASSANDRA-9325)
+  * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
 - * Backport CASSANDRA-10859 (CASSANDRA-11415)
 - * COPY FROM fails when importing blob (CASSANDRA-11375)
 + * Dont mark sstables as repairing with sub range repairs (CASSANDRA-11451)
 + * Fix use of NullUpdater for 2i during compaction (CASSANDRA-11450)
 + * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
 + * cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
 + * Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
 + * Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
 + * Clarify that now() function is calculated on the coordinator node in CQL documentation (CASSANDRA-10900)
 + * Fix bloom filter sizing with LCS (CASSANDRA-11344)
 + * (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
 + * Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
 + * Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
 + * Add missing newline at end of bin/cqlsh (CASSANDRA-11325)
 + * Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297)
 + * Unresolved hostname leads to replace being ignored (CASSANDRA-11210)
 + * Fix filtering on non-primary key columns for thrift static column families
 +   (CASSANDRA-6377)
 + * Only log yaml config once, at startup (CASSANDRA-11217)
 + * Preserve order for preferred SSL cipher suites (CASSANDRA-11164)
 + * Reference leak with parallel repairs on the same table (CASSANDRA-11215)
 + * Range.compareTo() violates the contract of Comparable (CASSANDRA-11216)
 + * Avoid NPE when serializing ErrorMessage with null message (CASSANDRA-11167)
 + * Replacing an aggregate with a new version doesn't reset INITCOND (CASSANDRA-10840)
 + * (cqlsh) cqlsh cannot be called through symlink (CASSANDRA-11037)
 + * fix ohc and java-driver pom dependencies in build.xml (CASSANDRA-10793)
 + * Protect from keyspace dropped during repair (CASSANDRA-11065)
 + * Handle adding fields to a UDT in SELECT JSON and toJson() (CASSANDRA-11146)
 + * Better error message for cleanup (CASSANDRA-10991)
 + * cqlsh pg-style-strings broken if line ends with ';' (CASSANDRA-11123)
 + * Use cloned TokenMetadata in size estimates to avoid race against membership check
 +   (CASSANDRA-10736)
 + * Always persist upsampled index summaries (CASSANDRA-10512)
 + * (cqlsh) Fix inconsistent auto-complete (CASSANDRA-10733)
 + * Make SELECT JSON and toJson() threadsafe (CASSANDRA-11048)
 + * Fix SELECT on tuple relations for mixed ASC/DESC clustering order (CASSANDRA-7281)
 + * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
 + * Fix paging on DISTINCT queries repeats result when first row in partition changes
 +   (CASSANDRA-10010)
 +Merged from 2.1:
 + * Add a -j parameter to scrub/cleanup/upgradesstables to state how
 +   many threads to use (CASSANDRA-11179)
   * Backport CASSANDRA-10679 (CASSANDRA-9598)
   * Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342)
   * Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 2ca2fea,6838de6..636c73b
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@@ -34,9 -35,9 +34,10 @@@ import java.util.concurrent.LinkedBlock
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicLong;
  
+ import com.google.common.annotations.VisibleForTesting;
  import com.google.common.collect.Iterables;
  import com.google.common.util.concurrent.*;
 +
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  


[05/15] cassandra git commit: Fix out-of-space error treatment in memtable flushing

Posted by jm...@apache.org.
Fix out-of-space error treatment in memtable flushing

Patch by Branimir Lambov; reviewed by Joshua McKenzie for
CASSANDRA-11448


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f3b3c410
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f3b3c410
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f3b3c410

Branch: refs/heads/cassandra-3.5
Commit: f3b3c410a0d84a4348cf05954b38df6b087762a7
Parents: 105fbb3
Author: Branimir Lambov <br...@datastax.com>
Authored: Wed Mar 30 16:29:55 2016 +0300
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:45:18 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af2518c..50bc894 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.14
+ * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
  * Backport CASSANDRA-10859 (CASSANDRA-11415)
  * COPY FROM fails when importing blob (CASSANDRA-11375)
  * Backport CASSANDRA-10679 (CASSANDRA-9598)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 3d66d3a..edc3564 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -967,6 +967,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         final OpOrder.Barrier writeBarrier;
         final CountDownLatch latch = new CountDownLatch(1);
         final ReplayPosition lastReplayPosition;
+        volatile FSWriteError flushFailure = null;
 
         private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
         {
@@ -1009,12 +1010,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             // must check lastReplayPosition != null because Flush may find that all memtables are clean
             // and so not set a lastReplayPosition
-            if (lastReplayPosition != null)
+            // If a flush errored out but the error was ignored, make sure we don't discard the commit log.
+            if (lastReplayPosition != null && flushFailure == null)
             {
                 CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
             }
 
             metric.pendingFlushes.dec();
+
+            if (flushFailure != null)
+                throw flushFailure;
         }
     }
 
@@ -1114,11 +1119,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             metric.memtableSwitchCount.inc();
 
-            for (Memtable memtable : memtables)
+            try
+            {
+                for (Memtable memtable : memtables)
+                {
+                    // flush the memtable
+                    MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
+                    reclaim(memtable);
+                }
+            }
+            catch (FSWriteError e)
             {
-                // flush the memtable
-                MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
-                reclaim(memtable);
+                JVMStabilityInspector.inspectThrowable(e);
+                // If we weren't killed, try to continue work but do not allow CommitLog to be discarded.
+                postFlush.flushFailure = e;
             }
 
             // signal the post-flush we've done our work

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 810c336..35aa447 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -310,7 +310,7 @@ public class Directories
             if (tooBig)
                 return null;
             else
-                throw new IOError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"));
+                throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
 
         // shortcut for single data directory systems
         if (candidates.size() == 1)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 0ea54ab..6838de6 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@ -35,6 +35,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.*;
 import org.slf4j.Logger;
@@ -554,7 +555,8 @@ public class CommitLogSegmentManager
     /**
      * @return a read-only collection of the active commit log segments
      */
-    Collection<CommitLogSegment> getActiveSegments()
+    @VisibleForTesting
+    public Collection<CommitLogSegment> getActiveSegments()
     {
         return Collections.unmodifiableCollection(activeSegments);
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
index 925efd6..1a15d6f 100644
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
@@ -17,7 +17,10 @@
  */
 package org.apache.cassandra.io.util;
 
+import java.io.IOException;
+
 import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.utils.WrappedRunnable;
 
 public abstract class DiskAwareRunnable extends WrappedRunnable
@@ -26,7 +29,7 @@ public abstract class DiskAwareRunnable extends WrappedRunnable
     {
         Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
         if (directory == null)
-            throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
+            throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
 
         return directory;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
new file mode 100644
index 0000000..8304aff
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import static junit.framework.Assert.fail;
+
+import java.io.IOError;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config.DiskFailurePolicy;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.BlacklistedDirectories;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories.DataDirectory;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.commitlog.CommitLogSegment;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.KillerForTests;
+
+/**
+ * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
+ */
+public class OutOfSpaceTest extends CQLTester
+{
+    @Test
+    public void testFlushUnwriteableDie() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        KillerForTests killerForTests = new KillerForTests();
+        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.die);
+            flushAndExpectError();
+            Assert.assertTrue(killerForTests.wasKilled());
+            Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+            JVMStabilityInspector.replaceKiller(originalKiller);
+        }
+    }
+
+    @Test
+    public void testFlushUnwriteableStop() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.stop);
+            flushAndExpectError();
+            Assert.assertFalse(Gossiper.instance.isEnabled());
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+    }
+
+    @Test
+    public void testFlushUnwriteableIgnore() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
+            flushAndExpectError();
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+
+        // Next flush should succeed.
+        makeTable();
+        flush();
+    }
+
+    public void makeTable() throws Throwable
+    {
+        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+
+        // insert exactly the amount of tombstones that shouldn't trigger an exception
+        for (int i = 0; i < 10; i++)
+            execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+    }
+
+    public void markDirectoriesUnwriteable()
+    {
+        ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+        try
+        {
+            for ( ; ; )
+            {
+                DataDirectory dir = cfs.directories.getWriteableLocation(1);
+                BlacklistedDirectories.maybeMarkUnwritable(cfs.directories.getLocationForDisk(dir));
+            }
+        }
+        catch (IOError e)
+        {
+            // Expected -- marked all directories as unwritable
+        }
+    }
+
+    public void flushAndExpectError() throws InterruptedException, ExecutionException
+    {
+        try
+        {
+            Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).forceFlush().get();
+            fail("FSWriteError expected.");
+        }
+        catch (ExecutionException e)
+        {
+            // Correct path.
+            Assert.assertTrue(e.getCause() instanceof FSWriteError);
+        }
+
+        // Make sure commit log wasn't discarded.
+        UUID cfid = currentTableMetadata().cfId;
+        for (CommitLogSegment segment : CommitLog.instance.allocator.getActiveSegments())
+            if (segment.getDirtyCFIDs().contains(cfid))
+                return;
+        fail("Expected commit log to remain dirty for the affected table.");
+    }
+}


[11/15] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0

Posted by jm...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bd4cab24
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bd4cab24
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bd4cab24

Branch: refs/heads/cassandra-3.5
Commit: bd4cab24bb9582d15d6b4cf251e7da21230f1460
Parents: be38081 caaa9fc
Author: Josh McKenzie <jo...@datastax.com>
Authored: Fri Apr 1 11:47:39 2016 -0400
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:49:41 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 7fc628e,78ea961..b9376bc
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -75,8 -38,10 +75,9 @@@ Merged from 2.2
   * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
   * Fix paging on DISTINCT queries repeats result when first row in partition changes
     (CASSANDRA-10010)
 + * cqlsh: change default encoding to UTF-8 (CASSANDRA-11124)
  Merged from 2.1:
 - * Add a -j parameter to scrub/cleanup/upgradesstables to state how
 -   many threads to use (CASSANDRA-11179)
 - * Backport CASSANDRA-10679 (CASSANDRA-9598)
++ * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
   * Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342)
   * Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286)
   * Avoid dropping message too quickly due to missing unit conversion (CASSANDRA-11302)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 8a8d0e7,636c73b..2ee4eed
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@@ -34,8 -34,10 +34,9 @@@ import java.util.concurrent.LinkedBlock
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicLong;
  
+ import com.google.common.annotations.VisibleForTesting;
  import com.google.common.collect.Iterables;
  import com.google.common.util.concurrent.*;
 -
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
index 0000000,8304aff..1527b1e
mode 000000,100644..100644
--- a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@@ -1,0 -1,157 +1,157 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.cassandra.cql3;
+ 
+ import static junit.framework.Assert.fail;
+ 
+ import java.io.IOError;
+ import java.util.UUID;
+ import java.util.concurrent.ExecutionException;
+ 
+ import org.junit.Assert;
+ import org.junit.Test;
+ 
+ import org.apache.cassandra.config.Config.DiskFailurePolicy;
+ import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.db.BlacklistedDirectories;
+ import org.apache.cassandra.db.ColumnFamilyStore;
+ import org.apache.cassandra.db.Directories.DataDirectory;
+ import org.apache.cassandra.db.commitlog.CommitLog;
+ import org.apache.cassandra.db.commitlog.CommitLogSegment;
+ import org.apache.cassandra.db.Keyspace;
+ import org.apache.cassandra.gms.Gossiper;
+ import org.apache.cassandra.io.FSWriteError;
+ import org.apache.cassandra.utils.JVMStabilityInspector;
+ import org.apache.cassandra.utils.KillerForTests;
+ 
+ /**
+  * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
+  */
+ public class OutOfSpaceTest extends CQLTester
+ {
+     @Test
+     public void testFlushUnwriteableDie() throws Throwable
+     {
+         makeTable();
+         markDirectoriesUnwriteable();
+ 
+         KillerForTests killerForTests = new KillerForTests();
+         JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+         try
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.die);
+             flushAndExpectError();
+             Assert.assertTrue(killerForTests.wasKilled());
+             Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+         }
+         finally
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+             JVMStabilityInspector.replaceKiller(originalKiller);
+         }
+     }
+ 
+     @Test
+     public void testFlushUnwriteableStop() throws Throwable
+     {
+         makeTable();
+         markDirectoriesUnwriteable();
+ 
+         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+         try
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.stop);
+             flushAndExpectError();
+             Assert.assertFalse(Gossiper.instance.isEnabled());
+         }
+         finally
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+         }
+     }
+ 
+     @Test
+     public void testFlushUnwriteableIgnore() throws Throwable
+     {
+         makeTable();
+         markDirectoriesUnwriteable();
+ 
+         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+         try
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
+             flushAndExpectError();
+         }
+         finally
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+         }
+ 
+         // Next flush should succeed.
+         makeTable();
+         flush();
+     }
+ 
+     public void makeTable() throws Throwable
+     {
+         createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+ 
+         // insert exactly the amount of tombstones that shouldn't trigger an exception
+         for (int i = 0; i < 10; i++)
+             execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+     }
+ 
+     public void markDirectoriesUnwriteable()
+     {
+         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+         try
+         {
+             for ( ; ; )
+             {
 -                DataDirectory dir = cfs.directories.getWriteableLocation(1);
 -                BlacklistedDirectories.maybeMarkUnwritable(cfs.directories.getLocationForDisk(dir));
++                DataDirectory dir = cfs.getDirectories().getWriteableLocation(1);
++                BlacklistedDirectories.maybeMarkUnwritable(cfs.getDirectories().getLocationForDisk(dir));
+             }
+         }
+         catch (IOError e)
+         {
+             // Expected -- marked all directories as unwritable
+         }
+     }
+ 
+     public void flushAndExpectError() throws InterruptedException, ExecutionException
+     {
+         try
+         {
+             Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).forceFlush().get();
+             fail("FSWriteError expected.");
+         }
+         catch (ExecutionException e)
+         {
+             // Correct path.
+             Assert.assertTrue(e.getCause() instanceof FSWriteError);
+         }
+ 
+         // Make sure commit log wasn't discarded.
+         UUID cfid = currentTableMetadata().cfId;
+         for (CommitLogSegment segment : CommitLog.instance.allocator.getActiveSegments())
+             if (segment.getDirtyCFIDs().contains(cfid))
+                 return;
+         fail("Expected commit log to remain dirty for the affected table.");
+     }
+ }


[12/15] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0

Posted by jm...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bd4cab24
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bd4cab24
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bd4cab24

Branch: refs/heads/cassandra-3.0
Commit: bd4cab24bb9582d15d6b4cf251e7da21230f1460
Parents: be38081 caaa9fc
Author: Josh McKenzie <jo...@datastax.com>
Authored: Fri Apr 1 11:47:39 2016 -0400
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:49:41 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 7fc628e,78ea961..b9376bc
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -75,8 -38,10 +75,9 @@@ Merged from 2.2
   * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
   * Fix paging on DISTINCT queries repeats result when first row in partition changes
     (CASSANDRA-10010)
 + * cqlsh: change default encoding to UTF-8 (CASSANDRA-11124)
  Merged from 2.1:
 - * Add a -j parameter to scrub/cleanup/upgradesstables to state how
 -   many threads to use (CASSANDRA-11179)
 - * Backport CASSANDRA-10679 (CASSANDRA-9598)
++ * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
   * Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342)
   * Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286)
   * Avoid dropping message too quickly due to missing unit conversion (CASSANDRA-11302)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 8a8d0e7,636c73b..2ee4eed
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@@ -34,8 -34,10 +34,9 @@@ import java.util.concurrent.LinkedBlock
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicLong;
  
+ import com.google.common.annotations.VisibleForTesting;
  import com.google.common.collect.Iterables;
  import com.google.common.util.concurrent.*;
 -
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
index 0000000,8304aff..1527b1e
mode 000000,100644..100644
--- a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@@ -1,0 -1,157 +1,157 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.cassandra.cql3;
+ 
+ import static junit.framework.Assert.fail;
+ 
+ import java.io.IOError;
+ import java.util.UUID;
+ import java.util.concurrent.ExecutionException;
+ 
+ import org.junit.Assert;
+ import org.junit.Test;
+ 
+ import org.apache.cassandra.config.Config.DiskFailurePolicy;
+ import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.db.BlacklistedDirectories;
+ import org.apache.cassandra.db.ColumnFamilyStore;
+ import org.apache.cassandra.db.Directories.DataDirectory;
+ import org.apache.cassandra.db.commitlog.CommitLog;
+ import org.apache.cassandra.db.commitlog.CommitLogSegment;
+ import org.apache.cassandra.db.Keyspace;
+ import org.apache.cassandra.gms.Gossiper;
+ import org.apache.cassandra.io.FSWriteError;
+ import org.apache.cassandra.utils.JVMStabilityInspector;
+ import org.apache.cassandra.utils.KillerForTests;
+ 
+ /**
+  * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
+  */
+ public class OutOfSpaceTest extends CQLTester
+ {
+     @Test
+     public void testFlushUnwriteableDie() throws Throwable
+     {
+         makeTable();
+         markDirectoriesUnwriteable();
+ 
+         KillerForTests killerForTests = new KillerForTests();
+         JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+         try
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.die);
+             flushAndExpectError();
+             Assert.assertTrue(killerForTests.wasKilled());
+             Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+         }
+         finally
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+             JVMStabilityInspector.replaceKiller(originalKiller);
+         }
+     }
+ 
+     @Test
+     public void testFlushUnwriteableStop() throws Throwable
+     {
+         makeTable();
+         markDirectoriesUnwriteable();
+ 
+         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+         try
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.stop);
+             flushAndExpectError();
+             Assert.assertFalse(Gossiper.instance.isEnabled());
+         }
+         finally
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+         }
+     }
+ 
+     @Test
+     public void testFlushUnwriteableIgnore() throws Throwable
+     {
+         makeTable();
+         markDirectoriesUnwriteable();
+ 
+         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+         try
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
+             flushAndExpectError();
+         }
+         finally
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+         }
+ 
+         // Next flush should succeed.
+         makeTable();
+         flush();
+     }
+ 
+     public void makeTable() throws Throwable
+     {
+         createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+ 
+         // insert exactly the amount of tombstones that shouldn't trigger an exception
+         for (int i = 0; i < 10; i++)
+             execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+     }
+ 
+     public void markDirectoriesUnwriteable()
+     {
+         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+         try
+         {
+             for ( ; ; )
+             {
 -                DataDirectory dir = cfs.directories.getWriteableLocation(1);
 -                BlacklistedDirectories.maybeMarkUnwritable(cfs.directories.getLocationForDisk(dir));
++                DataDirectory dir = cfs.getDirectories().getWriteableLocation(1);
++                BlacklistedDirectories.maybeMarkUnwritable(cfs.getDirectories().getLocationForDisk(dir));
+             }
+         }
+         catch (IOError e)
+         {
+             // Expected -- marked all directories as unwritable
+         }
+     }
+ 
+     public void flushAndExpectError() throws InterruptedException, ExecutionException
+     {
+         try
+         {
+             Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).forceFlush().get();
+             fail("FSWriteError expected.");
+         }
+         catch (ExecutionException e)
+         {
+             // Correct path.
+             Assert.assertTrue(e.getCause() instanceof FSWriteError);
+         }
+ 
+         // Make sure commit log wasn't discarded.
+         UUID cfid = currentTableMetadata().cfId;
+         for (CommitLogSegment segment : CommitLog.instance.allocator.getActiveSegments())
+             if (segment.getDirtyCFIDs().contains(cfid))
+                 return;
+         fail("Expected commit log to remain dirty for the affected table.");
+     }
+ }


[09/15] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2

Posted by jm...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/caaa9fc8
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/caaa9fc8
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/caaa9fc8

Branch: refs/heads/cassandra-3.5
Commit: caaa9fc82cd51b7321da4dad0715ebc561d180dd
Parents: 106750c f3b3c41
Author: Josh McKenzie <jo...@datastax.com>
Authored: Fri Apr 1 11:46:15 2016 -0400
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:46:15 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 3e51935,50bc894..78ea961
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,45 -1,7 +1,46 @@@
 -2.1.14
 +2.2.6
 + * Add cassandra-stress keystore option (CASSANDRA-9325)
+  * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
 - * Backport CASSANDRA-10859 (CASSANDRA-11415)
 - * COPY FROM fails when importing blob (CASSANDRA-11375)
 + * Dont mark sstables as repairing with sub range repairs (CASSANDRA-11451)
 + * Fix use of NullUpdater for 2i during compaction (CASSANDRA-11450)
 + * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
 + * cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
 + * Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
 + * Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
 + * Clarify that now() function is calculated on the coordinator node in CQL documentation (CASSANDRA-10900)
 + * Fix bloom filter sizing with LCS (CASSANDRA-11344)
 + * (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
 + * Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
 + * Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
 + * Add missing newline at end of bin/cqlsh (CASSANDRA-11325)
 + * Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297)
 + * Unresolved hostname leads to replace being ignored (CASSANDRA-11210)
 + * Fix filtering on non-primary key columns for thrift static column families
 +   (CASSANDRA-6377)
 + * Only log yaml config once, at startup (CASSANDRA-11217)
 + * Preserve order for preferred SSL cipher suites (CASSANDRA-11164)
 + * Reference leak with parallel repairs on the same table (CASSANDRA-11215)
 + * Range.compareTo() violates the contract of Comparable (CASSANDRA-11216)
 + * Avoid NPE when serializing ErrorMessage with null message (CASSANDRA-11167)
 + * Replacing an aggregate with a new version doesn't reset INITCOND (CASSANDRA-10840)
 + * (cqlsh) cqlsh cannot be called through symlink (CASSANDRA-11037)
 + * fix ohc and java-driver pom dependencies in build.xml (CASSANDRA-10793)
 + * Protect from keyspace dropped during repair (CASSANDRA-11065)
 + * Handle adding fields to a UDT in SELECT JSON and toJson() (CASSANDRA-11146)
 + * Better error message for cleanup (CASSANDRA-10991)
 + * cqlsh pg-style-strings broken if line ends with ';' (CASSANDRA-11123)
 + * Use cloned TokenMetadata in size estimates to avoid race against membership check
 +   (CASSANDRA-10736)
 + * Always persist upsampled index summaries (CASSANDRA-10512)
 + * (cqlsh) Fix inconsistent auto-complete (CASSANDRA-10733)
 + * Make SELECT JSON and toJson() threadsafe (CASSANDRA-11048)
 + * Fix SELECT on tuple relations for mixed ASC/DESC clustering order (CASSANDRA-7281)
 + * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
 + * Fix paging on DISTINCT queries repeats result when first row in partition changes
 +   (CASSANDRA-10010)
 +Merged from 2.1:
 + * Add a -j parameter to scrub/cleanup/upgradesstables to state how
 +   many threads to use (CASSANDRA-11179)
   * Backport CASSANDRA-10679 (CASSANDRA-9598)
   * Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342)
   * Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 2ca2fea,6838de6..636c73b
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@@ -34,9 -35,9 +34,10 @@@ import java.util.concurrent.LinkedBlock
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicLong;
  
+ import com.google.common.annotations.VisibleForTesting;
  import com.google.common.collect.Iterables;
  import com.google.common.util.concurrent.*;
 +
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  


[02/15] cassandra git commit: Fix out-of-space error treatment in memtable flushing

Posted by jm...@apache.org.
Fix out-of-space error treatment in memtable flushing

Patch by Branimir Lambov; reviewed by Joshua McKenzie for
CASSANDRA-11448


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f3b3c410
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f3b3c410
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f3b3c410

Branch: refs/heads/cassandra-2.2
Commit: f3b3c410a0d84a4348cf05954b38df6b087762a7
Parents: 105fbb3
Author: Branimir Lambov <br...@datastax.com>
Authored: Wed Mar 30 16:29:55 2016 +0300
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:45:18 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af2518c..50bc894 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.14
+ * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
  * Backport CASSANDRA-10859 (CASSANDRA-11415)
  * COPY FROM fails when importing blob (CASSANDRA-11375)
  * Backport CASSANDRA-10679 (CASSANDRA-9598)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 3d66d3a..edc3564 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -967,6 +967,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         final OpOrder.Barrier writeBarrier;
         final CountDownLatch latch = new CountDownLatch(1);
         final ReplayPosition lastReplayPosition;
+        volatile FSWriteError flushFailure = null;
 
         private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
         {
@@ -1009,12 +1010,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             // must check lastReplayPosition != null because Flush may find that all memtables are clean
             // and so not set a lastReplayPosition
-            if (lastReplayPosition != null)
+            // If a flush errored out but the error was ignored, make sure we don't discard the commit log.
+            if (lastReplayPosition != null && flushFailure == null)
             {
                 CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
             }
 
             metric.pendingFlushes.dec();
+
+            if (flushFailure != null)
+                throw flushFailure;
         }
     }
 
@@ -1114,11 +1119,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             metric.memtableSwitchCount.inc();
 
-            for (Memtable memtable : memtables)
+            try
+            {
+                for (Memtable memtable : memtables)
+                {
+                    // flush the memtable
+                    MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
+                    reclaim(memtable);
+                }
+            }
+            catch (FSWriteError e)
             {
-                // flush the memtable
-                MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
-                reclaim(memtable);
+                JVMStabilityInspector.inspectThrowable(e);
+                // If we weren't killed, try to continue work but do not allow CommitLog to be discarded.
+                postFlush.flushFailure = e;
             }
 
             // signal the post-flush we've done our work

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 810c336..35aa447 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -310,7 +310,7 @@ public class Directories
             if (tooBig)
                 return null;
             else
-                throw new IOError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"));
+                throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
 
         // shortcut for single data directory systems
         if (candidates.size() == 1)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 0ea54ab..6838de6 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@ -35,6 +35,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.*;
 import org.slf4j.Logger;
@@ -554,7 +555,8 @@ public class CommitLogSegmentManager
     /**
      * @return a read-only collection of the active commit log segments
      */
-    Collection<CommitLogSegment> getActiveSegments()
+    @VisibleForTesting
+    public Collection<CommitLogSegment> getActiveSegments()
     {
         return Collections.unmodifiableCollection(activeSegments);
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
index 925efd6..1a15d6f 100644
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
@@ -17,7 +17,10 @@
  */
 package org.apache.cassandra.io.util;
 
+import java.io.IOException;
+
 import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.utils.WrappedRunnable;
 
 public abstract class DiskAwareRunnable extends WrappedRunnable
@@ -26,7 +29,7 @@ public abstract class DiskAwareRunnable extends WrappedRunnable
     {
         Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
         if (directory == null)
-            throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
+            throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
 
         return directory;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
new file mode 100644
index 0000000..8304aff
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import static junit.framework.Assert.fail;
+
+import java.io.IOError;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config.DiskFailurePolicy;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.BlacklistedDirectories;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories.DataDirectory;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.commitlog.CommitLogSegment;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.KillerForTests;
+
+/**
+ * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
+ */
+public class OutOfSpaceTest extends CQLTester
+{
+    @Test
+    public void testFlushUnwriteableDie() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        KillerForTests killerForTests = new KillerForTests();
+        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.die);
+            flushAndExpectError();
+            Assert.assertTrue(killerForTests.wasKilled());
+            Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+            JVMStabilityInspector.replaceKiller(originalKiller);
+        }
+    }
+
+    @Test
+    public void testFlushUnwriteableStop() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.stop);
+            flushAndExpectError();
+            Assert.assertFalse(Gossiper.instance.isEnabled());
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+    }
+
+    @Test
+    public void testFlushUnwriteableIgnore() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
+            flushAndExpectError();
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+
+        // Next flush should succeed.
+        makeTable();
+        flush();
+    }
+
+    public void makeTable() throws Throwable
+    {
+        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+
+        // insert exactly the amount of tombstones that shouldn't trigger an exception
+        for (int i = 0; i < 10; i++)
+            execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+    }
+
+    public void markDirectoriesUnwriteable()
+    {
+        ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+        try
+        {
+            for ( ; ; )
+            {
+                DataDirectory dir = cfs.directories.getWriteableLocation(1);
+                BlacklistedDirectories.maybeMarkUnwritable(cfs.directories.getLocationForDisk(dir));
+            }
+        }
+        catch (IOError e)
+        {
+            // Expected -- marked all directories as unwritable
+        }
+    }
+
+    public void flushAndExpectError() throws InterruptedException, ExecutionException
+    {
+        try
+        {
+            Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).forceFlush().get();
+            fail("FSWriteError expected.");
+        }
+        catch (ExecutionException e)
+        {
+            // Correct path.
+            Assert.assertTrue(e.getCause() instanceof FSWriteError);
+        }
+
+        // Make sure commit log wasn't discarded.
+        UUID cfid = currentTableMetadata().cfId;
+        for (CommitLogSegment segment : CommitLog.instance.allocator.getActiveSegments())
+            if (segment.getDirtyCFIDs().contains(cfid))
+                return;
+        fail("Expected commit log to remain dirty for the affected table.");
+    }
+}


[15/15] cassandra git commit: Merge branch 'cassandra-3.5' into trunk

Posted by jm...@apache.org.
Merge branch 'cassandra-3.5' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6d1254ac
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6d1254ac
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6d1254ac

Branch: refs/heads/trunk
Commit: 6d1254ac238604f5fe6c5660eb72d40959626651
Parents: bdae452 af9b9cd
Author: Josh McKenzie <jo...@datastax.com>
Authored: Fri Apr 1 11:54:51 2016 -0400
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:54:51 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  | 134 ++++++++++------
 .../org/apache/cassandra/db/Directories.java    |  16 +-
 src/java/org/apache/cassandra/db/Memtable.java  |  29 +++-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |  56 -------
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 7 files changed, 276 insertions(+), 121 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/6d1254ac/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/6d1254ac/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index a6645b5,6e01b34..b47cf85
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -1128,17 -1149,10 +1160,15 @@@ public class ColumnFamilyStore implemen
                          }
                      }
                  }
-                 memtable.cfs.replaceFlushed(memtable, sstables);
-                 reclaim(memtable);
+             }
+             memtable.cfs.replaceFlushed(memtable, sstables);
+             reclaim(memtable);
 -            logger.debug("Flushed to {} ({} sstables, {} bytes), biggest {} bytes, smallest {} bytes", sstables, sstables.size(), totalBytesOnDisk, maxBytesOnDisk, minBytesOnDisk);
 +                logger.debug("Flushed to {} ({} sstables, {}), biggest {}, smallest {}",
 +                             sstables,
 +                             sstables.size(),
 +                             FBUtilities.prettyPrintMemory(totalBytesOnDisk),
 +                             FBUtilities.prettyPrintMemory(maxBytesOnDisk),
 +                             FBUtilities.prettyPrintMemory(minBytesOnDisk));
-             }
-             // signal the post-flush we've done our work
-             postFlush.latch.countDown();
          }
  
          private void reclaim(final Memtable memtable)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/6d1254ac/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------


[07/15] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2

Posted by jm...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/caaa9fc8
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/caaa9fc8
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/caaa9fc8

Branch: refs/heads/cassandra-2.2
Commit: caaa9fc82cd51b7321da4dad0715ebc561d180dd
Parents: 106750c f3b3c41
Author: Josh McKenzie <jo...@datastax.com>
Authored: Fri Apr 1 11:46:15 2016 -0400
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:46:15 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 3e51935,50bc894..78ea961
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,45 -1,7 +1,46 @@@
 -2.1.14
 +2.2.6
 + * Add cassandra-stress keystore option (CASSANDRA-9325)
+  * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
 - * Backport CASSANDRA-10859 (CASSANDRA-11415)
 - * COPY FROM fails when importing blob (CASSANDRA-11375)
 + * Dont mark sstables as repairing with sub range repairs (CASSANDRA-11451)
 + * Fix use of NullUpdater for 2i during compaction (CASSANDRA-11450)
 + * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
 + * cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
 + * Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
 + * Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
 + * Clarify that now() function is calculated on the coordinator node in CQL documentation (CASSANDRA-10900)
 + * Fix bloom filter sizing with LCS (CASSANDRA-11344)
 + * (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
 + * Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
 + * Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
 + * Add missing newline at end of bin/cqlsh (CASSANDRA-11325)
 + * Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297)
 + * Unresolved hostname leads to replace being ignored (CASSANDRA-11210)
 + * Fix filtering on non-primary key columns for thrift static column families
 +   (CASSANDRA-6377)
 + * Only log yaml config once, at startup (CASSANDRA-11217)
 + * Preserve order for preferred SSL cipher suites (CASSANDRA-11164)
 + * Reference leak with parallel repairs on the same table (CASSANDRA-11215)
 + * Range.compareTo() violates the contract of Comparable (CASSANDRA-11216)
 + * Avoid NPE when serializing ErrorMessage with null message (CASSANDRA-11167)
 + * Replacing an aggregate with a new version doesn't reset INITCOND (CASSANDRA-10840)
 + * (cqlsh) cqlsh cannot be called through symlink (CASSANDRA-11037)
 + * fix ohc and java-driver pom dependencies in build.xml (CASSANDRA-10793)
 + * Protect from keyspace dropped during repair (CASSANDRA-11065)
 + * Handle adding fields to a UDT in SELECT JSON and toJson() (CASSANDRA-11146)
 + * Better error message for cleanup (CASSANDRA-10991)
 + * cqlsh pg-style-strings broken if line ends with ';' (CASSANDRA-11123)
 + * Use cloned TokenMetadata in size estimates to avoid race against membership check
 +   (CASSANDRA-10736)
 + * Always persist upsampled index summaries (CASSANDRA-10512)
 + * (cqlsh) Fix inconsistent auto-complete (CASSANDRA-10733)
 + * Make SELECT JSON and toJson() threadsafe (CASSANDRA-11048)
 + * Fix SELECT on tuple relations for mixed ASC/DESC clustering order (CASSANDRA-7281)
 + * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
 + * Fix paging on DISTINCT queries repeats result when first row in partition changes
 +   (CASSANDRA-10010)
 +Merged from 2.1:
 + * Add a -j parameter to scrub/cleanup/upgradesstables to state how
 +   many threads to use (CASSANDRA-11179)
   * Backport CASSANDRA-10679 (CASSANDRA-9598)
   * Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342)
   * Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 2ca2fea,6838de6..636c73b
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@@ -34,9 -35,9 +34,10 @@@ import java.util.concurrent.LinkedBlock
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicLong;
  
+ import com.google.common.annotations.VisibleForTesting;
  import com.google.common.collect.Iterables;
  import com.google.common.util.concurrent.*;
 +
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  


[08/15] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2

Posted by jm...@apache.org.
Merge branch 'cassandra-2.1' into cassandra-2.2


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/caaa9fc8
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/caaa9fc8
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/caaa9fc8

Branch: refs/heads/cassandra-3.0
Commit: caaa9fc82cd51b7321da4dad0715ebc561d180dd
Parents: 106750c f3b3c41
Author: Josh McKenzie <jo...@datastax.com>
Authored: Fri Apr 1 11:46:15 2016 -0400
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:46:15 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 3e51935,50bc894..78ea961
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,45 -1,7 +1,46 @@@
 -2.1.14
 +2.2.6
 + * Add cassandra-stress keystore option (CASSANDRA-9325)
+  * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
 - * Backport CASSANDRA-10859 (CASSANDRA-11415)
 - * COPY FROM fails when importing blob (CASSANDRA-11375)
 + * Dont mark sstables as repairing with sub range repairs (CASSANDRA-11451)
 + * Fix use of NullUpdater for 2i during compaction (CASSANDRA-11450)
 + * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
 + * cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
 + * Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
 + * Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
 + * Clarify that now() function is calculated on the coordinator node in CQL documentation (CASSANDRA-10900)
 + * Fix bloom filter sizing with LCS (CASSANDRA-11344)
 + * (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
 + * Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
 + * Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
 + * Add missing newline at end of bin/cqlsh (CASSANDRA-11325)
 + * Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297)
 + * Unresolved hostname leads to replace being ignored (CASSANDRA-11210)
 + * Fix filtering on non-primary key columns for thrift static column families
 +   (CASSANDRA-6377)
 + * Only log yaml config once, at startup (CASSANDRA-11217)
 + * Preserve order for preferred SSL cipher suites (CASSANDRA-11164)
 + * Reference leak with parallel repairs on the same table (CASSANDRA-11215)
 + * Range.compareTo() violates the contract of Comparable (CASSANDRA-11216)
 + * Avoid NPE when serializing ErrorMessage with null message (CASSANDRA-11167)
 + * Replacing an aggregate with a new version doesn't reset INITCOND (CASSANDRA-10840)
 + * (cqlsh) cqlsh cannot be called through symlink (CASSANDRA-11037)
 + * fix ohc and java-driver pom dependencies in build.xml (CASSANDRA-10793)
 + * Protect from keyspace dropped during repair (CASSANDRA-11065)
 + * Handle adding fields to a UDT in SELECT JSON and toJson() (CASSANDRA-11146)
 + * Better error message for cleanup (CASSANDRA-10991)
 + * cqlsh pg-style-strings broken if line ends with ';' (CASSANDRA-11123)
 + * Use cloned TokenMetadata in size estimates to avoid race against membership check
 +   (CASSANDRA-10736)
 + * Always persist upsampled index summaries (CASSANDRA-10512)
 + * (cqlsh) Fix inconsistent auto-complete (CASSANDRA-10733)
 + * Make SELECT JSON and toJson() threadsafe (CASSANDRA-11048)
 + * Fix SELECT on tuple relations for mixed ASC/DESC clustering order (CASSANDRA-7281)
 + * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
 + * Fix paging on DISTINCT queries repeats result when first row in partition changes
 +   (CASSANDRA-10010)
 +Merged from 2.1:
 + * Add a -j parameter to scrub/cleanup/upgradesstables to state how
 +   many threads to use (CASSANDRA-11179)
   * Backport CASSANDRA-10679 (CASSANDRA-9598)
   * Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342)
   * Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/caaa9fc8/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 2ca2fea,6838de6..636c73b
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@@ -34,9 -35,9 +34,10 @@@ import java.util.concurrent.LinkedBlock
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicLong;
  
+ import com.google.common.annotations.VisibleForTesting;
  import com.google.common.collect.Iterables;
  import com.google.common.util.concurrent.*;
 +
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  


[04/15] cassandra git commit: Fix out-of-space error treatment in memtable flushing

Posted by jm...@apache.org.
Fix out-of-space error treatment in memtable flushing

Patch by Branimir Lambov; reviewed by Joshua McKenzie for
CASSANDRA-11448


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f3b3c410
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f3b3c410
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f3b3c410

Branch: refs/heads/cassandra-3.0
Commit: f3b3c410a0d84a4348cf05954b38df6b087762a7
Parents: 105fbb3
Author: Branimir Lambov <br...@datastax.com>
Authored: Wed Mar 30 16:29:55 2016 +0300
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:45:18 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af2518c..50bc894 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.14
+ * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
  * Backport CASSANDRA-10859 (CASSANDRA-11415)
  * COPY FROM fails when importing blob (CASSANDRA-11375)
  * Backport CASSANDRA-10679 (CASSANDRA-9598)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 3d66d3a..edc3564 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -967,6 +967,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         final OpOrder.Barrier writeBarrier;
         final CountDownLatch latch = new CountDownLatch(1);
         final ReplayPosition lastReplayPosition;
+        volatile FSWriteError flushFailure = null;
 
         private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
         {
@@ -1009,12 +1010,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             // must check lastReplayPosition != null because Flush may find that all memtables are clean
             // and so not set a lastReplayPosition
-            if (lastReplayPosition != null)
+            // If a flush errored out but the error was ignored, make sure we don't discard the commit log.
+            if (lastReplayPosition != null && flushFailure == null)
             {
                 CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
             }
 
             metric.pendingFlushes.dec();
+
+            if (flushFailure != null)
+                throw flushFailure;
         }
     }
 
@@ -1114,11 +1119,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             metric.memtableSwitchCount.inc();
 
-            for (Memtable memtable : memtables)
+            try
+            {
+                for (Memtable memtable : memtables)
+                {
+                    // flush the memtable
+                    MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
+                    reclaim(memtable);
+                }
+            }
+            catch (FSWriteError e)
             {
-                // flush the memtable
-                MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
-                reclaim(memtable);
+                JVMStabilityInspector.inspectThrowable(e);
+                // If we weren't killed, try to continue work but do not allow CommitLog to be discarded.
+                postFlush.flushFailure = e;
             }
 
             // signal the post-flush we've done our work

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 810c336..35aa447 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -310,7 +310,7 @@ public class Directories
             if (tooBig)
                 return null;
             else
-                throw new IOError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"));
+                throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
 
         // shortcut for single data directory systems
         if (candidates.size() == 1)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 0ea54ab..6838de6 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@ -35,6 +35,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.*;
 import org.slf4j.Logger;
@@ -554,7 +555,8 @@ public class CommitLogSegmentManager
     /**
      * @return a read-only collection of the active commit log segments
      */
-    Collection<CommitLogSegment> getActiveSegments()
+    @VisibleForTesting
+    public Collection<CommitLogSegment> getActiveSegments()
     {
         return Collections.unmodifiableCollection(activeSegments);
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
index 925efd6..1a15d6f 100644
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
@@ -17,7 +17,10 @@
  */
 package org.apache.cassandra.io.util;
 
+import java.io.IOException;
+
 import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.utils.WrappedRunnable;
 
 public abstract class DiskAwareRunnable extends WrappedRunnable
@@ -26,7 +29,7 @@ public abstract class DiskAwareRunnable extends WrappedRunnable
     {
         Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
         if (directory == null)
-            throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
+            throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
 
         return directory;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
new file mode 100644
index 0000000..8304aff
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import static junit.framework.Assert.fail;
+
+import java.io.IOError;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config.DiskFailurePolicy;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.BlacklistedDirectories;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories.DataDirectory;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.commitlog.CommitLogSegment;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.KillerForTests;
+
+/**
+ * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
+ */
+public class OutOfSpaceTest extends CQLTester
+{
+    @Test
+    public void testFlushUnwriteableDie() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        KillerForTests killerForTests = new KillerForTests();
+        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.die);
+            flushAndExpectError();
+            Assert.assertTrue(killerForTests.wasKilled());
+            Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+            JVMStabilityInspector.replaceKiller(originalKiller);
+        }
+    }
+
+    @Test
+    public void testFlushUnwriteableStop() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.stop);
+            flushAndExpectError();
+            Assert.assertFalse(Gossiper.instance.isEnabled());
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+    }
+
+    @Test
+    public void testFlushUnwriteableIgnore() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
+            flushAndExpectError();
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+
+        // Next flush should succeed.
+        makeTable();
+        flush();
+    }
+
+    public void makeTable() throws Throwable
+    {
+        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+
+        // insert exactly the amount of tombstones that shouldn't trigger an exception
+        for (int i = 0; i < 10; i++)
+            execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+    }
+
+    public void markDirectoriesUnwriteable()
+    {
+        ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+        try
+        {
+            for ( ; ; )
+            {
+                DataDirectory dir = cfs.directories.getWriteableLocation(1);
+                BlacklistedDirectories.maybeMarkUnwritable(cfs.directories.getLocationForDisk(dir));
+            }
+        }
+        catch (IOError e)
+        {
+            // Expected -- marked all directories as unwritable
+        }
+    }
+
+    public void flushAndExpectError() throws InterruptedException, ExecutionException
+    {
+        try
+        {
+            Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).forceFlush().get();
+            fail("FSWriteError expected.");
+        }
+        catch (ExecutionException e)
+        {
+            // Correct path.
+            Assert.assertTrue(e.getCause() instanceof FSWriteError);
+        }
+
+        // Make sure commit log wasn't discarded.
+        UUID cfid = currentTableMetadata().cfId;
+        for (CommitLogSegment segment : CommitLog.instance.allocator.getActiveSegments())
+            if (segment.getDirtyCFIDs().contains(cfid))
+                return;
+        fail("Expected commit log to remain dirty for the affected table.");
+    }
+}


[13/15] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.5

Posted by jm...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.5


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/af9b9cd5
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/af9b9cd5
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/af9b9cd5

Branch: refs/heads/cassandra-3.5
Commit: af9b9cd5ae248746475ada5434d08370148bbce0
Parents: 4111e4f bd4cab2
Author: Josh McKenzie <jo...@datastax.com>
Authored: Fri Apr 1 11:51:13 2016 -0400
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:52:45 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  | 134 ++++++++++------
 .../org/apache/cassandra/db/Directories.java    |  16 +-
 src/java/org/apache/cassandra/db/Memtable.java  |  29 +++-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |  54 -------
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 7 files changed, 276 insertions(+), 119 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b542a67,b9376bc..73d9ab5
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -109,9 -75,9 +109,10 @@@ Merged from 2.2
   * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
   * Fix paging on DISTINCT queries repeats result when first row in partition changes
     (CASSANDRA-10010)
 + * (cqlsh) Support timezone conversion using pytz (CASSANDRA-10397)
   * cqlsh: change default encoding to UTF-8 (CASSANDRA-11124)
  Merged from 2.1:
+  * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
   * Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342)
   * Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286)
   * Avoid dropping message too quickly due to missing unit conversion (CASSANDRA-11302)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index f193c4d,aa12b80..6e01b34
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -910,6 -893,7 +910,7 @@@ public class ColumnFamilyStore implemen
          final OpOrder.Barrier writeBarrier;
          final CountDownLatch latch = new CountDownLatch(1);
          final ReplayPosition lastReplayPosition;
 -        volatile FSWriteError flushFailure = null;
++        volatile Throwable flushFailure = null;
  
          private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
          {
@@@ -951,6 -936,9 +953,9 @@@
              }
  
              metric.pendingFlushes.dec();
+ 
+             if (flushFailure != null)
 -                throw flushFailure;
++                throw Throwables.propagate(flushFailure);
          }
      }
  
@@@ -1050,81 -1038,26 +1055,106 @@@
  
              metric.memtableSwitchCount.inc();
  
-             for (Memtable memtable : memtables)
+             try
              {
-                 List<Future<SSTableMultiWriter>> futures = new ArrayList<>();
-                 long totalBytesOnDisk = 0;
-                 long maxBytesOnDisk = 0;
-                 long minBytesOnDisk = Long.MAX_VALUE;
-                 List<SSTableReader> sstables = new ArrayList<>();
-                 try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH))
+                 for (Memtable memtable : memtables)
+                 {
 -                    // flush the memtable
 -                    MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
 -                    reclaim(memtable);
++                    flushMemtable(memtable);
+                 }
+             }
 -            catch (FSWriteError e)
++            catch (Throwable t)
+             {
 -                JVMStabilityInspector.inspectThrowable(e);
 -                // If we weren't killed, try to continue work but do not allow CommitLog to be discarded.
 -                postFlush.flushFailure = e;
++                JVMStabilityInspector.inspectThrowable(t);
++                postFlush.flushFailure = t;
+             }
 -
+             // signal the post-flush we've done our work
+             postFlush.latch.countDown();
+         }
+ 
++        public void flushMemtable(Memtable memtable)
++        {
++            List<Future<SSTableMultiWriter>> futures = new ArrayList<>();
++            long totalBytesOnDisk = 0;
++            long maxBytesOnDisk = 0;
++            long minBytesOnDisk = Long.MAX_VALUE;
++            List<SSTableReader> sstables = new ArrayList<>();
++            try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH))
++            {
++                List<Memtable.FlushRunnable> flushRunnables = null;
++                List<SSTableMultiWriter> flushResults = null;
++
++                try
 +                {
 +                    // flush the memtable
-                     List<Memtable.FlushRunnable> flushRunnables = memtable.flushRunnables(txn);
++                    flushRunnables = memtable.flushRunnables(txn);
 +
 +                    for (int i = 0; i < flushRunnables.size(); i++)
 +                        futures.add(perDiskflushExecutors[i].submit(flushRunnables.get(i)));
 +
-                     List<SSTableMultiWriter> flushResults = Lists.newArrayList(FBUtilities.waitOnFutures(futures));
++                    flushResults = Lists.newArrayList(FBUtilities.waitOnFutures(futures));
++                }
++                catch (Throwable t)
++                {
++                    t = memtable.abortRunnables(flushRunnables, t);
++                    t = txn.abort(t);
++                    throw Throwables.propagate(t);
++                }
 +
-                     try
++                try
++                {
++                    Iterator<SSTableMultiWriter> writerIterator = flushResults.iterator();
++                    while (writerIterator.hasNext())
 +                    {
-                         Iterator<SSTableMultiWriter> writerIterator = flushResults.iterator();
-                         while (writerIterator.hasNext())
++                        @SuppressWarnings("resource")
++                        SSTableMultiWriter writer = writerIterator.next();
++                        if (writer.getFilePointer() > 0)
 +                        {
-                             @SuppressWarnings("resource")
-                             SSTableMultiWriter writer = writerIterator.next();
-                             if (writer.getFilePointer() > 0)
-                             {
-                                 writer.setOpenResult(true).prepareToCommit();
-                             }
-                             else
-                             {
-                                 maybeFail(writer.abort(null));
-                                 writerIterator.remove();
-                             }
++                            writer.setOpenResult(true).prepareToCommit();
++                        }
++                        else
++                        {
++                            maybeFail(writer.abort(null));
++                            writerIterator.remove();
 +                        }
 +                    }
-                     catch (Throwable t)
-                     {
-                         for (SSTableMultiWriter writer : flushResults)
-                             t = writer.abort(t);
-                         t = txn.abort(t);
-                         Throwables.propagate(t);
-                     }
++                }
++                catch (Throwable t)
++                {
++                    for (SSTableMultiWriter writer : flushResults)
++                        t = writer.abort(t);
++                    t = txn.abort(t);
++                    Throwables.propagate(t);
++                }
 +
-                     txn.prepareToCommit();
++                txn.prepareToCommit();
 +
-                     Throwable accumulate = null;
-                     for (SSTableMultiWriter writer : flushResults)
-                         accumulate = writer.commit(accumulate);
++                Throwable accumulate = null;
++                for (SSTableMultiWriter writer : flushResults)
++                    accumulate = writer.commit(accumulate);
 +
-                     maybeFail(txn.commit(accumulate));
++                maybeFail(txn.commit(accumulate));
 +
-                     for (SSTableMultiWriter writer : flushResults)
++                for (SSTableMultiWriter writer : flushResults)
++                {
++                    Collection<SSTableReader> flushedSSTables = writer.finished();
++                    for (SSTableReader sstable : flushedSSTables)
 +                    {
-                         Collection<SSTableReader> flushedSSTables = writer.finished();
-                         for (SSTableReader sstable : flushedSSTables)
++                        if (sstable != null)
 +                        {
-                             if (sstable != null)
-                             {
-                                 sstables.add(sstable);
-                                 long size = sstable.bytesOnDisk();
-                                 totalBytesOnDisk += size;
-                                 maxBytesOnDisk = Math.max(maxBytesOnDisk, size);
-                                 minBytesOnDisk = Math.min(minBytesOnDisk, size);
-                             }
++                            sstables.add(sstable);
++                            long size = sstable.bytesOnDisk();
++                            totalBytesOnDisk += size;
++                            maxBytesOnDisk = Math.max(maxBytesOnDisk, size);
++                            minBytesOnDisk = Math.min(minBytesOnDisk, size);
 +                        }
 +                    }
 +                }
-                 memtable.cfs.replaceFlushed(memtable, sstables);
-                 reclaim(memtable);
-                 logger.debug("Flushed to {} ({} sstables, {} bytes), biggest {} bytes, smallest {} bytes", sstables, sstables.size(), totalBytesOnDisk, maxBytesOnDisk, minBytesOnDisk);
 +            }
-             // signal the post-flush we've done our work
-             postFlush.latch.countDown();
++            memtable.cfs.replaceFlushed(memtable, sstables);
++            reclaim(memtable);
++            logger.debug("Flushed to {} ({} sstables, {} bytes), biggest {} bytes, smallest {} bytes", sstables, sstables.size(), totalBytesOnDisk, maxBytesOnDisk, minBytesOnDisk);
 +        }
 +
          private void reclaim(final Memtable memtable)
          {
              // issue a read barrier for reclaiming the memory, and offload the wait to another thread

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Directories.java
index 73c6007,5f23baa..5bd2cf0
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@@ -320,7 -306,7 +320,7 @@@ public class Directorie
       * which may return any non-blacklisted directory - even a data directory that has no usable space.
       * Do not use this method in production code.
       *
--     * @throws IOError if all directories are blacklisted.
++     * @throws FSWriteError if all directories are blacklisted.
       */
      public File getDirectoryForNewSSTables()
      {
@@@ -330,11 -316,11 +330,14 @@@
      /**
       * Returns a non-blacklisted data directory that _currently_ has {@code writeSize} bytes as usable space.
       *
--     * @throws IOError if all directories are blacklisted.
++     * @throws FSWriteError if all directories are blacklisted.
       */
      public File getWriteableLocationAsFile(long writeSize)
      {
--        return getLocationForDisk(getWriteableLocation(writeSize));
++        File location = getLocationForDisk(getWriteableLocation(writeSize));
++        if (location == null)
++            throw new FSWriteError(new IOException("No configured data directory contains enough space to write " + writeSize + " bytes"), "");
++        return location;
      }
  
      /**
@@@ -366,9 -352,9 +369,10 @@@
      }
  
      /**
--     * Returns a non-blacklisted data directory that _currently_ has {@code writeSize} bytes as usable space.
++     * Returns a non-blacklisted data directory that _currently_ has {@code writeSize} bytes as usable space, null if
++     * there is not enough space left in all directories.
       *
--     * @throws IOError if all directories are blacklisted.
++     * @throws FSWriteError if all directories are blacklisted.
       */
      public DataDirectory getWriteableLocation(long writeSize)
      {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Memtable.java
index 244c7b6,5d5f7bf..bc054b4
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@@ -24,6 -25,6 +24,7 @@@ import java.util.concurrent.atomic.Atom
  import java.util.concurrent.atomic.AtomicReference;
  
  import com.google.common.annotations.VisibleForTesting;
++import com.google.common.base.Throwables;
  
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
@@@ -44,10 -44,11 +45,11 @@@ import org.apache.cassandra.db.rows.Unf
  import org.apache.cassandra.dht.*;
  import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
  import org.apache.cassandra.index.transactions.UpdateTransaction;
++import org.apache.cassandra.io.FSWriteError;
  import org.apache.cassandra.io.sstable.Descriptor;
 +import org.apache.cassandra.io.sstable.SSTableMultiWriter;
  import org.apache.cassandra.io.sstable.SSTableTxnWriter;
 -import org.apache.cassandra.io.sstable.format.SSTableReader;
  import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 -import org.apache.cassandra.io.util.DiskAwareRunnable;
  import org.apache.cassandra.service.ActiveRepairService;
  import org.apache.cassandra.utils.ByteBufferUtil;
  import org.apache.cassandra.utils.FBUtilities;
@@@ -255,32 -254,9 +257,47 @@@ public class Memtable implements Compar
          return partitions.size();
      }
  
 -    public FlushRunnable flushRunnable()
 +    public List<FlushRunnable> flushRunnables(LifecycleTransaction txn)
      {
 -        return new FlushRunnable(lastReplayPosition.get());
 +        List<Range<Token>> localRanges = Range.sort(StorageService.instance.getLocalRanges(cfs.keyspace.getName()));
 +
 +        if (!cfs.getPartitioner().splitter().isPresent() || localRanges.isEmpty())
 +            return Collections.singletonList(new FlushRunnable(lastReplayPosition.get(), txn));
 +
 +        return createFlushRunnables(localRanges, txn);
 +    }
 +
 +    private List<FlushRunnable> createFlushRunnables(List<Range<Token>> localRanges, LifecycleTransaction txn)
 +    {
 +        assert cfs.getPartitioner().splitter().isPresent();
 +
 +        Directories.DataDirectory[] locations = cfs.getDirectories().getWriteableLocations();
 +        List<PartitionPosition> boundaries = StorageService.getDiskBoundaries(localRanges, cfs.getPartitioner(), locations);
 +        List<FlushRunnable> runnables = new ArrayList<>(boundaries.size());
 +        PartitionPosition rangeStart = cfs.getPartitioner().getMinimumToken().minKeyBound();
 +        ReplayPosition context = lastReplayPosition.get();
-         for (int i = 0; i < boundaries.size(); i++)
++        try
 +        {
-             PartitionPosition t = boundaries.get(i);
-             runnables.add(new FlushRunnable(context, rangeStart, t, locations[i], txn));
-             rangeStart = t;
++            for (int i = 0; i < boundaries.size(); i++)
++            {
++                PartitionPosition t = boundaries.get(i);
++                runnables.add(new FlushRunnable(context, rangeStart, t, locations[i], txn));
++                rangeStart = t;
++            }
++            return runnables;
++        }
++        catch (Throwable e)
++        {
++            throw Throwables.propagate(abortRunnables(runnables, e));
 +        }
-         return runnables;
++    }
++
++    public Throwable abortRunnables(List<FlushRunnable> runnables, Throwable t)
++    {
++        if (runnables != null)
++            for (FlushRunnable runnable : runnables)
++                t = runnable.writer.abort(t);
++        return t;
      }
  
      public String toString()
@@@ -387,12 -345,21 +404,12 @@@
                                      * 1.2); // bloom filter and row index overhead
  
              this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
 -        }
  
 -        public long getExpectedWriteSize()
 -        {
 -            return estimatedSize;
 -        }
 +            if (flushLocation == null)
-                 writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getLocationForDisk(getDirectories().getWriteableLocation(estimatedSize))), columnsCollector.get(), statsCollector.get());
++                writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getWriteableLocationAsFile(estimatedSize)), columnsCollector.get(), statsCollector.get());
 +            else
 +                writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getLocationForDisk(flushLocation)), columnsCollector.get(), statsCollector.get());
  
 -        protected void runMayThrow() throws Exception
 -        {
 -            long writeSize = getExpectedWriteSize();
 -            Directories.DataDirectory dataDirectory = getWriteDirectory(writeSize);
 -            File sstableDirectory = cfs.getDirectories().getLocationForDisk(dataDirectory);
 -            assert sstableDirectory != null : "Flush task is not bound to any disk";
 -            Collection<SSTableReader> sstables = writeSortedContents(context, sstableDirectory);
 -            cfs.replaceFlushed(Memtable.this, sstables);
          }
  
          protected Directories getDirectories()

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
index a083218,1a15d6f..0000000
deleted file mode 100644,100644
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ /dev/null
@@@ -1,54 -1,42 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements.  See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership.  The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License.  You may obtain a copy of the License at
-- *
-- *     http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.cassandra.io.util;
 -
 -import java.io.IOException;
--
--import org.apache.cassandra.db.Directories;
 -import org.apache.cassandra.io.FSWriteError;
--import org.apache.cassandra.utils.WrappedRunnable;
--
--public abstract class DiskAwareRunnable extends WrappedRunnable
--{
--    protected Directories.DataDirectory getWriteDirectory(long writeSize)
--    {
-         Directories.DataDirectory directory;
-         directory = getDirectory();
- 
-         if (directory == null) // ok panic - write anywhere
-             directory = getDirectories().getWriteableLocation(writeSize);
- 
 -        Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
--        if (directory == null)
-             throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
 -            throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
--
--        return directory;
--    }
--
--    /**
--     * Get sstable directories for the CF.
--     * @return Directories instance for the CF.
--     */
--    protected abstract Directories getDirectories();
-     protected abstract Directories.DataDirectory getDirectory();
- 
-     /**
-      * Called if no disk is available with free space for the full write size.
-      * @return true if the scope of the task was successfully reduced.
-      */
-     public boolean reduceScopeForLimitedSpace()
-     {
-         return false;
-     }
--}


[10/15] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0

Posted by jm...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bd4cab24
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bd4cab24
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bd4cab24

Branch: refs/heads/trunk
Commit: bd4cab24bb9582d15d6b4cf251e7da21230f1460
Parents: be38081 caaa9fc
Author: Josh McKenzie <jo...@datastax.com>
Authored: Fri Apr 1 11:47:39 2016 -0400
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:49:41 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 7fc628e,78ea961..b9376bc
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -75,8 -38,10 +75,9 @@@ Merged from 2.2
   * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
   * Fix paging on DISTINCT queries repeats result when first row in partition changes
     (CASSANDRA-10010)
 + * cqlsh: change default encoding to UTF-8 (CASSANDRA-11124)
  Merged from 2.1:
 - * Add a -j parameter to scrub/cleanup/upgradesstables to state how
 -   many threads to use (CASSANDRA-11179)
 - * Backport CASSANDRA-10679 (CASSANDRA-9598)
++ * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
   * Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342)
   * Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286)
   * Avoid dropping message too quickly due to missing unit conversion (CASSANDRA-11302)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 8a8d0e7,636c73b..2ee4eed
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@@ -34,8 -34,10 +34,9 @@@ import java.util.concurrent.LinkedBlock
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicLong;
  
+ import com.google.common.annotations.VisibleForTesting;
  import com.google.common.collect.Iterables;
  import com.google.common.util.concurrent.*;
 -
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd4cab24/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
index 0000000,8304aff..1527b1e
mode 000000,100644..100644
--- a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@@ -1,0 -1,157 +1,157 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.cassandra.cql3;
+ 
+ import static junit.framework.Assert.fail;
+ 
+ import java.io.IOError;
+ import java.util.UUID;
+ import java.util.concurrent.ExecutionException;
+ 
+ import org.junit.Assert;
+ import org.junit.Test;
+ 
+ import org.apache.cassandra.config.Config.DiskFailurePolicy;
+ import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.db.BlacklistedDirectories;
+ import org.apache.cassandra.db.ColumnFamilyStore;
+ import org.apache.cassandra.db.Directories.DataDirectory;
+ import org.apache.cassandra.db.commitlog.CommitLog;
+ import org.apache.cassandra.db.commitlog.CommitLogSegment;
+ import org.apache.cassandra.db.Keyspace;
+ import org.apache.cassandra.gms.Gossiper;
+ import org.apache.cassandra.io.FSWriteError;
+ import org.apache.cassandra.utils.JVMStabilityInspector;
+ import org.apache.cassandra.utils.KillerForTests;
+ 
+ /**
+  * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
+  */
+ public class OutOfSpaceTest extends CQLTester
+ {
+     @Test
+     public void testFlushUnwriteableDie() throws Throwable
+     {
+         makeTable();
+         markDirectoriesUnwriteable();
+ 
+         KillerForTests killerForTests = new KillerForTests();
+         JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+         try
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.die);
+             flushAndExpectError();
+             Assert.assertTrue(killerForTests.wasKilled());
+             Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+         }
+         finally
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+             JVMStabilityInspector.replaceKiller(originalKiller);
+         }
+     }
+ 
+     @Test
+     public void testFlushUnwriteableStop() throws Throwable
+     {
+         makeTable();
+         markDirectoriesUnwriteable();
+ 
+         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+         try
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.stop);
+             flushAndExpectError();
+             Assert.assertFalse(Gossiper.instance.isEnabled());
+         }
+         finally
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+         }
+     }
+ 
+     @Test
+     public void testFlushUnwriteableIgnore() throws Throwable
+     {
+         makeTable();
+         markDirectoriesUnwriteable();
+ 
+         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+         try
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
+             flushAndExpectError();
+         }
+         finally
+         {
+             DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+         }
+ 
+         // Next flush should succeed.
+         makeTable();
+         flush();
+     }
+ 
+     public void makeTable() throws Throwable
+     {
+         createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+ 
+         // insert exactly the amount of tombstones that shouldn't trigger an exception
+         for (int i = 0; i < 10; i++)
+             execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+     }
+ 
+     public void markDirectoriesUnwriteable()
+     {
+         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+         try
+         {
+             for ( ; ; )
+             {
 -                DataDirectory dir = cfs.directories.getWriteableLocation(1);
 -                BlacklistedDirectories.maybeMarkUnwritable(cfs.directories.getLocationForDisk(dir));
++                DataDirectory dir = cfs.getDirectories().getWriteableLocation(1);
++                BlacklistedDirectories.maybeMarkUnwritable(cfs.getDirectories().getLocationForDisk(dir));
+             }
+         }
+         catch (IOError e)
+         {
+             // Expected -- marked all directories as unwritable
+         }
+     }
+ 
+     public void flushAndExpectError() throws InterruptedException, ExecutionException
+     {
+         try
+         {
+             Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).forceFlush().get();
+             fail("FSWriteError expected.");
+         }
+         catch (ExecutionException e)
+         {
+             // Correct path.
+             Assert.assertTrue(e.getCause() instanceof FSWriteError);
+         }
+ 
+         // Make sure commit log wasn't discarded.
+         UUID cfid = currentTableMetadata().cfId;
+         for (CommitLogSegment segment : CommitLog.instance.allocator.getActiveSegments())
+             if (segment.getDirtyCFIDs().contains(cfid))
+                 return;
+         fail("Expected commit log to remain dirty for the affected table.");
+     }
+ }


[03/15] cassandra git commit: Fix out-of-space error treatment in memtable flushing

Posted by jm...@apache.org.
Fix out-of-space error treatment in memtable flushing

Patch by Branimir Lambov; reviewed by Joshua McKenzie for
CASSANDRA-11448


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f3b3c410
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f3b3c410
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f3b3c410

Branch: refs/heads/trunk
Commit: f3b3c410a0d84a4348cf05954b38df6b087762a7
Parents: 105fbb3
Author: Branimir Lambov <br...@datastax.com>
Authored: Wed Mar 30 16:29:55 2016 +0300
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:45:18 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  24 ++-
 .../org/apache/cassandra/db/Directories.java    |   2 +-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |   5 +-
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 6 files changed, 185 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af2518c..50bc894 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.14
+ * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
  * Backport CASSANDRA-10859 (CASSANDRA-11415)
  * COPY FROM fails when importing blob (CASSANDRA-11375)
  * Backport CASSANDRA-10679 (CASSANDRA-9598)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 3d66d3a..edc3564 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -967,6 +967,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         final OpOrder.Barrier writeBarrier;
         final CountDownLatch latch = new CountDownLatch(1);
         final ReplayPosition lastReplayPosition;
+        volatile FSWriteError flushFailure = null;
 
         private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
         {
@@ -1009,12 +1010,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             // must check lastReplayPosition != null because Flush may find that all memtables are clean
             // and so not set a lastReplayPosition
-            if (lastReplayPosition != null)
+            // If a flush errored out but the error was ignored, make sure we don't discard the commit log.
+            if (lastReplayPosition != null && flushFailure == null)
             {
                 CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
             }
 
             metric.pendingFlushes.dec();
+
+            if (flushFailure != null)
+                throw flushFailure;
         }
     }
 
@@ -1114,11 +1119,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
             metric.memtableSwitchCount.inc();
 
-            for (Memtable memtable : memtables)
+            try
+            {
+                for (Memtable memtable : memtables)
+                {
+                    // flush the memtable
+                    MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
+                    reclaim(memtable);
+                }
+            }
+            catch (FSWriteError e)
             {
-                // flush the memtable
-                MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
-                reclaim(memtable);
+                JVMStabilityInspector.inspectThrowable(e);
+                // If we weren't killed, try to continue work but do not allow CommitLog to be discarded.
+                postFlush.flushFailure = e;
             }
 
             // signal the post-flush we've done our work

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 810c336..35aa447 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -310,7 +310,7 @@ public class Directories
             if (tooBig)
                 return null;
             else
-                throw new IOError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"));
+                throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
 
         // shortcut for single data directory systems
         if (candidates.size() == 1)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 0ea54ab..6838de6 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@ -35,6 +35,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.*;
 import org.slf4j.Logger;
@@ -554,7 +555,8 @@ public class CommitLogSegmentManager
     /**
      * @return a read-only collection of the active commit log segments
      */
-    Collection<CommitLogSegment> getActiveSegments()
+    @VisibleForTesting
+    public Collection<CommitLogSegment> getActiveSegments()
     {
         return Collections.unmodifiableCollection(activeSegments);
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
index 925efd6..1a15d6f 100644
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
@@ -17,7 +17,10 @@
  */
 package org.apache.cassandra.io.util;
 
+import java.io.IOException;
+
 import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.utils.WrappedRunnable;
 
 public abstract class DiskAwareRunnable extends WrappedRunnable
@@ -26,7 +29,7 @@ public abstract class DiskAwareRunnable extends WrappedRunnable
     {
         Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
         if (directory == null)
-            throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
+            throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
 
         return directory;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f3b3c410/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
new file mode 100644
index 0000000..8304aff
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+import static junit.framework.Assert.fail;
+
+import java.io.IOError;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config.DiskFailurePolicy;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.BlacklistedDirectories;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories.DataDirectory;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.db.commitlog.CommitLogSegment;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.utils.JVMStabilityInspector;
+import org.apache.cassandra.utils.KillerForTests;
+
+/**
+ * Test that TombstoneOverwhelmingException gets thrown when it should be and doesn't when it shouldn't be.
+ */
+public class OutOfSpaceTest extends CQLTester
+{
+    @Test
+    public void testFlushUnwriteableDie() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        KillerForTests killerForTests = new KillerForTests();
+        JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.die);
+            flushAndExpectError();
+            Assert.assertTrue(killerForTests.wasKilled());
+            Assert.assertFalse(killerForTests.wasKilledQuietly()); //only killed quietly on startup failure
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+            JVMStabilityInspector.replaceKiller(originalKiller);
+        }
+    }
+
+    @Test
+    public void testFlushUnwriteableStop() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.stop);
+            flushAndExpectError();
+            Assert.assertFalse(Gossiper.instance.isEnabled());
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+    }
+
+    @Test
+    public void testFlushUnwriteableIgnore() throws Throwable
+    {
+        makeTable();
+        markDirectoriesUnwriteable();
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
+            flushAndExpectError();
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+
+        // Next flush should succeed.
+        makeTable();
+        flush();
+    }
+
+    public void makeTable() throws Throwable
+    {
+        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+
+        // insert exactly the amount of tombstones that shouldn't trigger an exception
+        for (int i = 0; i < 10; i++)
+            execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
+    }
+
+    public void markDirectoriesUnwriteable()
+    {
+        ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+        try
+        {
+            for ( ; ; )
+            {
+                DataDirectory dir = cfs.directories.getWriteableLocation(1);
+                BlacklistedDirectories.maybeMarkUnwritable(cfs.directories.getLocationForDisk(dir));
+            }
+        }
+        catch (IOError e)
+        {
+            // Expected -- marked all directories as unwritable
+        }
+    }
+
+    public void flushAndExpectError() throws InterruptedException, ExecutionException
+    {
+        try
+        {
+            Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).forceFlush().get();
+            fail("FSWriteError expected.");
+        }
+        catch (ExecutionException e)
+        {
+            // Correct path.
+            Assert.assertTrue(e.getCause() instanceof FSWriteError);
+        }
+
+        // Make sure commit log wasn't discarded.
+        UUID cfid = currentTableMetadata().cfId;
+        for (CommitLogSegment segment : CommitLog.instance.allocator.getActiveSegments())
+            if (segment.getDirtyCFIDs().contains(cfid))
+                return;
+        fail("Expected commit log to remain dirty for the affected table.");
+    }
+}


[14/15] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.5

Posted by jm...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.5


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/af9b9cd5
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/af9b9cd5
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/af9b9cd5

Branch: refs/heads/trunk
Commit: af9b9cd5ae248746475ada5434d08370148bbce0
Parents: 4111e4f bd4cab2
Author: Josh McKenzie <jo...@datastax.com>
Authored: Fri Apr 1 11:51:13 2016 -0400
Committer: Josh McKenzie <jo...@datastax.com>
Committed: Fri Apr 1 11:52:45 2016 -0400

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  | 134 ++++++++++------
 .../org/apache/cassandra/db/Directories.java    |  16 +-
 src/java/org/apache/cassandra/db/Memtable.java  |  29 +++-
 .../db/commitlog/CommitLogSegmentManager.java   |   4 +-
 .../cassandra/io/util/DiskAwareRunnable.java    |  54 -------
 .../apache/cassandra/cql3/OutOfSpaceTest.java   | 157 +++++++++++++++++++
 7 files changed, 276 insertions(+), 119 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b542a67,b9376bc..73d9ab5
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -109,9 -75,9 +109,10 @@@ Merged from 2.2
   * (cqlsh) Support utf-8/cp65001 encoding on Windows (CASSANDRA-11030)
   * Fix paging on DISTINCT queries repeats result when first row in partition changes
     (CASSANDRA-10010)
 + * (cqlsh) Support timezone conversion using pytz (CASSANDRA-10397)
   * cqlsh: change default encoding to UTF-8 (CASSANDRA-11124)
  Merged from 2.1:
+  * Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
   * Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342)
   * Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286)
   * Avoid dropping message too quickly due to missing unit conversion (CASSANDRA-11302)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index f193c4d,aa12b80..6e01b34
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -910,6 -893,7 +910,7 @@@ public class ColumnFamilyStore implemen
          final OpOrder.Barrier writeBarrier;
          final CountDownLatch latch = new CountDownLatch(1);
          final ReplayPosition lastReplayPosition;
 -        volatile FSWriteError flushFailure = null;
++        volatile Throwable flushFailure = null;
  
          private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
          {
@@@ -951,6 -936,9 +953,9 @@@
              }
  
              metric.pendingFlushes.dec();
+ 
+             if (flushFailure != null)
 -                throw flushFailure;
++                throw Throwables.propagate(flushFailure);
          }
      }
  
@@@ -1050,81 -1038,26 +1055,106 @@@
  
              metric.memtableSwitchCount.inc();
  
-             for (Memtable memtable : memtables)
+             try
              {
-                 List<Future<SSTableMultiWriter>> futures = new ArrayList<>();
-                 long totalBytesOnDisk = 0;
-                 long maxBytesOnDisk = 0;
-                 long minBytesOnDisk = Long.MAX_VALUE;
-                 List<SSTableReader> sstables = new ArrayList<>();
-                 try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH))
+                 for (Memtable memtable : memtables)
+                 {
 -                    // flush the memtable
 -                    MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
 -                    reclaim(memtable);
++                    flushMemtable(memtable);
+                 }
+             }
 -            catch (FSWriteError e)
++            catch (Throwable t)
+             {
 -                JVMStabilityInspector.inspectThrowable(e);
 -                // If we weren't killed, try to continue work but do not allow CommitLog to be discarded.
 -                postFlush.flushFailure = e;
++                JVMStabilityInspector.inspectThrowable(t);
++                postFlush.flushFailure = t;
+             }
 -
+             // signal the post-flush we've done our work
+             postFlush.latch.countDown();
+         }
+ 
++        public void flushMemtable(Memtable memtable)
++        {
++            List<Future<SSTableMultiWriter>> futures = new ArrayList<>();
++            long totalBytesOnDisk = 0;
++            long maxBytesOnDisk = 0;
++            long minBytesOnDisk = Long.MAX_VALUE;
++            List<SSTableReader> sstables = new ArrayList<>();
++            try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH))
++            {
++                List<Memtable.FlushRunnable> flushRunnables = null;
++                List<SSTableMultiWriter> flushResults = null;
++
++                try
 +                {
 +                    // flush the memtable
-                     List<Memtable.FlushRunnable> flushRunnables = memtable.flushRunnables(txn);
++                    flushRunnables = memtable.flushRunnables(txn);
 +
 +                    for (int i = 0; i < flushRunnables.size(); i++)
 +                        futures.add(perDiskflushExecutors[i].submit(flushRunnables.get(i)));
 +
-                     List<SSTableMultiWriter> flushResults = Lists.newArrayList(FBUtilities.waitOnFutures(futures));
++                    flushResults = Lists.newArrayList(FBUtilities.waitOnFutures(futures));
++                }
++                catch (Throwable t)
++                {
++                    t = memtable.abortRunnables(flushRunnables, t);
++                    t = txn.abort(t);
++                    throw Throwables.propagate(t);
++                }
 +
-                     try
++                try
++                {
++                    Iterator<SSTableMultiWriter> writerIterator = flushResults.iterator();
++                    while (writerIterator.hasNext())
 +                    {
-                         Iterator<SSTableMultiWriter> writerIterator = flushResults.iterator();
-                         while (writerIterator.hasNext())
++                        @SuppressWarnings("resource")
++                        SSTableMultiWriter writer = writerIterator.next();
++                        if (writer.getFilePointer() > 0)
 +                        {
-                             @SuppressWarnings("resource")
-                             SSTableMultiWriter writer = writerIterator.next();
-                             if (writer.getFilePointer() > 0)
-                             {
-                                 writer.setOpenResult(true).prepareToCommit();
-                             }
-                             else
-                             {
-                                 maybeFail(writer.abort(null));
-                                 writerIterator.remove();
-                             }
++                            writer.setOpenResult(true).prepareToCommit();
++                        }
++                        else
++                        {
++                            maybeFail(writer.abort(null));
++                            writerIterator.remove();
 +                        }
 +                    }
-                     catch (Throwable t)
-                     {
-                         for (SSTableMultiWriter writer : flushResults)
-                             t = writer.abort(t);
-                         t = txn.abort(t);
-                         Throwables.propagate(t);
-                     }
++                }
++                catch (Throwable t)
++                {
++                    for (SSTableMultiWriter writer : flushResults)
++                        t = writer.abort(t);
++                    t = txn.abort(t);
++                    Throwables.propagate(t);
++                }
 +
-                     txn.prepareToCommit();
++                txn.prepareToCommit();
 +
-                     Throwable accumulate = null;
-                     for (SSTableMultiWriter writer : flushResults)
-                         accumulate = writer.commit(accumulate);
++                Throwable accumulate = null;
++                for (SSTableMultiWriter writer : flushResults)
++                    accumulate = writer.commit(accumulate);
 +
-                     maybeFail(txn.commit(accumulate));
++                maybeFail(txn.commit(accumulate));
 +
-                     for (SSTableMultiWriter writer : flushResults)
++                for (SSTableMultiWriter writer : flushResults)
++                {
++                    Collection<SSTableReader> flushedSSTables = writer.finished();
++                    for (SSTableReader sstable : flushedSSTables)
 +                    {
-                         Collection<SSTableReader> flushedSSTables = writer.finished();
-                         for (SSTableReader sstable : flushedSSTables)
++                        if (sstable != null)
 +                        {
-                             if (sstable != null)
-                             {
-                                 sstables.add(sstable);
-                                 long size = sstable.bytesOnDisk();
-                                 totalBytesOnDisk += size;
-                                 maxBytesOnDisk = Math.max(maxBytesOnDisk, size);
-                                 minBytesOnDisk = Math.min(minBytesOnDisk, size);
-                             }
++                            sstables.add(sstable);
++                            long size = sstable.bytesOnDisk();
++                            totalBytesOnDisk += size;
++                            maxBytesOnDisk = Math.max(maxBytesOnDisk, size);
++                            minBytesOnDisk = Math.min(minBytesOnDisk, size);
 +                        }
 +                    }
 +                }
-                 memtable.cfs.replaceFlushed(memtable, sstables);
-                 reclaim(memtable);
-                 logger.debug("Flushed to {} ({} sstables, {} bytes), biggest {} bytes, smallest {} bytes", sstables, sstables.size(), totalBytesOnDisk, maxBytesOnDisk, minBytesOnDisk);
 +            }
-             // signal the post-flush we've done our work
-             postFlush.latch.countDown();
++            memtable.cfs.replaceFlushed(memtable, sstables);
++            reclaim(memtable);
++            logger.debug("Flushed to {} ({} sstables, {} bytes), biggest {} bytes, smallest {} bytes", sstables, sstables.size(), totalBytesOnDisk, maxBytesOnDisk, minBytesOnDisk);
 +        }
 +
          private void reclaim(final Memtable memtable)
          {
              // issue a read barrier for reclaiming the memory, and offload the wait to another thread

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Directories.java
index 73c6007,5f23baa..5bd2cf0
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@@ -320,7 -306,7 +320,7 @@@ public class Directorie
       * which may return any non-blacklisted directory - even a data directory that has no usable space.
       * Do not use this method in production code.
       *
--     * @throws IOError if all directories are blacklisted.
++     * @throws FSWriteError if all directories are blacklisted.
       */
      public File getDirectoryForNewSSTables()
      {
@@@ -330,11 -316,11 +330,14 @@@
      /**
       * Returns a non-blacklisted data directory that _currently_ has {@code writeSize} bytes as usable space.
       *
--     * @throws IOError if all directories are blacklisted.
++     * @throws FSWriteError if all directories are blacklisted.
       */
      public File getWriteableLocationAsFile(long writeSize)
      {
--        return getLocationForDisk(getWriteableLocation(writeSize));
++        File location = getLocationForDisk(getWriteableLocation(writeSize));
++        if (location == null)
++            throw new FSWriteError(new IOException("No configured data directory contains enough space to write " + writeSize + " bytes"), "");
++        return location;
      }
  
      /**
@@@ -366,9 -352,9 +369,10 @@@
      }
  
      /**
--     * Returns a non-blacklisted data directory that _currently_ has {@code writeSize} bytes as usable space.
++     * Returns a non-blacklisted data directory that _currently_ has {@code writeSize} bytes as usable space, null if
++     * there is not enough space left in all directories.
       *
--     * @throws IOError if all directories are blacklisted.
++     * @throws FSWriteError if all directories are blacklisted.
       */
      public DataDirectory getWriteableLocation(long writeSize)
      {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Memtable.java
index 244c7b6,5d5f7bf..bc054b4
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@@ -24,6 -25,6 +24,7 @@@ import java.util.concurrent.atomic.Atom
  import java.util.concurrent.atomic.AtomicReference;
  
  import com.google.common.annotations.VisibleForTesting;
++import com.google.common.base.Throwables;
  
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
@@@ -44,10 -44,11 +45,11 @@@ import org.apache.cassandra.db.rows.Unf
  import org.apache.cassandra.dht.*;
  import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
  import org.apache.cassandra.index.transactions.UpdateTransaction;
++import org.apache.cassandra.io.FSWriteError;
  import org.apache.cassandra.io.sstable.Descriptor;
 +import org.apache.cassandra.io.sstable.SSTableMultiWriter;
  import org.apache.cassandra.io.sstable.SSTableTxnWriter;
 -import org.apache.cassandra.io.sstable.format.SSTableReader;
  import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 -import org.apache.cassandra.io.util.DiskAwareRunnable;
  import org.apache.cassandra.service.ActiveRepairService;
  import org.apache.cassandra.utils.ByteBufferUtil;
  import org.apache.cassandra.utils.FBUtilities;
@@@ -255,32 -254,9 +257,47 @@@ public class Memtable implements Compar
          return partitions.size();
      }
  
 -    public FlushRunnable flushRunnable()
 +    public List<FlushRunnable> flushRunnables(LifecycleTransaction txn)
      {
 -        return new FlushRunnable(lastReplayPosition.get());
 +        List<Range<Token>> localRanges = Range.sort(StorageService.instance.getLocalRanges(cfs.keyspace.getName()));
 +
 +        if (!cfs.getPartitioner().splitter().isPresent() || localRanges.isEmpty())
 +            return Collections.singletonList(new FlushRunnable(lastReplayPosition.get(), txn));
 +
 +        return createFlushRunnables(localRanges, txn);
 +    }
 +
 +    private List<FlushRunnable> createFlushRunnables(List<Range<Token>> localRanges, LifecycleTransaction txn)
 +    {
 +        assert cfs.getPartitioner().splitter().isPresent();
 +
 +        Directories.DataDirectory[] locations = cfs.getDirectories().getWriteableLocations();
 +        List<PartitionPosition> boundaries = StorageService.getDiskBoundaries(localRanges, cfs.getPartitioner(), locations);
 +        List<FlushRunnable> runnables = new ArrayList<>(boundaries.size());
 +        PartitionPosition rangeStart = cfs.getPartitioner().getMinimumToken().minKeyBound();
 +        ReplayPosition context = lastReplayPosition.get();
-         for (int i = 0; i < boundaries.size(); i++)
++        try
 +        {
-             PartitionPosition t = boundaries.get(i);
-             runnables.add(new FlushRunnable(context, rangeStart, t, locations[i], txn));
-             rangeStart = t;
++            for (int i = 0; i < boundaries.size(); i++)
++            {
++                PartitionPosition t = boundaries.get(i);
++                runnables.add(new FlushRunnable(context, rangeStart, t, locations[i], txn));
++                rangeStart = t;
++            }
++            return runnables;
++        }
++        catch (Throwable e)
++        {
++            throw Throwables.propagate(abortRunnables(runnables, e));
 +        }
-         return runnables;
++    }
++
++    public Throwable abortRunnables(List<FlushRunnable> runnables, Throwable t)
++    {
++        if (runnables != null)
++            for (FlushRunnable runnable : runnables)
++                t = runnable.writer.abort(t);
++        return t;
      }
  
      public String toString()
@@@ -387,12 -345,21 +404,12 @@@
                                      * 1.2); // bloom filter and row index overhead
  
              this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
 -        }
  
 -        public long getExpectedWriteSize()
 -        {
 -            return estimatedSize;
 -        }
 +            if (flushLocation == null)
-                 writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getLocationForDisk(getDirectories().getWriteableLocation(estimatedSize))), columnsCollector.get(), statsCollector.get());
++                writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getWriteableLocationAsFile(estimatedSize)), columnsCollector.get(), statsCollector.get());
 +            else
 +                writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getLocationForDisk(flushLocation)), columnsCollector.get(), statsCollector.get());
  
 -        protected void runMayThrow() throws Exception
 -        {
 -            long writeSize = getExpectedWriteSize();
 -            Directories.DataDirectory dataDirectory = getWriteDirectory(writeSize);
 -            File sstableDirectory = cfs.getDirectories().getLocationForDisk(dataDirectory);
 -            assert sstableDirectory != null : "Flush task is not bound to any disk";
 -            Collection<SSTableReader> sstables = writeSortedContents(context, sstableDirectory);
 -            cfs.replaceFlushed(Memtable.this, sstables);
          }
  
          protected Directories getDirectories()

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af9b9cd5/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
index a083218,1a15d6f..0000000
deleted file mode 100644,100644
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ /dev/null
@@@ -1,54 -1,42 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements.  See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership.  The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License.  You may obtain a copy of the License at
-- *
-- *     http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
-- */
--package org.apache.cassandra.io.util;
 -
 -import java.io.IOException;
--
--import org.apache.cassandra.db.Directories;
 -import org.apache.cassandra.io.FSWriteError;
--import org.apache.cassandra.utils.WrappedRunnable;
--
--public abstract class DiskAwareRunnable extends WrappedRunnable
--{
--    protected Directories.DataDirectory getWriteDirectory(long writeSize)
--    {
-         Directories.DataDirectory directory;
-         directory = getDirectory();
- 
-         if (directory == null) // ok panic - write anywhere
-             directory = getDirectories().getWriteableLocation(writeSize);
- 
 -        Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
--        if (directory == null)
-             throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
 -            throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
--
--        return directory;
--    }
--
--    /**
--     * Get sstable directories for the CF.
--     * @return Directories instance for the CF.
--     */
--    protected abstract Directories getDirectories();
-     protected abstract Directories.DataDirectory getDirectory();
- 
-     /**
-      * Called if no disk is available with free space for the full write size.
-      * @return true if the scope of the task was successfully reduced.
-      */
-     public boolean reduceScopeForLimitedSpace()
-     {
-         return false;
-     }
--}