You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by xe...@apache.org on 2014/03/25 22:43:43 UTC

[2/2] git commit: Add multiple memory allocation options for memtables patch by benedict and xedin; reviewed by marcuse and xedin for CASSANDRA-6689

Add multiple memory allocation options for memtables
patch by benedict and xedin; reviewed by marcuse and xedin for CASSANDRA-6689


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1a3b5dbc
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1a3b5dbc
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1a3b5dbc

Branch: refs/heads/cassandra-2.1
Commit: 1a3b5dbcb5fbee5066ceaa2467536119893f7e7f
Parents: 48847b5
Author: Pavel Yaskevich <xe...@apache.org>
Authored: Tue Mar 25 14:02:22 2014 -0700
Committer: Pavel Yaskevich <xe...@apache.org>
Committed: Tue Mar 25 14:21:07 2014 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 conf/cassandra.yaml                             |   6 +
 .../org/apache/cassandra/config/Config.java     |  13 +-
 .../cassandra/config/DatabaseDescriptor.java    |  68 ++++--
 .../apache/cassandra/cql3/ColumnCondition.java  |   1 -
 .../cassandra/db/ArrayBackedSortedColumns.java  |  34 ++-
 .../cassandra/db/CollationController.java       |  46 +++-
 .../apache/cassandra/db/ColumnFamilyStore.java  |  87 +++++--
 .../cassandra/db/ColumnFamilyStoreMBean.java    |   2 +-
 src/java/org/apache/cassandra/db/Memtable.java  |  72 +++---
 .../apache/cassandra/db/RowIteratorFactory.java |  10 +-
 .../db/commitlog/CommitLogSegment.java          |   3 +-
 .../db/composites/BoundedComposite.java         |   2 +-
 .../cassandra/db/composites/Composite.java      |   2 +-
 .../cassandra/db/composites/Composites.java     |   2 +-
 .../db/composites/CompoundComposite.java        |   2 +-
 .../db/composites/CompoundSparseCellName.java   |   2 +-
 .../composites/CompoundSparseCellNameType.java  |   2 +-
 .../db/composites/SimpleComposite.java          |   2 +-
 .../db/composites/SimpleSparseCellName.java     |   2 +-
 .../SimpleSparseInternedCellName.java           |   2 +-
 .../apache/cassandra/db/filter/ColumnSlice.java |   2 +-
 .../apache/cassandra/db/filter/QueryFilter.java |  31 ++-
 .../AbstractSimplePerColumnSecondaryIndex.java  |  21 +-
 .../cassandra/db/index/SecondaryIndex.java      |  23 +-
 .../db/index/composites/CompositesIndex.java    |  18 +-
 .../db/index/composites/CompositesSearcher.java |  37 ++-
 .../cassandra/db/index/keys/KeysSearcher.java   |  18 +-
 .../cassandra/io/sstable/SSTableWriter.java     |  10 +
 .../org/apache/cassandra/io/util/FileUtils.java |  51 ++--
 .../cassandra/metrics/ColumnFamilyMetrics.java  |  33 ++-
 .../apache/cassandra/utils/ByteBufferUtil.java  |   2 +-
 .../utils/memory/AbstractAllocator.java         |  23 +-
 .../utils/memory/ContextAllocator.java          |  20 +-
 .../apache/cassandra/utils/memory/HeapPool.java |  38 ++-
 .../utils/memory/HeapPoolAllocator.java         |  47 ----
 .../utils/memory/HeapSlabAllocator.java         | 204 ----------------
 .../cassandra/utils/memory/HeapSlabPool.java    |  34 ---
 .../org/apache/cassandra/utils/memory/Pool.java | 232 +++++++++++--------
 .../cassandra/utils/memory/PoolAllocator.java   | 223 +++++++++++-------
 .../utils/memory/PoolCleanerThread.java         |   4 +-
 .../cassandra/utils/memory/SlabAllocator.java   | 223 ++++++++++++++++++
 .../apache/cassandra/utils/memory/SlabPool.java |  41 ++++
 test/conf/cassandra.yaml                        |   1 +
 .../cassandra/db/CollationControllerTest.java   |  13 +-
 .../apache/cassandra/db/RangeTombstoneTest.java |  32 ++-
 .../db/SecondaryIndexCellSizeTest.java          |  13 +-
 .../db/context/CounterContextTest.java          |  25 +-
 .../db/index/PerRowSecondaryIndexTest.java      |  14 +-
 49 files changed, 1047 insertions(+), 747 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d51e1bd..3ca737a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -31,6 +31,7 @@
  * Fix off-by-one error in stress (CASSANDRA-6883)
  * Make OpOrder AutoCloseable (CASSANDRA-6901)
  * Remove sync repair JMX interface (CASSANDRA-6900)
+ * Add multiple memory allocation options for memtables (CASSANDRA-6689)
 Merged from 2.0:
  * Fix race processing range scan responses (CASSANDRA-6820)
  * Allow deleting snapshots from dropped keyspaces (CASSANDRA-6821)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 41bfb40..07afd05 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -278,6 +278,12 @@ concurrent_counter_writes: 32
 # that will trigger a flush of the largest memtable.
 memtable_cleanup_threshold: 0.4
 
+# Specify the way Cassandra allocates and manages memtable memory.
+# Options are:
+#   heap_buffers:    on heap nio buffers
+#   offheap_buffers: off heap (direct) nio buffers
+memtable_allocation_type: heap_buffers
+
 # Total space to use for commitlogs.  Since commitlog segments are
 # mmapped, and hence use up address space, the default size is 32
 # on 32-bit JVMs, and 1024 on 64-bit JVMs.

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 2ce22f3..400dad5 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -92,8 +92,8 @@ public class Config
     // we don't want a lot of contention, but we also don't want to starve all other tables
     // if a big one flushes. OS buffering should be able to minimize contention with 2 threads.
     public int memtable_flush_writers = 2;
-
-    public Integer memtable_total_space_in_mb;
+    public Integer memtable_heap_space_in_mb;
+    public Integer memtable_offheap_space_in_mb;
     public float memtable_cleanup_threshold = 0.4f;
 
     public Integer storage_port = 7000;
@@ -204,7 +204,7 @@ public class Config
 
     public boolean inter_dc_tcp_nodelay = true;
 
-    public String memtable_allocator = "HeapSlabPool";
+    public MemtableAllocationType memtable_allocation_type = MemtableAllocationType.heap_buffers;
 
     private static boolean outboundBindAny = false;
 
@@ -287,6 +287,13 @@ public class Config
         standard,
     }
 
+    public static enum MemtableAllocationType
+    {
+        unslabbed_heap_buffers,
+        heap_buffers,
+        offheap_buffers
+    }
+
     public static enum DiskFailurePolicy
     {
         best_effort,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 25b05ff..1e0ac0d 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -22,7 +22,15 @@ import java.io.FileFilter;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
@@ -30,7 +38,12 @@ import com.google.common.primitives.Longs;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.auth.*;
+import org.apache.cassandra.auth.AllowAllAuthenticator;
+import org.apache.cassandra.auth.AllowAllAuthorizer;
+import org.apache.cassandra.auth.AllowAllInternodeAuthenticator;
+import org.apache.cassandra.auth.IAuthenticator;
+import org.apache.cassandra.auth.IAuthorizer;
+import org.apache.cassandra.auth.IInternodeAuthenticator;
 import org.apache.cassandra.config.Config.RequestSchedulerId;
 import org.apache.cassandra.config.EncryptionOptions.ClientEncryptionOptions;
 import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
@@ -52,7 +65,9 @@ import org.apache.cassandra.scheduler.NoScheduler;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.memory.HeapPool;
 import org.apache.cassandra.utils.memory.Pool;
+import org.apache.cassandra.utils.memory.SlabPool;
 
 public class DatabaseDescriptor
 {
@@ -95,8 +110,6 @@ public class DatabaseDescriptor
     private static String localDC;
     private static Comparator<InetAddress> localComparator;
 
-    private static Class<? extends Pool> memtablePool;
-
     static
     {
         // In client mode, we use a default configuration. Note that the fields of this class will be
@@ -260,12 +273,20 @@ public class DatabaseDescriptor
         if (conf.file_cache_size_in_mb == null)
             conf.file_cache_size_in_mb = Math.min(512, (int) (Runtime.getRuntime().maxMemory() / (4 * 1048576)));
 
-        if (conf.memtable_total_space_in_mb == null)
-            conf.memtable_total_space_in_mb = (int) (Runtime.getRuntime().maxMemory() / (4 * 1048576));
-        if (conf.memtable_total_space_in_mb <= 0)
+        if (conf.memtable_offheap_space_in_mb == null)
+            conf.memtable_offheap_space_in_mb = (int) (Runtime.getRuntime().maxMemory() / (4 * 1048576));
+        if (conf.memtable_offheap_space_in_mb < 0)
+            throw new ConfigurationException("memtable_offheap_space_in_mb must be positive");
+        // for the moment, we default to twice as much on-heap space as off-heap, as heap overhead is very large
+        if (conf.memtable_heap_space_in_mb == null)
+            conf.memtable_heap_space_in_mb = (int) (Runtime.getRuntime().maxMemory() / (4 * 1048576));
+        if (conf.memtable_heap_space_in_mb <= 0)
             throw new ConfigurationException("memtable_heap_space_in_mb must be positive");
-        logger.info("Global memtable heap threshold is enabled at {}MB", conf.memtable_total_space_in_mb);
-
+        logger.info("Global memtable on-heap threshold is enabled at {}MB", conf.memtable_heap_space_in_mb);
+        if (conf.memtable_offheap_space_in_mb == 0)
+            logger.info("Global memtable off-heap threshold is disabled, HeapAllocator will be used instead");
+        else
+            logger.info("Global memtable off-heap threshold is enabled at {}MB", conf.memtable_offheap_space_in_mb);
         if (conf.memtable_cleanup_threshold < 0.01f)
             throw new ConfigurationException("memtable_cleanup_threshold must be >= 0.01");
         if (conf.memtable_cleanup_threshold > 0.99f)
@@ -520,11 +541,6 @@ public class DatabaseDescriptor
             conf.server_encryption_options = conf.encryption_options;
         }
 
-        String allocatorPoolClass = conf.memtable_allocator;
-        if (!allocatorPoolClass.contains("."))
-            allocatorPoolClass = "org.apache.cassandra.utils.memory." + allocatorPoolClass;
-        memtablePool = FBUtilities.classForName(allocatorPoolClass, "allocator pool");
-
         // Hardcoded system keyspaces
         List<KSMetaData> systemKeyspaces = Arrays.asList(KSMetaData.systemKeyspace());
         assert systemKeyspaces.size() == Schema.systemKeyspaceNames.size();
@@ -1462,15 +1478,23 @@ public class DatabaseDescriptor
 
     public static Pool getMemtableAllocatorPool()
     {
-        try
+        long heapLimit = ((long) conf.memtable_heap_space_in_mb) << 20;
+        long offHeapLimit = ((long) conf.memtable_offheap_space_in_mb) << 20;
+        switch (conf.memtable_allocation_type)
         {
-            return memtablePool
-                   .getConstructor(long.class, float.class, Runnable.class)
-                   .newInstance(((long) conf.memtable_total_space_in_mb) << 20, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
-        }
-        catch (Exception e)
-        {
-            throw new AssertionError(e);
+            case unslabbed_heap_buffers:
+                return new HeapPool(heapLimit, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
+            case heap_buffers:
+                return new SlabPool(heapLimit, 0, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
+            case offheap_buffers:
+                if (!FileUtils.isCleanerAvailable())
+                {
+                    logger.error("Could not free direct byte buffer: offheap_buffers is not a safe memtable_allocation_type without this ability, please adjust your config. This feature is only guaranteed to work on an Oracle JVM. Refusing to start.");
+                    System.exit(-1);
+                }
+                return new SlabPool(heapLimit, offHeapLimit, conf.memtable_cleanup_threshold, new ColumnFamilyStore.FlushLargestColumnFamily());
+            default:
+                throw new AssertionError();
         }
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/cql3/ColumnCondition.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/ColumnCondition.java b/src/java/org/apache/cassandra/cql3/ColumnCondition.java
index 735388d..0c69a60 100644
--- a/src/java/org/apache/cassandra/cql3/ColumnCondition.java
+++ b/src/java/org/apache/cassandra/cql3/ColumnCondition.java
@@ -31,7 +31,6 @@ import org.apache.cassandra.db.composites.Composite;
 import org.apache.cassandra.db.filter.ColumnSlice;
 import org.apache.cassandra.db.marshal.CollectionType;
 import org.apache.cassandra.db.marshal.CounterColumnType;
-import org.apache.cassandra.db.marshal.CompositeType;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 
 /**

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java b/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java
index 1fdebd8..34bc388 100644
--- a/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java
+++ b/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java
@@ -17,7 +17,12 @@
  */
 package org.apache.cassandra.db;
 
-import java.util.*;
+import java.util.AbstractCollection;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
 
 import com.google.common.base.Function;
 import com.google.common.collect.AbstractIterator;
@@ -29,6 +34,7 @@ import org.apache.cassandra.db.composites.CellName;
 import org.apache.cassandra.db.composites.CellNameType;
 import org.apache.cassandra.db.composites.Composite;
 import org.apache.cassandra.db.filter.ColumnSlice;
+import org.apache.cassandra.utils.memory.HeapAllocator;
 
 /**
  * A ColumnFamily backed by an array.
@@ -54,19 +60,19 @@ public class ArrayBackedSortedColumns extends ColumnFamily
     {
         public ArrayBackedSortedColumns create(CFMetaData metadata, boolean insertReversed, int initialCapacity)
         {
-            return new ArrayBackedSortedColumns(metadata, insertReversed, initialCapacity);
+            return new ArrayBackedSortedColumns(metadata, insertReversed, initialCapacity == 0 ? EMPTY_ARRAY : new Cell[initialCapacity], 0, 0);
         }
     };
 
-    private ArrayBackedSortedColumns(CFMetaData metadata, boolean reversed, int initialCapacity)
+    private ArrayBackedSortedColumns(CFMetaData metadata, boolean reversed, Cell[] cells, int size, int sortedSize)
     {
         super(metadata);
         this.reversed = reversed;
         this.deletionInfo = DeletionInfo.live();
-        this.cells = initialCapacity == 0 ? EMPTY_ARRAY : new Cell[initialCapacity];
-        this.size = 0;
-        this.sortedSize = 0;
-        this.isSorted = true;
+        this.cells = cells;
+        this.size = size;
+        this.sortedSize = sortedSize;
+        this.isSorted = size == sortedSize;
     }
 
     private ArrayBackedSortedColumns(ArrayBackedSortedColumns original)
@@ -80,6 +86,20 @@ public class ArrayBackedSortedColumns extends ColumnFamily
         this.isSorted = original.isSorted;
     }
 
+    public static ColumnFamily cloneToHeap(ColumnFamily value, ColumnFamilyStore cfs)
+    {
+        if (value.getColumnCount() == 0)
+            return value;
+        // we skip anything
+        final Cell[] cells = new Cell[value.getColumnCount()];
+        int i = 0;
+        for (Cell cell : value)
+            cells[i++] = cell.localCopy(cfs, HeapAllocator.instance);
+        ColumnFamily r = new ArrayBackedSortedColumns(cfs.metadata, value.isInsertReversed(), cells, i, i);
+        r.delete(value);
+        return r;
+    }
+
     public ColumnFamily.Factory getFactory()
     {
         return factory;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/CollationController.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/CollationController.java b/src/java/org/apache/cassandra/db/CollationController.java
index 0df3619..e839ac1 100644
--- a/src/java/org/apache/cassandra/db/CollationController.java
+++ b/src/java/org/apache/cassandra/db/CollationController.java
@@ -17,7 +17,11 @@
  */
 package org.apache.cassandra.db;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.TreeSet;
 
 import com.google.common.collect.Iterables;
 
@@ -30,6 +34,7 @@ import org.apache.cassandra.db.marshal.CounterColumnType;
 import org.apache.cassandra.io.sstable.SSTableReader;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.memory.HeapAllocator;
 
 public class CollationController
 {
@@ -46,12 +51,12 @@ public class CollationController
         this.gcBefore = gcBefore;
     }
 
-    public ColumnFamily getTopLevelColumns()
+    public ColumnFamily getTopLevelColumns(boolean copyOnHeap)
     {
         return filter.filter instanceof NamesQueryFilter
                && cfs.metadata.getDefaultValidator() != CounterColumnType.instance
-               ? collectTimeOrderedData()
-               : collectAllData();
+               ? collectTimeOrderedData(copyOnHeap)
+               : collectAllData(copyOnHeap);
     }
 
     /**
@@ -59,7 +64,7 @@ public class CollationController
      * Once we have data for all requests columns that is newer than the newest remaining maxtimestamp,
      * we stop.
      */
-    private ColumnFamily collectTimeOrderedData()
+    private ColumnFamily collectTimeOrderedData(boolean copyOnHeap)
     {
         final ColumnFamily container = ArrayBackedSortedColumns.factory.create(cfs.metadata, filter.filter.isReversed());
         List<OnDiskAtomIterator> iterators = new ArrayList<>();
@@ -71,13 +76,19 @@ public class CollationController
             Tracing.trace("Merging memtable contents");
             for (Memtable memtable : view.memtables)
             {
-                OnDiskAtomIterator iter = filter.getMemtableColumnIterator(memtable);
-                if (iter != null)
+                ColumnFamily cf = memtable.getColumnFamily(filter.key);
+                if (cf != null)
                 {
+                    OnDiskAtomIterator iter = filter.getColumnFamilyIterator(cf);
                     iterators.add(iter);
                     filter.delete(container.deletionInfo(), iter.getColumnFamily());
                     while (iter.hasNext())
-                        container.addAtom(iter.next());
+                    {
+                        OnDiskAtom atom = iter.next();
+                        if (copyOnHeap)
+                            atom = ((Cell) atom).localCopy(cfs, HeapAllocator.instance);
+                        container.addAtom(atom);
+                    }
                 }
             }
 
@@ -173,7 +184,7 @@ public class CollationController
      * Collects data the brute-force way: gets an iterator for the filter in question
      * from every memtable and sstable, then merges them together.
      */
-    private ColumnFamily collectAllData()
+    private ColumnFamily collectAllData(boolean copyOnHeap)
     {
         Tracing.trace("Acquiring sstable references");
         ColumnFamilyStore.ViewFragment view = cfs.markReferenced(filter.key);
@@ -185,10 +196,21 @@ public class CollationController
             Tracing.trace("Merging memtable tombstones");
             for (Memtable memtable : view.memtables)
             {
-                OnDiskAtomIterator iter = filter.getMemtableColumnIterator(memtable);
-                if (iter != null)
+                ColumnFamily cf = memtable.getColumnFamily(filter.key);
+                if (cf != null)
                 {
-                    filter.delete(returnDeletionInfo, iter.getColumnFamily());
+                    OnDiskAtomIterator iter = filter.getColumnFamilyIterator(cf);
+                    if (copyOnHeap)
+                    {
+                        ColumnFamily newCf = cf.cloneMeShallow(ArrayBackedSortedColumns.factory, false);
+                        for (Cell cell : cf)
+                        {
+                            newCf.addColumn(cell.localCopy(cfs, HeapAllocator.instance));
+                        }
+                        cf = newCf;
+                        iter = filter.getColumnFamilyIterator(cf);
+                    }
+                    filter.delete(returnDeletionInfo, cf);
                     iterators.add(iter);
                 }
             }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 1041812..ab0f588 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -94,6 +94,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                                                                                              new LinkedBlockingQueue<Runnable>(),
                                                                                              new NamedThreadFactory("MemtablePostFlush"),
                                                                                              "internal");
+    public static final ExecutorService reclaimExecutor = new JMXEnabledThreadPoolExecutor(1, StageManager.KEEPALIVE,
+                                                                                           TimeUnit.SECONDS,
+                                                                                           new LinkedBlockingQueue<Runnable>(),
+                                                                                           new NamedThreadFactory("MemtableReclaimMemory"),
+                                                                                           "internal");
 
     public final Keyspace keyspace;
     public final String name;
@@ -112,6 +117,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
      */
     private final DataTracker data;
 
+    /* The read order, used to track accesses to off-heap memtable storage */
+    public final OpOrder readOrdering = new OpOrder();
+
     /* This is used to generate the next index for a SSTable */
     private final AtomicInteger fileIndexGenerator = new AtomicInteger(0);
 
@@ -823,9 +831,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
      */
     public ListenableFuture<?> switchMemtable()
     {
-        logger.info("Enqueuing flush of {}", name);
         synchronized (data)
         {
+            logFlush();
             Flush flush = new Flush(false);
             flushExecutor.execute(flush);
             ListenableFutureTask<?> task = ListenableFutureTask.create(flush.postFlush, null);
@@ -834,6 +842,34 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         }
     }
 
+    // print out size of all memtables we're enqueuing
+    private void logFlush()
+    {
+        // reclaiming includes that which we are GC-ing;
+        float onHeapRatio = 0, offHeapRatio = 0;
+        long onHeapTotal = 0, offHeapTotal = 0;
+        Memtable memtable = getDataTracker().getView().getCurrentMemtable();
+        onHeapRatio +=  memtable.getAllocator().onHeap().ownershipRatio();
+        offHeapRatio += memtable.getAllocator().offHeap().ownershipRatio();
+        onHeapTotal += memtable.getAllocator().onHeap().owns();
+        offHeapTotal += memtable.getAllocator().offHeap().owns();
+
+        for (SecondaryIndex index : indexManager.getIndexes())
+        {
+            if (index.getAllocator() != null)
+            {
+                onHeapRatio += index.getAllocator().onHeap().ownershipRatio();
+                offHeapRatio += index.getAllocator().offHeap().ownershipRatio();
+                onHeapTotal += index.getAllocator().onHeap().owns();
+                offHeapTotal += index.getAllocator().offHeap().owns();
+            }
+        }
+
+        logger.info("Enqueuing flush of {}: {}", name, String.format("%d (%.0f%%) on-heap, %d (%.0f%%) off-heap",
+                                                                     onHeapTotal, onHeapRatio * 100, offHeapTotal, offHeapRatio * 100));
+    }
+
+
     public ListenableFuture<?> forceFlush()
     {
         return forceFlush(null);
@@ -946,7 +982,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
     /**
      * Should only be constructed/used from switchMemtable() or truncate(), with ownership of the DataTracker monitor.
-     * In the constructor the current memtable(s) are swapped, and a barrer on outstanding writes is issued;
+     * In the constructor the current memtable(s) are swapped, and a barrier on outstanding writes is issued;
      * when run by the flushWriter the barrier is waited on to ensure all outstanding writes have completed
      * before all memtables are immediately written, and the CL is either immediately marked clean or, if
      * there are custom secondary indexes, the post flush clean up is left to update those indexes and mark
@@ -1027,7 +1063,18 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
             {
                 // flush the memtable
                 MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
-                memtable.setDiscarded();
+
+                // issue a read barrier for reclaiming the memory, and offload the wait to another thread
+                final OpOrder.Barrier readBarrier = readOrdering.newBarrier();
+                readBarrier.issue();
+                reclaimExecutor.execute(new WrappedRunnable()
+                {
+                    public void runMayThrow() throws InterruptedException, ExecutionException
+                    {
+                        readBarrier.await();
+                        memtable.setDiscarded();
+                    }
+                });
             }
 
             // signal the post-flush we've done our work
@@ -1055,27 +1102,30 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
                 // find the total ownership ratio for the memtable and all SecondaryIndexes owned by this CF,
                 // both on- and off-heap, and select the largest of the two ratios to weight this CF
-                float onHeap = 0f;
-                onHeap += current.getAllocator().ownershipRatio();
+                float onHeap = 0f, offHeap = 0f;
+                onHeap += current.getAllocator().onHeap().ownershipRatio();
+                offHeap += current.getAllocator().offHeap().ownershipRatio();
 
                 for (SecondaryIndex index : cfs.indexManager.getIndexes())
                 {
-                    if (index.getOnHeapAllocator() != null)
-                        onHeap += index.getOnHeapAllocator().ownershipRatio();
+                    if (index.getAllocator() != null)
+                    {
+                        onHeap += index.getAllocator().onHeap().ownershipRatio();
+                        offHeap += index.getAllocator().offHeap().ownershipRatio();
+                    }
                 }
 
-                if (onHeap > largestRatio)
+                float ratio = Math.max(onHeap, offHeap);
+
+                if (ratio > largestRatio)
                 {
                     largest = current;
-                    largestRatio = onHeap;
+                    largestRatio = ratio;
                 }
             }
 
             if (largest != null)
-            {
                 largest.cfs.switchMemtableIfCurrent(largest);
-                logger.info("Reclaiming {} of {} retained memtable bytes", largest.getAllocator().reclaiming(), Memtable.memoryPool.used());
-            }
         }
     }
 
@@ -1348,7 +1398,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
     public long getMemtableDataSize()
     {
-        return metric.memtableHeapSize.value();
+        return metric.memtableOnHeapSize.value();
     }
 
     public int getMemtableSwitchCount()
@@ -1840,7 +1890,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     {
         Tracing.trace("Executing single-partition query on {}", name);
         CollationController controller = new CollationController(this, filter, gcBefore);
-        ColumnFamily columns = controller.getTopLevelColumns();
+        ColumnFamily columns;
+        try (OpOrder.Group op = readOrdering.start())
+        {
+            columns = controller.getTopLevelColumns(Memtable.memoryPool.needToCopyOnHeap());
+        }
         metric.updateSSTableIterated(controller.getSstablesIterated());
         return columns;
     }
@@ -2009,7 +2063,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
     public List<Row> getRangeSlice(ExtendedFilter filter)
     {
-        return filter(getSequentialIterator(filter.dataRange, filter.timestamp), filter);
+        try (OpOrder.Group op = readOrdering.start())
+        {
+            return filter(getSequentialIterator(filter.dataRange, filter.timestamp), filter);
+        }
     }
 
     @VisibleForTesting

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index 77cfe90..6fa208c 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -35,7 +35,7 @@ public interface ColumnFamilyStoreMBean
      * Returns the total amount of data stored in the memtable, including
      * column related overhead.
      *
-     * @see org.apache.cassandra.metrics.ColumnFamilyMetrics#memtableHeapSize
+     * @see org.apache.cassandra.metrics.ColumnFamilyMetrics#memtableOnHeapSize
      * @return The size in bytes.
      * @deprecated
      */

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java
index 1308fe9..91dcf2d 100644
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@ -18,35 +18,38 @@
 package org.apache.cassandra.db;
 
 import java.io.File;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.AbstractMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 import com.google.common.base.Throwables;
-
-import org.apache.cassandra.db.commitlog.CommitLog;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.concurrent.OpOrder;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.index.SecondaryIndexManager;
-import org.apache.cassandra.dht.LongToken;
-import org.apache.cassandra.io.util.DiskAwareRunnable;
-
-import org.apache.cassandra.utils.memory.AbstractAllocator;
-import org.apache.cassandra.utils.memory.ContextAllocator;
-import org.apache.cassandra.utils.ObjectSizes;
-import org.apache.cassandra.utils.memory.Pool;
-import org.apache.cassandra.utils.memory.PoolAllocator;
-import org.apache.cassandra.service.ActiveRepairService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.commitlog.ReplayPosition;
 import org.apache.cassandra.db.composites.CellNameType;
+import org.apache.cassandra.db.index.SecondaryIndexManager;
+import org.apache.cassandra.dht.LongToken;
 import org.apache.cassandra.io.sstable.SSTableReader;
 import org.apache.cassandra.io.sstable.SSTableWriter;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.DiskAwareRunnable;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.ContextAllocator;
+import org.apache.cassandra.utils.memory.HeapAllocator;
+import org.apache.cassandra.utils.memory.Pool;
+import org.apache.cassandra.utils.memory.PoolAllocator;
 
 public class Memtable
 {
@@ -82,12 +85,12 @@ public class Memtable
     public Memtable(ColumnFamilyStore cfs)
     {
         this.cfs = cfs;
-        this.allocator = memoryPool.newAllocator(cfs.keyspace.writeOrder);
+        this.allocator = memoryPool.newAllocator();
         this.initialComparator = cfs.metadata.comparator;
         this.cfs.scheduleFlush();
     }
 
-    public AbstractAllocator getAllocator()
+    public PoolAllocator getAllocator()
     {
         return allocator;
     }
@@ -225,37 +228,44 @@ public class Memtable
 
     public String toString()
     {
-        return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%% of heap limit)",
-                             cfs.name, hashCode(), liveDataSize, currentOperations, 100 * allocator.ownershipRatio());
+        return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
+                             cfs.name, hashCode(), liveDataSize, currentOperations, 100 * allocator.onHeap().ownershipRatio(), 100 * allocator.offHeap().ownershipRatio());
     }
 
     /**
      * @param startWith Include data in the result from and including this key and to the end of the memtable
      * @return An iterator of entries with the data from the start key
      */
-    public Iterator<Map.Entry<DecoratedKey, AtomicBTreeColumns>> getEntryIterator(final RowPosition startWith, final RowPosition stopAt)
+    public Iterator<Map.Entry<DecoratedKey, ColumnFamily>> getEntryIterator(final RowPosition startWith, final RowPosition stopAt)
     {
-        return new Iterator<Map.Entry<DecoratedKey, AtomicBTreeColumns>>()
+        return new Iterator<Map.Entry<DecoratedKey, ColumnFamily>>()
         {
-            private Iterator<? extends Map.Entry<RowPosition, AtomicBTreeColumns>> iter = stopAt.isMinimum(cfs.partitioner)
+            private Iterator<? extends Map.Entry<? extends RowPosition, AtomicBTreeColumns>> iter = stopAt.isMinimum(cfs.partitioner)
                                                                                         ? rows.tailMap(startWith).entrySet().iterator()
                                                                                         : rows.subMap(startWith, true, stopAt, true).entrySet().iterator();
-            private Map.Entry<RowPosition, AtomicBTreeColumns> currentEntry;
+            private Map.Entry<? extends RowPosition, ? extends ColumnFamily> currentEntry;
 
             public boolean hasNext()
             {
                 return iter.hasNext();
             }
 
-            public Map.Entry<DecoratedKey, AtomicBTreeColumns> next()
+            public Map.Entry<DecoratedKey, ColumnFamily> next()
             {
-                Map.Entry<RowPosition, AtomicBTreeColumns> entry = iter.next();
-                // Store the reference to the current entry so that remove() can update the current size.
-                currentEntry = entry;
+                Map.Entry<? extends RowPosition, ? extends ColumnFamily> entry = iter.next();
                 // Actual stored key should be true DecoratedKey
                 assert entry.getKey() instanceof DecoratedKey;
+                if (memoryPool.needToCopyOnHeap())
+                {
+                    DecoratedKey key = (DecoratedKey) entry.getKey();
+                    key = new DecoratedKey(key.token, HeapAllocator.instance.clone(key.key));
+                    ColumnFamily columns = ArrayBackedSortedColumns.cloneToHeap(entry.getValue(), cfs);
+                    entry = new AbstractMap.SimpleImmutableEntry<>(key, columns);
+                }
+                // Store the reference to the current entry so that remove() can update the current size.
+                currentEntry = entry;
                 // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
-                return (Map.Entry<DecoratedKey, AtomicBTreeColumns>) (Object)entry;
+                return (Map.Entry<DecoratedKey, ColumnFamily>) entry;
             }
 
             public void remove()
@@ -359,6 +369,8 @@ public class Memtable
 
                 if (writer.getFilePointer() > 0)
                 {
+                    writer.isolateReferences();
+
                     // temp sstables should contain non-repaired data.
                     ssTable = writer.closeAndOpenReader();
                     logger.info(String.format("Completed flushing %s (%d bytes) for commitlog position %s",

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/RowIteratorFactory.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/RowIteratorFactory.java b/src/java/org/apache/cassandra/db/RowIteratorFactory.java
index 31a2b38..464b9d1 100644
--- a/src/java/org/apache/cassandra/db/RowIteratorFactory.java
+++ b/src/java/org/apache/cassandra/db/RowIteratorFactory.java
@@ -62,7 +62,7 @@ public class RowIteratorFactory
         // memtables
         for (Memtable memtable : memtables)
         {
-            iterators.add(new ConvertToColumnIterator<>(range, memtable.getEntryIterator(range.startKey(), range.stopKey())));
+            iterators.add(new ConvertToColumnIterator(range, memtable.getEntryIterator(range.startKey(), range.stopKey())));
         }
 
         for (SSTableReader sstable : sstables)
@@ -120,12 +120,12 @@ public class RowIteratorFactory
     /**
      * Get a ColumnIterator for a specific key in the memtable.
      */
-    private static class ConvertToColumnIterator<T extends ColumnFamily> implements CloseableIterator<OnDiskAtomIterator>
+    private static class ConvertToColumnIterator implements CloseableIterator<OnDiskAtomIterator>
     {
         private final DataRange range;
-        private final Iterator<Map.Entry<DecoratedKey, T>> iter;
+        private final Iterator<Map.Entry<DecoratedKey, ColumnFamily>> iter;
 
-        public ConvertToColumnIterator(DataRange range, Iterator<Map.Entry<DecoratedKey, T>> iter)
+        public ConvertToColumnIterator(DataRange range, Iterator<Map.Entry<DecoratedKey, ColumnFamily>> iter)
         {
             this.range = range;
             this.iter = iter;
@@ -145,7 +145,7 @@ public class RowIteratorFactory
          */
         public OnDiskAtomIterator next()
         {
-            final Map.Entry<DecoratedKey, T> entry = iter.next();
+            final Map.Entry<DecoratedKey, ColumnFamily> entry = iter.next();
             return new LazyColumnIterator(entry.getKey(), new IColumnIteratorFactory()
             {
                 public OnDiskAtomIterator create()

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
index 9436a5a..e5c9b3e 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
@@ -390,7 +390,8 @@ public class CommitLogSegment
     {
         try
         {
-            FileUtils.clean(buffer);
+            if (FileUtils.isCleanerAvailable())
+                FileUtils.clean(buffer);
             logFileAccessor.close();
         }
         catch (IOException e)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/composites/BoundedComposite.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/BoundedComposite.java b/src/java/org/apache/cassandra/db/composites/BoundedComposite.java
index d988add..5ea39a4 100644
--- a/src/java/org/apache/cassandra/db/composites/BoundedComposite.java
+++ b/src/java/org/apache/cassandra/db/composites/BoundedComposite.java
@@ -100,7 +100,7 @@ public class BoundedComposite extends AbstractComposite
     }
 
     @Override
-    public void free(PoolAllocator<?> allocator)
+    public void free(PoolAllocator allocator)
     {
         wrapped.free(allocator);
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/composites/Composite.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/Composite.java b/src/java/org/apache/cassandra/db/composites/Composite.java
index 99a0d46..9208bee 100644
--- a/src/java/org/apache/cassandra/db/composites/Composite.java
+++ b/src/java/org/apache/cassandra/db/composites/Composite.java
@@ -76,5 +76,5 @@ public interface Composite extends IMeasurableMemory
 
     public int dataSize();
     public Composite copy(AbstractAllocator allocator);
-    public void free(PoolAllocator<?> allocator);
+    public void free(PoolAllocator allocator);
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/composites/Composites.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/Composites.java b/src/java/org/apache/cassandra/db/composites/Composites.java
index 902c82a..4f05cdb 100644
--- a/src/java/org/apache/cassandra/db/composites/Composites.java
+++ b/src/java/org/apache/cassandra/db/composites/Composites.java
@@ -114,7 +114,7 @@ public abstract class Composites
         }
 
         @Override
-        public void free(PoolAllocator<?> allocator)
+        public void free(PoolAllocator allocator)
         {
         }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/composites/CompoundComposite.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/CompoundComposite.java b/src/java/org/apache/cassandra/db/composites/CompoundComposite.java
index cae1c9e..5ba4c8a 100644
--- a/src/java/org/apache/cassandra/db/composites/CompoundComposite.java
+++ b/src/java/org/apache/cassandra/db/composites/CompoundComposite.java
@@ -87,7 +87,7 @@ public class CompoundComposite extends AbstractComposite
     }
 
     @Override
-    public void free(PoolAllocator<?> allocator)
+    public void free(PoolAllocator allocator)
     {
         for (ByteBuffer element : elements)
             allocator.free(element);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java
index 0be9433..753adbf 100644
--- a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java
+++ b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellName.java
@@ -166,7 +166,7 @@ public class CompoundSparseCellName extends CompoundComposite implements CellNam
         }
 
         @Override
-        public void free(PoolAllocator<?> allocator)
+        public void free(PoolAllocator allocator)
         {
             super.free(allocator);
             allocator.free(collectionElement);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java
index 4f58e48..88f6892 100644
--- a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java
+++ b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java
@@ -93,7 +93,7 @@ public class CompoundSparseCellNameType extends AbstractCompoundCellNameType
             }
 
             @Override
-            public void free(PoolAllocator<?> allocator)
+            public void free(PoolAllocator allocator)
             {
             }
         };

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/composites/SimpleComposite.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/SimpleComposite.java b/src/java/org/apache/cassandra/db/composites/SimpleComposite.java
index 7fd6ddb..78173c7 100644
--- a/src/java/org/apache/cassandra/db/composites/SimpleComposite.java
+++ b/src/java/org/apache/cassandra/db/composites/SimpleComposite.java
@@ -78,7 +78,7 @@ public class SimpleComposite extends AbstractComposite
     }
 
     @Override
-    public void free(PoolAllocator<?> allocator)
+    public void free(PoolAllocator allocator)
     {
         allocator.free(element);
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java b/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java
index 3f9fd7d..5c9e991 100644
--- a/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java
+++ b/src/java/org/apache/cassandra/db/composites/SimpleSparseCellName.java
@@ -102,7 +102,7 @@ public class SimpleSparseCellName extends AbstractComposite implements CellName
         return new SimpleSparseCellName(columnName.clone(allocator));
     }
 
-    public void free(PoolAllocator<?> allocator)
+    public void free(PoolAllocator allocator)
     {
         allocator.free(columnName.bytes);
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java b/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java
index 6cc29dc..a0395cf 100644
--- a/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java
+++ b/src/java/org/apache/cassandra/db/composites/SimpleSparseInternedCellName.java
@@ -50,7 +50,7 @@ public class SimpleSparseInternedCellName extends SimpleSparseCellName
     }
 
     @Override
-    public void free(PoolAllocator<?> allocator)
+    public void free(PoolAllocator allocator)
     {
         // no-op, never copied
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/filter/ColumnSlice.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/ColumnSlice.java b/src/java/org/apache/cassandra/db/filter/ColumnSlice.java
index 19e18bd..262ebae 100644
--- a/src/java/org/apache/cassandra/db/filter/ColumnSlice.java
+++ b/src/java/org/apache/cassandra/db/filter/ColumnSlice.java
@@ -267,7 +267,7 @@ public class ColumnSlice
         }
 
         @Override
-        public void free(PoolAllocator<?> allocator)
+        public void free(PoolAllocator allocator)
         {
             throw new UnsupportedOperationException();
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/filter/QueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/QueryFilter.java b/src/java/org/apache/cassandra/db/filter/QueryFilter.java
index 6435043..8236912 100644
--- a/src/java/org/apache/cassandra/db/filter/QueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/QueryFilter.java
@@ -17,15 +17,23 @@
  */
 package org.apache.cassandra.db.filter;
 
-import java.util.*;
-
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.SortedSet;
+
+import org.apache.cassandra.db.Cell;
+import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionInfo;
+import org.apache.cassandra.db.OnDiskAtom;
+import org.apache.cassandra.db.RangeTombstone;
 import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
 import org.apache.cassandra.db.composites.CellName;
 import org.apache.cassandra.db.composites.Composite;
 import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.util.FileDataInput;
 import org.apache.cassandra.utils.MergeIterator;
 
 public class QueryFilter
@@ -43,14 +51,6 @@ public class QueryFilter
         this.timestamp = timestamp;
     }
 
-    public OnDiskAtomIterator getMemtableColumnIterator(Memtable memtable)
-    {
-        ColumnFamily cf = memtable.getColumnFamily(key);
-        if (cf == null)
-            return null;
-        return getColumnFamilyIterator(cf);
-    }
-
     public OnDiskAtomIterator getColumnFamilyIterator(ColumnFamily cf)
     {
         assert cf != null;
@@ -62,11 +62,6 @@ public class QueryFilter
         return filter.getSSTableColumnIterator(sstable, key);
     }
 
-    public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry)
-    {
-        return filter.getSSTableColumnIterator(sstable, file, key, indexEntry);
-    }
-
     public void collateOnDiskAtom(final ColumnFamily returnCF, List<? extends Iterator<? extends OnDiskAtom>> toCollate, final int gcBefore)
     {
         collateOnDiskAtom(returnCF, toCollate, filter, gcBefore, timestamp);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java b/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java
index 9788168..4604c16 100644
--- a/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java
+++ b/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java
@@ -20,17 +20,24 @@ package org.apache.cassandra.db.index;
 import java.nio.ByteBuffer;
 import java.util.concurrent.Future;
 
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.concurrent.OpOrder;
 import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.ArrayBackedSortedColumns;
+import org.apache.cassandra.db.Cell;
+import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ExpiringCell;
+import org.apache.cassandra.db.IndexExpression;
 import org.apache.cassandra.db.composites.CellName;
 import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.db.marshal.*;
-import org.apache.cassandra.dht.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.dht.LocalToken;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.memory.AbstractAllocator;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.PoolAllocator;
 
 /**
  * Implements a secondary index for a column family using a second column family
@@ -164,7 +171,7 @@ public abstract class AbstractSimplePerColumnSecondaryIndex extends PerColumnSec
         return indexCfs.name;
     }
 
-    public AbstractAllocator getOnHeapAllocator()
+    public PoolAllocator getAllocator()
     {
         return indexCfs.getDataTracker().getView().getCurrentMemtable().getAllocator();
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
index 7d9338a..88d8905 100644
--- a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
+++ b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
@@ -18,33 +18,40 @@
 package org.apache.cassandra.db.index;
 
 import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import java.util.concurrent.FutureTask;
 
-import org.apache.cassandra.utils.memory.AbstractAllocator;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.db.composites.*;
 import org.apache.cassandra.db.Cell;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.index.keys.KeysIndex;
+import org.apache.cassandra.db.composites.CellName;
+import org.apache.cassandra.db.composites.CellNameType;
+import org.apache.cassandra.db.composites.SimpleDenseCellNameType;
 import org.apache.cassandra.db.index.composites.CompositesIndex;
+import org.apache.cassandra.db.index.keys.KeysIndex;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.BytesType;
 import org.apache.cassandra.db.marshal.LocalByPartionerType;
-import org.apache.cassandra.dht.*;
+import org.apache.cassandra.dht.LocalToken;
+import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.ReducingKeyIterator;
 import org.apache.cassandra.io.sstable.SSTableReader;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.memory.PoolAllocator;
 
 /**
  * Abstract base class for different types of secondary indexes.
@@ -145,7 +152,7 @@ public abstract class SecondaryIndex
     /**
      * Get current amount of memory this index is consuming (in bytes)
      */
-    public abstract AbstractAllocator getOnHeapAllocator();
+    public abstract PoolAllocator getAllocator();
 
     /**
      * Allow access to the underlying column family store if there is one

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/index/composites/CompositesIndex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesIndex.java b/src/java/org/apache/cassandra/db/index/composites/CompositesIndex.java
index 52b76ea..f69f716 100644
--- a/src/java/org/apache/cassandra/db/index/composites/CompositesIndex.java
+++ b/src/java/org/apache/cassandra/db/index/composites/CompositesIndex.java
@@ -129,18 +129,14 @@ public abstract class CompositesIndex extends AbstractSimplePerColumnSecondaryIn
 
     public abstract boolean isStale(IndexedEntry entry, ColumnFamily data, long now);
 
-    public void delete(IndexedEntry entry)
+    public void delete(IndexedEntry entry, OpOrder.Group opGroup)
     {
-        // start a mini-transaction for this delete, to ensure safe memtable updates
-        try (OpOrder.Group opGroup = baseCfs.keyspace.writeOrder.start())
-        {
-            int localDeletionTime = (int) (System.currentTimeMillis() / 1000);
-            ColumnFamily cfi = ArrayBackedSortedColumns.factory.create(indexCfs.metadata);
-            cfi.addTombstone(entry.indexEntry, localDeletionTime, entry.timestamp);
-            indexCfs.apply(entry.indexValue, cfi, SecondaryIndexManager.nullUpdater, opGroup, null);
-            if (logger.isDebugEnabled())
-                logger.debug("removed index entry for cleaned-up value {}:{}", entry.indexValue, cfi);
-        }
+        int localDeletionTime = (int) (System.currentTimeMillis() / 1000);
+        ColumnFamily cfi = ArrayBackedSortedColumns.factory.create(indexCfs.metadata);
+        cfi.addTombstone(entry.indexEntry, localDeletionTime, entry.timestamp);
+        indexCfs.apply(entry.indexValue, cfi, SecondaryIndexManager.nullUpdater, opGroup, null);
+        if (logger.isDebugEnabled())
+            logger.debug("removed index entry for cleaned-up value {}:{}", entry.indexValue, cfi);
     }
 
     protected AbstractType<?> getExpressionComparator()

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java b/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java
index e09b2d1..e52ab17 100644
--- a/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java
+++ b/src/java/org/apache/cassandra/db/index/composites/CompositesSearcher.java
@@ -19,20 +19,36 @@ package org.apache.cassandra.db.index.composites;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.List;
+import java.util.Set;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.ArrayBackedSortedColumns;
+import org.apache.cassandra.db.Cell;
+import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.IndexExpression;
+import org.apache.cassandra.db.Row;
+import org.apache.cassandra.db.RowPosition;
 import org.apache.cassandra.db.composites.CellNameType;
 import org.apache.cassandra.db.composites.Composite;
 import org.apache.cassandra.db.composites.Composites;
-import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.filter.ColumnSlice;
+import org.apache.cassandra.db.filter.ExtendedFilter;
+import org.apache.cassandra.db.filter.IDiskAtomFilter;
+import org.apache.cassandra.db.filter.QueryFilter;
+import org.apache.cassandra.db.filter.SliceQueryFilter;
 import org.apache.cassandra.db.index.SecondaryIndexManager;
 import org.apache.cassandra.db.index.SecondaryIndexSearcher;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.concurrent.OpOrder;
 
 public class CompositesSearcher extends SecondaryIndexSearcher
 {
@@ -47,7 +63,14 @@ public class CompositesSearcher extends SecondaryIndexSearcher
     public List<Row> search(ExtendedFilter filter)
     {
         assert filter.getClause() != null && !filter.getClause().isEmpty();
-        return baseCfs.filter(getIndexedIterator(filter), filter);
+        final IndexExpression primary = highestSelectivityPredicate(filter.getClause());
+        final CompositesIndex index = (CompositesIndex)indexManager.getIndexForColumn(primary.column);
+        // TODO: this should perhaps not open and maintain a writeOp for the full duration, but instead only *try* to delete stale entries, without blocking if there's no room
+        // as it stands, we open a writeOp and keep it open for the duration to ensure that should this CF get flushed to make room we don't block the reclamation of any room being made
+        try (OpOrder.Group writeOp = baseCfs.keyspace.writeOrder.start(); OpOrder.Group baseOp = baseCfs.readOrdering.start(); OpOrder.Group indexOp = index.getIndexCfs().readOrdering.start())
+        {
+            return baseCfs.filter(getIndexedIterator(writeOp, filter, primary, index), filter);
+        }
     }
 
     private Composite makePrefix(CompositesIndex index, ByteBuffer key, ExtendedFilter filter, boolean isStart)
@@ -69,13 +92,11 @@ public class CompositesSearcher extends SecondaryIndexSearcher
         return isStart ? prefix.start() : prefix.end();
     }
 
-    private ColumnFamilyStore.AbstractScanIterator getIndexedIterator(final ExtendedFilter filter)
+    private ColumnFamilyStore.AbstractScanIterator getIndexedIterator(final OpOrder.Group writeOp, final ExtendedFilter filter, final IndexExpression primary, final CompositesIndex index)
     {
         // Start with the most-restrictive indexed clause, then apply remaining clauses
         // to each row matching that clause.
         // TODO: allow merge join instead of just one index + loop
-        final IndexExpression primary = highestSelectivityPredicate(filter.getClause());
-        final CompositesIndex index = (CompositesIndex)indexManager.getIndexForColumn(primary.column);
         assert index != null;
         assert index.getIndexCfs() != null;
         final DecoratedKey indexKey = index.getIndexKeyFor(primary.value);
@@ -264,7 +285,7 @@ public class CompositesSearcher extends SecondaryIndexSearcher
                         ColumnFamily newData = baseCfs.getColumnFamily(new QueryFilter(dk, baseCfs.name, dataFilter, filter.timestamp));
                         if (newData == null || index.isStale(entry, newData, filter.timestamp))
                         {
-                            index.delete(entry);
+                            index.delete(entry, writeOp);
                             continue;
                         }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/db/index/keys/KeysSearcher.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/keys/KeysSearcher.java b/src/java/org/apache/cassandra/db/index/keys/KeysSearcher.java
index af780d3..27b61e4 100644
--- a/src/java/org/apache/cassandra/db/index/keys/KeysSearcher.java
+++ b/src/java/org/apache/cassandra/db/index/keys/KeysSearcher.java
@@ -53,17 +53,22 @@ public class KeysSearcher extends SecondaryIndexSearcher
     public List<Row> search(ExtendedFilter filter)
     {
         assert filter.getClause() != null && !filter.getClause().isEmpty();
-        return baseCfs.filter(getIndexedIterator(filter), filter);
+        final IndexExpression primary = highestSelectivityPredicate(filter.getClause());
+        final SecondaryIndex index = indexManager.getIndexForColumn(primary.column);
+        // TODO: this should perhaps not open and maintain a writeOp for the full duration, but instead only *try* to delete stale entries, without blocking if there's no room
+        // as it stands, we open a writeOp and keep it open for the duration to ensure that should this CF get flushed to make room we don't block the reclamation of any room  being made
+        try (OpOrder.Group writeOp = baseCfs.keyspace.writeOrder.start(); OpOrder.Group baseOp = baseCfs.readOrdering.start(); OpOrder.Group indexOp = index.getIndexCfs().readOrdering.start())
+        {
+            return baseCfs.filter(getIndexedIterator(writeOp, filter, primary, index), filter);
+        }
     }
 
-    private ColumnFamilyStore.AbstractScanIterator getIndexedIterator(final ExtendedFilter filter)
+    private ColumnFamilyStore.AbstractScanIterator getIndexedIterator(final OpOrder.Group writeOp, final ExtendedFilter filter, final IndexExpression primary, final SecondaryIndex index)
     {
 
         // Start with the most-restrictive indexed clause, then apply remaining clauses
         // to each row matching that clause.
         // TODO: allow merge join instead of just one index + loop
-        final IndexExpression primary = highestSelectivityPredicate(filter.getClause());
-        final SecondaryIndex index = indexManager.getIndexForColumn(primary.column);
         assert index != null;
         assert index.getIndexCfs() != null;
         final DecoratedKey indexKey = index.getIndexKeyFor(primary.value);
@@ -187,10 +192,7 @@ public class KeysSearcher extends SecondaryIndexSearcher
                         {
                             // delete the index entry w/ its own timestamp
                             Cell dummyCell = new Cell(primaryColumn, indexKey.key, cell.timestamp());
-                            try (OpOrder.Group opGroup = baseCfs.keyspace.writeOrder.start())
-                            {
-                                ((PerColumnSecondaryIndex) index).delete(dk.key, dummyCell, opGroup);
-                            }
+                            ((PerColumnSecondaryIndex)index).delete(dk.key, dummyCell, writeOp);
                             continue;
                         }
                         return new Row(dk, data);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
index 173bba8..3ed129f 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
@@ -321,6 +321,16 @@ public class SSTableWriter extends SSTable
         }
     }
 
+    // we use this method to ensure any managed data we may have retained references to during the write are no
+    // longer referenced, so that we do not need to enclose the expensive call to closeAndOpenReader() in a transaction
+    public void isolateReferences()
+    {
+        // currently we only maintain references to first/last/lastWrittenKey from the data provided; all other
+        // data retention is done through copying
+        first = getMinimalKey(first);
+        last = lastWrittenKey = getMinimalKey(last);
+    }
+
     public SSTableReader closeAndOpenReader()
     {
         return closeAndOpenReader(System.currentTimeMillis());

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/io/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java
index 2f22e32..15d890c 100644
--- a/src/java/org/apache/cassandra/io/util/FileUtils.java
+++ b/src/java/org/apache/cassandra/io/util/FileUtils.java
@@ -17,12 +17,17 @@
  */
 package org.apache.cassandra.io.util;
 
-import java.io.*;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
-import java.nio.file.Files;
 import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.text.DecimalFormat;
@@ -40,6 +45,7 @@ import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.service.StorageService;
+import sun.nio.ch.DirectBuffer;
 
 public class FileUtils
 {
@@ -50,23 +56,22 @@ public class FileUtils
     private static final double TB = 1024*1024*1024*1024d;
 
     private static final DecimalFormat df = new DecimalFormat("#.##");
-
-    private static final Method cleanerMethod;
+    private static final boolean canCleanDirectBuffers;
 
     static
     {
-        Method m;
+        boolean canClean = false;
         try
         {
-            m = Class.forName("sun.nio.ch.DirectBuffer").getMethod("cleaner");
+            ByteBuffer buf = ByteBuffer.allocateDirect(1);
+            ((DirectBuffer) buf).cleaner().clean();
+            canClean = true;
         }
-        catch (Exception e)
+        catch (Throwable t)
         {
-            // Perhaps a non-sun-derived JVM - contributions welcome
-            logger.info("Cannot initialize un-mmaper.  (Are you using a non-SUN JVM?)  Compacted data files will not be removed promptly.  Consider using a SUN JVM or using standard disk access mode");
-            m = null;
+            logger.info("Cannot initialize un-mmaper.  (Are you using a non-Oracle JVM?)  Compacted data files will not be removed promptly.  Consider using an Oracle JVM or using standard disk access mode");
         }
-        cleanerMethod = m;
+        canCleanDirectBuffers = canClean;
     }
 
     public static void createHardLink(File from, File to)
@@ -268,28 +273,12 @@ public class FileUtils
 
     public static boolean isCleanerAvailable()
     {
-        return cleanerMethod != null;
+        return canCleanDirectBuffers;
     }
 
     public static void clean(MappedByteBuffer buffer)
     {
-        try
-        {
-            Object cleaner = cleanerMethod.invoke(buffer);
-            cleaner.getClass().getMethod("clean").invoke(cleaner);
-        }
-        catch (IllegalAccessException e)
-        {
-            throw new RuntimeException(e);
-        }
-        catch (InvocationTargetException e)
-        {
-            throw new RuntimeException(e);
-        }
-        catch (NoSuchMethodException e)
-        {
-            throw new RuntimeException(e);
-        }
+        ((DirectBuffer) buffer).cleaner().clean();
     }
 
     public static void createDirectory(String directory)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index 288bec9..efbf60e 100644
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@ -34,11 +34,15 @@ import org.apache.cassandra.utils.EstimatedHistogram;
 public class ColumnFamilyMetrics
 {
     /** Total amount of data stored in the memtable that resides on-heap, including column related overhead and overwritten rows. */
-    public final Gauge<Long> memtableHeapSize;
+    public final Gauge<Long> memtableOnHeapSize;
+    /** Total amount of data stored in the memtable that resides off-heap, including column related overhead and overwritten rows. */
+    public final Gauge<Long> memtableOffHeapSize;
     /** Total amount of live data stored in the memtable, excluding any data structure overhead */
     public final Gauge<Long> memtableLiveDataSize;
     /** Total amount of data stored in the memtables (2i and pending flush memtables included) that resides on-heap. */
-    public final Gauge<Long> allMemtablesHeapSize;
+    public final Gauge<Long> allMemtablesOnHeapSize;
+    /** Total amount of data stored in the memtables (2i and pending flush memtables included) that resides off-heap. */
+    public final Gauge<Long> allMemtablesOffHeapSize;
     /** Total amount of live data stored in the memtables (2i and pending flush memtables included) that resides off-heap, excluding any data structure overhead */
     public final Gauge<Long> allMemtablesLiveDataSize;
     /** Total number of columns present in the memtable. */
@@ -127,11 +131,18 @@ public class ColumnFamilyMetrics
                 return cfs.getDataTracker().getView().getCurrentMemtable().getOperations();
             }
         });
-        memtableHeapSize = Metrics.newGauge(factory.createMetricName("MemtableHeapSize"), new Gauge<Long>()
+        memtableOnHeapSize = Metrics.newGauge(factory.createMetricName("MemtableHeapSize"), new Gauge<Long>()
         {
             public Long value()
             {
-                return cfs.getDataTracker().getView().getCurrentMemtable().getAllocator().owns();
+                return cfs.getDataTracker().getView().getCurrentMemtable().getAllocator().onHeap().owns();
+            }
+        });
+        memtableOffHeapSize = Metrics.newGauge(factory.createMetricName("MemtableHeapSize"), new Gauge<Long>()
+        {
+            public Long value()
+            {
+                return cfs.getDataTracker().getView().getCurrentMemtable().getAllocator().offHeap().owns();
             }
         });
         memtableLiveDataSize = Metrics.newGauge(factory.createMetricName("MemtableLiveDataSize"), new Gauge<Long>()
@@ -141,13 +152,23 @@ public class ColumnFamilyMetrics
                 return cfs.getDataTracker().getView().getCurrentMemtable().getLiveDataSize();
             }
         });
-        allMemtablesHeapSize = Metrics.newGauge(factory.createMetricName("AllMemtablesHeapSize"), new Gauge<Long>()
+        allMemtablesOnHeapSize = Metrics.newGauge(factory.createMetricName("AllMemtablesHeapSize"), new Gauge<Long>()
+        {
+            public Long value()
+            {
+                long size = 0;
+                for (ColumnFamilyStore cfs2 : cfs.concatWithIndexes())
+                    size += cfs2.getDataTracker().getView().getCurrentMemtable().getAllocator().onHeap().owns();
+                return size;
+            }
+        });
+        allMemtablesOffHeapSize = Metrics.newGauge(factory.createMetricName("AllMemtablesHeapSize"), new Gauge<Long>()
         {
             public Long value()
             {
                 long size = 0;
                 for (ColumnFamilyStore cfs2 : cfs.concatWithIndexes())
-                    size += cfs2.getDataTracker().getView().getCurrentMemtable().getAllocator().owns();
+                    size += cfs2.getDataTracker().getView().getCurrentMemtable().getAllocator().offHeap().owns();
                 return size;
             }
         });

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
index 1bd5d4e..f20a46a 100644
--- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
+++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
@@ -513,7 +513,7 @@ public class ByteBufferUtil
     /** trims size of bytebuffer to exactly number of bytes in it, to do not hold too much memory */
     public static ByteBuffer minimalBufferFor(ByteBuffer buf)
     {
-        return buf.capacity() > buf.remaining() ? ByteBuffer.wrap(getArray(buf)) : buf;
+        return buf.capacity() > buf.remaining() || !buf.hasArray() ? ByteBuffer.wrap(getArray(buf)) : buf;
     }
 
     // Doesn't change bb position

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/utils/memory/AbstractAllocator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/memory/AbstractAllocator.java b/src/java/org/apache/cassandra/utils/memory/AbstractAllocator.java
index 8ebdf30..0735d6e 100644
--- a/src/java/org/apache/cassandra/utils/memory/AbstractAllocator.java
+++ b/src/java/org/apache/cassandra/utils/memory/AbstractAllocator.java
@@ -17,10 +17,10 @@
  */
 package org.apache.cassandra.utils.memory;
 
-import org.apache.cassandra.utils.ByteBufferUtil;
-
 import java.nio.ByteBuffer;
 
+import org.apache.cassandra.utils.ByteBufferUtil;
+
 public abstract class AbstractAllocator
 {
     /**
@@ -40,23 +40,4 @@ public abstract class AbstractAllocator
     }
 
     public abstract ByteBuffer allocate(int size);
-
-    //
-    // only really applicable to Pooled subclasses, but we provide default implementations here
-    //
-
-    public long owns()
-    {
-        return 0;
-    }
-
-    public float ownershipRatio()
-    {
-        return 0;
-    }
-
-    public long reclaiming()
-    {
-        return 0;
-    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/utils/memory/ContextAllocator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/memory/ContextAllocator.java b/src/java/org/apache/cassandra/utils/memory/ContextAllocator.java
index a09fbe2..a2b1446 100644
--- a/src/java/org/apache/cassandra/utils/memory/ContextAllocator.java
+++ b/src/java/org/apache/cassandra/utils/memory/ContextAllocator.java
@@ -20,8 +20,9 @@ package org.apache.cassandra.utils.memory;
 
 import java.nio.ByteBuffer;
 
-import com.google.common.base.*;
 import org.apache.cassandra.db.Cell;
+import com.google.common.base.Function;
+
 import org.apache.cassandra.utils.concurrent.OpOrder;
 
 /**
@@ -54,21 +55,4 @@ public final class ContextAllocator extends AbstractAllocator implements Functio
     {
         return column.localCopy(this);
     }
-
-    public long owns()
-    {
-        return allocator.owns();
-    }
-
-    @Override
-    public float ownershipRatio()
-    {
-        return allocator.ownershipRatio();
-    }
-
-    @Override
-    public long reclaiming()
-    {
-        return allocator.reclaiming();
-    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/utils/memory/HeapPool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/memory/HeapPool.java b/src/java/org/apache/cassandra/utils/memory/HeapPool.java
index 425a0b4..9d08698 100644
--- a/src/java/org/apache/cassandra/utils/memory/HeapPool.java
+++ b/src/java/org/apache/cassandra/utils/memory/HeapPool.java
@@ -18,17 +18,49 @@
  */
 package org.apache.cassandra.utils.memory;
 
+import java.nio.ByteBuffer;
+
 import org.apache.cassandra.utils.concurrent.OpOrder;
 
 public class HeapPool extends Pool
 {
     public HeapPool(long maxOnHeapMemory, float cleanupThreshold, Runnable cleaner)
     {
-        super(maxOnHeapMemory, cleanupThreshold, cleaner);
+        super(maxOnHeapMemory, 0, cleanupThreshold, cleaner);
+    }
+
+    public boolean needToCopyOnHeap()
+    {
+        return false;
+    }
+
+    public Allocator newAllocator()
+    {
+        return new Allocator(this);
     }
 
-    public HeapPoolAllocator newAllocator(OpOrder writes)
+    public final class Allocator extends PoolAllocator
     {
-        return new HeapPoolAllocator(this);
+        Allocator(HeapPool pool)
+        {
+            super(pool.onHeap.newAllocator(), pool.offHeap.newAllocator());
+        }
+
+        public ByteBuffer allocate(int size)
+        {
+            return allocate(size, null);
+        }
+
+        public ByteBuffer allocate(int size, OpOrder.Group opGroup)
+        {
+            onHeap().allocate(size, opGroup);
+            // must loop trying to acquire
+            return ByteBuffer.allocate(size);
+        }
+
+        public void free(ByteBuffer buffer)
+        {
+            onHeap().release(buffer.remaining());
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1a3b5dbc/src/java/org/apache/cassandra/utils/memory/HeapPoolAllocator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/memory/HeapPoolAllocator.java b/src/java/org/apache/cassandra/utils/memory/HeapPoolAllocator.java
deleted file mode 100644
index cf798d0..0000000
--- a/src/java/org/apache/cassandra/utils/memory/HeapPoolAllocator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.utils.memory;
-
-import org.apache.cassandra.utils.concurrent.OpOrder;
-
-import java.nio.ByteBuffer;
-
-public final class HeapPoolAllocator extends PoolAllocator
-{
-    HeapPoolAllocator(HeapPool pool)
-    {
-        super(pool);
-    }
-
-    public ByteBuffer allocate(int size)
-    {
-        return allocate(size, null);
-    }
-
-    public ByteBuffer allocate(int size, OpOrder.Group opGroup)
-    {
-        markAllocated(size, opGroup);
-        // must loop trying to acquire
-        return ByteBuffer.allocate(size);
-    }
-
-    public void free(ByteBuffer name)
-    {
-        release(name.remaining());
-    }
-}