You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2014/06/25 05:14:37 UTC
[2/3] git commit: clean out in_memory_compaction_limit patch by
jbellis; reviewed by ayeschenko for CASSANDRA-7428
clean out in_memory_compaction_limit
patch by jbellis; reviewed by ayeschenko for CASSANDRA-7428
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1569bd9c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1569bd9c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1569bd9c
Branch: refs/heads/trunk
Commit: 1569bd9c4b7af1099a922f16e6873cdff6bf4f07
Parents: a224e58
Author: Jonathan Ellis <jb...@apache.org>
Authored: Mon Jun 23 13:25:59 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Tue Jun 24 22:13:53 2014 -0500
----------------------------------------------------------------------
conf/cassandra.yaml | 5 -
.../org/apache/cassandra/config/Config.java | 1 -
.../cassandra/config/DatabaseDescriptor.java | 15 --
.../cassandra/db/HintedHandOffManager.java | 2 +-
.../cassandra/thrift/CassandraServer.java | 2 +-
test/conf/cassandra.yaml | 1 -
.../apache/cassandra/db/RangeTombstoneTest.java | 13 +-
.../db/compaction/CompactionsTest.java | 142 +++++++++----------
8 files changed, 69 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1569bd9c/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 102e304..69d3107 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -483,11 +483,6 @@ column_index_size_in_kb: 64
# Caution should be taken on increasing the size of this threshold as it can lead to node instability.
batch_size_warn_threshold_in_kb: 5
-# Size limit for rows being compacted in memory. Larger rows will spill
-# over to disk and use a slower two-pass compaction process. A message
-# will be logged specifying the row key.
-in_memory_compaction_limit_in_mb: 64
-
# Number of simultaneous compactions to allow, NOT including
# validation "compactions" for anti-entropy repair. Simultaneous
# compactions can help preserve read performance in a mixed read/write
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1569bd9c/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 4069b15..c5e2b76 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -131,7 +131,6 @@ public class Config
/* if the size of columns or super-columns are more than this, indexing will kick in */
public Integer column_index_size_in_kb = 64;
public Integer batch_size_warn_threshold_in_kb = 5;
- public Integer in_memory_compaction_limit_in_mb = 64;
public Integer concurrent_compactors;
public volatile Integer compaction_throughput_mb_per_sec = 16;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1569bd9c/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 54e27dd..1e1f7b1 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -447,11 +447,6 @@ public class DatabaseDescriptor
requestSchedulerId = RequestSchedulerId.keyspace;
}
- if (conf.in_memory_compaction_limit_in_mb != null && conf.in_memory_compaction_limit_in_mb <= 0)
- {
- throw new ConfigurationException("in_memory_compaction_limit_in_mb must be a positive integer");
- }
-
// if data dirs, commitlog dir, or saved caches dir are set in cassandra.yaml, use that. Otherwise,
// use -Dcassandra.storagedir (set in cassandra-env.sh) as the parent dir for data/, commitlog/, and saved_caches/
if (conf.commitlog_directory == null)
@@ -976,16 +971,6 @@ public class DatabaseDescriptor
return conf.memtable_flush_writers;
}
- public static int getInMemoryCompactionLimit()
- {
- return conf.in_memory_compaction_limit_in_mb * 1024 * 1024;
- }
-
- public static void setInMemoryCompactionLimit(int sizeInMB)
- {
- conf.in_memory_compaction_limit_in_mb = sizeInMB;
- }
-
public static int getConcurrentCompactors()
{
return conf.concurrent_compactors;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1569bd9c/src/java/org/apache/cassandra/db/HintedHandOffManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManager.java b/src/java/org/apache/cassandra/db/HintedHandOffManager.java
index 20fa0cd..76c0b99 100644
--- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java
+++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java
@@ -494,7 +494,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean
return PAGE_SIZE;
// page size of 1 does not allow actual paging b/c of >= behavior on startColumn
- return Math.max(2, Math.min(PAGE_SIZE, DatabaseDescriptor.getInMemoryCompactionLimit() / averageColumnSize));
+ return Math.max(2, Math.min(PAGE_SIZE, 4 * 1024 * 1024 / averageColumnSize));
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1569bd9c/src/java/org/apache/cassandra/thrift/CassandraServer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/CassandraServer.java b/src/java/org/apache/cassandra/thrift/CassandraServer.java
index 1a77ffa..86b5bee 100644
--- a/src/java/org/apache/cassandra/thrift/CassandraServer.java
+++ b/src/java/org/apache/cassandra/thrift/CassandraServer.java
@@ -554,7 +554,7 @@ public class CassandraServer implements Cassandra.Iface
if (cfs.getMeanColumns() > 0)
{
int averageColumnSize = (int) (cfs.getMeanRowSize() / cfs.getMeanColumns());
- pageSize = Math.min(COUNT_PAGE_SIZE, DatabaseDescriptor.getInMemoryCompactionLimit() / averageColumnSize);
+ pageSize = Math.min(COUNT_PAGE_SIZE, 4 * 1024 * 1024 / averageColumnSize);
pageSize = Math.max(2, pageSize);
logger.debug("average row column size is {}; using pageSize of {}", averageColumnSize, pageSize);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1569bd9c/test/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml
index 72a357d..26d0ee5 100644
--- a/test/conf/cassandra.yaml
+++ b/test/conf/cassandra.yaml
@@ -4,7 +4,6 @@
#
cluster_name: Test Cluster
memtable_allocation_type: offheap_objects
-in_memory_compaction_limit_in_mb: 1
commitlog_sync: batch
commitlog_sync_batch_window_in_ms: 1.0
partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1569bd9c/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
index ace0ae3..3314696 100644
--- a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
+++ b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
@@ -312,22 +312,13 @@ public class RangeTombstoneTest extends SchemaLoader
}
@Test
- public void testPreCompactedRowWithRangeTombstonesUpdatesSecondaryIndex() throws Exception
+ public void testRowWithRangeTombstonesUpdatesSecondaryIndex() throws Exception
{
- // nothing special to do here, just run the test
runCompactionWithRangeTombstoneAndCheckSecondaryIndex();
}
@Test
- public void testLazilyCompactedRowWithRangeTombstonesUpdatesSecondaryIndex() throws Exception
- {
- // make sure we use LazilyCompactedRow by exceeding in_memory_compaction_limit
- DatabaseDescriptor.setInMemoryCompactionLimit(0);
- runCompactionWithRangeTombstoneAndCheckSecondaryIndex();
- }
-
- @Test
- public void testLazilyCompactedRowGeneratesSameSSTablesAsPreCompactedRow() throws Exception
+ public void testRangeTombstoneCompaction() throws Exception
{
Keyspace table = Keyspace.open(KSNAME);
ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1569bd9c/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index c60b650..979b079 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -352,93 +352,81 @@ public class CompactionsTest extends SchemaLoader
@Test
public void testRangeTombstones() throws IOException, ExecutionException, InterruptedException
{
- boolean lazy = false;
+ Keyspace keyspace = Keyspace.open(KEYSPACE1);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");
+ cfs.clearUnsafe();
- do
- {
- Keyspace keyspace = Keyspace.open(KEYSPACE1);
- ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");
- cfs.clearUnsafe();
+ // disable compaction while flushing
+ cfs.disableAutoCompaction();
- // disable compaction while flushing
- cfs.disableAutoCompaction();
+ final CFMetaData cfmeta = cfs.metadata;
+ Directories dir = cfs.directories;
- final CFMetaData cfmeta = cfs.metadata;
- Directories dir = cfs.directories;
+ ArrayList<DecoratedKey> keys = new ArrayList<DecoratedKey>();
- ArrayList<DecoratedKey> keys = new ArrayList<DecoratedKey>();
+ for (int i=0; i < 4; i++)
+ {
+ keys.add(Util.dk(""+i));
+ }
- for (int i=0; i < 4; i++)
- {
- keys.add(Util.dk(""+i));
- }
+ ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfmeta);
+ cf.addColumn(Util.column("01", "a", 1)); // this must not resurrect
+ cf.addColumn(Util.column("a", "a", 3));
+ cf.deletionInfo().add(new RangeTombstone(Util.cellname("0"), Util.cellname("b"), 2, (int) (System.currentTimeMillis()/1000)),cfmeta.comparator);
+
+ SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()),
+ 0,
+ 0,
+ cfs.metadata,
+ StorageService.getPartitioner(),
+ new MetadataCollector(cfs.metadata.comparator));
+
+
+ writer.append(Util.dk("0"), cf);
+ writer.append(Util.dk("1"), cf);
+ writer.append(Util.dk("3"), cf);
+
+ cfs.addSSTable(writer.closeAndOpenReader());
+ writer = new SSTableWriter(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()),
+ 0,
+ 0,
+ cfs.metadata,
+ StorageService.getPartitioner(),
+ new MetadataCollector(cfs.metadata.comparator));
+
+ writer.append(Util.dk("0"), cf);
+ writer.append(Util.dk("1"), cf);
+ writer.append(Util.dk("2"), cf);
+ writer.append(Util.dk("3"), cf);
+ cfs.addSSTable(writer.closeAndOpenReader());
- ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfmeta);
- cf.addColumn(Util.column("01", "a", 1)); // this must not resurrect
- cf.addColumn(Util.column("a", "a", 3));
- cf.deletionInfo().add(new RangeTombstone(Util.cellname("0"), Util.cellname("b"), 2, (int) (System.currentTimeMillis()/1000)),cfmeta.comparator);
-
- SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()),
- 0,
- 0,
- cfs.metadata,
- StorageService.getPartitioner(),
- new MetadataCollector(cfs.metadata.comparator));
-
-
- writer.append(Util.dk("0"), cf);
- writer.append(Util.dk("1"), cf);
- writer.append(Util.dk("3"), cf);
-
- cfs.addSSTable(writer.closeAndOpenReader());
- writer = new SSTableWriter(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()),
- 0,
- 0,
- cfs.metadata,
- StorageService.getPartitioner(),
- new MetadataCollector(cfs.metadata.comparator));
-
- writer.append(Util.dk("0"), cf);
- writer.append(Util.dk("1"), cf);
- writer.append(Util.dk("2"), cf);
- writer.append(Util.dk("3"), cf);
- cfs.addSSTable(writer.closeAndOpenReader());
-
- Collection<SSTableReader> toCompact = cfs.getSSTables();
- assert toCompact.size() == 2;
-
- // forcing lazy comapction
- if (lazy)
- DatabaseDescriptor.setInMemoryCompactionLimit(0);
-
- // Force compaction on first sstables. Since each row is in only one sstable, we will be using EchoedRow.
- Util.compact(cfs, toCompact);
- assertEquals(1, cfs.getSSTables().size());
-
- // Now assert we do have the 4 keys
- assertEquals(4, Util.getRangeSlice(cfs).size());
-
- ArrayList<DecoratedKey> k = new ArrayList<DecoratedKey>();
- for (Row r : Util.getRangeSlice(cfs))
- {
- k.add(r.key);
- assertEquals(ByteBufferUtil.bytes("a"),r.cf.getColumn(Util.cellname("a")).value());
- assertNull(r.cf.getColumn(Util.cellname("01")));
- assertEquals(3,r.cf.getColumn(Util.cellname("a")).timestamp());
- }
+ Collection<SSTableReader> toCompact = cfs.getSSTables();
+ assert toCompact.size() == 2;
- for (SSTableReader sstable : cfs.getSSTables())
- {
- StatsMetadata stats = sstable.getSSTableMetadata();
- assertEquals(ByteBufferUtil.bytes("0"), stats.minColumnNames.get(0));
- assertEquals(ByteBufferUtil.bytes("b"), stats.maxColumnNames.get(0));
- }
+ // Force compaction on first sstables. Since each row is in only one sstable, we will be using EchoedRow.
+ Util.compact(cfs, toCompact);
+ assertEquals(1, cfs.getSSTables().size());
+
+ // Now assert we do have the 4 keys
+ assertEquals(4, Util.getRangeSlice(cfs).size());
- assertEquals(keys, k);
+ ArrayList<DecoratedKey> k = new ArrayList<DecoratedKey>();
+ for (Row r : Util.getRangeSlice(cfs))
+ {
+ k.add(r.key);
+ assertEquals(ByteBufferUtil.bytes("a"),r.cf.getColumn(Util.cellname("a")).value());
+ assertNull(r.cf.getColumn(Util.cellname("01")));
+ assertEquals(3,r.cf.getColumn(Util.cellname("a")).timestamp());
+ }
- lazy=!lazy;
+ for (SSTableReader sstable : cfs.getSSTables())
+ {
+ StatsMetadata stats = sstable.getSSTableMetadata();
+ assertEquals(ByteBufferUtil.bytes("0"), stats.minColumnNames.get(0));
+ assertEquals(ByteBufferUtil.bytes("b"), stats.maxColumnNames.get(0));
}
- while (lazy);
+
+ assertEquals(keys, k);
}
@Test