You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2016/03/16 09:37:51 UTC
[01/10] cassandra git commit: Fix bloom filter sizing with LCS
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.2 719caa676 -> a48b836b1
refs/heads/cassandra-3.0 a479fb0db -> b623e82c4
refs/heads/cassandra-3.5 950b1a3d2 -> faa9524a6
refs/heads/trunk 3dcbe90e0 -> f387d8824
Fix bloom filter sizing with LCS
Patch by marcuse; reviewed by Paulo Motta for CASSANDRA-11344
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a48b836b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a48b836b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a48b836b
Branch: refs/heads/cassandra-2.2
Commit: a48b836b15fe548334eb302051b34bd5554fa8f9
Parents: 719caa6
Author: Marcus Eriksson <ma...@apache.org>
Authored: Fri Mar 11 09:36:05 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Mar 16 09:17:30 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../writers/MaxSSTableSizeWriter.java | 27 ++++++++++++--------
2 files changed, 17 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a48b836b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b37ef84..a69164e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.6
+ * Fix bloom filter sizing with LCS (CASSANDRA-11344)
* (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
* Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
* Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a48b836b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index d30a612..2bae3b8 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@ -32,7 +32,6 @@ import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
public class MaxSSTableSizeWriter extends CompactionAwareWriter
{
- private final long estimatedTotalKeys;
private final long expectedWriteSize;
private final long maxSSTableSize;
private final int level;
@@ -46,10 +45,9 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
this.allSSTables = txn.originals();
this.level = level;
this.maxSSTableSize = maxSSTableSize;
- long totalSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType);
+ long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, compactionType);
expectedWriteSize = Math.min(maxSSTableSize, totalSize);
- estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
- estimatedSSTables = Math.max(1, estimatedTotalKeys / maxSSTableSize);
+ estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
@SuppressWarnings("resource")
SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
@@ -61,6 +59,19 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
sstableWriter.switchWriter(writer);
}
+ /**
+ * Gets the estimated total amount of data to write during compaction
+ */
+ private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+ {
+ long estimatedKeysBeforeCompaction = 0;
+ for (SSTableReader sstable : nonExpiredSSTables)
+ estimatedKeysBeforeCompaction += sstable.estimatedKeys();
+ estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
+ double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
+ return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
+ }
+
@Override
public boolean append(AbstractCompactedRow row)
{
@@ -80,10 +91,4 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
}
return rie != null;
}
-
- @Override
- public long estimatedKeys()
- {
- return estimatedTotalKeys;
- }
-}
\ No newline at end of file
+}
[08/10] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.5
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.5
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/faa9524a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/faa9524a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/faa9524a
Branch: refs/heads/cassandra-3.5
Commit: faa9524a6858b55cf61209505902612dc90f310c
Parents: 950b1a3 b623e82
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed Mar 16 09:23:52 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Mar 16 09:23:52 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../writers/MaxSSTableSizeWriter.java | 27 +++++++++++++-------
2 files changed, 19 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/faa9524a/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 4a060d3,13c682f..70e65e4
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -7,8 -6,10 +7,9 @@@ Merged from 3.0
* Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377)
* Fix sstableloader fail when using materialized view (CASSANDRA-11275)
Merged from 2.2:
+ * Fix bloom filter sizing with LCS (CASSANDRA-11344)
* (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
* Add missing newline at end of bin/cqlsh (CASSANDRA-11325)
- * Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297)
* Unresolved hostname leads to replace being ignored (CASSANDRA-11210)
* Only log yaml config once, at startup (CASSANDRA-11217)
* Reference leak with parallel repairs on the same table (CASSANDRA-11215)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/faa9524a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 609c898,b206498..8c3ada7
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@@ -32,7 -33,7 +33,6 @@@ import org.apache.cassandra.io.sstable.
public class MaxSSTableSizeWriter extends CompactionAwareWriter
{
- private final long estimatedTotalKeys;
- private final long expectedWriteSize;
private final long maxSSTableSize;
private final int level;
private final long estimatedSSTables;
@@@ -74,11 -63,27 +74,26 @@@
this.allSSTables = txn.originals();
this.level = level;
this.maxSSTableSize = maxSSTableSize;
- estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
- estimatedSSTables = Math.max(1, estimatedTotalKeys / maxSSTableSize);
+
+ long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, txn.opType());
- expectedWriteSize = Math.min(maxSSTableSize, totalSize);
+ estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
+ }
+
+ /**
+ * Gets the estimated total amount of data to write during compaction
+ */
+ private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+ {
+ long estimatedKeysBeforeCompaction = 0;
+ for (SSTableReader sstable : nonExpiredSSTables)
+ estimatedKeysBeforeCompaction += sstable.estimatedKeys();
+ estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
+ double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
++
+ return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
}
- @Override
- public boolean realAppend(UnfilteredRowIterator partition)
+ protected boolean realAppend(UnfilteredRowIterator partition)
{
RowIndexEntry rie = sstableWriter.append(partition);
if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize)
@@@ -103,11 -103,6 +118,5 @@@
txn);
sstableWriter.switchWriter(writer);
-
}
-
- @Override
- public long estimatedKeys()
- {
- return estimatedTotalKeys;
- }
}
[05/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b623e82c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b623e82c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b623e82c
Branch: refs/heads/cassandra-3.5
Commit: b623e82c4527aeb15f12ae975adb5d3cf864b34d
Parents: a479fb0 a48b836
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed Mar 16 09:22:44 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Mar 16 09:23:04 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../writers/MaxSSTableSizeWriter.java | 27 ++++++++++++--------
2 files changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b623e82c/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 3b5e51f,a69164e..13c682f
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,12 -1,8 +1,13 @@@
-2.2.6
+3.0.5
+ * Fix sstabledump to include missing info in debug output (CASSANDRA-11321)
+ * Establish and implement canonical bulk reading workload(s) (CASSANDRA-10331)
+ * Fix paging for IN queries on tables without clustering columns (CASSANDRA-11208)
+ * Remove recursive call from CompositesSearcher (CASSANDRA-11304)
+ * Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377)
+ * Fix sstableloader fail when using materialized view (CASSANDRA-11275)
+Merged from 2.2:
+ * Fix bloom filter sizing with LCS (CASSANDRA-11344)
* (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
- * Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
- * Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
* Add missing newline at end of bin/cqlsh (CASSANDRA-11325)
* Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297)
* Unresolved hostname leads to replace being ignored (CASSANDRA-11210)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b623e82c/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 142fe87,2bae3b8..b206498
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@@ -20,10 -20,10 +20,11 @@@ package org.apache.cassandra.db.compact
import java.util.Set;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.db.compaction.AbstractCompactedRow;
+import org.apache.cassandra.db.SerializationHeader;
+ import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@@@ -63,39 -45,50 +63,46 @@@ public class MaxSSTableSizeWriter exten
this.allSSTables = txn.originals();
this.level = level;
this.maxSSTableSize = maxSSTableSize;
- long totalSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, txn.opType());
- long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, compactionType);
++
++ long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, txn.opType());
expectedWriteSize = Math.min(maxSSTableSize, totalSize);
- estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
- estimatedSSTables = Math.max(1, estimatedTotalKeys / maxSSTableSize);
+ estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
- File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
- @SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
- estimatedTotalKeys / estimatedSSTables,
- minRepairedAt,
- cfs.metadata,
- cfs.partitioner,
- new MetadataCollector(allSSTables, cfs.metadata.comparator, level));
- sstableWriter.switchWriter(writer);
+ }
+
+ /**
+ * Gets the estimated total amount of data to write during compaction
+ */
+ private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+ {
+ long estimatedKeysBeforeCompaction = 0;
+ for (SSTableReader sstable : nonExpiredSSTables)
+ estimatedKeysBeforeCompaction += sstable.estimatedKeys();
+ estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
+ double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
+ return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
}
@Override
- public boolean append(AbstractCompactedRow row)
+ public boolean realAppend(UnfilteredRowIterator partition)
{
- RowIndexEntry rie = sstableWriter.append(row);
+ RowIndexEntry rie = sstableWriter.append(partition);
if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize)
- {
- File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
- @SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
- estimatedTotalKeys / estimatedSSTables,
- minRepairedAt,
- cfs.metadata,
- cfs.partitioner,
- new MetadataCollector(allSSTables, cfs.metadata.comparator, level));
-
- sstableWriter.switchWriter(writer);
- }
+ switchCompactionLocation(getWriteDirectory(expectedWriteSize));
return rie != null;
}
+
+ public void switchCompactionLocation(Directories.DataDirectory location)
+ {
+ @SuppressWarnings("resource")
+ SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(location))),
+ estimatedTotalKeys / estimatedSSTables,
+ minRepairedAt,
+ cfs.metadata,
+ new MetadataCollector(allSSTables, cfs.metadata.comparator, level),
+ SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
+ txn);
+
+ sstableWriter.switchWriter(writer);
+
+ }
-
- @Override
- public long estimatedKeys()
- {
- return estimatedTotalKeys;
- }
}
[04/10] cassandra git commit: Fix bloom filter sizing with LCS
Posted by ma...@apache.org.
Fix bloom filter sizing with LCS
Patch by marcuse; reviewed by Paulo Motta for CASSANDRA-11344
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a48b836b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a48b836b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a48b836b
Branch: refs/heads/trunk
Commit: a48b836b15fe548334eb302051b34bd5554fa8f9
Parents: 719caa6
Author: Marcus Eriksson <ma...@apache.org>
Authored: Fri Mar 11 09:36:05 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Mar 16 09:17:30 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../writers/MaxSSTableSizeWriter.java | 27 ++++++++++++--------
2 files changed, 17 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a48b836b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b37ef84..a69164e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.6
+ * Fix bloom filter sizing with LCS (CASSANDRA-11344)
* (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
* Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
* Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a48b836b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index d30a612..2bae3b8 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@ -32,7 +32,6 @@ import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
public class MaxSSTableSizeWriter extends CompactionAwareWriter
{
- private final long estimatedTotalKeys;
private final long expectedWriteSize;
private final long maxSSTableSize;
private final int level;
@@ -46,10 +45,9 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
this.allSSTables = txn.originals();
this.level = level;
this.maxSSTableSize = maxSSTableSize;
- long totalSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType);
+ long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, compactionType);
expectedWriteSize = Math.min(maxSSTableSize, totalSize);
- estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
- estimatedSSTables = Math.max(1, estimatedTotalKeys / maxSSTableSize);
+ estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
@SuppressWarnings("resource")
SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
@@ -61,6 +59,19 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
sstableWriter.switchWriter(writer);
}
+ /**
+ * Gets the estimated total amount of data to write during compaction
+ */
+ private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+ {
+ long estimatedKeysBeforeCompaction = 0;
+ for (SSTableReader sstable : nonExpiredSSTables)
+ estimatedKeysBeforeCompaction += sstable.estimatedKeys();
+ estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
+ double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
+ return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
+ }
+
@Override
public boolean append(AbstractCompactedRow row)
{
@@ -80,10 +91,4 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
}
return rie != null;
}
-
- @Override
- public long estimatedKeys()
- {
- return estimatedTotalKeys;
- }
-}
\ No newline at end of file
+}
[06/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b623e82c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b623e82c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b623e82c
Branch: refs/heads/cassandra-3.0
Commit: b623e82c4527aeb15f12ae975adb5d3cf864b34d
Parents: a479fb0 a48b836
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed Mar 16 09:22:44 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Mar 16 09:23:04 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../writers/MaxSSTableSizeWriter.java | 27 ++++++++++++--------
2 files changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b623e82c/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 3b5e51f,a69164e..13c682f
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,12 -1,8 +1,13 @@@
-2.2.6
+3.0.5
+ * Fix sstabledump to include missing info in debug output (CASSANDRA-11321)
+ * Establish and implement canonical bulk reading workload(s) (CASSANDRA-10331)
+ * Fix paging for IN queries on tables without clustering columns (CASSANDRA-11208)
+ * Remove recursive call from CompositesSearcher (CASSANDRA-11304)
+ * Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377)
+ * Fix sstableloader fail when using materialized view (CASSANDRA-11275)
+Merged from 2.2:
+ * Fix bloom filter sizing with LCS (CASSANDRA-11344)
* (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
- * Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
- * Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
* Add missing newline at end of bin/cqlsh (CASSANDRA-11325)
* Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297)
* Unresolved hostname leads to replace being ignored (CASSANDRA-11210)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b623e82c/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 142fe87,2bae3b8..b206498
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@@ -20,10 -20,10 +20,11 @@@ package org.apache.cassandra.db.compact
import java.util.Set;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.db.compaction.AbstractCompactedRow;
+import org.apache.cassandra.db.SerializationHeader;
+ import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@@@ -63,39 -45,50 +63,46 @@@ public class MaxSSTableSizeWriter exten
this.allSSTables = txn.originals();
this.level = level;
this.maxSSTableSize = maxSSTableSize;
- long totalSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, txn.opType());
- long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, compactionType);
++
++ long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, txn.opType());
expectedWriteSize = Math.min(maxSSTableSize, totalSize);
- estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
- estimatedSSTables = Math.max(1, estimatedTotalKeys / maxSSTableSize);
+ estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
- File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
- @SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
- estimatedTotalKeys / estimatedSSTables,
- minRepairedAt,
- cfs.metadata,
- cfs.partitioner,
- new MetadataCollector(allSSTables, cfs.metadata.comparator, level));
- sstableWriter.switchWriter(writer);
+ }
+
+ /**
+ * Gets the estimated total amount of data to write during compaction
+ */
+ private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+ {
+ long estimatedKeysBeforeCompaction = 0;
+ for (SSTableReader sstable : nonExpiredSSTables)
+ estimatedKeysBeforeCompaction += sstable.estimatedKeys();
+ estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
+ double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
+ return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
}
@Override
- public boolean append(AbstractCompactedRow row)
+ public boolean realAppend(UnfilteredRowIterator partition)
{
- RowIndexEntry rie = sstableWriter.append(row);
+ RowIndexEntry rie = sstableWriter.append(partition);
if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize)
- {
- File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
- @SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
- estimatedTotalKeys / estimatedSSTables,
- minRepairedAt,
- cfs.metadata,
- cfs.partitioner,
- new MetadataCollector(allSSTables, cfs.metadata.comparator, level));
-
- sstableWriter.switchWriter(writer);
- }
+ switchCompactionLocation(getWriteDirectory(expectedWriteSize));
return rie != null;
}
+
+ public void switchCompactionLocation(Directories.DataDirectory location)
+ {
+ @SuppressWarnings("resource")
+ SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(location))),
+ estimatedTotalKeys / estimatedSSTables,
+ minRepairedAt,
+ cfs.metadata,
+ new MetadataCollector(allSSTables, cfs.metadata.comparator, level),
+ SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
+ txn);
+
+ sstableWriter.switchWriter(writer);
+
+ }
-
- @Override
- public long estimatedKeys()
- {
- return estimatedTotalKeys;
- }
}
[10/10] cassandra git commit: Merge branch 'cassandra-3.5' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-3.5' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f387d882
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f387d882
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f387d882
Branch: refs/heads/trunk
Commit: f387d88249c2e43f651f5e7a14e0f39402c6f445
Parents: 3dcbe90 faa9524
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed Mar 16 09:24:03 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Mar 16 09:24:03 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../writers/MaxSSTableSizeWriter.java | 27 +++++++++++++-------
2 files changed, 19 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f387d882/CHANGES.txt
----------------------------------------------------------------------
[09/10] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.5
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.5
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/faa9524a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/faa9524a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/faa9524a
Branch: refs/heads/trunk
Commit: faa9524a6858b55cf61209505902612dc90f310c
Parents: 950b1a3 b623e82
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed Mar 16 09:23:52 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Mar 16 09:23:52 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../writers/MaxSSTableSizeWriter.java | 27 +++++++++++++-------
2 files changed, 19 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/faa9524a/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 4a060d3,13c682f..70e65e4
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -7,8 -6,10 +7,9 @@@ Merged from 3.0
* Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377)
* Fix sstableloader fail when using materialized view (CASSANDRA-11275)
Merged from 2.2:
+ * Fix bloom filter sizing with LCS (CASSANDRA-11344)
* (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
* Add missing newline at end of bin/cqlsh (CASSANDRA-11325)
- * Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297)
* Unresolved hostname leads to replace being ignored (CASSANDRA-11210)
* Only log yaml config once, at startup (CASSANDRA-11217)
* Reference leak with parallel repairs on the same table (CASSANDRA-11215)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/faa9524a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 609c898,b206498..8c3ada7
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@@ -32,7 -33,7 +33,6 @@@ import org.apache.cassandra.io.sstable.
public class MaxSSTableSizeWriter extends CompactionAwareWriter
{
- private final long estimatedTotalKeys;
- private final long expectedWriteSize;
private final long maxSSTableSize;
private final int level;
private final long estimatedSSTables;
@@@ -74,11 -63,27 +74,26 @@@
this.allSSTables = txn.originals();
this.level = level;
this.maxSSTableSize = maxSSTableSize;
- estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
- estimatedSSTables = Math.max(1, estimatedTotalKeys / maxSSTableSize);
+
+ long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, txn.opType());
- expectedWriteSize = Math.min(maxSSTableSize, totalSize);
+ estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
+ }
+
+ /**
+ * Gets the estimated total amount of data to write during compaction
+ */
+ private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+ {
+ long estimatedKeysBeforeCompaction = 0;
+ for (SSTableReader sstable : nonExpiredSSTables)
+ estimatedKeysBeforeCompaction += sstable.estimatedKeys();
+ estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
+ double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
++
+ return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
}
- @Override
- public boolean realAppend(UnfilteredRowIterator partition)
+ protected boolean realAppend(UnfilteredRowIterator partition)
{
RowIndexEntry rie = sstableWriter.append(partition);
if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize)
@@@ -103,11 -103,6 +118,5 @@@
txn);
sstableWriter.switchWriter(writer);
-
}
-
- @Override
- public long estimatedKeys()
- {
- return estimatedTotalKeys;
- }
}
[02/10] cassandra git commit: Fix bloom filter sizing with LCS
Posted by ma...@apache.org.
Fix bloom filter sizing with LCS
Patch by marcuse; reviewed by Paulo Motta for CASSANDRA-11344
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a48b836b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a48b836b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a48b836b
Branch: refs/heads/cassandra-3.0
Commit: a48b836b15fe548334eb302051b34bd5554fa8f9
Parents: 719caa6
Author: Marcus Eriksson <ma...@apache.org>
Authored: Fri Mar 11 09:36:05 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Mar 16 09:17:30 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../writers/MaxSSTableSizeWriter.java | 27 ++++++++++++--------
2 files changed, 17 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a48b836b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b37ef84..a69164e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.6
+ * Fix bloom filter sizing with LCS (CASSANDRA-11344)
* (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
* Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
* Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a48b836b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index d30a612..2bae3b8 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@ -32,7 +32,6 @@ import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
public class MaxSSTableSizeWriter extends CompactionAwareWriter
{
- private final long estimatedTotalKeys;
private final long expectedWriteSize;
private final long maxSSTableSize;
private final int level;
@@ -46,10 +45,9 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
this.allSSTables = txn.originals();
this.level = level;
this.maxSSTableSize = maxSSTableSize;
- long totalSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType);
+ long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, compactionType);
expectedWriteSize = Math.min(maxSSTableSize, totalSize);
- estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
- estimatedSSTables = Math.max(1, estimatedTotalKeys / maxSSTableSize);
+ estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
@SuppressWarnings("resource")
SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
@@ -61,6 +59,19 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
sstableWriter.switchWriter(writer);
}
+ /**
+ * Gets the estimated total amount of data to write during compaction
+ */
+ private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+ {
+ long estimatedKeysBeforeCompaction = 0;
+ for (SSTableReader sstable : nonExpiredSSTables)
+ estimatedKeysBeforeCompaction += sstable.estimatedKeys();
+ estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
+ double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
+ return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
+ }
+
@Override
public boolean append(AbstractCompactedRow row)
{
@@ -80,10 +91,4 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
}
return rie != null;
}
-
- @Override
- public long estimatedKeys()
- {
- return estimatedTotalKeys;
- }
-}
\ No newline at end of file
+}
[07/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b623e82c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b623e82c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b623e82c
Branch: refs/heads/trunk
Commit: b623e82c4527aeb15f12ae975adb5d3cf864b34d
Parents: a479fb0 a48b836
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed Mar 16 09:22:44 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Mar 16 09:23:04 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../writers/MaxSSTableSizeWriter.java | 27 ++++++++++++--------
2 files changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b623e82c/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 3b5e51f,a69164e..13c682f
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,12 -1,8 +1,13 @@@
-2.2.6
+3.0.5
+ * Fix sstabledump to include missing info in debug output (CASSANDRA-11321)
+ * Establish and implement canonical bulk reading workload(s) (CASSANDRA-10331)
+ * Fix paging for IN queries on tables without clustering columns (CASSANDRA-11208)
+ * Remove recursive call from CompositesSearcher (CASSANDRA-11304)
+ * Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377)
+ * Fix sstableloader fail when using materialized view (CASSANDRA-11275)
+Merged from 2.2:
+ * Fix bloom filter sizing with LCS (CASSANDRA-11344)
* (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
- * Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
- * Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
* Add missing newline at end of bin/cqlsh (CASSANDRA-11325)
* Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297)
* Unresolved hostname leads to replace being ignored (CASSANDRA-11210)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b623e82c/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 142fe87,2bae3b8..b206498
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@@ -20,10 -20,10 +20,11 @@@ package org.apache.cassandra.db.compact
import java.util.Set;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.db.compaction.AbstractCompactedRow;
+import org.apache.cassandra.db.SerializationHeader;
+ import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@@@ -63,39 -45,50 +63,46 @@@ public class MaxSSTableSizeWriter exten
this.allSSTables = txn.originals();
this.level = level;
this.maxSSTableSize = maxSSTableSize;
- long totalSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, txn.opType());
- long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, compactionType);
++
++ long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, txn.opType());
expectedWriteSize = Math.min(maxSSTableSize, totalSize);
- estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
- estimatedSSTables = Math.max(1, estimatedTotalKeys / maxSSTableSize);
+ estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
- File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
- @SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
- estimatedTotalKeys / estimatedSSTables,
- minRepairedAt,
- cfs.metadata,
- cfs.partitioner,
- new MetadataCollector(allSSTables, cfs.metadata.comparator, level));
- sstableWriter.switchWriter(writer);
+ }
+
+ /**
+ * Gets the estimated total amount of data to write during compaction
+ */
+ private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+ {
+ long estimatedKeysBeforeCompaction = 0;
+ for (SSTableReader sstable : nonExpiredSSTables)
+ estimatedKeysBeforeCompaction += sstable.estimatedKeys();
+ estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
+ double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
+ return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
}
@Override
- public boolean append(AbstractCompactedRow row)
+ public boolean realAppend(UnfilteredRowIterator partition)
{
- RowIndexEntry rie = sstableWriter.append(row);
+ RowIndexEntry rie = sstableWriter.append(partition);
if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize)
- {
- File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
- @SuppressWarnings("resource")
- SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
- estimatedTotalKeys / estimatedSSTables,
- minRepairedAt,
- cfs.metadata,
- cfs.partitioner,
- new MetadataCollector(allSSTables, cfs.metadata.comparator, level));
-
- sstableWriter.switchWriter(writer);
- }
+ switchCompactionLocation(getWriteDirectory(expectedWriteSize));
return rie != null;
}
+
+ public void switchCompactionLocation(Directories.DataDirectory location)
+ {
+ @SuppressWarnings("resource")
+ SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(location))),
+ estimatedTotalKeys / estimatedSSTables,
+ minRepairedAt,
+ cfs.metadata,
+ new MetadataCollector(allSSTables, cfs.metadata.comparator, level),
+ SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
+ txn);
+
+ sstableWriter.switchWriter(writer);
+
+ }
-
- @Override
- public long estimatedKeys()
- {
- return estimatedTotalKeys;
- }
}
[03/10] cassandra git commit: Fix bloom filter sizing with LCS
Posted by ma...@apache.org.
Fix bloom filter sizing with LCS
Patch by marcuse; reviewed by Paulo Motta for CASSANDRA-11344
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a48b836b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a48b836b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a48b836b
Branch: refs/heads/cassandra-3.5
Commit: a48b836b15fe548334eb302051b34bd5554fa8f9
Parents: 719caa6
Author: Marcus Eriksson <ma...@apache.org>
Authored: Fri Mar 11 09:36:05 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Mar 16 09:17:30 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../writers/MaxSSTableSizeWriter.java | 27 ++++++++++++--------
2 files changed, 17 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a48b836b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b37ef84..a69164e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.6
+ * Fix bloom filter sizing with LCS (CASSANDRA-11344)
* (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
* Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
* Non-obsoleting compaction operations over compressed files can impose rate limit on normal reads (CASSANDRA-11301)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a48b836b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index d30a612..2bae3b8 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@ -32,7 +32,6 @@ import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
public class MaxSSTableSizeWriter extends CompactionAwareWriter
{
- private final long estimatedTotalKeys;
private final long expectedWriteSize;
private final long maxSSTableSize;
private final int level;
@@ -46,10 +45,9 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
this.allSSTables = txn.originals();
this.level = level;
this.maxSSTableSize = maxSSTableSize;
- long totalSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType);
+ long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs, compactionType);
expectedWriteSize = Math.min(maxSSTableSize, totalSize);
- estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
- estimatedSSTables = Math.max(1, estimatedTotalKeys / maxSSTableSize);
+ estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
@SuppressWarnings("resource")
SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
@@ -61,6 +59,19 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
sstableWriter.switchWriter(writer);
}
+ /**
+ * Gets the estimated total amount of data to write during compaction
+ */
+ private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables, long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+ {
+ long estimatedKeysBeforeCompaction = 0;
+ for (SSTableReader sstable : nonExpiredSSTables)
+ estimatedKeysBeforeCompaction += sstable.estimatedKeys();
+ estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
+ double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
+ return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
+ }
+
@Override
public boolean append(AbstractCompactedRow row)
{
@@ -80,10 +91,4 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
}
return rie != null;
}
-
- @Override
- public long estimatedKeys()
- {
- return estimatedTotalKeys;
- }
-}
\ No newline at end of file
+}