You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2016/05/11 08:46:39 UTC
cassandra git commit: Estimate compressed on disk size when deciding
if sstable size limit reached
Repository: cassandra
Updated Branches:
refs/heads/trunk 2d76bd4c6 -> 0f1e838d1
Estimate compressed on disk size when deciding if sstable size limit reached
Patch by Tom Petracca; reviewed by marcuse for CASSANDRA-11623
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0f1e838d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0f1e838d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0f1e838d
Branch: refs/heads/trunk
Commit: 0f1e838d17ac90d4d44e3369309b245afa969f33
Parents: 2d76bd4
Author: Tom Petracca <tp...@palantir.com>
Authored: Mon Apr 18 16:37:00 2016 -0400
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed May 11 10:43:44 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../compaction/writers/MajorLeveledCompactionWriter.java | 7 +++----
.../db/compaction/writers/MaxSSTableSizeWriter.java | 2 +-
.../writers/SplittingSizeTieredCompactionWriter.java | 3 +--
.../io/compress/CompressedSequentialWriter.java | 11 +++++++++++
.../cassandra/io/sstable/format/SSTableWriter.java | 5 +++++
.../cassandra/io/sstable/format/big/BigTableWriter.java | 5 +++++
.../org/apache/cassandra/io/util/SequentialWriter.java | 5 +++++
8 files changed, 32 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0b70d77..a9421b6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.8
+ * Estimate compressed on disk size when deciding if sstable size limit reached (CASSANDRA-11623)
* cassandra-stress profiles should support case sensitive schemas (CASSANDRA-11546)
* Remove DatabaseDescriptor dependency from FileUtils (CASSANDRA-11578)
* Faster streaming (CASSANDRA-9766)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
index 91a18da..6cccfcb 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.db.compaction.writers;
-import java.util.List;
import java.util.Set;
import org.apache.cassandra.db.ColumnFamilyStore;
@@ -82,12 +81,12 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
@SuppressWarnings("resource")
public boolean realAppend(UnfilteredRowIterator partition)
{
- long posBefore = sstableWriter.currentWriter().getOnDiskFilePointer();
RowIndexEntry rie = sstableWriter.append(partition);
- totalWrittenInLevel += sstableWriter.currentWriter().getOnDiskFilePointer() - posBefore;
partitionsWritten++;
- if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize)
+ long totalWrittenInCurrentWriter = sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten();
+ if (totalWrittenInCurrentWriter > maxSSTableSize)
{
+ totalWrittenInLevel += totalWrittenInCurrentWriter;
if (totalWrittenInLevel > LeveledManifest.maxBytesForLevel(currentLevel, maxSSTableSize))
{
totalWrittenInLevel = 0;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 8c3ada7..864185e 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@ -96,7 +96,7 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
protected boolean realAppend(UnfilteredRowIterator partition)
{
RowIndexEntry rie = sstableWriter.append(partition);
- if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize)
+ if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > maxSSTableSize)
{
switchCompactionLocation(sstableDirectory);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
index 290e88b..46cb891 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
@@ -18,7 +18,6 @@
package org.apache.cassandra.db.compaction.writers;
import java.util.Arrays;
-import java.util.List;
import java.util.Set;
import org.slf4j.Logger;
@@ -89,7 +88,7 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter
public boolean realAppend(UnfilteredRowIterator partition)
{
RowIndexEntry rie = sstableWriter.append(partition);
- if (sstableWriter.currentWriter().getOnDiskFilePointer() > currentBytesToWrite && currentRatioIndex < ratios.length - 1) // if we underestimate how many keys we have, the last sstable might get more than we expect
+ if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > currentBytesToWrite && currentRatioIndex < ratios.length - 1) // if we underestimate how many keys we have, the last sstable might get more than we expect
{
currentRatioIndex++;
currentBytesToWrite = Math.round(totalSize * ratios[currentRatioIndex]);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
index 9bd1145..1f33d53 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
@@ -92,6 +92,17 @@ public class CompressedSequentialWriter extends SequentialWriter
}
}
+ /**
+ * Get a quick estimation on how many bytes have been written to disk
+ *
+ * It should for the most part be exactly the same as getOnDiskFilePointer()
+ */
+ @Override
+ public long getEstimatedOnDiskBytesWritten()
+ {
+ return chunkOffset;
+ }
+
@Override
public void flush()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
index 6aaf776..9f2e159 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
@@ -212,6 +212,11 @@ public abstract class SSTableWriter extends SSTable implements Transactional
public abstract long getOnDiskFilePointer();
+ public long getEstimatedOnDiskBytesWritten()
+ {
+ return getOnDiskFilePointer();
+ }
+
public abstract void resetAndTruncate();
public SSTableWriter setRepairedAt(long repairedAt)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 39dc889..8645158 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -394,6 +394,11 @@ public class BigTableWriter extends SSTableWriter
return dataFile.getOnDiskFilePointer();
}
+ public long getEstimatedOnDiskBytesWritten()
+ {
+ return dataFile.getEstimatedOnDiskBytesWritten();
+ }
+
/**
* Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed.
*/
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/util/SequentialWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/SequentialWriter.java b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
index f153b4d..45e4cfa 100644
--- a/src/java/org/apache/cassandra/io/util/SequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
@@ -276,6 +276,11 @@ public class SequentialWriter extends BufferedDataOutputStreamPlus implements Tr
return position();
}
+ public long getEstimatedOnDiskBytesWritten()
+ {
+ return getOnDiskFilePointer();
+ }
+
public long length()
{
try