You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2011/11/09 18:03:03 UTC
svn commit: r1199850 - in /cassandra/branches/cassandra-1.0: ./
src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/compress/
src/java/org/apache/cassandra/io/sstable/
src/java/org/apache/cassandra/tools/ test/unit/org/apache/cassandra/i...
Author: jbellis
Date: Wed Nov 9 17:03:03 2011
New Revision: 1199850
URL: http://svn.apache.org/viewvc?rev=1199850&view=rev
Log:
revert #3393 temporarily
Modified:
cassandra/branches/cassandra-1.0/CHANGES.txt
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java
cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
Modified: cassandra/branches/cassandra-1.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/CHANGES.txt?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0/CHANGES.txt Wed Nov 9 17:03:03 2011
@@ -14,9 +14,9 @@ Merged from 0.8:
* Make counter shard merging thread safe (CASSANDRA-3178)
* Revert CASSANDRA-2855
* Fix bug preventing the use of efficient cross-DC writes (CASSANDRA-3472)
- * Report compression ratio in CFSMBean (CASSANDRA-3393)
* `describe ring` command for CLI (CASSANDRA-3220)
+
1.0.2
* "defragment" rows for name-based queries under STCS (CASSANDRA-2503)
* cleanup usage of StorageService.setMode() (CASANDRA-3388)
Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Wed Nov 9 17:03:03 2011
@@ -1784,12 +1784,6 @@ public class ColumnFamilyStore implement
return data.getEstimatedColumnCountHistogram();
}
- @Override
- public double getCompressionRatio()
- {
- return data.getCompressionRatio();
- }
-
/** true if this CFS contains secondary index data */
public boolean isIndex()
{
Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java Wed Nov 9 17:03:03 2011
@@ -213,7 +213,6 @@ public interface ColumnFamilyStoreMBean
public long[] getEstimatedRowSizeHistogram();
public long[] getEstimatedColumnCountHistogram();
- public double getCompressionRatio();
/**
* Returns a list of the names of the built column indexes for current store
Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java Wed Nov 9 17:03:03 2011
@@ -364,21 +364,6 @@ public class DataTracker
return histogram;
}
- public double getCompressionRatio()
- {
- double sum = 0;
- int total = 0;
- for (SSTableReader sstable : getSSTables())
- {
- if (sstable.getCompressionRatio() != Double.MIN_VALUE)
- {
- sum += sstable.getCompressionRatio();
- total++;
- }
- }
- return total != 0 ? (double)sum/total: 0;
- }
-
public long getMinRowSize()
{
long min = 0;
Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java Wed Nov 9 17:03:03 2011
@@ -23,15 +23,14 @@ import java.io.IOException;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
-import org.apache.cassandra.io.sstable.SSTableMetadata.Collector;
import org.apache.cassandra.io.util.FileMark;
import org.apache.cassandra.io.util.SequentialWriter;
public class CompressedSequentialWriter extends SequentialWriter
{
- public static SequentialWriter open(String dataFilePath, String indexFilePath, boolean skipIOCache, CompressionParameters parameters, Collector sstableMetadataCollector) throws IOException
+ public static SequentialWriter open(String dataFilePath, String indexFilePath, boolean skipIOCache, CompressionParameters parameters) throws IOException
{
- return new CompressedSequentialWriter(new File(dataFilePath), indexFilePath, skipIOCache, parameters, sstableMetadataCollector);
+ return new CompressedSequentialWriter(new File(dataFilePath), indexFilePath, skipIOCache, parameters);
}
// holds offset in the file where current chunk should be written
@@ -50,11 +49,7 @@ public class CompressedSequentialWriter
private final Checksum checksum = new CRC32();
- private long originalSize = 0, compressedSize = 0;
-
- private Collector sstableMetadataCollector;
-
- public CompressedSequentialWriter(File file, String indexFilePath, boolean skipIOCache, CompressionParameters parameters, Collector sstableMetadataCollector) throws IOException
+ public CompressedSequentialWriter(File file, String indexFilePath, boolean skipIOCache, CompressionParameters parameters) throws IOException
{
super(file, parameters.chunkLength(), skipIOCache);
this.compressor = parameters.sstableCompressor;
@@ -65,7 +60,6 @@ public class CompressedSequentialWriter
/* Index File (-CompressionInfo.db component) and it's header */
metadataWriter = new CompressionMetadata.Writer(indexFilePath);
metadataWriter.writeHeader(parameters);
- this.sstableMetadataCollector = sstableMetadataCollector;
}
@Override
@@ -88,9 +82,6 @@ public class CompressedSequentialWriter
// compressing data with buffer re-use
int compressedLength = compressor.compress(buffer, 0, validBufferBytes, compressed, 0);
- originalSize += validBufferBytes;
- compressedSize += compressedLength;
-
// update checksum
checksum.update(buffer, 0, validBufferBytes);
@@ -188,7 +179,7 @@ public class CompressedSequentialWriter
return; // already closed
super.close();
- sstableMetadataCollector.addCompressionRatio(compressedSize, originalSize);
+
metadataWriter.finalizeHeader(current, chunkCount);
metadataWriter.close();
}
Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java Wed Nov 9 17:03:03 2011
@@ -48,7 +48,7 @@ public class Descriptor
// f (0.7.0): switched bloom filter implementations in data component
// g (0.8): tracks flushed-at context in metadata component
// h (1.0): tracks max client timestamp in metadata component
- public static final String CURRENT_VERSION = "i";
+ public static final String CURRENT_VERSION = "h";
public final File directory;
public final String version;
@@ -112,11 +112,6 @@ public class Descriptor
return version.compareTo("g") >= 0;
}
- public boolean hasCompressionRatio()
- {
- return version.compareTo("i") >= 0;
- }
-
public String filenameFor(Component component)
{
return filenameFor(component.name());
Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java Wed Nov 9 17:03:03 2011
@@ -53,7 +53,6 @@ public class SSTableMetadata
protected final EstimatedHistogram estimatedColumnCount;
protected final ReplayPosition replayPosition;
protected final long maxTimestamp;
- protected final double compressionRatio;
public static final SSTableMetadataSerializer serializer = new SSTableMetadataSerializer();
private SSTableMetadata()
@@ -64,16 +63,15 @@ public class SSTableMetadata
// when there is no max timestamp recorded, default to max long
private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts, ReplayPosition replayPosition)
{
- this(rowSizes, columnCounts, replayPosition, Long.MAX_VALUE, Double.MIN_VALUE);
+ this(rowSizes, columnCounts, replayPosition, Long.MAX_VALUE);
}
- private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts, ReplayPosition replayPosition, long maxTimestamp, double cr)
+ private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts, ReplayPosition replayPosition, long maxTimestamp)
{
this.estimatedRowSize = rowSizes;
this.estimatedColumnCount = columnCounts;
this.replayPosition = replayPosition;
this.maxTimestamp = maxTimestamp;
- this.compressionRatio = cr;
}
public static SSTableMetadata createDefaultInstance()
@@ -106,11 +104,6 @@ public class SSTableMetadata
return maxTimestamp;
}
- public double getCompressionRatio()
- {
- return compressionRatio;
- }
-
static EstimatedHistogram defaultColumnCountHistogram()
{
// EH of 114 can track a max value of 2395318855, i.e., > 2B columns
@@ -129,7 +122,6 @@ public class SSTableMetadata
protected EstimatedHistogram estimatedColumnCount;
protected ReplayPosition replayPosition;
protected long maxTimestamp;
- protected double compressionRatio;
private Collector()
{
@@ -137,7 +129,6 @@ public class SSTableMetadata
this.estimatedColumnCount = defaultColumnCountHistogram();
this.replayPosition = ReplayPosition.NONE;
this.maxTimestamp = Long.MIN_VALUE;
- this.compressionRatio = Double.MIN_VALUE;
}
public void addRowSize(long rowSize)
@@ -150,15 +141,6 @@ public class SSTableMetadata
estimatedColumnCount.add(columnCount);
}
- /**
- * Ratio is compressed/uncompressed and it is
- * if you have 1.x then compression isn't helping
- */
- public void addCompressionRatio(long compressed, long uncompressed)
- {
- compressionRatio = (double) compressed/uncompressed;
- }
-
public void updateMaxTimestamp(long potentialMax)
{
maxTimestamp = Math.max(maxTimestamp, potentialMax);
@@ -166,7 +148,7 @@ public class SSTableMetadata
public SSTableMetadata finalizeMetadata()
{
- return new SSTableMetadata(estimatedRowSize, estimatedColumnCount, replayPosition, maxTimestamp, compressionRatio);
+ return new SSTableMetadata(estimatedRowSize, estimatedColumnCount, replayPosition, maxTimestamp);
}
public Collector estimatedRowSize(EstimatedHistogram estimatedRowSize)
@@ -198,7 +180,6 @@ public class SSTableMetadata
EstimatedHistogram.serializer.serialize(sstableStats.getEstimatedColumnCount(), dos);
ReplayPosition.serializer.serialize(sstableStats.getReplayPosition(), dos);
dos.writeLong(sstableStats.getMaxTimestamp());
- dos.writeDouble(sstableStats.getCompressionRatio());
}
public SSTableMetadata deserialize(Descriptor descriptor) throws IOException
@@ -224,14 +205,8 @@ public class SSTableMetadata
ReplayPosition replayPosition = descriptor.hasReplayPosition()
? ReplayPosition.serializer.deserialize(dis)
: ReplayPosition.NONE;
- long maxTimestamp = Long.MAX_VALUE;
- double compressionRatio = Double.MIN_VALUE;
- if (descriptor.hasCompressionRatio())
- {
- maxTimestamp = dis.readLong();
- compressionRatio = dis.readDouble();
- }
- return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp, compressionRatio);
+
+ return new SSTableMetadata(rowSizes, columnCounts, replayPosition);
}
finally
{
@@ -245,7 +220,7 @@ public class SSTableMetadata
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(dis);
ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(dis);
long maxTimestamp = dis.readLong();
- return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp, Double.MIN_VALUE);
+ return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp);
}
public long serializedSize(SSTableMetadata object)
Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Wed Nov 9 17:03:03 2011
@@ -894,11 +894,6 @@ public class SSTableReader extends SSTab
return sstableMetadata.getEstimatedColumnCount();
}
- public double getCompressionRatio()
- {
- return sstableMetadata.getCompressionRatio();
- }
-
public ReplayPosition getReplayPosition()
{
return sstableMetadata.getReplayPosition();
Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Wed Nov 9 17:03:03 2011
@@ -89,8 +89,7 @@ public class SSTableWriter extends SSTab
dataFile = CompressedSequentialWriter.open(getFilename(),
descriptor.filenameFor(Component.COMPRESSION_INFO),
true,
- metadata.compressionParameters(),
- sstableMetadataCollector);
+ metadata.compressionParameters());
}
else
{
Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java Wed Nov 9 17:03:03 2011
@@ -489,8 +489,6 @@ public class NodeCmd
outs.println("\t\tCompacted row minimum size: " + cfstore.getMinRowSize());
outs.println("\t\tCompacted row maximum size: " + cfstore.getMaxRowSize());
outs.println("\t\tCompacted row mean size: " + cfstore.getMeanRowSize());
- if (cfstore.getCompressionRatio() != 0)
- outs.println("\t\tSSTable Compression Ratio: " + cfstore.getCompressionRatio());
outs.println("");
}
Modified: cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java (original)
+++ cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java Wed Nov 9 17:03:03 2011
@@ -24,7 +24,6 @@ import java.util.concurrent.Callable;
import org.junit.Test;
-import org.apache.cassandra.io.sstable.SSTableMetadata;
import org.apache.cassandra.io.util.*;
import static org.junit.Assert.assertEquals;
@@ -55,9 +54,8 @@ public class CompressedRandomAccessReade
try
{
- SSTableMetadata.Collector sstableMetadataCollector = SSTableMetadata.createCollector().replayPosition(null);
SequentialWriter writer = compressed
- ? new CompressedSequentialWriter(f, filename + ".metadata", false, new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector)
+ ? new CompressedSequentialWriter(f, filename + ".metadata", false, new CompressionParameters(SnappyCompressor.instance))
: new SequentialWriter(f, CompressionParameters.DEFAULT_CHUNK_LENGTH, false);
writer.write("The quick ".getBytes());
@@ -105,8 +103,8 @@ public class CompressedRandomAccessReade
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- SSTableMetadata.Collector sstableMetadataCollector = SSTableMetadata.createCollector().replayPosition(null);
- SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), false, new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector);
+
+ SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), false, new CompressionParameters(SnappyCompressor.instance));
writer.write(CONTENT.getBytes());
writer.close();