You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2011/11/03 15:20:56 UTC
svn commit: r1197131 - in /cassandra/trunk: ./ contrib/
interface/thrift/gen-java/org/apache/cassandra/thrift/
src/java/org/apache/cassandra/io/compress/
src/java/org/apache/cassandra/io/sstable/
src/java/org/apache/cassandra/io/util/ src/java/org/apac...
Author: slebresne
Date: Thu Nov 3 14:20:56 2011
New Revision: 1197131
URL: http://svn.apache.org/viewvc?rev=1197131&view=rev
Log:
merge from 1.0
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/contrib/ (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed)
cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java
cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/SegmentedFile.java
cassandra/trunk/src/java/org/apache/cassandra/streaming/FileStreamTask.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 3 14:20:56 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1196956
/cassandra/branches/cassandra-0.8.0:1125021-1130369
/cassandra/branches/cassandra-0.8.1:1101014-1125018
-/cassandra/branches/cassandra-1.0:1167085-1196958,1197123
+/cassandra/branches/cassandra-1.0:1167085-1196958,1197123,1197130
/cassandra/branches/cassandra-1.0.0:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1:1102511-1125020
Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1197131&r1=1197130&r2=1197131&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Thu Nov 3 14:20:56 2011
@@ -7,7 +7,7 @@
1.0.2
* "defragment" rows for name-based queries under STCS (CASSANDRA-2503)
* Add timing information to cassandra-cli GET/SET/LIST queries (CASSANDRA-3326)
- * Cache for CompressionMetadata objects (CASSANDRA-3427)
+ * Only create one CompressionMetadata object per sstable (CASSANDRA-3427)
* cleanup usage of StorageService.setMode() (CASANDRA-3388)
* synchronize BiMap of bootstrapping tokens (CASSANDRA-3417)
* Avoid large array allocation for compressed chunk offsets (CASSANDRA-3432)
Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 3 14:20:56 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1196956
/cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
/cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
-/cassandra/branches/cassandra-1.0/contrib:1167085-1196958,1197123
+/cassandra/branches/cassandra-1.0/contrib:1167085-1196958,1197123,1197130
/cassandra/branches/cassandra-1.0.0/contrib:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/contrib:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 3 14:20:56 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1196956
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1196958,1197123
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1196958,1197123,1197130
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 3 14:20:56 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1196956
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1196958,1197123
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1196958,1197123,1197130
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 3 14:20:56 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1196956
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1196958,1197123
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1196958,1197123,1197130
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 3 14:20:56 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1196956
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1196958,1197123
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1196958,1197123,1197130
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 3 14:20:56 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1196956
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1196958,1197123
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1196958,1197123,1197130
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1102511-1125020
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java?rev=1197131&r1=1197130&r2=1197131&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java Thu Nov 3 14:20:56 2011
@@ -34,11 +34,6 @@ public class CompressedRandomAccessReade
{
private static final Logger logger = LoggerFactory.getLogger(CompressedRandomAccessReader.class);
- public static RandomAccessReader open(String dataFilePath, boolean skipIOCache) throws IOException
- {
- return open(dataFilePath, CompressionMetadata.get(dataFilePath), skipIOCache);
- }
-
public static RandomAccessReader open(String dataFilePath, CompressionMetadata metadata) throws IOException
{
return open(dataFilePath, metadata, false);
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java?rev=1197131&r1=1197130&r2=1197131&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java Thu Nov 3 14:20:56 2011
@@ -21,7 +21,6 @@ package org.apache.cassandra.io.compress
import java.io.*;
import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.io.sstable.Component;
@@ -41,43 +40,23 @@ public class CompressionMetadata
public final CompressionParameters parameters;
/**
- * Caches instances of CompressionMetadata.
- * Each metada holds the chunk offsets index, which is reasonably big for
- * enough data, so it's an expensive structure. We thus only want one
- * CompressionMetadata created for each sstable.
- * Note that we could have a compressionMetadata field in SSTableReader,
- * but CompressedSegmentFile.Builder needs it before the reader is
- * created, so it's easier that way.
- */
- private final static Map<String, CompressionMetadata> cache = new ConcurrentHashMap<String, CompressionMetadata>();
-
- /**
- * Get metadata about given compressed file including uncompressed data length, chunk size
+ * Create metadata about given compressed file including uncompressed data length, chunk size
* and list of the chunk offsets of the compressed data.
*
+ * This is an expensive operation! Don't create more than one for each
+ * sstable.
+ *
* @param dataFilePath Path to the compressed file
*
* @return metadata about given compressed file.
*/
- public static CompressionMetadata get(String dataFilePath)
+ public static CompressionMetadata create(String dataFilePath)
{
- CompressionMetadata metadata = cache.get(dataFilePath);
- if (metadata != null)
- return metadata;
-
- // We want this to be relatively fast, because it's called often (for each
- // range query). On the side, we don't care too much if the initial
- // creation is no thread-safe, because we'll call this when the
- // SSTableReader is loaded/created, so we're pretty sure there won't
- // be any contention. Besides, if we really do create two
- // CompressionMetadata, it's not the end of the world, so we don't
- // bother with synchronization
Descriptor desc = Descriptor.fromFilename(dataFilePath);
+
try
{
- metadata = new CompressionMetadata(desc.filenameFor(Component.COMPRESSION_INFO), new File(dataFilePath).length());
- cache.put(dataFilePath, metadata);
- return metadata;
+ return new CompressionMetadata(desc.filenameFor(Component.COMPRESSION_INFO), new File(dataFilePath).length());
}
catch (IOException e)
{
@@ -85,7 +64,7 @@ public class CompressionMetadata
}
}
- // This is package protected because of the tests. Don't use, use get() instead.
+ // This is package protected because of the tests.
CompressionMetadata(String indexFilePath, long compressedLength) throws IOException
{
this.indexFilePath = indexFilePath;
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1197131&r1=1197130&r2=1197131&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Thu Nov 3 14:20:56 2011
@@ -44,6 +44,7 @@ import org.apache.cassandra.db.filter.Qu
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.io.compress.CompressionMetadata;
import org.apache.cassandra.io.util.*;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.*;
@@ -312,7 +313,7 @@ public class SSTableReader extends SSTab
{
boolean cacheLoading = keyCache != null && !keysToLoadInCache.isEmpty();
SegmentedFile.Builder ibuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
- SegmentedFile.Builder dbuilder = (components.contains(Component.COMPRESSION_INFO))
+ SegmentedFile.Builder dbuilder = compression
? SegmentedFile.getCompressedBuilder()
: SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
@@ -400,6 +401,18 @@ public class SSTableReader extends SSTab
}
/**
+ * Returns the compression metadata for this sstable.
+ * @throws IllegalStateException if the sstable is not compressed
+ */
+ public CompressionMetadata getCompressionMetadata()
+ {
+ if (!compression)
+ throw new IllegalStateException(this + " is not compressed");
+
+ return ((CompressedSegmentedFile)dfile).metadata;
+ }
+
+ /**
* For testing purposes only.
*/
public void forceFilterFailures()
@@ -899,7 +912,7 @@ public class SSTableReader extends SSTab
public RandomAccessReader openDataReader(boolean skipIOCache) throws IOException
{
return compression
- ? CompressedRandomAccessReader.open(getFilename(), skipIOCache)
+ ? CompressedRandomAccessReader.open(getFilename(), getCompressionMetadata(), skipIOCache)
: RandomAccessReader.open(new File(getFilename()), skipIOCache);
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1197131&r1=1197130&r2=1197131&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java Thu Nov 3 14:20:56 2011
@@ -26,7 +26,7 @@ import org.apache.cassandra.io.compress.
public class CompressedSegmentedFile extends SegmentedFile
{
- private final CompressionMetadata metadata;
+ public final CompressionMetadata metadata;
public CompressedSegmentedFile(String path, CompressionMetadata metadata)
{
@@ -52,7 +52,7 @@ public class CompressedSegmentedFile ext
*/
public SegmentedFile complete(String path)
{
- return new CompressedSegmentedFile(path, CompressionMetadata.get(path));
+ return new CompressedSegmentedFile(path, CompressionMetadata.create(path));
}
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/util/SegmentedFile.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1197131&r1=1197130&r2=1197131&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/util/SegmentedFile.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/util/SegmentedFile.java Thu Nov 3 14:20:56 2011
@@ -26,6 +26,7 @@ import java.util.Iterator;
import java.util.NoSuchElementException;
import org.apache.cassandra.config.Config;
+import org.apache.cassandra.io.compress.CompressionMetadata;
import org.apache.cassandra.utils.Pair;
/**
Modified: cassandra/trunk/src/java/org/apache/cassandra/streaming/FileStreamTask.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/streaming/FileStreamTask.java?rev=1197131&r1=1197130&r2=1197131&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/streaming/FileStreamTask.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/streaming/FileStreamTask.java Thu Nov 3 14:20:56 2011
@@ -121,7 +121,7 @@ public class FileStreamTask extends Wrap
// TODO just use a raw RandomAccessFile since we're managing our own buffer here
RandomAccessReader file = (header.file.sstable.compression) // try to skip kernel page cache if possible
- ? CompressedRandomAccessReader.open(header.file.getFilename(), true)
+ ? CompressedRandomAccessReader.open(header.file.getFilename(), header.file.sstable.getCompressionMetadata(), true)
: RandomAccessReader.open(new File(header.file.getFilename()), true);
// setting up data compression stream