You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@cassandra.apache.org by Norman Maurer <no...@googlemail.com> on 2011/10/10 16:03:34 UTC
Re: svn commit: r1180970 - in /cassandra/branches/cassandra-1.0.0: ./
src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/
src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/io/util/
Hi there,
did you alter the snappy-java version by mistake or was this part of
the commit ?
Bye,
Norman
2011/10/10 <sl...@apache.org>:
> Author: slebresne
> Date: Mon Oct 10 13:56:27 2011
> New Revision: 1180970
>
> URL: http://svn.apache.org/viewvc?rev=1180970&view=rev
> Log:
> Fix places where uncompressed sstable size is used in place of the compressed one.
> patch by slebresne; reviewed by jbellis for CASSANDRA-3338
>
> Modified:
> cassandra/branches/cassandra-1.0.0/CHANGES.txt
> cassandra/branches/cassandra-1.0.0/build.xml
> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>
> Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original)
> +++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Mon Oct 10 13:56:27 2011
> @@ -12,6 +12,8 @@
> * run compaction and hinted handoff threads at MIN_PRIORITY (CASSANDRA-3308)
> * default hsha thrift server to cpu core count in rpc pool (CASSANDRA-3329)
> * add bin\daemon to binary tarball for Windows service (CASSANDRA-3331)
> + * Fix places where uncompressed size of sstables was use in place of the
> + compressed one (CASSANDRA-3338)
> Fixes merged from 0.8 below:
> * Fix tool .bat files when CASSANDRA_HOME contains spaces (CASSANDRA-3258)
> * Force flush of status table when removing/updating token (CASSANDRA-3243)
>
> Modified: cassandra/branches/cassandra-1.0.0/build.xml
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/build.xml?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/build.xml (original)
> +++ cassandra/branches/cassandra-1.0.0/build.xml Mon Oct 10 13:56:27 2011
> @@ -350,7 +350,7 @@ url=${svn.entry.url}?pathrev=${svn.entry
> <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt"/>
> <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}" url="${scm.url}"/>
> <dependencyManagement>
> - <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3.3"/>
> + <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3"/>
> <dependency groupId="com.ning" artifactId="compress-lzf" version="0.8.4"/>
> <dependency groupId="com.google.guava" artifactId="guava" version="r08"/>
> <dependency groupId="commons-cli" artifactId="commons-cli" version="1.1"/>
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon Oct 10 13:56:27 2011
> @@ -915,7 +915,7 @@ public class ColumnFamilyStore implement
> long expectedFileSize = 0;
> for (SSTableReader sstable : sstables)
> {
> - long size = sstable.length();
> + long size = sstable.onDiskLength();
> expectedFileSize = expectedFileSize + size;
> }
> return expectedFileSize;
> @@ -930,9 +930,9 @@ public class ColumnFamilyStore implement
> SSTableReader maxFile = null;
> for (SSTableReader sstable : sstables)
> {
> - if (sstable.length() > maxSize)
> + if (sstable.onDiskLength() > maxSize)
> {
> - maxSize = sstable.length();
> + maxSize = sstable.onDiskLength();
> maxFile = sstable;
> }
> }
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java (original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java Mon Oct 10 13:56:27 2011
> @@ -472,7 +472,7 @@ public class CompactionManager implement
> boolean isCommutative = cfs.metadata.getDefaultValidator().isCommutative();
>
> // Calculate the expected compacted filesize
> - String compactionFileLocation = cfs.table.getDataFileLocation(sstable.length());
> + String compactionFileLocation = cfs.table.getDataFileLocation(sstable.onDiskLength());
> if (compactionFileLocation == null)
> throw new IOException("disk full");
> int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(),
> @@ -765,8 +765,8 @@ public class CompactionManager implement
>
> String format = "Cleaned up to %s. %,d to %,d (~%d%% of original) bytes for %,d keys. Time: %,dms.";
> long dTime = System.currentTimeMillis() - startTime;
> - long startsize = sstable.length();
> - long endsize = newSstable.length();
> + long startsize = sstable.onDiskLength();
> + long endsize = newSstable.onDiskLength();
> double ratio = (double)endsize / (double)startsize;
> logger.info(String.format(format, writer.getFilename(), startsize, endsize, (int)(ratio*100), totalkeysWritten, dTime));
> }
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java (original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java Mon Oct 10 13:56:27 2011
> @@ -100,7 +100,7 @@ public class SizeTieredCompactionStrateg
> {
> List<Pair<SSTableReader, Long>> tableLengthPairs = new ArrayList<Pair<SSTableReader, Long>>();
> for(SSTableReader table: collection)
> - tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.length()));
> + tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.onDiskLength()));
> return tableLengthPairs;
> }
>
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java (original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java Mon Oct 10 13:56:27 2011
> @@ -257,7 +257,7 @@ public abstract class SSTable
> long sum = 0;
> for (SSTableReader sstable : sstables)
> {
> - sum += sstable.length();
> + sum += sstable.onDiskLength();
> }
> return sum;
> }
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Mon Oct 10 13:56:27 2011
> @@ -555,7 +555,7 @@ public class SSTableReader extends SSTab
> long right = getPosition(new DecoratedKey(range.right, null), Operator.GT);
> if (right == -1 || Range.isWrapAround(range.left, range.right))
> // right is past the end of the file, or it wraps
> - right = length();
> + right = uncompressedLength();
> if (left == right)
> // empty range
> continue;
> @@ -669,13 +669,25 @@ public class SSTableReader extends SSTab
> }
>
> /**
> - * @return The length in bytes of the data file for this SSTable.
> + * @return The length in bytes of the data for this SSTable. For
> + * compressed files, this is not the same thing as the on disk size (see
> + * onDiskLength())
> */
> - public long length()
> + public long uncompressedLength()
> {
> return dfile.length;
> }
>
> + /**
> + * @return The length in bytes of the on disk size for this SSTable. For
> + * compressed files, this is not the same thing as the data length (see
> + * length())
> + */
> + public long onDiskLength()
> + {
> + return dfile.onDiskLength;
> + }
> +
> public boolean acquireReference()
> {
> while (true)
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java (original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java Mon Oct 10 13:56:27 2011
> @@ -30,7 +30,7 @@ public class CompressedSegmentedFile ext
>
> public CompressedSegmentedFile(String path, CompressionMetadata metadata)
> {
> - super(path, metadata.dataLength);
> + super(path, metadata.dataLength, metadata.compressedFileLength);
> this.metadata = metadata;
> }
>
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java (original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java Mon Oct 10 13:56:27 2011
> @@ -42,13 +42,23 @@ public abstract class SegmentedFile
> public final String path;
> public final long length;
>
> + // This differs from length for compressed files (but we still need length for
> + // SegmentIterator because offsets in the file are relative to the uncompressed size)
> + public final long onDiskLength;
> +
> /**
> * Use getBuilder to get a Builder to construct a SegmentedFile.
> */
> SegmentedFile(String path, long length)
> {
> + this(path, length, length);
> + }
> +
> + protected SegmentedFile(String path, long length, long onDiskLength)
> + {
> this.path = path;
> this.length = length;
> + this.onDiskLength = onDiskLength;
> }
>
> /**
>
>
>
Re: svn commit: r1180970 - in /cassandra/branches/cassandra-1.0.0: ./
src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/
src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/io/util/
Posted by Stephen Connolly <st...@gmail.com>.
Yeah it was my bad... the patch for the build versions included an extra .3...
when reviewing the pom for the 1.0.0 vote I caught the mis-match...
pinged Sylvain on IRC...
should probably have raised a JIRA...
but the issue could at best affect only windows users running
cassandra embedded in their build via maven...
not even sure it would affect them given the changes between 1.0.3 and 1.0.3.3
but safer to use the _exact_ same version
-Stephen
On 10 October 2011 15:19, Norman Maurer <no...@googlemail.com> wrote:
> No problem... It just catched my attention ;)
>
> Keep on coding,
> Norman
>
> 2011/10/10 Sylvain Lebresne <sy...@datastax.com>:
>> On Mon, Oct 10, 2011 at 4:03 PM, Norman Maurer
>> <no...@googlemail.com> wrote:
>>> Hi there,
>>>
>>> did you alter the snappy-java version by mistake or was this part of
>>> the commit ?
>>
>> Sorry, that was intentional but I should probably not have been lazy
>> and made a separate commit.
>> I just updated the version referenced in the pom so that it matches
>> what we are using. i.e, I
>> haven't updated the lib, I've just made it so that user of maven
>> central will get the 'right' version.
>>
>> --
>> Sylvain
>>
>>>
>>> Bye,
>>> Norman
>>>
>>>
>>> 2011/10/10 <sl...@apache.org>:
>>>> Author: slebresne
>>>> Date: Mon Oct 10 13:56:27 2011
>>>> New Revision: 1180970
>>>>
>>>> URL: http://svn.apache.org/viewvc?rev=1180970&view=rev
>>>> Log:
>>>> Fix places where uncompressed sstable size is used in place of the compressed one.
>>>> patch by slebresne; reviewed by jbellis for CASSANDRA-3338
>>>>
>>>> Modified:
>>>> cassandra/branches/cassandra-1.0.0/CHANGES.txt
>>>> cassandra/branches/cassandra-1.0.0/build.xml
>>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>>>>
>>>> Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt
>>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1180970&r1=1180969&r2=1180970&view=diff
>>>> ==============================================================================
>>>> --- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original)
>>>> +++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Mon Oct 10 13:56:27 2011
>>>> @@ -12,6 +12,8 @@
>>>> * run compaction and hinted handoff threads at MIN_PRIORITY (CASSANDRA-3308)
>>>> * default hsha thrift server to cpu core count in rpc pool (CASSANDRA-3329)
>>>> * add bin\daemon to binary tarball for Windows service (CASSANDRA-3331)
>>>> + * Fix places where uncompressed size of sstables was use in place of the
>>>> + compressed one (CASSANDRA-3338)
>>>> Fixes merged from 0.8 below:
>>>> * Fix tool .bat files when CASSANDRA_HOME contains spaces (CASSANDRA-3258)
>>>> * Force flush of status table when removing/updating token (CASSANDRA-3243)
>>>>
>>>> Modified: cassandra/branches/cassandra-1.0.0/build.xml
>>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/build.xml?rev=1180970&r1=1180969&r2=1180970&view=diff
>>>> ==============================================================================
>>>> --- cassandra/branches/cassandra-1.0.0/build.xml (original)
>>>> +++ cassandra/branches/cassandra-1.0.0/build.xml Mon Oct 10 13:56:27 2011
>>>> @@ -350,7 +350,7 @@ url=${svn.entry.url}?pathrev=${svn.entry
>>>> <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt"/>
>>>> <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}" url="${scm.url}"/>
>>>> <dependencyManagement>
>>>> - <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3.3"/>
>>>> + <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3"/>
>>>> <dependency groupId="com.ning" artifactId="compress-lzf" version="0.8.4"/>
>>>> <dependency groupId="com.google.guava" artifactId="guava" version="r08"/>
>>>> <dependency groupId="commons-cli" artifactId="commons-cli" version="1.1"/>
>>>>
>>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>>> ==============================================================================
>>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
>>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon Oct 10 13:56:27 2011
>>>> @@ -915,7 +915,7 @@ public class ColumnFamilyStore implement
>>>> long expectedFileSize = 0;
>>>> for (SSTableReader sstable : sstables)
>>>> {
>>>> - long size = sstable.length();
>>>> + long size = sstable.onDiskLength();
>>>> expectedFileSize = expectedFileSize + size;
>>>> }
>>>> return expectedFileSize;
>>>> @@ -930,9 +930,9 @@ public class ColumnFamilyStore implement
>>>> SSTableReader maxFile = null;
>>>> for (SSTableReader sstable : sstables)
>>>> {
>>>> - if (sstable.length() > maxSize)
>>>> + if (sstable.onDiskLength() > maxSize)
>>>> {
>>>> - maxSize = sstable.length();
>>>> + maxSize = sstable.onDiskLength();
>>>> maxFile = sstable;
>>>> }
>>>> }
>>>>
>>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>>> ==============================================================================
>>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java (original)
>>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java Mon Oct 10 13:56:27 2011
>>>> @@ -472,7 +472,7 @@ public class CompactionManager implement
>>>> boolean isCommutative = cfs.metadata.getDefaultValidator().isCommutative();
>>>>
>>>> // Calculate the expected compacted filesize
>>>> - String compactionFileLocation = cfs.table.getDataFileLocation(sstable.length());
>>>> + String compactionFileLocation = cfs.table.getDataFileLocation(sstable.onDiskLength());
>>>> if (compactionFileLocation == null)
>>>> throw new IOException("disk full");
>>>> int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(),
>>>> @@ -765,8 +765,8 @@ public class CompactionManager implement
>>>>
>>>> String format = "Cleaned up to %s. %,d to %,d (~%d%% of original) bytes for %,d keys. Time: %,dms.";
>>>> long dTime = System.currentTimeMillis() - startTime;
>>>> - long startsize = sstable.length();
>>>> - long endsize = newSstable.length();
>>>> + long startsize = sstable.onDiskLength();
>>>> + long endsize = newSstable.onDiskLength();
>>>> double ratio = (double)endsize / (double)startsize;
>>>> logger.info(String.format(format, writer.getFilename(), startsize, endsize, (int)(ratio*100), totalkeysWritten, dTime));
>>>> }
>>>>
>>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>>> ==============================================================================
>>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java (original)
>>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java Mon Oct 10 13:56:27 2011
>>>> @@ -100,7 +100,7 @@ public class SizeTieredCompactionStrateg
>>>> {
>>>> List<Pair<SSTableReader, Long>> tableLengthPairs = new ArrayList<Pair<SSTableReader, Long>>();
>>>> for(SSTableReader table: collection)
>>>> - tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.length()));
>>>> + tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.onDiskLength()));
>>>> return tableLengthPairs;
>>>> }
>>>>
>>>>
>>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>>> ==============================================================================
>>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java (original)
>>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java Mon Oct 10 13:56:27 2011
>>>> @@ -257,7 +257,7 @@ public abstract class SSTable
>>>> long sum = 0;
>>>> for (SSTableReader sstable : sstables)
>>>> {
>>>> - sum += sstable.length();
>>>> + sum += sstable.onDiskLength();
>>>> }
>>>> return sum;
>>>> }
>>>>
>>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>>> ==============================================================================
>>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original)
>>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Mon Oct 10 13:56:27 2011
>>>> @@ -555,7 +555,7 @@ public class SSTableReader extends SSTab
>>>> long right = getPosition(new DecoratedKey(range.right, null), Operator.GT);
>>>> if (right == -1 || Range.isWrapAround(range.left, range.right))
>>>> // right is past the end of the file, or it wraps
>>>> - right = length();
>>>> + right = uncompressedLength();
>>>> if (left == right)
>>>> // empty range
>>>> continue;
>>>> @@ -669,13 +669,25 @@ public class SSTableReader extends SSTab
>>>> }
>>>>
>>>> /**
>>>> - * @return The length in bytes of the data file for this SSTable.
>>>> + * @return The length in bytes of the data for this SSTable. For
>>>> + * compressed files, this is not the same thing as the on disk size (see
>>>> + * onDiskLength())
>>>> */
>>>> - public long length()
>>>> + public long uncompressedLength()
>>>> {
>>>> return dfile.length;
>>>> }
>>>>
>>>> + /**
>>>> + * @return The length in bytes of the on disk size for this SSTable. For
>>>> + * compressed files, this is not the same thing as the data length (see
>>>> + * length())
>>>> + */
>>>> + public long onDiskLength()
>>>> + {
>>>> + return dfile.onDiskLength;
>>>> + }
>>>> +
>>>> public boolean acquireReference()
>>>> {
>>>> while (true)
>>>>
>>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>>> ==============================================================================
>>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java (original)
>>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java Mon Oct 10 13:56:27 2011
>>>> @@ -30,7 +30,7 @@ public class CompressedSegmentedFile ext
>>>>
>>>> public CompressedSegmentedFile(String path, CompressionMetadata metadata)
>>>> {
>>>> - super(path, metadata.dataLength);
>>>> + super(path, metadata.dataLength, metadata.compressedFileLength);
>>>> this.metadata = metadata;
>>>> }
>>>>
>>>>
>>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>>> ==============================================================================
>>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java (original)
>>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java Mon Oct 10 13:56:27 2011
>>>> @@ -42,13 +42,23 @@ public abstract class SegmentedFile
>>>> public final String path;
>>>> public final long length;
>>>>
>>>> + // This differs from length for compressed files (but we still need length for
>>>> + // SegmentIterator because offsets in the file are relative to the uncompressed size)
>>>> + public final long onDiskLength;
>>>> +
>>>> /**
>>>> * Use getBuilder to get a Builder to construct a SegmentedFile.
>>>> */
>>>> SegmentedFile(String path, long length)
>>>> {
>>>> + this(path, length, length);
>>>> + }
>>>> +
>>>> + protected SegmentedFile(String path, long length, long onDiskLength)
>>>> + {
>>>> this.path = path;
>>>> this.length = length;
>>>> + this.onDiskLength = onDiskLength;
>>>> }
>>>>
>>>> /**
>>>>
>>>>
>>>>
>>>
>>
>
Re: svn commit: r1180970 - in /cassandra/branches/cassandra-1.0.0: ./
src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/
src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/io/util/
Posted by Norman Maurer <no...@googlemail.com>.
No problem... It just catched my attention ;)
Keep on coding,
Norman
2011/10/10 Sylvain Lebresne <sy...@datastax.com>:
> On Mon, Oct 10, 2011 at 4:03 PM, Norman Maurer
> <no...@googlemail.com> wrote:
>> Hi there,
>>
>> did you alter the snappy-java version by mistake or was this part of
>> the commit ?
>
> Sorry, that was intentional but I should probably not have been lazy
> and made a separate commit.
> I just updated the version referenced in the pom so that it matches
> what we are using. i.e, I
> haven't updated the lib, I've just made it so that user of maven
> central will get the 'right' version.
>
> --
> Sylvain
>
>>
>> Bye,
>> Norman
>>
>>
>> 2011/10/10 <sl...@apache.org>:
>>> Author: slebresne
>>> Date: Mon Oct 10 13:56:27 2011
>>> New Revision: 1180970
>>>
>>> URL: http://svn.apache.org/viewvc?rev=1180970&view=rev
>>> Log:
>>> Fix places where uncompressed sstable size is used in place of the compressed one.
>>> patch by slebresne; reviewed by jbellis for CASSANDRA-3338
>>>
>>> Modified:
>>> cassandra/branches/cassandra-1.0.0/CHANGES.txt
>>> cassandra/branches/cassandra-1.0.0/build.xml
>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original)
>>> +++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Mon Oct 10 13:56:27 2011
>>> @@ -12,6 +12,8 @@
>>> * run compaction and hinted handoff threads at MIN_PRIORITY (CASSANDRA-3308)
>>> * default hsha thrift server to cpu core count in rpc pool (CASSANDRA-3329)
>>> * add bin\daemon to binary tarball for Windows service (CASSANDRA-3331)
>>> + * Fix places where uncompressed size of sstables was use in place of the
>>> + compressed one (CASSANDRA-3338)
>>> Fixes merged from 0.8 below:
>>> * Fix tool .bat files when CASSANDRA_HOME contains spaces (CASSANDRA-3258)
>>> * Force flush of status table when removing/updating token (CASSANDRA-3243)
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/build.xml
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/build.xml?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/build.xml (original)
>>> +++ cassandra/branches/cassandra-1.0.0/build.xml Mon Oct 10 13:56:27 2011
>>> @@ -350,7 +350,7 @@ url=${svn.entry.url}?pathrev=${svn.entry
>>> <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt"/>
>>> <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}" url="${scm.url}"/>
>>> <dependencyManagement>
>>> - <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3.3"/>
>>> + <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3"/>
>>> <dependency groupId="com.ning" artifactId="compress-lzf" version="0.8.4"/>
>>> <dependency groupId="com.google.guava" artifactId="guava" version="r08"/>
>>> <dependency groupId="commons-cli" artifactId="commons-cli" version="1.1"/>
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon Oct 10 13:56:27 2011
>>> @@ -915,7 +915,7 @@ public class ColumnFamilyStore implement
>>> long expectedFileSize = 0;
>>> for (SSTableReader sstable : sstables)
>>> {
>>> - long size = sstable.length();
>>> + long size = sstable.onDiskLength();
>>> expectedFileSize = expectedFileSize + size;
>>> }
>>> return expectedFileSize;
>>> @@ -930,9 +930,9 @@ public class ColumnFamilyStore implement
>>> SSTableReader maxFile = null;
>>> for (SSTableReader sstable : sstables)
>>> {
>>> - if (sstable.length() > maxSize)
>>> + if (sstable.onDiskLength() > maxSize)
>>> {
>>> - maxSize = sstable.length();
>>> + maxSize = sstable.onDiskLength();
>>> maxFile = sstable;
>>> }
>>> }
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java (original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java Mon Oct 10 13:56:27 2011
>>> @@ -472,7 +472,7 @@ public class CompactionManager implement
>>> boolean isCommutative = cfs.metadata.getDefaultValidator().isCommutative();
>>>
>>> // Calculate the expected compacted filesize
>>> - String compactionFileLocation = cfs.table.getDataFileLocation(sstable.length());
>>> + String compactionFileLocation = cfs.table.getDataFileLocation(sstable.onDiskLength());
>>> if (compactionFileLocation == null)
>>> throw new IOException("disk full");
>>> int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(),
>>> @@ -765,8 +765,8 @@ public class CompactionManager implement
>>>
>>> String format = "Cleaned up to %s. %,d to %,d (~%d%% of original) bytes for %,d keys. Time: %,dms.";
>>> long dTime = System.currentTimeMillis() - startTime;
>>> - long startsize = sstable.length();
>>> - long endsize = newSstable.length();
>>> + long startsize = sstable.onDiskLength();
>>> + long endsize = newSstable.onDiskLength();
>>> double ratio = (double)endsize / (double)startsize;
>>> logger.info(String.format(format, writer.getFilename(), startsize, endsize, (int)(ratio*100), totalkeysWritten, dTime));
>>> }
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java (original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java Mon Oct 10 13:56:27 2011
>>> @@ -100,7 +100,7 @@ public class SizeTieredCompactionStrateg
>>> {
>>> List<Pair<SSTableReader, Long>> tableLengthPairs = new ArrayList<Pair<SSTableReader, Long>>();
>>> for(SSTableReader table: collection)
>>> - tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.length()));
>>> + tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.onDiskLength()));
>>> return tableLengthPairs;
>>> }
>>>
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java (original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java Mon Oct 10 13:56:27 2011
>>> @@ -257,7 +257,7 @@ public abstract class SSTable
>>> long sum = 0;
>>> for (SSTableReader sstable : sstables)
>>> {
>>> - sum += sstable.length();
>>> + sum += sstable.onDiskLength();
>>> }
>>> return sum;
>>> }
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Mon Oct 10 13:56:27 2011
>>> @@ -555,7 +555,7 @@ public class SSTableReader extends SSTab
>>> long right = getPosition(new DecoratedKey(range.right, null), Operator.GT);
>>> if (right == -1 || Range.isWrapAround(range.left, range.right))
>>> // right is past the end of the file, or it wraps
>>> - right = length();
>>> + right = uncompressedLength();
>>> if (left == right)
>>> // empty range
>>> continue;
>>> @@ -669,13 +669,25 @@ public class SSTableReader extends SSTab
>>> }
>>>
>>> /**
>>> - * @return The length in bytes of the data file for this SSTable.
>>> + * @return The length in bytes of the data for this SSTable. For
>>> + * compressed files, this is not the same thing as the on disk size (see
>>> + * onDiskLength())
>>> */
>>> - public long length()
>>> + public long uncompressedLength()
>>> {
>>> return dfile.length;
>>> }
>>>
>>> + /**
>>> + * @return The length in bytes of the on disk size for this SSTable. For
>>> + * compressed files, this is not the same thing as the data length (see
>>> + * length())
>>> + */
>>> + public long onDiskLength()
>>> + {
>>> + return dfile.onDiskLength;
>>> + }
>>> +
>>> public boolean acquireReference()
>>> {
>>> while (true)
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java (original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java Mon Oct 10 13:56:27 2011
>>> @@ -30,7 +30,7 @@ public class CompressedSegmentedFile ext
>>>
>>> public CompressedSegmentedFile(String path, CompressionMetadata metadata)
>>> {
>>> - super(path, metadata.dataLength);
>>> + super(path, metadata.dataLength, metadata.compressedFileLength);
>>> this.metadata = metadata;
>>> }
>>>
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java (original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java Mon Oct 10 13:56:27 2011
>>> @@ -42,13 +42,23 @@ public abstract class SegmentedFile
>>> public final String path;
>>> public final long length;
>>>
>>> + // This differs from length for compressed files (but we still need length for
>>> + // SegmentIterator because offsets in the file are relative to the uncompressed size)
>>> + public final long onDiskLength;
>>> +
>>> /**
>>> * Use getBuilder to get a Builder to construct a SegmentedFile.
>>> */
>>> SegmentedFile(String path, long length)
>>> {
>>> + this(path, length, length);
>>> + }
>>> +
>>> + protected SegmentedFile(String path, long length, long onDiskLength)
>>> + {
>>> this.path = path;
>>> this.length = length;
>>> + this.onDiskLength = onDiskLength;
>>> }
>>>
>>> /**
>>>
>>>
>>>
>>
>
Re: svn commit: r1180970 - in /cassandra/branches/cassandra-1.0.0: ./
src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/
src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/io/util/
Posted by Sylvain Lebresne <sy...@datastax.com>.
On Mon, Oct 10, 2011 at 4:03 PM, Norman Maurer
<no...@googlemail.com> wrote:
> Hi there,
>
> did you alter the snappy-java version by mistake or was this part of
> the commit ?
Sorry, that was intentional but I should probably not have been lazy
and made a separate commit.
I just updated the version referenced in the pom so that it matches
what we are using. i.e, I
haven't updated the lib, I've just made it so that user of maven
central will get the 'right' version.
--
Sylvain
>
> Bye,
> Norman
>
>
> 2011/10/10 <sl...@apache.org>:
>> Author: slebresne
>> Date: Mon Oct 10 13:56:27 2011
>> New Revision: 1180970
>>
>> URL: http://svn.apache.org/viewvc?rev=1180970&view=rev
>> Log:
>> Fix places where uncompressed sstable size is used in place of the compressed one.
>> patch by slebresne; reviewed by jbellis for CASSANDRA-3338
>>
>> Modified:
>> cassandra/branches/cassandra-1.0.0/CHANGES.txt
>> cassandra/branches/cassandra-1.0.0/build.xml
>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>>
>> Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original)
>> +++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Mon Oct 10 13:56:27 2011
>> @@ -12,6 +12,8 @@
>> * run compaction and hinted handoff threads at MIN_PRIORITY (CASSANDRA-3308)
>> * default hsha thrift server to cpu core count in rpc pool (CASSANDRA-3329)
>> * add bin\daemon to binary tarball for Windows service (CASSANDRA-3331)
>> + * Fix places where uncompressed size of sstables was use in place of the
>> + compressed one (CASSANDRA-3338)
>> Fixes merged from 0.8 below:
>> * Fix tool .bat files when CASSANDRA_HOME contains spaces (CASSANDRA-3258)
>> * Force flush of status table when removing/updating token (CASSANDRA-3243)
>>
>> Modified: cassandra/branches/cassandra-1.0.0/build.xml
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/build.xml?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/build.xml (original)
>> +++ cassandra/branches/cassandra-1.0.0/build.xml Mon Oct 10 13:56:27 2011
>> @@ -350,7 +350,7 @@ url=${svn.entry.url}?pathrev=${svn.entry
>> <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt"/>
>> <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}" url="${scm.url}"/>
>> <dependencyManagement>
>> - <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3.3"/>
>> + <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3"/>
>> <dependency groupId="com.ning" artifactId="compress-lzf" version="0.8.4"/>
>> <dependency groupId="com.google.guava" artifactId="guava" version="r08"/>
>> <dependency groupId="commons-cli" artifactId="commons-cli" version="1.1"/>
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon Oct 10 13:56:27 2011
>> @@ -915,7 +915,7 @@ public class ColumnFamilyStore implement
>> long expectedFileSize = 0;
>> for (SSTableReader sstable : sstables)
>> {
>> - long size = sstable.length();
>> + long size = sstable.onDiskLength();
>> expectedFileSize = expectedFileSize + size;
>> }
>> return expectedFileSize;
>> @@ -930,9 +930,9 @@ public class ColumnFamilyStore implement
>> SSTableReader maxFile = null;
>> for (SSTableReader sstable : sstables)
>> {
>> - if (sstable.length() > maxSize)
>> + if (sstable.onDiskLength() > maxSize)
>> {
>> - maxSize = sstable.length();
>> + maxSize = sstable.onDiskLength();
>> maxFile = sstable;
>> }
>> }
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java (original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java Mon Oct 10 13:56:27 2011
>> @@ -472,7 +472,7 @@ public class CompactionManager implement
>> boolean isCommutative = cfs.metadata.getDefaultValidator().isCommutative();
>>
>> // Calculate the expected compacted filesize
>> - String compactionFileLocation = cfs.table.getDataFileLocation(sstable.length());
>> + String compactionFileLocation = cfs.table.getDataFileLocation(sstable.onDiskLength());
>> if (compactionFileLocation == null)
>> throw new IOException("disk full");
>> int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(),
>> @@ -765,8 +765,8 @@ public class CompactionManager implement
>>
>> String format = "Cleaned up to %s. %,d to %,d (~%d%% of original) bytes for %,d keys. Time: %,dms.";
>> long dTime = System.currentTimeMillis() - startTime;
>> - long startsize = sstable.length();
>> - long endsize = newSstable.length();
>> + long startsize = sstable.onDiskLength();
>> + long endsize = newSstable.onDiskLength();
>> double ratio = (double)endsize / (double)startsize;
>> logger.info(String.format(format, writer.getFilename(), startsize, endsize, (int)(ratio*100), totalkeysWritten, dTime));
>> }
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java (original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java Mon Oct 10 13:56:27 2011
>> @@ -100,7 +100,7 @@ public class SizeTieredCompactionStrateg
>> {
>> List<Pair<SSTableReader, Long>> tableLengthPairs = new ArrayList<Pair<SSTableReader, Long>>();
>> for(SSTableReader table: collection)
>> - tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.length()));
>> + tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.onDiskLength()));
>> return tableLengthPairs;
>> }
>>
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java (original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java Mon Oct 10 13:56:27 2011
>> @@ -257,7 +257,7 @@ public abstract class SSTable
>> long sum = 0;
>> for (SSTableReader sstable : sstables)
>> {
>> - sum += sstable.length();
>> + sum += sstable.onDiskLength();
>> }
>> return sum;
>> }
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Mon Oct 10 13:56:27 2011
>> @@ -555,7 +555,7 @@ public class SSTableReader extends SSTab
>> long right = getPosition(new DecoratedKey(range.right, null), Operator.GT);
>> if (right == -1 || Range.isWrapAround(range.left, range.right))
>> // right is past the end of the file, or it wraps
>> - right = length();
>> + right = uncompressedLength();
>> if (left == right)
>> // empty range
>> continue;
>> @@ -669,13 +669,25 @@ public class SSTableReader extends SSTab
>> }
>>
>> /**
>> - * @return The length in bytes of the data file for this SSTable.
>> + * @return The length in bytes of the data for this SSTable. For
>> + * compressed files, this is not the same thing as the on disk size (see
>> + * onDiskLength())
>> */
>> - public long length()
>> + public long uncompressedLength()
>> {
>> return dfile.length;
>> }
>>
>> + /**
>> + * @return The length in bytes of the on disk size for this SSTable. For
>> + * compressed files, this is not the same thing as the data length (see
>> + * length())
>> + */
>> + public long onDiskLength()
>> + {
>> + return dfile.onDiskLength;
>> + }
>> +
>> public boolean acquireReference()
>> {
>> while (true)
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java (original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java Mon Oct 10 13:56:27 2011
>> @@ -30,7 +30,7 @@ public class CompressedSegmentedFile ext
>>
>> public CompressedSegmentedFile(String path, CompressionMetadata metadata)
>> {
>> - super(path, metadata.dataLength);
>> + super(path, metadata.dataLength, metadata.compressedFileLength);
>> this.metadata = metadata;
>> }
>>
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java (original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java Mon Oct 10 13:56:27 2011
>> @@ -42,13 +42,23 @@ public abstract class SegmentedFile
>> public final String path;
>> public final long length;
>>
>> + // This differs from length for compressed files (but we still need length for
>> + // SegmentIterator because offsets in the file are relative to the uncompressed size)
>> + public final long onDiskLength;
>> +
>> /**
>> * Use getBuilder to get a Builder to construct a SegmentedFile.
>> */
>> SegmentedFile(String path, long length)
>> {
>> + this(path, length, length);
>> + }
>> +
>> + protected SegmentedFile(String path, long length, long onDiskLength)
>> + {
>> this.path = path;
>> this.length = length;
>> + this.onDiskLength = onDiskLength;
>> }
>>
>> /**
>>
>>
>>
>