You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2015/03/05 21:05:41 UTC
[2/3] cassandra git commit: Fix Adler32 digest for compressed sstables
Fix Adler32 digest for compressed sstables
patch by jeff jirsa; reviewed by benedict for CASSANDRA-8778
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0e621317
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0e621317
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0e621317
Branch: refs/heads/trunk
Commit: 0e621317edb2cffac2a02d0ac13bd4a370373c73
Parents: 59ed491
Author: Jeff Jirsa <je...@jeffjirsa.net>
Authored: Thu Mar 5 20:05:05 2015 +0000
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Thu Mar 5 20:05:05 2015 +0000
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../io/compress/CompressedSequentialWriter.java | 2 +-
.../io/util/ChecksummedSequentialWriter.java | 2 +-
.../io/util/DataIntegrityMetadata.java | 22 ++++++++++++++++++--
4 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0e621317/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 57dd97e..59f9207 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -30,6 +30,7 @@
* markCompacting only succeeds if the exact SSTableReader instances being
marked are in the live set (CASSANDRA-8689)
* cassandra-stress support for varint (CASSANDRA-8882)
+ * Fix Adler32 digest for compressed sstables (CASSANDRA-8778)
Merged from 2.0:
* Fix regression in mixed single and multi-column relation support for
SELECT statements (CASSANDRA-8613)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0e621317/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
index ad53e83..6152c5f 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
@@ -132,7 +132,7 @@ public class CompressedSequentialWriter extends SequentialWriter
// write data itself
out.write(compressed.buffer, 0, compressedLength);
// write corresponding checksum
- crcMetadata.append(compressed.buffer, 0, compressedLength);
+ crcMetadata.append(compressed.buffer, 0, compressedLength, true);
lastFlushOffset += compressedLength + 4;
}
catch (IOException e)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0e621317/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java b/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java
index f4281b2..526347b 100644
--- a/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java
@@ -37,7 +37,7 @@ public class ChecksummedSequentialWriter extends SequentialWriter
protected void flushData()
{
super.flushData();
- crcMetadata.append(buffer, 0, validBufferBytes);
+ crcMetadata.append(buffer, 0, validBufferBytes, false);
}
public void writeFullChecksum(Descriptor descriptor)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0e621317/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
index 797b964..e6f5083 100644
--- a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
+++ b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
@@ -23,6 +23,7 @@ import java.io.DataOutput;
import java.io.File;
import java.io.IOError;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.zip.Adler32;
import java.util.zip.Checksum;
@@ -107,15 +108,32 @@ public class DataIntegrityMetadata
}
}
- public void append(byte[] buffer, int start, int end)
+ // checksumIncrementalResult indicates if the checksum we compute for this buffer should itself be
+ // included in the full checksum, translating to if the partial checksum is serialized along with the
+ // data it checksums (in which case the file checksum as calculated by external tools would mismatch if
+ // we did not include it), or independently.
+
+ // CompressedSequentialWriters serialize the partial checksums inline with the compressed data chunks they
+ // corroborate, whereas ChecksummedSequentialWriters serialize them to a different file.
+ public void append(byte[] buffer, int start, int end, boolean checksumIncrementalResult)
{
try
{
+ int incrementalChecksumValue;
+
incrementalChecksum.update(buffer, start, end);
- incrementalOut.writeInt((int) incrementalChecksum.getValue());
+ incrementalChecksumValue = (int) incrementalChecksum.getValue();
+ incrementalOut.writeInt((int) incrementalChecksumValue);
incrementalChecksum.reset();
fullChecksum.update(buffer, start, end);
+
+ if (checksumIncrementalResult)
+ {
+ ByteBuffer byteBuffer = ByteBuffer.allocate(4);
+ byteBuffer.putInt((int) incrementalChecksumValue);
+ fullChecksum.update(byteBuffer.array(), 0, byteBuffer.array().length);
+ }
}
catch (IOException e)
{