You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2011/11/09 15:05:52 UTC
svn commit: r1199764 - in /cassandra/trunk: ./ contrib/
interface/thrift/gen-java/org/apache/cassandra/thrift/
src/java/org/apache/cassandra/io/sstable/
src/java/org/apache/cassandra/io/util/
Author: slebresne
Date: Wed Nov 9 14:05:50 2011
New Revision: 1199764
URL: http://svn.apache.org/viewvc?rev=1199764&view=rev
Log:
merge from 1.0
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/contrib/ (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed)
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Component.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/SequentialWriter.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 9 14:05:50 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1198724,1198726-1199259,1199284
/cassandra/branches/cassandra-0.8.0:1125021-1130369
/cassandra/branches/cassandra-0.8.1:1101014-1125018
-/cassandra/branches/cassandra-1.0:1167085-1199374,1199379
+/cassandra/branches/cassandra-1.0:1167085-1199374,1199379,1199763
/cassandra/branches/cassandra-1.0.0:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1:1102511-1125020
Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1199764&r1=1199763&r2=1199764&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Wed Nov 9 14:05:50 2011
@@ -13,6 +13,7 @@
* (CQL) fix handling of rows with no columns (CASSANDRA-3424, 3473)
* fix querying supercolumns by name returning only a subset of
subcolumns or old subcolumn versions (CASSANDRA-3446)
+ * automatically compute sha1 sum for uncompressed data files (CASSANDRA-3456)
Merged from 0.8:
* Make counter shard merging thread safe (CASSANDRA-3178)
* fix updating CF row_cache_provider (CASSANDRA-3414)
Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 9 14:05:50 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1198724,1198726-1199259,1199284
/cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
/cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
-/cassandra/branches/cassandra-1.0/contrib:1167085-1199374,1199379
+/cassandra/branches/cassandra-1.0/contrib:1167085-1199374,1199379,1199763
/cassandra/branches/cassandra-1.0.0/contrib:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/contrib:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 9 14:05:50 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1199374,1199379
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1199374,1199379,1199763
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 9 14:05:50 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1199374,1199379
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1199374,1199379,1199763
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 9 14:05:50 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1199374,1199379
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1199374,1199379,1199763
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 9 14:05:50 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1199374,1199379
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1199374,1199379,1199763
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 9 14:05:50 2011
@@ -4,7 +4,7 @@
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1199374,1199379
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1199374,1199379,1199763
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1102511-1125020
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Component.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Component.java?rev=1199764&r1=1199763&r2=1199764&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Component.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Component.java Wed Nov 9 14:05:50 2011
@@ -54,7 +54,9 @@ public class Component
// statistical metadata about the content of the sstable
STATS("Statistics.db"),
// a bitmap secondary index: many of these may exist per sstable
- BITMAP_INDEX("Bitidx.db");
+ BITMAP_INDEX("Bitidx.db"),
+ // holds sha1 sum of the data file (to be checked by sha1sum)
+ DIGEST("Digest.sha1");
final String repr;
Type(String repr)
@@ -78,6 +80,7 @@ public class Component
public final static Component COMPACTED_MARKER = new Component(Type.COMPACTED_MARKER, -1);
public final static Component COMPRESSION_INFO = new Component(Type.COMPRESSION_INFO, -1);
public final static Component STATS = new Component(Type.STATS, -1);
+ public final static Component DIGEST = new Component(Type.DIGEST, -1);
public final Type type;
public final int id;
@@ -108,6 +111,7 @@ public class Component
case COMPACTED_MARKER:
case COMPRESSION_INFO:
case STATS:
+ case DIGEST:
return type.repr;
case BITMAP_INDEX:
return String.format("%d%c%s", id, separator, type.repr);
@@ -145,6 +149,7 @@ public class Component
case COMPACTED_MARKER: component = Component.COMPACTED_MARKER; break;
case COMPRESSION_INFO: component = Component.COMPRESSION_INFO; break;
case STATS: component = Component.STATS; break;
+ case DIGEST: component = Component.DIGEST; break;
case BITMAP_INDEX:
component = new Component(type, id);
break;
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1199764&r1=1199763&r2=1199764&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java Wed Nov 9 14:05:50 2011
@@ -58,6 +58,7 @@ public abstract class SSTable
public static final String COMPONENT_INDEX = Component.Type.PRIMARY_INDEX.repr;
public static final String COMPONENT_FILTER = Component.Type.FILTER.repr;
public static final String COMPONENT_STATS = Component.Type.STATS.repr;
+ public static final String COMPONENT_DIGEST = Component.Type.DIGEST.repr;
public static final String TEMPFILE_MARKER = "tmp";
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=1199764&r1=1199763&r2=1199764&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Wed Nov 9 14:05:50 2011
@@ -37,9 +37,7 @@ import org.apache.cassandra.dht.IPartiti
import org.apache.cassandra.io.compress.CompressedSequentialWriter;
import org.apache.cassandra.io.util.*;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.BloomFilter;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.*;
public class SSTableWriter extends SSTable
{
@@ -66,6 +64,10 @@ public class SSTableWriter extends SSTab
Set<Component> components = new HashSet<Component>(Arrays.asList(Component.DATA, Component.FILTER, Component.PRIMARY_INDEX, Component.STATS));
if (metadata.compressionParameters().sstableCompressor != null)
components.add(Component.COMPRESSION_INFO);
+ else
+ // it would feel safer to actually add this component later in maybeWriteDigest(),
+ // but the components are unmodifiable after construction
+ components.add(Component.DIGEST);
return components;
}
@@ -93,6 +95,7 @@ public class SSTableWriter extends SSTab
{
dbuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
dataFile = SequentialWriter.open(new File(getFilename()), true);
+ dataFile.setComputeDigest();
}
this.sstableMetadataCollector = sstableMetadataCollector;
@@ -299,6 +302,7 @@ public class SSTableWriter extends SSTab
// write sstable statistics
SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata();
writeMetadata(descriptor, sstableMetadata);
+ maybeWriteDigest();
// remove the 'tmp' marker from all components
final Descriptor newdesc = rename(descriptor, components);
@@ -323,6 +327,21 @@ public class SSTableWriter extends SSTab
return sstable;
}
+ private void maybeWriteDigest() throws IOException
+ {
+ byte[] digest = dataFile.digest();
+ if (digest == null)
+ return;
+
+ SequentialWriter out = SequentialWriter.open(new File(descriptor.filenameFor(SSTable.COMPONENT_DIGEST)), true);
+ // Writting output compatible with sha1sum
+ Descriptor newdesc = descriptor.asTemporary(false);
+ String[] tmp = newdesc.filenameFor(SSTable.COMPONENT_DATA).split(new Character(File.separatorChar).toString());
+ String dataFileName = tmp[tmp.length - 1];
+ out.write(String.format("%s %s", Hex.bytesToHex(digest), dataFileName).getBytes());
+ out.close();
+ }
+
private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata) throws IOException
{
SequentialWriter out = SequentialWriter.open(new File(desc.filenameFor(SSTable.COMPONENT_STATS)), true);
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/util/SequentialWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/SequentialWriter.java?rev=1199764&r1=1199763&r2=1199764&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/util/SequentialWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/util/SequentialWriter.java Wed Nov 9 14:05:50 2011
@@ -20,6 +20,8 @@ package org.apache.cassandra.io.util;
import java.io.*;
import java.nio.channels.ClosedChannelException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import org.apache.cassandra.utils.CLibrary;
@@ -47,6 +49,7 @@ public class SequentialWriter extends Ou
private long ioCacheStartOffset = 0, bytesSinceCacheFlush = 0;
public final DataOutputStream stream;
+ private MessageDigest digest;
public SequentialWriter(File file, int bufferSize, boolean skipIOCache) throws IOException
{
@@ -199,6 +202,8 @@ public class SequentialWriter extends Ou
protected void flushData() throws IOException
{
out.write(buffer, 0, validBufferBytes);
+ if (digest != null)
+ digest.update(buffer, 0, validBufferBytes);
}
public long getFilePointer()
@@ -286,6 +291,41 @@ public class SequentialWriter extends Ou
}
/**
+ * Turn on digest computation on this writer.
+ * This can only be called before any data is written to this write,
+ * otherwise an IllegalStateException is thrown.
+ */
+ public void setComputeDigest()
+ {
+ if (current != 0)
+ throw new IllegalStateException();
+
+ try
+ {
+ digest = MessageDigest.getInstance("SHA-1");
+ }
+ catch (NoSuchAlgorithmException e)
+ {
+ // SHA-1 is standard in java 6
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Return the digest associated to this file or null if no digest was
+ * created.
+ * This can only be called once the file is fully created, i.e. after
+ * close() has been called. Otherwise an IllegalStateException is thrown.
+ */
+ public byte[] digest()
+ {
+ if (buffer != null)
+ throw new IllegalStateException();
+
+ return digest == null ? null : digest.digest();
+ }
+
+ /**
* Class to hold a mark to the position of the file
*/
protected static class BufferedFileWriterMark implements FileMark