You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by st...@apache.org on 2016/05/26 02:57:23 UTC
[06/10] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.7
Merge branch 'cassandra-3.0' into cassandra-3.7
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/68d20edb
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/68d20edb
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/68d20edb
Branch: refs/heads/trunk
Commit: 68d20edb2ba89349303dee4ae3be75f8d363c3ff
Parents: f39a352 85cc390
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Thu May 26 10:51:46 2016 +0800
Committer: Stefania Alborghetti <st...@datastax.com>
Committed: Thu May 26 10:52:31 2016 +0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 11 +
conf/cassandra.yaml | 8 +-
.../org/apache/cassandra/config/Config.java | 6 +
.../cassandra/config/DatabaseDescriptor.java | 8 +
.../apache/cassandra/db/ClusteringPrefix.java | 4 +-
.../columniterator/AbstractSSTableIterator.java | 2 +-
.../cassandra/db/marshal/AbstractType.java | 46 +++-
src/java/org/apache/cassandra/db/rows/Rows.java | 2 +-
.../cassandra/db/rows/UnfilteredSerializer.java | 13 +-
.../io/sstable/SSTableIdentityIterator.java | 5 +
.../io/sstable/format/big/BigTableScanner.java | 2 +-
.../io/util/RebufferingInputStream.java | 2 +-
.../compaction/BlacklistingCompactionsTest.java | 45 +++-
.../cassandra/index/sasi/SASIIndexTest.java | 8 +-
.../sstable/SSTableCorruptionDetectionTest.java | 247 +++++++++++++++++++
.../cassandra/io/sstable/SSTableWriterTest.java | 49 +++-
.../io/sstable/SSTableWriterTestBase.java | 10 +-
18 files changed, 439 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index d77aa1e,ddfb24f..b2d31bb
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,6 -1,5 +1,7 @@@
-3.0.7
+3.7
+ * Don't use static dataDirectories field in Directories instances (CASSANDRA-11647)
+Merged from 3.0:
+ * Prevent OOM failures on SSTable corruption, improve tests for corruption detection (CASSANDRA-9530)
* Use CFS.initialDirectories when clearing snapshots (CASSANDRA-11705)
* Allow compaction strategies to disable early open (CASSANDRA-11754)
* Refactor Materialized View code (CASSANDRA-11475)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/NEWS.txt
----------------------------------------------------------------------
diff --cc NEWS.txt
index ac33a60,ac1ef17..2f5e3b5
--- a/NEWS.txt
+++ b/NEWS.txt
@@@ -13,7 -13,18 +13,18 @@@ restore snapshots created with the prev
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
-3.0.7
-=====
++3.7
++===
+
+ Upgrading
+ ---------
+ - A maximum size for SSTables values has been introduced, to prevent out of memory
+ exceptions when reading corrupt SSTables. This maximum size can be set via
+ max_value_size_in_mb in cassandra.yaml. The default is 256MB, which matches the default
+ value of native_transport_max_frame_size_in_mb. SSTables will be considered corrupt if
+ they contain values whose size exceeds this limit. See CASSANDRA-9530 for more details.
+
-3.0.6
+3.6
=====
New features
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/conf/cassandra.yaml
----------------------------------------------------------------------
diff --cc conf/cassandra.yaml
index 959d855,4b92f64..dcd5278
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@@ -979,65 -946,7 +980,70 @@@ enable_scripted_user_defined_functions
# setting.
windows_timer_interval: 1
+
+# Enables encrypting data at-rest (on disk). Different key providers can be plugged in, but the default reads from
+# a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced by
+# the "key_alias" is the only key that will be used for encrypt opertaions; previously used keys
+# can still (and should!) be in the keystore and will be used on decrypt operations
+# (to handle the case of key rotation).
+#
+# It is strongly recommended to download and install Java Cryptography Extension (JCE)
+# Unlimited Strength Jurisdiction Policy Files for your version of the JDK.
+# (current link: http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html)
+#
+# Currently, only the following file types are supported for transparent data encryption, although
+# more are coming in future cassandra releases: commitlog, hints
+transparent_data_encryption_options:
+ enabled: false
+ chunk_length_kb: 64
+ cipher: AES/CBC/PKCS5Padding
+ key_alias: testing:1
+ # CBC IV length for AES needs to be 16 bytes (which is also the default size)
+ # iv_length: 16
+ key_provider:
+ - class_name: org.apache.cassandra.security.JKSKeyProvider
+ parameters:
+ - keystore: conf/.keystore
+ keystore_password: cassandra
+ store_type: JCEKS
+ key_password: cassandra
+
+
+#####################
+# SAFETY THRESHOLDS #
+#####################
+
+# When executing a scan, within or across a partition, we need to keep the
+# tombstones seen in memory so we can return them to the coordinator, which
+# will use them to make sure other replicas also know about the deleted rows.
+# With workloads that generate a lot of tombstones, this can cause performance
+# problems and even exaust the server heap.
+# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets)
+# Adjust the thresholds here if you understand the dangers and want to
+# scan more tombstones anyway. These thresholds may also be adjusted at runtime
+# using the StorageService mbean.
+tombstone_warn_threshold: 1000
+tombstone_failure_threshold: 100000
+
+# Log WARN on any batch size exceeding this value. 5kb per batch by default.
+# Caution should be taken on increasing the size of this threshold as it can lead to node instability.
+batch_size_warn_threshold_in_kb: 5
+
+# Fail any batch exceeding this value. 50kb (10x warn threshold) by default.
+batch_size_fail_threshold_in_kb: 50
+
+# Log WARN on any batches not of type LOGGED than span across more partitions than this limit
+unlogged_batch_across_partitions_warn_threshold: 10
+
+# Log a warning when compacting partitions larger than this value
+compaction_large_partition_warning_threshold_mb: 100
+
+# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level
+# Adjust the threshold based on your application throughput requirement
+# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level
+gc_warn_threshold_in_ms: 1000
++
+ # Maximum size of any value in SSTables. Safety measure to detect SSTable corruption
+ # early. Any value size larger than this threshold will result into marking an SSTable
+ # as corrupted.
+ # max_value_size_in_mb: 256
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/db/ClusteringPrefix.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/db/marshal/AbstractType.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
index b442df0,21ce450..bd52f2f
--- a/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
@@@ -33,7 -36,7 +36,8 @@@ import static org.junit.Assert.assertNo
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
+import org.apache.cassandra.cache.ChunkCache;
+ import org.apache.cassandra.config.*;
import org.apache.cassandra.db.*;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@@@ -147,10 -169,8 +170,10 @@@ public class BlacklistingCompactionsTes
// We want to write something large enough that the corruption cannot get undetected
// (even without compression)
byte[] corruption = new byte[corruptionSize];
- Arrays.fill(corruption, (byte)0xFF);
+ random.nextBytes(corruption);
raf.write(corruption);
+ if (ChunkCache.instance != null)
+ ChunkCache.instance.invalidateFile(sstable.getFilename());
}
finally