You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2013/12/03 14:57:13 UTC
[1/4] git commit: Warn when a read collection has > 64k elements
Updated Branches:
refs/heads/trunk d12a0d7b0 -> b34d43f97
Warn when a read collection has > 64k elements
patch by slebresne; reviewed by iamaleksey for CASSANDRA-5428
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f634ac7e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f634ac7e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f634ac7e
Branch: refs/heads/trunk
Commit: f634ac7eae468b944d22951fc7c9d05aa6c7f447
Parents: ecd9422
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Tue Dec 3 14:53:33 2013 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Tue Dec 3 14:53:33 2013 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/marshal/CollectionType.java | 17 +++++++++++++++++
.../org/apache/cassandra/db/marshal/ListType.java | 2 ++
.../org/apache/cassandra/db/marshal/MapType.java | 2 ++
.../org/apache/cassandra/db/marshal/SetType.java | 2 ++
5 files changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f634ac7e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c80a00a..8e6cffa 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,6 +8,7 @@
* Throw IRE if a prepared has more markers than supported (CASSANDRA-5598)
* Expose Thread metrics for the native protocol server (CASSANDRA-6234)
* Change snapshot response message verb (CASSANDRA-6415)
+ * Warn when collection read has > 65K elements (CASSANDRA-5428)
1.2.12
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f634ac7e/src/java/org/apache/cassandra/db/marshal/CollectionType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/marshal/CollectionType.java b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
index ad2ea67..a34a2b7 100644
--- a/src/java/org/apache/cassandra/db/marshal/CollectionType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
@@ -20,6 +20,9 @@ package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.db.IColumn;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -33,6 +36,10 @@ import org.apache.cassandra.utils.Pair;
*/
public abstract class CollectionType<T> extends AbstractType<T>
{
+ private static final Logger logger = LoggerFactory.getLogger(CollectionType.class);
+
+ public static final int MAX_ELEMENTS = 65535;
+
public enum Kind
{
MAP, SET, LIST
@@ -105,6 +112,16 @@ public abstract class CollectionType<T> extends AbstractType<T>
return (ByteBuffer)result.flip();
}
+ protected List<Pair<ByteBuffer, IColumn>> enforceLimit(List<Pair<ByteBuffer, IColumn>> columns)
+ {
+ if (columns.size() <= MAX_ELEMENTS)
+ return columns;
+
+ logger.error("Detected collection with {} elements, more than the {} limit. Only the first {} elements will be returned to the client. "
+ + "Please see http://cassandra.apache.org/doc/cql3/CQL.html#collections for more details.", columns.size(), MAX_ELEMENTS, MAX_ELEMENTS);
+ return columns.subList(0, MAX_ELEMENTS);
+ }
+
public static ByteBuffer pack(List<ByteBuffer> buffers, int elements)
{
int size = 0;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f634ac7e/src/java/org/apache/cassandra/db/marshal/ListType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/marshal/ListType.java b/src/java/org/apache/cassandra/db/marshal/ListType.java
index b6613ae..b219af1 100644
--- a/src/java/org/apache/cassandra/db/marshal/ListType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ListType.java
@@ -120,6 +120,8 @@ public class ListType<T> extends CollectionType<List<T>>
public ByteBuffer serialize(List<Pair<ByteBuffer, IColumn>> columns)
{
+ columns = enforceLimit(columns);
+
List<ByteBuffer> bbs = new ArrayList<ByteBuffer>(columns.size());
int size = 0;
for (Pair<ByteBuffer, IColumn> p : columns)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f634ac7e/src/java/org/apache/cassandra/db/marshal/MapType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/marshal/MapType.java b/src/java/org/apache/cassandra/db/marshal/MapType.java
index 19310df..750851e 100644
--- a/src/java/org/apache/cassandra/db/marshal/MapType.java
+++ b/src/java/org/apache/cassandra/db/marshal/MapType.java
@@ -137,6 +137,8 @@ public class MapType<K, V> extends CollectionType<Map<K, V>>
*/
public ByteBuffer serialize(List<Pair<ByteBuffer, IColumn>> columns)
{
+ columns = enforceLimit(columns);
+
List<ByteBuffer> bbs = new ArrayList<ByteBuffer>(2 * columns.size());
int size = 0;
for (Pair<ByteBuffer, IColumn> p : columns)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f634ac7e/src/java/org/apache/cassandra/db/marshal/SetType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/marshal/SetType.java b/src/java/org/apache/cassandra/db/marshal/SetType.java
index 73f25e0..a1ba74d 100644
--- a/src/java/org/apache/cassandra/db/marshal/SetType.java
+++ b/src/java/org/apache/cassandra/db/marshal/SetType.java
@@ -120,6 +120,8 @@ public class SetType<T> extends CollectionType<Set<T>>
public ByteBuffer serialize(List<Pair<ByteBuffer, IColumn>> columns)
{
+ columns = enforceLimit(columns);
+
List<ByteBuffer> bbs = new ArrayList<ByteBuffer>(columns.size());
int size = 0;
for (Pair<ByteBuffer, IColumn> p : columns)
[4/4] git commit: Merge branch 'cassandra-2.0' into trunk
Posted by sl...@apache.org.
Merge branch 'cassandra-2.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b34d43f9
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b34d43f9
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b34d43f9
Branch: refs/heads/trunk
Commit: b34d43f9747d2ebc1feb516d9675801bcd293d8b
Parents: d12a0d7 1334f94
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Tue Dec 3 14:57:06 2013 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Tue Dec 3 14:57:06 2013 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/marshal/CollectionType.java | 17 +++++++++++++++++
.../org/apache/cassandra/db/marshal/ListType.java | 2 ++
.../org/apache/cassandra/db/marshal/MapType.java | 2 ++
.../org/apache/cassandra/db/marshal/SetType.java | 2 ++
5 files changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b34d43f9/CHANGES.txt
----------------------------------------------------------------------
[3/4] git commit: Fix merge
Posted by sl...@apache.org.
Fix merge
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1334f94e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1334f94e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1334f94e
Branch: refs/heads/trunk
Commit: 1334f94e40ce5dbed7270808abb2330ea6d37c51
Parents: b2da839
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Tue Dec 3 14:56:19 2013 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Tue Dec 3 14:56:19 2013 +0100
----------------------------------------------------------------------
src/java/org/apache/cassandra/db/marshal/CollectionType.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1334f94e/src/java/org/apache/cassandra/db/marshal/CollectionType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/marshal/CollectionType.java b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
index 9408980..07c86e0 100644
--- a/src/java/org/apache/cassandra/db/marshal/CollectionType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
@@ -113,7 +113,7 @@ public abstract class CollectionType<T> extends AbstractType<T>
return (ByteBuffer)result.flip();
}
- protected List<Pair<ByteBuffer, IColumn>> enforceLimit(List<Pair<ByteBuffer, IColumn>> columns)
+ protected List<Pair<ByteBuffer, Column>> enforceLimit(List<Pair<ByteBuffer, Column>> columns)
{
if (columns.size() <= MAX_ELEMENTS)
return columns;
[2/4] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0
Posted by sl...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b2da839f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b2da839f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b2da839f
Branch: refs/heads/trunk
Commit: b2da839f076f14f35c5591b39736c8d7241974ee
Parents: 6724964 f634ac7
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Tue Dec 3 14:54:41 2013 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Tue Dec 3 14:54:41 2013 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/marshal/CollectionType.java | 17 +++++++++++++++++
.../org/apache/cassandra/db/marshal/ListType.java | 2 ++
.../org/apache/cassandra/db/marshal/MapType.java | 2 ++
.../org/apache/cassandra/db/marshal/SetType.java | 2 ++
5 files changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2da839f/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 11f4c09,8e6cffa..a7ab215
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -13,44 -8,10 +13,45 @@@ Merged from 1.2
* Throw IRE if a prepared has more markers than supported (CASSANDRA-5598)
* Expose Thread metrics for the native protocol server (CASSANDRA-6234)
* Change snapshot response message verb (CASSANDRA-6415)
+ * Warn when collection read has > 65K elements (CASSANDRA-5428)
-1.2.12
+2.0.3
+ * Fix FD leak on slice read path (CASSANDRA-6275)
+ * Cancel read meter task when closing SSTR (CASSANDRA-6358)
+ * free off-heap IndexSummary during bulk (CASSANDRA-6359)
+ * Recover from IOException in accept() thread (CASSANDRA-6349)
+ * Improve Gossip tolerance of abnormally slow tasks (CASSANDRA-6338)
+ * Fix trying to hint timed out counter writes (CASSANDRA-6322)
+ * Allow restoring specific columnfamilies from archived CL (CASSANDRA-4809)
+ * Avoid flushing compaction_history after each operation (CASSANDRA-6287)
+ * Fix repair assertion error when tombstones expire (CASSANDRA-6277)
+ * Skip loading corrupt key cache (CASSANDRA-6260)
+ * Fixes for compacting larger-than-memory rows (CASSANDRA-6274)
+ * Compact hottest sstables first and optionally omit coldest from
+ compaction entirely (CASSANDRA-6109)
+ * Fix modifying column_metadata from thrift (CASSANDRA-6182)
+ * cqlsh: fix LIST USERS output (CASSANDRA-6242)
+ * Add IRequestSink interface (CASSANDRA-6248)
+ * Update memtable size while flushing (CASSANDRA-6249)
+ * Provide hooks around CQL2/CQL3 statement execution (CASSANDRA-6252)
+ * Require Permission.SELECT for CAS updates (CASSANDRA-6247)
+ * New CQL-aware SSTableWriter (CASSANDRA-5894)
+ * Reject CAS operation when the protocol v1 is used (CASSANDRA-6270)
+ * Correctly throw error when frame too large (CASSANDRA-5981)
+ * Fix serialization bug in PagedRange with 2ndary indexes (CASSANDRA-6299)
+ * Fix CQL3 table validation in Thrift (CASSANDRA-6140)
+ * Fix bug missing results with IN clauses (CASSANDRA-6327)
+ * Fix paging with reversed slices (CASSANDRA-6343)
+ * Set minTimestamp correctly to be able to drop expired sstables (CASSANDRA-6337)
+ * Support NaN and Infinity as float literals (CASSANDRA-6003)
+ * Remove RF from nodetool ring output (CASSANDRA-6289)
+ * Fix attempting to flush empty rows (CASSANDRA-6374)
+ * Fix potential out of bounds exception when paging (CASSANDRA-6333)
+Merged from 1.2:
+ * Optimize FD phi calculation (CASSANDRA-6386)
+ * Improve initial FD phi estimate when starting up (CASSANDRA-6385)
+ * Don't list CQL3 table in CLI describe even if named explicitely (CASSANDRA-5750)
* Invalidate row cache when dropping CF (CASSANDRA-6351)
* add non-jamm path for cached statements (CASSANDRA-6293)
* (Hadoop) Require CFRR batchSize to be at least 2 (CASSANDRA-6114)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2da839f/src/java/org/apache/cassandra/db/marshal/CollectionType.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/marshal/CollectionType.java
index f922d56,a34a2b7..9408980
--- a/src/java/org/apache/cassandra/db/marshal/CollectionType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
@@@ -20,9 -20,11 +20,12 @@@ package org.apache.cassandra.db.marshal
import java.nio.ByteBuffer;
import java.util.List;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.cql3.CQL3Type;
-import org.apache.cassandra.db.IColumn;
+import org.apache.cassandra.db.Column;
+import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2da839f/src/java/org/apache/cassandra/db/marshal/ListType.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/marshal/ListType.java
index 8331897,b219af1..808ba45
--- a/src/java/org/apache/cassandra/db/marshal/ListType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ListType.java
@@@ -82,11 -118,13 +82,13 @@@ public class ListType<T> extends Collec
sb.append(getClass().getName()).append(TypeParser.stringifyTypeParameters(Collections.<AbstractType<?>>singletonList(elements)));
}
- public ByteBuffer serialize(List<Pair<ByteBuffer, IColumn>> columns)
+ public ByteBuffer serialize(List<Pair<ByteBuffer, Column>> columns)
{
+ columns = enforceLimit(columns);
+
List<ByteBuffer> bbs = new ArrayList<ByteBuffer>(columns.size());
int size = 0;
- for (Pair<ByteBuffer, IColumn> p : columns)
+ for (Pair<ByteBuffer, Column> p : columns)
{
bbs.add(p.right.value());
size += 2 + p.right.value().remaining();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2da839f/src/java/org/apache/cassandra/db/marshal/MapType.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/marshal/MapType.java
index 76f7a6d,750851e..fd96da7
--- a/src/java/org/apache/cassandra/db/marshal/MapType.java
+++ b/src/java/org/apache/cassandra/db/marshal/MapType.java
@@@ -87,13 -133,15 +87,15 @@@ public class MapType<K, V> extends Coll
}
/**
- * Creates the same output than decompose, but from the internal representation.
+ * Creates the same output than serialize, but from the internal representation.
*/
- public ByteBuffer serialize(List<Pair<ByteBuffer, IColumn>> columns)
+ public ByteBuffer serialize(List<Pair<ByteBuffer, Column>> columns)
{
+ columns = enforceLimit(columns);
+
List<ByteBuffer> bbs = new ArrayList<ByteBuffer>(2 * columns.size());
int size = 0;
- for (Pair<ByteBuffer, IColumn> p : columns)
+ for (Pair<ByteBuffer, Column> p : columns)
{
bbs.add(p.left);
bbs.add(p.right.value());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2da839f/src/java/org/apache/cassandra/db/marshal/SetType.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/marshal/SetType.java
index 0f36ec1,a1ba74d..c947d26
--- a/src/java/org/apache/cassandra/db/marshal/SetType.java
+++ b/src/java/org/apache/cassandra/db/marshal/SetType.java
@@@ -82,11 -118,13 +82,13 @@@ public class SetType<T> extends Collect
sb.append(getClass().getName()).append(TypeParser.stringifyTypeParameters(Collections.<AbstractType<?>>singletonList(elements)));
}
- public ByteBuffer serialize(List<Pair<ByteBuffer, IColumn>> columns)
+ public ByteBuffer serialize(List<Pair<ByteBuffer, Column>> columns)
{
+ columns = enforceLimit(columns);
+
List<ByteBuffer> bbs = new ArrayList<ByteBuffer>(columns.size());
int size = 0;
- for (Pair<ByteBuffer, IColumn> p : columns)
+ for (Pair<ByteBuffer, Column> p : columns)
{
bbs.add(p.left);
size += 2 + p.left.remaining();