You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by bl...@apache.org on 2016/09/27 12:13:57 UTC
[1/3] cassandra git commit: Make Collections deserialization more
robust
Repository: cassandra
Updated Branches:
refs/heads/trunk 577701336 -> 7bef41856
Make Collections deserialization more robust
patch by Benjamin Lerer; reviewed by Alex Petrov for CASSANDRA-12618
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6dc595dd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6dc595dd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6dc595dd
Branch: refs/heads/trunk
Commit: 6dc595dd2ebf8477b4775473d1d2417fa3a0fcfd
Parents: e9fe96f
Author: Benjamin Lerer <b....@gmail.com>
Authored: Tue Sep 27 14:02:10 2016 +0200
Committer: Benjamin Lerer <b....@gmail.com>
Committed: Tue Sep 27 14:02:10 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 4 ++
.../apache/cassandra/db/marshal/UserType.java | 8 ++--
.../cassandra/serializers/ListSerializer.java | 10 ++++-
.../cassandra/serializers/MapSerializer.java | 10 ++++-
.../cassandra/serializers/SetSerializer.java | 13 ++++++-
.../cassandra/serializers/UTF8Serializer.java | 3 ++
.../validation/entities/CollectionsTest.java | 41 ++++++++++++++++++++
.../cql3/validation/entities/TupleTypeTest.java | 10 +++++
.../cql3/validation/entities/UserTypesTest.java | 11 ++++++
9 files changed, 101 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6dc595dd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bc9fc5b..998849e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,7 @@
+2.2.9
+ * Make Collections deserialization more robust (CASSANDRA-12618)
+
+
2.2.8
* Fix exceptions when enabling gossip on nodes that haven't joined the ring (CASSANDRA-12253)
* Fix authentication problem when invoking clqsh copy from a SOURCE command (CASSANDRA-12642)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6dc595dd/src/java/org/apache/cassandra/db/marshal/UserType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/marshal/UserType.java b/src/java/org/apache/cassandra/db/marshal/UserType.java
index 6102d26..187deeb 100644
--- a/src/java/org/apache/cassandra/db/marshal/UserType.java
+++ b/src/java/org/apache/cassandra/db/marshal/UserType.java
@@ -19,7 +19,6 @@ package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
-import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.*;
@@ -28,8 +27,7 @@ import com.google.common.base.Objects;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
-import org.apache.cassandra.serializers.*;
-import org.apache.cassandra.transport.Server;
+import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
@@ -123,7 +121,7 @@ public class UserType extends TupleType
return;
if (input.remaining() < 4)
- throw new MarshalException(String.format("Not enough bytes to read size of %dth field %s", i, fieldName(i)));
+ throw new MarshalException(String.format("Not enough bytes to read size of %dth field %s", i, fieldNameAsString(i)));
int size = input.getInt();
@@ -132,7 +130,7 @@ public class UserType extends TupleType
continue;
if (input.remaining() < size)
- throw new MarshalException(String.format("Not enough bytes to read %dth field %s", i, fieldName(i)));
+ throw new MarshalException(String.format("Not enough bytes to read %dth field %s", i, fieldNameAsString(i)));
ByteBuffer field = ByteBufferUtil.readBytes(input, size);
types.get(i).validate(field);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6dc595dd/src/java/org/apache/cassandra/serializers/ListSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/serializers/ListSerializer.java b/src/java/org/apache/cassandra/serializers/ListSerializer.java
index aeee2b9..d2d0610 100644
--- a/src/java/org/apache/cassandra/serializers/ListSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/ListSerializer.java
@@ -84,7 +84,15 @@ public class ListSerializer<T> extends CollectionSerializer<List<T>>
{
ByteBuffer input = bytes.duplicate();
int n = readCollectionSize(input, version);
- List<T> l = new ArrayList<T>(n);
+
+ if (n < 0)
+ throw new MarshalException("The data cannot be deserialized as a list");
+
+ // If the received bytes are not corresponding to a list, n might be a huge number.
+ // In such a case we do not want to initialize the list with that size as it can result
+ // in an OOM (see CASSANDRA-12618). On the other hand we do not want to have to resize the list
+ // if we can avoid it, so we put a reasonable limit on the initialCapacity.
+ List<T> l = new ArrayList<T>(Math.min(n, 256));
for (int i = 0; i < n; i++)
{
// We can have nulls in lists that are used for IN values
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6dc595dd/src/java/org/apache/cassandra/serializers/MapSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/serializers/MapSerializer.java b/src/java/org/apache/cassandra/serializers/MapSerializer.java
index 8350f66..66831a3 100644
--- a/src/java/org/apache/cassandra/serializers/MapSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/MapSerializer.java
@@ -94,7 +94,15 @@ public class MapSerializer<K, V> extends CollectionSerializer<Map<K, V>>
{
ByteBuffer input = bytes.duplicate();
int n = readCollectionSize(input, version);
- Map<K, V> m = new LinkedHashMap<K, V>(n);
+
+ if (n < 0)
+ throw new MarshalException("The data cannot be deserialized as a map");
+
+ // If the received bytes are not corresponding to a map, n might be a huge number.
+ // In such a case we do not want to initialize the map with that initialCapacity as it can result
+ // in an OOM when put is called (see CASSANDRA-12618). On the other hand we do not want to have to resize
+ // the map if we can avoid it, so we put a reasonable limit on the initialCapacity.
+ Map<K, V> m = new LinkedHashMap<K, V>(Math.min(n, 256));
for (int i = 0; i < n; i++)
{
ByteBuffer kbb = readValue(input, version);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6dc595dd/src/java/org/apache/cassandra/serializers/SetSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/serializers/SetSerializer.java b/src/java/org/apache/cassandra/serializers/SetSerializer.java
index 21f5075..4aaf36a 100644
--- a/src/java/org/apache/cassandra/serializers/SetSerializer.java
+++ b/src/java/org/apache/cassandra/serializers/SetSerializer.java
@@ -81,7 +81,16 @@ public class SetSerializer<T> extends CollectionSerializer<Set<T>>
{
ByteBuffer input = bytes.duplicate();
int n = readCollectionSize(input, version);
- Set<T> l = new LinkedHashSet<T>(n);
+
+ if (n < 0)
+ throw new MarshalException("The data cannot be deserialized as a set");
+
+ // If the received bytes are not corresponding to a set, n might be a huge number.
+ // In such a case we do not want to initialize the set with that initialCapacity as it can result
+ // in an OOM when add is called (see CASSANDRA-12618). On the other hand we do not want to have to resize
+ // the set if we can avoid it, so we put a reasonable limit on the initialCapacity.
+ Set<T> l = new LinkedHashSet<T>(Math.min(n, 256));
+
for (int i = 0; i < n; i++)
{
ByteBuffer databb = readValue(input, version);
@@ -94,7 +103,7 @@ public class SetSerializer<T> extends CollectionSerializer<Set<T>>
}
catch (BufferUnderflowException e)
{
- throw new MarshalException("Not enough bytes to read a list");
+ throw new MarshalException("Not enough bytes to read a set");
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6dc595dd/src/java/org/apache/cassandra/serializers/UTF8Serializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/serializers/UTF8Serializer.java b/src/java/org/apache/cassandra/serializers/UTF8Serializer.java
index e3ea2d5..7c41b94 100644
--- a/src/java/org/apache/cassandra/serializers/UTF8Serializer.java
+++ b/src/java/org/apache/cassandra/serializers/UTF8Serializer.java
@@ -52,6 +52,9 @@ public class UTF8Serializer extends AbstractTextSerializer
// buf has already been sliced/duplicated.
static boolean validate(ByteBuffer buf)
{
+ if (buf == null)
+ return false;
+
buf = buf.slice();
int b = 0;
State state = State.START;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6dc595dd/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
index 6266fe7..115b755 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
@@ -607,4 +607,45 @@ public class CollectionsTest extends CQLTester
assertRows(execute("select s_list from %s where k1='a'"), row(list(0)));
}
+ @Test
+ public void testInvalidInputForList() throws Throwable
+ {
+ createTable("CREATE TABLE %s(pk int PRIMARY KEY, l list<text>)");
+ assertInvalidMessage("Not enough bytes to read a list",
+ "INSERT INTO %s (pk, l) VALUES (?, ?)", 1, "test");
+ assertInvalidMessage("Not enough bytes to read a list",
+ "INSERT INTO %s (pk, l) VALUES (?, ?)", 1, Long.MAX_VALUE);
+ assertInvalidMessage("Not enough bytes to read a list",
+ "INSERT INTO %s (pk, l) VALUES (?, ?)", 1, "");
+ assertInvalidMessage("The data cannot be deserialized as a list",
+ "INSERT INTO %s (pk, l) VALUES (?, ?)", 1, -1);
+ }
+
+ @Test
+ public void testInvalidInputForSet() throws Throwable
+ {
+ createTable("CREATE TABLE %s(pk int PRIMARY KEY, s set<text>)");
+ assertInvalidMessage("Not enough bytes to read a set",
+ "INSERT INTO %s (pk, s) VALUES (?, ?)", 1, "test");
+ assertInvalidMessage("String didn't validate.",
+ "INSERT INTO %s (pk, s) VALUES (?, ?)", 1, Long.MAX_VALUE);
+ assertInvalidMessage("Not enough bytes to read a set",
+ "INSERT INTO %s (pk, s) VALUES (?, ?)", 1, "");
+ assertInvalidMessage("The data cannot be deserialized as a set",
+ "INSERT INTO %s (pk, s) VALUES (?, ?)", 1, -1);
+ }
+
+ @Test
+ public void testInvalidInputForMap() throws Throwable
+ {
+ createTable("CREATE TABLE %s(pk int PRIMARY KEY, m map<text, text>)");
+ assertInvalidMessage("Not enough bytes to read a map",
+ "INSERT INTO %s (pk, m) VALUES (?, ?)", 1, "test");
+ assertInvalidMessage("String didn't validate.",
+ "INSERT INTO %s (pk, m) VALUES (?, ?)", 1, Long.MAX_VALUE);
+ assertInvalidMessage("Not enough bytes to read a map",
+ "INSERT INTO %s (pk, m) VALUES (?, ?)", 1, "");
+ assertInvalidMessage("The data cannot be deserialized as a map",
+ "INSERT INTO %s (pk, m) VALUES (?, ?)", 1, -1);
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6dc595dd/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
index 0e7084f..0783dd1 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
@@ -193,4 +193,14 @@ public class TupleTypeTest extends CQLTester
row(0, 0, "b"),
row(0, 0, "c"));
}
+
+ @Test
+ public void testInvalidInputForTuple() throws Throwable
+ {
+ createTable("CREATE TABLE %s(pk int PRIMARY KEY, t tuple<text, text>)");
+ assertInvalidMessage("Not enough bytes to read 0th component",
+ "INSERT INTO %s (pk, t) VALUES (?, ?)", 1, "test");
+ assertInvalidMessage("Not enough bytes to read 0th component",
+ "INSERT INTO %s (pk, t) VALUES (?, ?)", 1, Long.MAX_VALUE);
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6dc595dd/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
index cd87eda..3803e5c 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
@@ -45,6 +45,17 @@ public class UserTypesTest extends CQLTester
}
@Test
+ public void testInvalidInputForUserType() throws Throwable
+ {
+ String myType = createType("CREATE TYPE %s (f int)");
+ createTable("CREATE TABLE %s(pk int PRIMARY KEY, t frozen<" + myType + ">)");
+ assertInvalidMessage("Not enough bytes to read 0th field f",
+ "INSERT INTO %s (pk, t) VALUES (?, ?)", 1, "test");
+ assertInvalidMessage("Not enough bytes to read 0th field f",
+ "INSERT INTO %s (pk, t) VALUES (?, ?)", 1, Long.MAX_VALUE);
+ }
+
+ @Test
public void testCassandra8105() throws Throwable
{
String ut1 = createType("CREATE TYPE %s (a int, b int)");
[2/3] cassandra git commit: Merge branch cassandra-2.2 into
cassandra-3.0
Posted by bl...@apache.org.
Merge branch cassandra-2.2 into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b7fc5dc1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b7fc5dc1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b7fc5dc1
Branch: refs/heads/trunk
Commit: b7fc5dc1c74b20fc1250c738631487db98b72e50
Parents: b7da003 6dc595d
Author: Benjamin Lerer <b....@gmail.com>
Authored: Tue Sep 27 14:06:08 2016 +0200
Committer: Benjamin Lerer <b....@gmail.com>
Committed: Tue Sep 27 14:08:50 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/marshal/UserType.java | 6 +--
.../cassandra/serializers/ListSerializer.java | 10 ++++-
.../cassandra/serializers/MapSerializer.java | 10 ++++-
.../cassandra/serializers/SetSerializer.java | 11 ++++-
.../cassandra/serializers/UTF8Serializer.java | 3 ++
.../validation/entities/CollectionsTest.java | 42 ++++++++++++++++++++
.../cql3/validation/entities/TupleTypeTest.java | 10 +++++
.../cql3/validation/entities/UserTypesTest.java | 11 +++++
9 files changed, 98 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7fc5dc1/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 0524e49,998849e..576dfb5
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -9,64 -7,6 +9,65 @@@ Merged from 2.2
* Fix authentication problem when invoking clqsh copy from a SOURCE command (CASSANDRA-12642)
* Decrement pending range calculator jobs counter in finally block
(CASSANDRA-12554)
+Merged from 2.1:
++ * Make Collections deserialization more robust (CASSANDRA-12618)
+ * Add system property to set the max number of native transport requests in queue (CASSANDRA-11363)
+
+
+3.0.9
+ * Handle composite prefixes with final EOC=0 as in 2.x and refactor LegacyLayout.decodeBound (CASSANDRA-12423)
+ * Fix paging for 2.x to 3.x upgrades (CASSANDRA-11195)
+ * select_distinct_with_deletions_test failing on non-vnode environments (CASSANDRA-11126)
+ * Stack Overflow returned to queries while upgrading (CASSANDRA-12527)
+ * Fix legacy regex for temporary files from 2.2 (CASSANDRA-12565)
+ * Add option to state current gc_grace_seconds to tools/bin/sstablemetadata (CASSANDRA-12208)
+ * Fix file system race condition that may cause LogAwareFileLister to fail to classify files (CASSANDRA-11889)
+ * Fix file handle leaks due to simultaneous compaction/repair and
+ listing snapshots, calculating snapshot sizes, or making schema
+ changes (CASSANDRA-11594)
+ * Fix nodetool repair exits with 0 for some errors (CASSANDRA-12508)
+ * Do not shut down BatchlogManager twice during drain (CASSANDRA-12504)
+ * Disk failure policy should not be invoked on out of space (CASSANDRA-12385)
+ * Calculate last compacted key on startup (CASSANDRA-6216)
+ * Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements (CASSANDRA-7190)
+ * Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436)
+ * Fix potential resource leak in RMIServerSocketFactoryImpl (CASSANDRA-12331)
+ * Backport CASSANDRA-12002 (CASSANDRA-12177)
+ * Make sure compaction stats are updated when compaction is interrupted (CASSANDRA-12100)
+ * Fix potential bad messaging service message for paged range reads
+ within mixed-version 3.x clusters (CASSANDRA-12249)
+ * Change commitlog and sstables to track dirty and clean intervals (CASSANDRA-11828)
+ * NullPointerException during compaction on table with static columns (CASSANDRA-12336)
+ * Fixed ConcurrentModificationException when reading metrics in GraphiteReporter (CASSANDRA-11823)
+ * Fix upgrade of super columns on thrift (CASSANDRA-12335)
+ * Fixed flacky BlacklistingCompactionsTest, switched to fixed size types and increased corruption size (CASSANDRA-12359)
+ * Rerun ReplicationAwareTokenAllocatorTest on failure to avoid flakiness (CASSANDRA-12277)
+ * Exception when computing read-repair for range tombstones (CASSANDRA-12263)
+ * Lost counter writes in compact table and static columns (CASSANDRA-12219)
+ * AssertionError with MVs on updating a row that isn't indexed due to a null value (CASSANDRA-12247)
+ * Disable RR and speculative retry with EACH_QUORUM reads (CASSANDRA-11980)
+ * Add option to override compaction space check (CASSANDRA-12180)
+ * Faster startup by only scanning each directory for temporary files once (CASSANDRA-12114)
+ * Respond with v1/v2 protocol header when responding to driver that attempts
+ to connect with too low of a protocol version (CASSANDRA-11464)
+ * NullPointerExpception when reading/compacting table (CASSANDRA-11988)
+ * Fix problem with undeleteable rows on upgrade to new sstable format (CASSANDRA-12144)
+ * Fix paging logic for deleted partitions with static columns (CASSANDRA-12107)
+ * Wait until the message is being send to decide which serializer must be used (CASSANDRA-11393)
+ * Fix migration of static thrift column names with non-text comparators (CASSANDRA-12147)
+ * Fix upgrading sparse tables that are incorrectly marked as dense (CASSANDRA-11315)
+ * Fix reverse queries ignoring range tombstones (CASSANDRA-11733)
+ * Avoid potential race when rebuilding CFMetaData (CASSANDRA-12098)
+ * Avoid missing sstables when getting the canonical sstables (CASSANDRA-11996)
+ * Always select the live sstables when getting sstables in bounds (CASSANDRA-11944)
+ * Fix column ordering of results with static columns for Thrift requests in
+ a mixed 2.x/3.x cluster, also fix potential non-resolved duplication of
+ those static columns in query results (CASSANDRA-12123)
+ * Avoid digest mismatch with empty but static rows (CASSANDRA-12090)
+ * Fix EOF exception when altering column type (CASSANDRA-11820)
+ * Fix JsonTransformer output of partition with deletion info (CASSANDRA-12418)
+ * Fix NPE in SSTableLoader when specifying partial directory path (CASSANDRA-12609)
+Merged from 2.2:
* Add local address entry in PropertyFileSnitch (CASSANDRA-11332)
* cqlshlib tests: increase default execute timeout (CASSANDRA-12481)
* Forward writes to replacement node when replace_address != broadcast_address (CASSANDRA-8523)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7fc5dc1/src/java/org/apache/cassandra/db/marshal/UserType.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7fc5dc1/src/java/org/apache/cassandra/serializers/ListSerializer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7fc5dc1/src/java/org/apache/cassandra/serializers/MapSerializer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7fc5dc1/src/java/org/apache/cassandra/serializers/SetSerializer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7fc5dc1/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
index a0a6e73,115b755..3d26f5a
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
@@@ -690,178 -608,44 +690,220 @@@ public class CollectionsTest extends CQ
}
@Test
+ public void testListWithElementsBiggerThan64K() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, l list<text>)");
+
+ byte[] bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
+ Arrays.fill(bytes, (byte) 1);
+ String largeText = new String(bytes);
+
+ bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
+ Arrays.fill(bytes, (byte) 2);
+ String largeText2 = new String(bytes);
+
+ execute("INSERT INTO %s(k, l) VALUES (0, ?)", list(largeText, "v2"));
+ flush();
+
+ assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list(largeText, "v2")));
+
+ execute("DELETE l[?] FROM %s WHERE k = 0", 0);
+
+ assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list("v2")));
+
+ execute("UPDATE %s SET l[?] = ? WHERE k = 0", 0, largeText);
+
+ assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list(largeText)));
+
+ // Full overwrite
+ execute("UPDATE %s SET l = ? WHERE k = 0", list("v1", largeText));
+ flush();
+
+ assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list("v1", largeText)));
+
+ execute("UPDATE %s SET l = l + ? WHERE k = 0", list("v2", largeText2));
+
+ assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list("v1", largeText, "v2", largeText2)));
+
+ execute("UPDATE %s SET l = l - ? WHERE k = 0", list(largeText, "v2"));
+
+ assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list("v1", largeText2)));
+
+ execute("DELETE l FROM %s WHERE k = 0");
+
+ assertRows(execute("SELECT l FROM %s WHERE k = 0"), row((Object) null));
+
+ execute("INSERT INTO %s(k, l) VALUES (0, ['" + largeText + "', 'v2'])");
+ flush();
+
+ assertRows(execute("SELECT l FROM %s WHERE k = 0"), row(list(largeText, "v2")));
+ }
+
+ @Test
+ public void testMapsWithElementsBiggerThan64K() throws Throwable
+ {
+ byte[] bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
+ Arrays.fill(bytes, (byte) 1);
+ String largeText = new String(bytes);
+ bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
+ Arrays.fill(bytes, (byte) 2);
+ String largeText2 = new String(bytes);
+
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, m map<text, text>)");
+
+ execute("INSERT INTO %s(k, m) VALUES (0, ?)", map("k1", largeText, largeText, "v2"));
+ flush();
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map("k1", largeText, largeText, "v2")));
+
+ execute("UPDATE %s SET m[?] = ? WHERE k = 0", "k3", largeText);
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map("k1", largeText, largeText, "v2", "k3", largeText)));
+
+ execute("UPDATE %s SET m[?] = ? WHERE k = 0", largeText2, "v4");
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map("k1", largeText, largeText, "v2", "k3", largeText, largeText2, "v4")));
+
+ execute("DELETE m[?] FROM %s WHERE k = 0", "k1");
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map(largeText, "v2", "k3", largeText, largeText2, "v4")));
+
+ execute("DELETE m[?] FROM %s WHERE k = 0", largeText2);
+ flush();
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map(largeText, "v2", "k3", largeText)));
+
+ // Full overwrite
+ execute("UPDATE %s SET m = ? WHERE k = 0", map("k5", largeText, largeText, "v6"));
+ flush();
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map("k5", largeText, largeText, "v6")));
+
+ execute("UPDATE %s SET m = m + ? WHERE k = 0", map("k7", largeText));
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map("k5", largeText, largeText, "v6", "k7", largeText)));
+
+ execute("UPDATE %s SET m = m + ? WHERE k = 0", map(largeText2, "v8"));
+ flush();
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map("k5", largeText, largeText, "v6", "k7", largeText, largeText2, "v8")));
+
+ execute("DELETE m FROM %s WHERE k = 0");
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"), row((Object) null));
+
+ execute("INSERT INTO %s(k, m) VALUES (0, {'" + largeText + "' : 'v1', 'k2' : '" + largeText + "'})");
+ flush();
+
+ assertRows(execute("SELECT m FROM %s WHERE k = 0"),
+ row(map(largeText, "v1", "k2", largeText)));
+ }
+
+ @Test
+ public void testSetsWithElementsBiggerThan64K() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, s set<text>)");
+
+ byte[] bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
+ Arrays.fill(bytes, (byte) 1);
+ String largeText = new String(bytes);
+
+ bytes = new byte[FBUtilities.MAX_UNSIGNED_SHORT + 10];
+ Arrays.fill(bytes, (byte) 2);
+ String largeText2 = new String(bytes);
+
+ execute("INSERT INTO %s(k, s) VALUES (0, ?)", set(largeText, "v2"));
+ flush();
+
+ assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set(largeText, "v2")));
+
+ execute("DELETE s[?] FROM %s WHERE k = 0", largeText);
+
+ assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set("v2")));
+
+ // Full overwrite
+ execute("UPDATE %s SET s = ? WHERE k = 0", set("v1", largeText));
+ flush();
+
+ assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set("v1", largeText)));
+
+ execute("UPDATE %s SET s = s + ? WHERE k = 0", set("v2", largeText2));
+
+ assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set("v1", largeText, "v2", largeText2)));
+
+ execute("UPDATE %s SET s = s - ? WHERE k = 0", set(largeText, "v2"));
+
+ assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set("v1", largeText2)));
+
+ execute("DELETE s FROM %s WHERE k = 0");
+
+ assertRows(execute("SELECT s FROM %s WHERE k = 0"), row((Object) null));
+
+ execute("INSERT INTO %s(k, s) VALUES (0, {'" + largeText + "', 'v2'})");
+ flush();
+
+ assertRows(execute("SELECT s FROM %s WHERE k = 0"), row(set(largeText, "v2")));
+ }
+
+ @Test
+ public void testRemovalThroughUpdate() throws Throwable
+ {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY, l list<int>)");
+
+ execute("INSERT INTO %s(k, l) VALUES(?, ?)", 0, list(1, 2, 3));
+ assertRows(execute("SELECT * FROM %s"), row(0, list(1, 2, 3)));
+
+ execute("UPDATE %s SET l[0] = null WHERE k=0");
+ assertRows(execute("SELECT * FROM %s"), row(0, list(2, 3)));
+ }
++
++ @Test
+ public void testInvalidInputForList() throws Throwable
+ {
+ createTable("CREATE TABLE %s(pk int PRIMARY KEY, l list<text>)");
+ assertInvalidMessage("Not enough bytes to read a list",
+ "INSERT INTO %s (pk, l) VALUES (?, ?)", 1, "test");
+ assertInvalidMessage("Not enough bytes to read a list",
+ "INSERT INTO %s (pk, l) VALUES (?, ?)", 1, Long.MAX_VALUE);
+ assertInvalidMessage("Not enough bytes to read a list",
+ "INSERT INTO %s (pk, l) VALUES (?, ?)", 1, "");
+ assertInvalidMessage("The data cannot be deserialized as a list",
+ "INSERT INTO %s (pk, l) VALUES (?, ?)", 1, -1);
+ }
+
+ @Test
+ public void testInvalidInputForSet() throws Throwable
+ {
+ createTable("CREATE TABLE %s(pk int PRIMARY KEY, s set<text>)");
+ assertInvalidMessage("Not enough bytes to read a set",
+ "INSERT INTO %s (pk, s) VALUES (?, ?)", 1, "test");
+ assertInvalidMessage("String didn't validate.",
+ "INSERT INTO %s (pk, s) VALUES (?, ?)", 1, Long.MAX_VALUE);
+ assertInvalidMessage("Not enough bytes to read a set",
+ "INSERT INTO %s (pk, s) VALUES (?, ?)", 1, "");
+ assertInvalidMessage("The data cannot be deserialized as a set",
+ "INSERT INTO %s (pk, s) VALUES (?, ?)", 1, -1);
+ }
+
+ @Test
+ public void testInvalidInputForMap() throws Throwable
+ {
+ createTable("CREATE TABLE %s(pk int PRIMARY KEY, m map<text, text>)");
+ assertInvalidMessage("Not enough bytes to read a map",
+ "INSERT INTO %s (pk, m) VALUES (?, ?)", 1, "test");
+ assertInvalidMessage("String didn't validate.",
+ "INSERT INTO %s (pk, m) VALUES (?, ?)", 1, Long.MAX_VALUE);
+ assertInvalidMessage("Not enough bytes to read a map",
+ "INSERT INTO %s (pk, m) VALUES (?, ?)", 1, "");
+ assertInvalidMessage("The data cannot be deserialized as a map",
+ "INSERT INTO %s (pk, m) VALUES (?, ?)", 1, -1);
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7fc5dc1/test/unit/org/apache/cassandra/cql3/validation/entities/TupleTypeTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7fc5dc1/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
----------------------------------------------------------------------
[3/3] cassandra git commit: Merge branch cassandra-3.0 into trunk
Posted by bl...@apache.org.
Merge branch cassandra-3.0 into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7bef4185
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7bef4185
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7bef4185
Branch: refs/heads/trunk
Commit: 7bef41856f7156a32c5b0cc066b43c697a52d69d
Parents: 5777013 b7fc5dc
Author: Benjamin Lerer <b....@gmail.com>
Authored: Tue Sep 27 14:10:37 2016 +0200
Committer: Benjamin Lerer <b....@gmail.com>
Committed: Tue Sep 27 14:13:22 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/marshal/TupleType.java | 7 +++-
.../apache/cassandra/db/marshal/UserType.java | 6 +--
.../cassandra/serializers/ListSerializer.java | 10 ++++-
.../cassandra/serializers/MapSerializer.java | 10 ++++-
.../cassandra/serializers/SetSerializer.java | 11 ++++-
.../cassandra/serializers/UTF8Serializer.java | 3 ++
.../validation/entities/CollectionsTest.java | 42 ++++++++++++++++++++
.../cql3/validation/entities/TupleTypeTest.java | 10 +++++
.../cql3/validation/entities/UserTypesTest.java | 11 +++++
10 files changed, 104 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7bef4185/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index e10b870,576dfb5..b6a687d
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -86,57 -30,12 +86,58 @@@ Merged from 3.0
* Disk failure policy should not be invoked on out of space (CASSANDRA-12385)
* Calculate last compacted key on startup (CASSANDRA-6216)
* Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements (CASSANDRA-7190)
+ * If CF has no clustering columns, any row cache is full partition cache (CASSANDRA-12499)
+Merged from 2.2:
++ * Make Collections deserialization more robust (CASSANDRA-12618)
+ * Fix exceptions when enabling gossip on nodes that haven't joined the ring (CASSANDRA-12253)
+ * Fix authentication problem when invoking clqsh copy from a SOURCE command (CASSANDRA-12642)
+ * Decrement pending range calculator jobs counter in finally block
+ * cqlshlib tests: increase default execute timeout (CASSANDRA-12481)
+ * Forward writes to replacement node when replace_address != broadcast_address (CASSANDRA-8523)
+ * Fail repair on non-existing table (CASSANDRA-12279)
+ * Enable repair -pr and -local together (fix regression of CASSANDRA-7450) (CASSANDRA-12522)
+
+
+3.8, 3.9
+ * Fix value skipping with counter columns (CASSANDRA-11726)
+ * Fix nodetool tablestats miss SSTable count (CASSANDRA-12205)
+ * Fixed flacky SSTablesIteratedTest (CASSANDRA-12282)
+ * Fixed flacky SSTableRewriterTest: check file counts before calling validateCFS (CASSANDRA-12348)
+ * cqlsh: Fix handling of $$-escaped strings (CASSANDRA-12189)
+ * Fix SSL JMX requiring truststore containing server cert (CASSANDRA-12109)
+ * RTE from new CDC column breaks in flight queries (CASSANDRA-12236)
+ * Fix hdr logging for single operation workloads (CASSANDRA-12145)
+ * Fix SASI PREFIX search in CONTAINS mode with partial terms (CASSANDRA-12073)
+ * Increase size of flushExecutor thread pool (CASSANDRA-12071)
+ * Partial revert of CASSANDRA-11971, cannot recycle buffer in SP.sendMessagesToNonlocalDC (CASSANDRA-11950)
+ * Upgrade netty to 4.0.39 (CASSANDRA-12032, CASSANDRA-12034)
+ * Improve details in compaction log message (CASSANDRA-12080)
+ * Allow unset values in CQLSSTableWriter (CASSANDRA-11911)
+ * Chunk cache to request compressor-compatible buffers if pool space is exhausted (CASSANDRA-11993)
+ * Remove DatabaseDescriptor dependencies from SequentialWriter (CASSANDRA-11579)
+ * Move skip_stop_words filter before stemming (CASSANDRA-12078)
+ * Support seek() in EncryptedFileSegmentInputStream (CASSANDRA-11957)
+ * SSTable tools mishandling LocalPartitioner (CASSANDRA-12002)
+ * When SEPWorker assigned work, set thread name to match pool (CASSANDRA-11966)
+ * Add cross-DC latency metrics (CASSANDRA-11596)
+ * Allow terms in selection clause (CASSANDRA-10783)
+ * Add bind variables to trace (CASSANDRA-11719)
+ * Switch counter shards' clock to timestamps (CASSANDRA-9811)
+ * Introduce HdrHistogram and response/service/wait separation to stress tool (CASSANDRA-11853)
+ * entry-weighers in QueryProcessor should respect partitionKeyBindIndexes field (CASSANDRA-11718)
+ * Support older ant versions (CASSANDRA-11807)
+ * Estimate compressed on disk size when deciding if sstable size limit reached (CASSANDRA-11623)
+ * cassandra-stress profiles should support case sensitive schemas (CASSANDRA-11546)
+ * Remove DatabaseDescriptor dependency from FileUtils (CASSANDRA-11578)
+ * Faster streaming (CASSANDRA-9766)
+ * Add prepared query parameter to trace for "Execute CQL3 prepared query" session (CASSANDRA-11425)
+ * Add repaired percentage metric (CASSANDRA-11503)
+ * Add Change-Data-Capture (CASSANDRA-8844)
+Merged from 3.0:
+ * Fix paging for 2.x to 3.x upgrades (CASSANDRA-11195)
* Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436)
* Fix potential resource leak in RMIServerSocketFactoryImpl (CASSANDRA-12331)
- * Backport CASSANDRA-12002 (CASSANDRA-12177)
* Make sure compaction stats are updated when compaction is interrupted (CASSANDRA-12100)
- * Fix potential bad messaging service message for paged range reads
- within mixed-version 3.x clusters (CASSANDRA-12249)
* Change commitlog and sstables to track dirty and clean intervals (CASSANDRA-11828)
* NullPointerException during compaction on table with static columns (CASSANDRA-12336)
* Fixed ConcurrentModificationException when reading metrics in GraphiteReporter (CASSANDRA-11823)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7bef4185/src/java/org/apache/cassandra/db/marshal/TupleType.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/marshal/TupleType.java
index eaf6653,5486183..24498aa
--- a/src/java/org/apache/cassandra/db/marshal/TupleType.java
+++ b/src/java/org/apache/cassandra/db/marshal/TupleType.java
@@@ -154,7 -125,7 +154,7 @@@ public class TupleType extends Abstract
if (!input.hasRemaining())
return;
-- if (input.remaining() < 4)
++ if (input.remaining() < Integer.BYTES)
throw new MarshalException(String.format("Not enough bytes to read size of %dth component", i));
int size = input.getInt();
@@@ -188,17 -159,8 +188,22 @@@
return Arrays.copyOfRange(components, 0, i);
int size = input.getInt();
++
++ if (input.remaining() < size)
++ throw new MarshalException(String.format("Not enough bytes to read %dth component", i));
++
++ // size < 0 means null value
components[i] = size < 0 ? null : ByteBufferUtil.readBytes(input, size);
}
+
+ // error out if we got more values in the tuple/UDT than we expected
+ if (input.hasRemaining())
+ {
+ throw new InvalidRequestException(String.format(
+ "Expected %s %s for %s column, but got more",
+ size(), size() == 1 ? "value" : "values", this.asCQL3Type()));
+ }
+
return components;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7bef4185/src/java/org/apache/cassandra/db/marshal/UserType.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/marshal/UserType.java
index 74378b2,f11c029..a9c02f3
--- a/src/java/org/apache/cassandra/db/marshal/UserType.java
+++ b/src/java/org/apache/cassandra/db/marshal/UserType.java
@@@ -24,15 -25,11 +24,15 @@@ import java.util.stream.Collectors
import com.google.common.base.Objects;
import org.apache.cassandra.cql3.*;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
- import org.apache.cassandra.serializers.*;
+ import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A user defined type.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7bef4185/src/java/org/apache/cassandra/serializers/UTF8Serializer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7bef4185/test/unit/org/apache/cassandra/cql3/validation/entities/CollectionsTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7bef4185/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
index 59383c6,850d8ee..1b2f3e9
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UserTypesTest.java
@@@ -48,6 -46,17 +48,17 @@@ public class UserTypesTest extends CQLT
}
@Test
+ public void testInvalidInputForUserType() throws Throwable
+ {
+ String myType = createType("CREATE TYPE %s (f int)");
+ createTable("CREATE TABLE %s(pk int PRIMARY KEY, t frozen<" + myType + ">)");
- assertInvalidMessage("Not enough bytes to read 0th field f",
++ assertInvalidMessage("Not enough bytes to read 0th component",
+ "INSERT INTO %s (pk, t) VALUES (?, ?)", 1, "test");
- assertInvalidMessage("Not enough bytes to read 0th field f",
++ assertInvalidMessage("Not enough bytes to read 0th component",
+ "INSERT INTO %s (pk, t) VALUES (?, ?)", 1, Long.MAX_VALUE);
+ }
+
+ @Test
public void testCassandra8105() throws Throwable
{
String ut1 = createType("CREATE TYPE %s (a int, b int)");