You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ty...@apache.org on 2015/05/08 20:39:08 UTC
[1/3] cassandra git commit: Add ks and table to err log about large
collections
Repository: cassandra
Updated Branches:
refs/heads/trunk 28132de16 -> 1f23bb4b3
Add ks and table to err log about large collections
Patch by Carl Yeksigian; reviewed by Tyler Hobbs for CASSANDRA-9286
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/15235ee6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/15235ee6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/15235ee6
Branch: refs/heads/trunk
Commit: 15235ee63be2d8f39486ff97dbbdd8ed24bb9812
Parents: 917068d
Author: Tyler Hobbs <ty...@apache.org>
Authored: Fri May 8 13:28:33 2015 -0500
Committer: Tyler Hobbs <ty...@apache.org>
Committed: Fri May 8 13:28:33 2015 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../cassandra/cql3/statements/SelectStatement.java | 2 +-
.../org/apache/cassandra/db/marshal/CollectionType.java | 11 +++++++----
src/java/org/apache/cassandra/db/marshal/ListType.java | 5 +++--
src/java/org/apache/cassandra/db/marshal/MapType.java | 5 +++--
src/java/org/apache/cassandra/db/marshal/SetType.java | 5 +++--
6 files changed, 19 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15235ee6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1540d28..d3715c4 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,6 @@
2.0.15:
+ * Include keyspace and table name in error log for collections over the size
+ limit (CASSANDRA-9286)
* Avoid potential overlap in LCS with single-partition sstables (CASSANDRA-9322)
* Log warning message when a table is queried before the schema has fully
propagated (CASSANDRA-9136)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15235ee6/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
index 6b3c781..8a4deb6 100644
--- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@ -1399,7 +1399,7 @@ public class SelectStatement implements CQLStatement, MeasurableForPreparedCache
if (name.type.isCollection())
{
List<Pair<ByteBuffer, Column>> collection = group.getCollection(name.name.key);
- result.add(collection == null ? null : ((CollectionType)name.type).serialize(collection));
+ result.add(collection == null ? null : ((CollectionType)name.type).serialize(name, collection));
}
else
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15235ee6/src/java/org/apache/cassandra/db/marshal/CollectionType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/marshal/CollectionType.java b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
index 29b77a0..d1ae130 100644
--- a/src/java/org/apache/cassandra/db/marshal/CollectionType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.cql3.CFDefinition;
import org.apache.cassandra.cql3.CQL3Type;
import org.apache.cassandra.db.Column;
import org.apache.cassandra.serializers.MarshalException;
@@ -58,7 +59,7 @@ public abstract class CollectionType<T> extends AbstractType<T>
protected abstract void appendToStringBuilder(StringBuilder sb);
- public abstract ByteBuffer serialize(List<Pair<ByteBuffer, Column>> columns);
+ public abstract ByteBuffer serialize(CFDefinition.Name name, List<Pair<ByteBuffer, Column>> columns);
@Override
public String toString()
@@ -108,13 +109,15 @@ public abstract class CollectionType<T> extends AbstractType<T>
return (ByteBuffer)result.flip();
}
- protected List<Pair<ByteBuffer, Column>> enforceLimit(List<Pair<ByteBuffer, Column>> columns)
+ protected List<Pair<ByteBuffer, Column>> enforceLimit(CFDefinition.Name name, List<Pair<ByteBuffer, Column>> columns)
{
if (columns.size() <= MAX_ELEMENTS)
return columns;
- logger.error("Detected collection with {} elements, more than the {} limit. Only the first {} elements will be returned to the client. "
- + "Please see http://cassandra.apache.org/doc/cql3/CQL.html#collections for more details.", columns.size(), MAX_ELEMENTS, MAX_ELEMENTS);
+ logger.error("Detected collection for table {}.{} with {} elements, more than the {} limit. Only the first {}"
+ + "elements will be returned to the client. Please see "
+ + "http://cassandra.apache.org/doc/cql3/CQL.html#collections for more details.",
+ name.ksName, name.cfName, columns.size(), MAX_ELEMENTS, MAX_ELEMENTS);
return columns.subList(0, MAX_ELEMENTS);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15235ee6/src/java/org/apache/cassandra/db/marshal/ListType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/marshal/ListType.java b/src/java/org/apache/cassandra/db/marshal/ListType.java
index 4b45bd7..dff98b7 100644
--- a/src/java/org/apache/cassandra/db/marshal/ListType.java
+++ b/src/java/org/apache/cassandra/db/marshal/ListType.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
import java.util.*;
+import org.apache.cassandra.cql3.CFDefinition;
import org.apache.cassandra.db.Column;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
@@ -115,9 +116,9 @@ public class ListType<T> extends CollectionType<List<T>>
sb.append(getClass().getName()).append(TypeParser.stringifyTypeParameters(Collections.<AbstractType<?>>singletonList(elements)));
}
- public ByteBuffer serialize(List<Pair<ByteBuffer, Column>> columns)
+ public ByteBuffer serialize(CFDefinition.Name name, List<Pair<ByteBuffer, Column>> columns)
{
- columns = enforceLimit(columns);
+ columns = enforceLimit(name, columns);
List<ByteBuffer> bbs = new ArrayList<ByteBuffer>(columns.size());
int size = 0;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15235ee6/src/java/org/apache/cassandra/db/marshal/MapType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/marshal/MapType.java b/src/java/org/apache/cassandra/db/marshal/MapType.java
index 08f795f..f6883ee 100644
--- a/src/java/org/apache/cassandra/db/marshal/MapType.java
+++ b/src/java/org/apache/cassandra/db/marshal/MapType.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
import java.util.*;
+import org.apache.cassandra.cql3.CFDefinition;
import org.apache.cassandra.db.Column;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
@@ -123,9 +124,9 @@ public class MapType<K, V> extends CollectionType<Map<K, V>>
/**
* Creates the same output than serialize, but from the internal representation.
*/
- public ByteBuffer serialize(List<Pair<ByteBuffer, Column>> columns)
+ public ByteBuffer serialize(CFDefinition.Name name, List<Pair<ByteBuffer, Column>> columns)
{
- columns = enforceLimit(columns);
+ columns = enforceLimit(name, columns);
List<ByteBuffer> bbs = new ArrayList<ByteBuffer>(2 * columns.size());
int size = 0;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15235ee6/src/java/org/apache/cassandra/db/marshal/SetType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/marshal/SetType.java b/src/java/org/apache/cassandra/db/marshal/SetType.java
index 614ecc7..b43c9b6 100644
--- a/src/java/org/apache/cassandra/db/marshal/SetType.java
+++ b/src/java/org/apache/cassandra/db/marshal/SetType.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
import java.util.*;
+import org.apache.cassandra.cql3.CFDefinition;
import org.apache.cassandra.db.Column;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
@@ -88,9 +89,9 @@ public class SetType<T> extends CollectionType<Set<T>>
sb.append(getClass().getName()).append(TypeParser.stringifyTypeParameters(Collections.<AbstractType<?>>singletonList(elements)));
}
- public ByteBuffer serialize(List<Pair<ByteBuffer, Column>> columns)
+ public ByteBuffer serialize(CFDefinition.Name name, List<Pair<ByteBuffer, Column>> columns)
{
- columns = enforceLimit(columns);
+ columns = enforceLimit(name, columns);
List<ByteBuffer> bbs = new ArrayList<ByteBuffer>(columns.size());
int size = 0;
[3/3] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by ty...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1f23bb4b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1f23bb4b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1f23bb4b
Branch: refs/heads/trunk
Commit: 1f23bb4b3d293ff614530884e5307a102c2ff677
Parents: 28132de d8b79d5
Author: Tyler Hobbs <ty...@apache.org>
Authored: Fri May 8 13:36:55 2015 -0500
Committer: Tyler Hobbs <ty...@apache.org>
Committed: Fri May 8 13:36:55 2015 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../cassandra/cql3/statements/SelectStatement.java | 2 +-
.../apache/cassandra/db/marshal/CollectionType.java | 13 ++++++++-----
3 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1f23bb4b/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1f23bb4b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1f23bb4b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
----------------------------------------------------------------------
[2/3] cassandra git commit: Merge branch 'cassandra-2.0' into
cassandra-2.1
Posted by ty...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d8b79d5a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d8b79d5a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d8b79d5a
Branch: refs/heads/trunk
Commit: d8b79d5aff93b75198bb60e5140ff3fea912d387
Parents: a431d84 15235ee
Author: Tyler Hobbs <ty...@apache.org>
Authored: Fri May 8 13:36:10 2015 -0500
Committer: Tyler Hobbs <ty...@apache.org>
Committed: Fri May 8 13:36:10 2015 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../cassandra/cql3/statements/SelectStatement.java | 2 +-
.../apache/cassandra/db/marshal/CollectionType.java | 13 ++++++++-----
3 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d8b79d5a/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 2b7a355,d3715c4..162b34f
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,21 -1,6 +1,23 @@@
-2.0.15:
+2.1.6
+ * Delete processed sstables in sstablesplit/sstableupgrade (CASSANDRA-8606)
+ * Improve sstable exclusion from partition tombstones (CASSANDRA-9298)
+ * Validate the indexed column rather than the cell's contents for 2i (CASSANDRA-9057)
+ * Add support for top-k custom 2i queries (CASSANDRA-8717)
+ * Fix error when dropping table during compaction (CASSANDRA-9251)
+ * cassandra-stress supports validation operations over user profiles (CASSANDRA-8773)
+ * Add support for rate limiting log messages (CASSANDRA-9029)
+ * Log the partition key with tombstone warnings (CASSANDRA-8561)
+ * Reduce runWithCompactionsDisabled poll interval to 1ms (CASSANDRA-9271)
+ * Fix PITR commitlog replay (CASSANDRA-9195)
+ * GCInspector logs very different times (CASSANDRA-9124)
+ * Fix deleting from an empty list (CASSANDRA-9198)
+ * Update tuple and collection types that use a user-defined type when that UDT
+ is modified (CASSANDRA-9148, CASSANDRA-9192)
+ * Use higher timeout for prepair and snapshot in repair (CASSANDRA-9261)
+ * Fix anticompaction blocking ANTI_ENTROPY stage (CASSANDRA-9151)
+Merged from 2.0:
+ * Include keyspace and table name in error log for collections over the size
+ limit (CASSANDRA-9286)
* Avoid potential overlap in LCS with single-partition sstables (CASSANDRA-9322)
* Log warning message when a table is queried before the schema has fully
propagated (CASSANDRA-9136)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d8b79d5a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
index 07e60d4,8a4deb6..51f4941
--- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@@ -1288,25 -1300,12 +1288,25 @@@ public class SelectStatement implement
}
}
- private boolean hasValueForQuery(ColumnGroupMap staticGroup)
+ private static void addValue(Selection.ResultSetBuilder result, ColumnDefinition def, CQL3Row row, QueryOptions options)
{
- for (CFDefinition.Name name : Iterables.filter(selection.getColumns(), isStaticFilter))
- if (staticGroup.hasValueFor(name.name.key))
- return true;
- return false;
+ if (row == null)
+ {
+ result.add((ByteBuffer)null);
+ return;
+ }
+
+ if (def.type.isMultiCell())
+ {
+ List<Cell> cells = row.getMultiCellColumn(def.name);
+ ByteBuffer buffer = cells == null
+ ? null
- : ((CollectionType)def.type).serializeForNativeProtocol(cells, options.getProtocolVersion());
++ : ((CollectionType)def.type).serializeForNativeProtocol(def, cells, options.getProtocolVersion());
+ result.add(buffer);
+ return;
+ }
+
+ result.add(row.getColumn(def.name));
}
private boolean hasNoClusteringColumnsRestriction()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d8b79d5a/src/java/org/apache/cassandra/db/marshal/CollectionType.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/marshal/CollectionType.java
index 24ad533,d1ae130..8662c1e
--- a/src/java/org/apache/cassandra/db/marshal/CollectionType.java
+++ b/src/java/org/apache/cassandra/db/marshal/CollectionType.java
@@@ -20,8 -20,6 +20,9 @@@ package org.apache.cassandra.db.marshal
import java.nio.ByteBuffer;
import java.util.List;
++import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.db.Cell;
+import org.apache.cassandra.transport.Server;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -76,102 -86,49 +77,104 @@@ public abstract class CollectionType<T
}
}
- public void validate(ByteBuffer bytes)
+ public boolean isCollection()
{
- valueComparator().validate(bytes);
+ return true;
}
- public boolean isCollection()
+ @Override
+ public void validateCellValue(ByteBuffer cellValue) throws MarshalException
{
- return true;
+ if (isMultiCell())
+ valueComparator().validate(cellValue);
+ else
+ super.validateCellValue(cellValue);
}
- // Utilitary method
- protected static ByteBuffer pack(List<ByteBuffer> buffers, int elements, int size)
+ /**
+ * Checks if this collection is Map.
+ * @return <code>true</code> if this collection is a Map, <code>false</code> otherwise.
+ */
+ public boolean isMap()
{
- ByteBuffer result = ByteBuffer.allocate(2 + size);
- result.putShort((short)elements);
- for (ByteBuffer bb : buffers)
- {
- result.putShort((short)bb.remaining());
- result.put(bb.duplicate());
- }
- return (ByteBuffer)result.flip();
+ return kind == Kind.MAP;
+ }
+
- public List<Cell> enforceLimit(List<Cell> cells, int version)
++ public List<Cell> enforceLimit(ColumnDefinition def, List<Cell> cells, int version)
+ {
+ assert isMultiCell();
+
+ if (version >= Server.VERSION_3 || cells.size() <= MAX_ELEMENTS)
+ return cells;
+
- logger.error("Detected collection with {} elements, more than the {} limit. Only the first {} elements will be returned to the client. "
- + "Please see http://cassandra.apache.org/doc/cql3/CQL.html#collections for more details.", cells.size(), MAX_ELEMENTS, MAX_ELEMENTS);
++ logger.error("Detected collection for table {}.{} with {} elements, more than the {} limit. Only the first {}" +
++ " elements will be returned to the client. Please see " +
++ "http://cassandra.apache.org/doc/cql3/CQL.html#collections for more details.",
++ def.ksName, def.cfName, cells.size(), MAX_ELEMENTS, MAX_ELEMENTS);
+ return cells.subList(0, MAX_ELEMENTS);
+ }
+
+ public abstract List<ByteBuffer> serializedValues(List<Cell> cells);
+
- public ByteBuffer serializeForNativeProtocol(List<Cell> cells, int version)
++ public ByteBuffer serializeForNativeProtocol(ColumnDefinition def, List<Cell> cells, int version)
+ {
+ assert isMultiCell();
- cells = enforceLimit(cells, version);
++ cells = enforceLimit(def, cells, version);
+ List<ByteBuffer> values = serializedValues(cells);
+ return CollectionSerializer.pack(values, cells.size(), version);
}
- protected List<Pair<ByteBuffer, Column>> enforceLimit(CFDefinition.Name name, List<Pair<ByteBuffer, Column>> columns)
+ @Override
+ public boolean isCompatibleWith(AbstractType<?> previous)
{
- if (columns.size() <= MAX_ELEMENTS)
- return columns;
-
- logger.error("Detected collection for table {}.{} with {} elements, more than the {} limit. Only the first {}"
- + "elements will be returned to the client. Please see "
- + "http://cassandra.apache.org/doc/cql3/CQL.html#collections for more details.",
- name.ksName, name.cfName, columns.size(), MAX_ELEMENTS, MAX_ELEMENTS);
- return columns.subList(0, MAX_ELEMENTS);
+ if (this == previous)
+ return true;
+
+ if (!getClass().equals(previous.getClass()))
+ return false;
+
+ CollectionType tprev = (CollectionType) previous;
+ if (this.isMultiCell() != tprev.isMultiCell())
+ return false;
+
+ // subclasses should handle compatibility checks for frozen collections
+ if (!this.isMultiCell())
+ return isCompatibleWithFrozen(tprev);
+
+ if (!this.nameComparator().isCompatibleWith(tprev.nameComparator()))
+ return false;
+
+ // the value comparator is only used for Cell values, so sorting doesn't matter
+ return this.valueComparator().isValueCompatibleWith(tprev.valueComparator());
}
- public static ByteBuffer pack(List<ByteBuffer> buffers, int elements)
+ @Override
+ public boolean isValueCompatibleWithInternal(AbstractType<?> previous)
{
- int size = 0;
- for (ByteBuffer bb : buffers)
- size += 2 + bb.remaining();
- return pack(buffers, elements, size);
+ // for multi-cell collections, compatibility and value-compatibility are the same
+ if (this.isMultiCell())
+ return isCompatibleWith(previous);
+
+ if (this == previous)
+ return true;
+
+ if (!getClass().equals(previous.getClass()))
+ return false;
+
+ CollectionType tprev = (CollectionType) previous;
+ if (this.isMultiCell() != tprev.isMultiCell())
+ return false;
+
+ // subclasses should handle compatibility checks for frozen collections
+ return isValueCompatibleWithFrozen(tprev);
}
+ /** A version of isCompatibleWith() to deal with non-multicell (frozen) collections */
+ protected abstract boolean isCompatibleWithFrozen(CollectionType<?> previous);
+
+ /** A version of isValueCompatibleWith() to deal with non-multicell (frozen) collections */
+ protected abstract boolean isValueCompatibleWithFrozen(CollectionType<?> previous);
+
public CQL3Type asCQL3Type()
{
return new CQL3Type.Collection(this);