You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2015/05/06 12:55:57 UTC
[1/2] cassandra git commit: Validate the indexed column rather than
the cell's contents.
Repository: cassandra
Updated Branches:
refs/heads/trunk 20c5ea02c -> 6b1276791
Validate the indexed column rather than the cell's contents.
Patch by Carl Yeksigian; reviewed by Sam Tunnicliffe for CASSANDRA-9057
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/04a2c242
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/04a2c242
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/04a2c242
Branch: refs/heads/trunk
Commit: 04a2c2420b086036e4b7cc585ac507b0b9bc9558
Parents: 2252773
Author: Carl Yeksigian <ca...@apache.org>
Authored: Wed May 6 12:39:42 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed May 6 12:47:36 2015 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cql3/statements/UpdateStatement.java | 13 +-
.../AbstractSimplePerColumnSecondaryIndex.java | 8 +-
.../db/index/PerColumnSecondaryIndex.java | 5 +
.../db/index/PerRowSecondaryIndex.java | 7 +
.../cassandra/db/index/SecondaryIndex.java | 2 +-
.../db/index/SecondaryIndexManager.java | 8 +-
.../cassandra/thrift/CassandraServer.java | 6 +-
.../cassandra/thrift/ThriftValidation.java | 18 +-
.../cql3/IndexedValuesValidationTest.java | 103 +++++++--
.../db/SecondaryIndexCellSizeTest.java | 224 -------------------
.../cassandra/thrift/ThriftValidationTest.java | 10 +-
12 files changed, 137 insertions(+), 268 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 05728d8..09bdfb1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.6
+ * Validate the indexed column rather than the cell's contents for 2i (CASSANDRA-9057)
* Add support for top-k custom 2i queries (CASSANDRA-8717)
* Fix error when dropping table during compaction (CASSANDRA-9251)
* cassandra-stress supports validation operations over user profiles (CASSANDRA-8773)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
index 63f87c0..06282ad 100644
--- a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
@@ -25,6 +25,7 @@ import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.composites.Composite;
+import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -112,16 +113,17 @@ public class UpdateStatement extends ModificationStatement
// validateIndexedColumns trigger a call to Keyspace.open() which we want to be able to avoid in some case
//(e.g. when using CQLSSTableWriter)
if (validateIndexedColumns)
- validateIndexedColumns(cf);
+ validateIndexedColumns(key, cf);
}
/**
* Checks that the value of the indexed columns is valid.
*
+ * @param key row key for the column family
* @param cf the column family
* @throws InvalidRequestException if one of the values is invalid
*/
- private void validateIndexedColumns(ColumnFamily cf) throws InvalidRequestException
+ private void validateIndexedColumns(ByteBuffer key, ColumnFamily cf) throws InvalidRequestException
{
SecondaryIndexManager indexManager = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfId).indexManager;
if (indexManager.hasIndexes())
@@ -129,12 +131,15 @@ public class UpdateStatement extends ModificationStatement
for (Cell cell : cf)
{
// Indexed values must be validated by any applicable index. See CASSANDRA-3057/4240/8081 for more details
- if (!indexManager.validate(cell))
+ SecondaryIndex failedIndex = indexManager.validate(key, cell);
+ if (failedIndex != null)
+ {
throw new InvalidRequestException(String.format("Can't index column value of size %d for index %s on %s.%s",
cell.value().remaining(),
- cfm.getColumnDefinition(cell.name()).getIndexName(),
+ failedIndex.getIndexName(),
cfm.ksName,
cfm.cfName));
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java b/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java
index 4c398c4..e80c7bd 100644
--- a/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java
+++ b/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java
@@ -185,5 +185,11 @@ public abstract class AbstractSimplePerColumnSecondaryIndex extends PerColumnSec
public long estimateResultRows()
{
return getIndexCfs().getMeanColumns();
- }
+ }
+
+ public boolean validate(ByteBuffer rowKey, Cell cell)
+ {
+ return getIndexedValue(rowKey, cell).remaining() < FBUtilities.MAX_UNSIGNED_SHORT
+ && makeIndexColumnName(rowKey, cell).toByteBuffer().remaining() < FBUtilities.MAX_UNSIGNED_SHORT;
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java b/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java
index 9740d09..ba902ec 100644
--- a/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java
+++ b/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java
@@ -64,6 +64,11 @@ public abstract class PerColumnSecondaryIndex extends SecondaryIndex
return getIndexName();
}
+ public boolean validate(ByteBuffer rowKey, Cell cell)
+ {
+ return validate(cell);
+ }
+
public boolean validate(Cell cell)
{
return cell.value().remaining() < FBUtilities.MAX_UNSIGNED_SHORT;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java b/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java
index d73d056..f6f0e8d 100644
--- a/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java
+++ b/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.db.index;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.db.Cell;
import org.apache.cassandra.db.ColumnFamily;
@@ -58,6 +59,12 @@ public abstract class PerRowSecondaryIndex extends SecondaryIndex
}
}
+
+ public boolean validate(ByteBuffer rowKey, Cell cell)
+ {
+ return validate(cell);
+ }
+
public boolean validate(Cell cell)
{
return true;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
index 3081016..20fdcec 100644
--- a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
+++ b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
@@ -354,7 +354,7 @@ public abstract class SecondaryIndex
return index;
}
- public abstract boolean validate(Cell cell);
+ public abstract boolean validate(ByteBuffer rowKey, Cell cell);
public abstract long estimateResultRows();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
index ab6df1e..bf3aea5 100644
--- a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
+++ b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
@@ -666,14 +666,14 @@ public class SecondaryIndexManager
index.setIndexRemoved();
}
- public boolean validate(Cell cell)
+ public SecondaryIndex validate(ByteBuffer rowKey, Cell cell)
{
for (SecondaryIndex index : indexFor(cell.name()))
{
- if (!index.validate(cell))
- return false;
+ if (!index.validate(rowKey, cell))
+ return index;
}
- return true;
+ return null;
}
static boolean shouldCleanupOldValue(Cell oldCell, Cell newCell)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/src/java/org/apache/cassandra/thrift/CassandraServer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/CassandraServer.java b/src/java/org/apache/cassandra/thrift/CassandraServer.java
index 6955e64..e8d08b5 100644
--- a/src/java/org/apache/cassandra/thrift/CassandraServer.java
+++ b/src/java/org/apache/cassandra/thrift/CassandraServer.java
@@ -673,7 +673,7 @@ public class CassandraServer implements Cassandra.Iface
throw new org.apache.cassandra.exceptions.InvalidRequestException("missing mandatory super column name for super CF " + column_parent.column_family);
}
ThriftValidation.validateColumnNames(metadata, column_parent, Arrays.asList(column.name));
- ThriftValidation.validateColumnData(metadata, column_parent.super_column, column);
+ ThriftValidation.validateColumnData(metadata, key, column_parent.super_column, column);
org.apache.cassandra.db.Mutation mutation;
try
@@ -766,7 +766,7 @@ public class CassandraServer implements Cassandra.Iface
});
ThriftValidation.validateColumnNames(metadata, new ColumnParent(column_family), names);
for (Column column : updates)
- ThriftValidation.validateColumnData(metadata, null, column);
+ ThriftValidation.validateColumnData(metadata, key, null, column);
CFMetaData cfm = Schema.instance.getCFMetaData(cState.getKeyspace(), column_family);
ColumnFamily cfUpdates = ArrayBackedSortedColumns.factory.create(cfm);
@@ -858,7 +858,7 @@ public class CassandraServer implements Cassandra.Iface
for (Mutation m : columnFamilyMutations.getValue())
{
- ThriftValidation.validateMutation(metadata, m);
+ ThriftValidation.validateMutation(metadata, key, m);
if (m.deletion != null)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/src/java/org/apache/cassandra/thrift/ThriftValidation.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/ThriftValidation.java b/src/java/org/apache/cassandra/thrift/ThriftValidation.java
index 3b5663b..d5d9f73 100644
--- a/src/java/org/apache/cassandra/thrift/ThriftValidation.java
+++ b/src/java/org/apache/cassandra/thrift/ThriftValidation.java
@@ -30,6 +30,7 @@ import org.apache.cassandra.db.composites.*;
import org.apache.cassandra.db.filter.IDiskAtomFilter;
import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.db.filter.SliceQueryFilter;
+import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.ColumnToCollectionType;
@@ -294,7 +295,7 @@ public class ThriftValidation
}
}
- public static void validateColumnOrSuperColumn(CFMetaData metadata, ColumnOrSuperColumn cosc)
+ public static void validateColumnOrSuperColumn(CFMetaData metadata, ByteBuffer key, ColumnOrSuperColumn cosc)
throws org.apache.cassandra.exceptions.InvalidRequestException
{
boolean isCommutative = metadata.isCounter();
@@ -315,7 +316,7 @@ public class ThriftValidation
validateTtl(cosc.column);
validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column((ByteBuffer)null).setColumn(cosc.column.name));
- validateColumnData(metadata, null, cosc.column);
+ validateColumnData(metadata, key, null, cosc.column);
}
if (cosc.super_column != null)
@@ -326,7 +327,7 @@ public class ThriftValidation
for (Column c : cosc.super_column.columns)
{
validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column(cosc.super_column.name).setColumn(c.name));
- validateColumnData(metadata, cosc.super_column.name, c);
+ validateColumnData(metadata, key, cosc.super_column.name, c);
}
}
@@ -365,7 +366,7 @@ public class ThriftValidation
}
}
- public static void validateMutation(CFMetaData metadata, Mutation mut)
+ public static void validateMutation(CFMetaData metadata, ByteBuffer key, Mutation mut)
throws org.apache.cassandra.exceptions.InvalidRequestException
{
ColumnOrSuperColumn cosc = mut.column_or_supercolumn;
@@ -382,7 +383,7 @@ public class ThriftValidation
if (cosc != null)
{
- validateColumnOrSuperColumn(metadata, cosc);
+ validateColumnOrSuperColumn(metadata, key, cosc);
}
else
{
@@ -431,7 +432,7 @@ public class ThriftValidation
/**
* Validates the data part of the column (everything in the column object but the name, which is assumed to be valid)
*/
- public static void validateColumnData(CFMetaData metadata, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException
+ public static void validateColumnData(CFMetaData metadata, ByteBuffer key, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException
{
validateTtl(column);
if (!column.isSetValue())
@@ -461,10 +462,11 @@ public class ThriftValidation
}
// Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details
- if (!Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(asDBColumn(cn, column)))
+ SecondaryIndex failedIndex = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(key, asDBColumn(cn, column));
+ if (failedIndex != null)
throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Can't index column value of size %d for index %s in CF %s of KS %s",
column.value.remaining(),
- metadata.getColumnDefinition(cn).getIndexName(),
+ failedIndex.getIndexName(),
metadata.cfName,
metadata.ksName));
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/test/unit/org/apache/cassandra/cql3/IndexedValuesValidationTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/IndexedValuesValidationTest.java b/test/unit/org/apache/cassandra/cql3/IndexedValuesValidationTest.java
index 05acf86..487fbc2 100644
--- a/test/unit/org/apache/cassandra/cql3/IndexedValuesValidationTest.java
+++ b/test/unit/org/apache/cassandra/cql3/IndexedValuesValidationTest.java
@@ -21,15 +21,18 @@
package org.apache.cassandra.cql3;
import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
import org.junit.Test;
-import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.fail;
public class IndexedValuesValidationTest extends CQLTester
{
+ private static final int TOO_BIG = 1024 * 65;
// CASSANDRA-8280/8081
// reject updates with indexed values where value > 64k
@Test
@@ -37,50 +40,110 @@ public class IndexedValuesValidationTest extends CQLTester
{
createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY (a))");
createIndex("CREATE INDEX ON %s(c)");
- performInsertWithIndexedValueOver64k("INSERT INTO %s (a, b, c) VALUES (0, 0, ?)");
+ failInsert("INSERT INTO %s (a, b, c) VALUES (0, 0, ?)", ByteBuffer.allocate(TOO_BIG));
}
@Test
- public void testIndexOnClusteringValueOver64k() throws Throwable
+ public void testIndexOnClusteringColumnInsertPartitionKeyAndClusteringsOver64k() throws Throwable
{
- createTable("CREATE TABLE %s(a int, b blob, c int, PRIMARY KEY (a, b))");
+ createTable("CREATE TABLE %s(a blob, b blob, c blob, d int, PRIMARY KEY (a, b, c))");
createIndex("CREATE INDEX ON %s(b)");
- performInsertWithIndexedValueOver64k("INSERT INTO %s (a, b, c) VALUES (0, ?, 0)");
+
+ // CompositeIndexOnClusteringKey creates index entries composed of the
+ // PK plus all of the non-indexed clustering columns from the primary row
+ // so we should reject where len(a) + len(c) > 65560 as this will form the
+ // total clustering in the index table
+ ByteBuffer a = ByteBuffer.allocate(100);
+ ByteBuffer b = ByteBuffer.allocate(10);
+ ByteBuffer c = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT - 99);
+
+ failInsert("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, 0)", a, b, c);
}
@Test
- public void testIndexOnPartitionKeyOver64k() throws Throwable
+ public void testCompactTableWithValueOver64k() throws Throwable
{
- createTable("CREATE TABLE %s(a blob, b int, c int, PRIMARY KEY ((a, b)))");
- createIndex("CREATE INDEX ON %s(a)");
- performInsertWithIndexedValueOver64k("INSERT INTO %s (a, b, c) VALUES (?, 0, 0)");
+ createTable("CREATE TABLE %s(a int, b blob, PRIMARY KEY (a)) WITH COMPACT STORAGE");
+ createIndex("CREATE INDEX ON %s(b)");
+ failInsert("INSERT INTO %s (a, b) VALUES (0, ?)", ByteBuffer.allocate(TOO_BIG));
}
@Test
- public void testCompactTableWithValueOver64k() throws Throwable
+ public void testIndexOnCollectionValueInsertPartitionKeyAndCollectionKeyOver64k() throws Throwable
{
- createTable("CREATE TABLE %s(a int, b blob, PRIMARY KEY (a)) WITH COMPACT STORAGE");
+ createTable("CREATE TABLE %s(a blob , b map<blob, int>, PRIMARY KEY (a))");
createIndex("CREATE INDEX ON %s(b)");
- performInsertWithIndexedValueOver64k("INSERT INTO %s (a, b) VALUES (0, ?)");
+
+ // A collection key > 64k by itself will be rejected from
+ // the primary table.
+ // To test index validation we need to ensure that
+ // len(b) < 64k, but len(a) + len(b) > 64k as that will
+ // form the clustering in the index table
+ ByteBuffer a = ByteBuffer.allocate(100);
+ ByteBuffer b = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT - 100);
+
+ failInsert("UPDATE %s SET b[?] = 0 WHERE a = ?", b, a);
}
- public void performInsertWithIndexedValueOver64k(String insertCQL) throws Throwable
+ @Test
+ public void testIndexOnCollectionKeyInsertPartitionKeyAndClusteringOver64k() throws Throwable
{
- ByteBuffer buf = ByteBuffer.allocate(1024 * 65);
- buf.clear();
+ createTable("CREATE TABLE %s(a blob, b blob, c map<blob, int>, PRIMARY KEY (a, b))");
+ createIndex("CREATE INDEX ON %s(KEYS(c))");
+
+ // Basically the same as the case with non-collection clustering
+ // CompositeIndexOnCollectionKeyy creates index entries composed of the
+ // PK plus all of the clustering columns from the primary row, except the
+ // collection element - which becomes the partition key in the index table
+ ByteBuffer a = ByteBuffer.allocate(100);
+ ByteBuffer b = ByteBuffer.allocate(FBUtilities.MAX_UNSIGNED_SHORT - 100);
+ ByteBuffer c = ByteBuffer.allocate(10);
- //read more than 64k
- for (int i=0; i<1024 + 1; i++)
- buf.put((byte)0);
+ failInsert("UPDATE %s SET c[?] = 0 WHERE a = ? and b = ?", c, a, b);
+ }
+ @Test
+ public void testIndexOnPartitionKeyInsertValueOver64k() throws Throwable
+ {
+ createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY ((a, b)))");
+ createIndex("CREATE INDEX ON %s(a)");
+ succeedInsert("INSERT INTO %s (a, b, c) VALUES (0, 0, ?)", ByteBuffer.allocate(TOO_BIG));
+ }
+
+ @Test
+ public void testIndexOnClusteringColumnInsertValueOver64k() throws Throwable
+ {
+ createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY (a, b))");
+ createIndex("CREATE INDEX ON %s(b)");
+ succeedInsert("INSERT INTO %s (a, b, c) VALUES (0, 0, ?)", ByteBuffer.allocate(TOO_BIG));
+ }
+
+ @Test
+ public void testIndexOnFullCollectionEntryInsertCollectionValueOver64k() throws Throwable
+ {
+ createTable("CREATE TABLE %s(a int, b frozen<map<int, blob>>, PRIMARY KEY (a))");
+ createIndex("CREATE INDEX ON %s(full(b))");
+ Map<Integer, ByteBuffer> map = new HashMap();
+ map.put(0, ByteBuffer.allocate(1024 * 65));
+ failInsert("INSERT INTO %s (a, b) VALUES (0, ?)", map);
+ }
+
+ public void failInsert(String insertCQL, Object...args) throws Throwable
+ {
try
{
- execute(insertCQL, buf);
+ execute(insertCQL, args);
fail("Expected statement to fail validation");
}
- catch (InvalidRequestException e)
+ catch (Exception e)
{
// as expected
}
}
+
+ public void succeedInsert(String insertCQL, Object...args) throws Throwable
+ {
+ execute(insertCQL, args);
+ flush();
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/test/unit/org/apache/cassandra/db/SecondaryIndexCellSizeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/SecondaryIndexCellSizeTest.java b/test/unit/org/apache/cassandra/db/SecondaryIndexCellSizeTest.java
deleted file mode 100644
index 08dcf24..0000000
--- a/test/unit/org/apache/cassandra/db/SecondaryIndexCellSizeTest.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*/
-package org.apache.cassandra.db;
-
-import java.nio.ByteBuffer;
-import java.util.Set;
-
-import org.junit.Test;
-
-import org.apache.cassandra.db.composites.CellName;
-import org.apache.cassandra.db.composites.CellNames;
-import org.apache.cassandra.db.index.PerColumnSecondaryIndex;
-import org.apache.cassandra.db.index.PerRowSecondaryIndex;
-import org.apache.cassandra.db.index.SecondaryIndexSearcher;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.concurrent.OpOrder;
-import org.apache.cassandra.utils.memory.MemtableAllocator;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class SecondaryIndexCellSizeTest
-{
- @Test
- public void test64kColumn()
- {
- // a byte buffer more than 64k
- ByteBuffer buffer = ByteBuffer.allocate(1024 * 65);
- buffer.clear();
-
- //read more than 64k
- for (int i=0; i<1024*64/4 + 1; i++)
- buffer.putInt(0);
-
- // for read
- buffer.flip();
- Cell cell = new BufferCell(CellNames.simpleDense(ByteBufferUtil.bytes("test")), buffer, 0);
-
- SecondaryIndexCellSizeTest.MockRowIndex mockRowIndex = new SecondaryIndexCellSizeTest.MockRowIndex();
- SecondaryIndexCellSizeTest.MockColumnIndex mockColumnIndex = new SecondaryIndexCellSizeTest.MockColumnIndex();
-
- assertTrue(mockRowIndex.validate(cell));
- assertFalse(mockColumnIndex.validate(cell));
-
- // test less than 64k value
- buffer.flip();
- buffer.clear();
- buffer.putInt(20);
- buffer.flip();
-
- assertTrue(mockRowIndex.validate(cell));
- assertTrue(mockColumnIndex.validate(cell));
- }
-
- private class MockRowIndex extends PerRowSecondaryIndex
- {
- public void init()
- {
- }
-
- public void validateOptions() throws ConfigurationException
- {
- }
-
- public String getIndexName()
- {
- return null;
- }
-
- protected SecondaryIndexSearcher createSecondaryIndexSearcher(Set<ByteBuffer> columns)
- {
- return null;
- }
-
- public void forceBlockingFlush()
- {
- }
-
- public ColumnFamilyStore getIndexCfs()
- {
- return null;
- }
-
- public void removeIndex(ByteBuffer columnName)
- {
- }
-
- public void invalidate()
- {
- }
-
- public void truncateBlocking(long truncatedAt)
- {
- }
-
- public void index(ByteBuffer rowKey, ColumnFamily cf)
- {
- }
-
- public void delete(DecoratedKey key, OpOrder.Group opGroup)
- {
- }
-
- public void index(ByteBuffer rowKey)
- {
- }
-
- public void reload()
- {
- }
-
- public boolean indexes(CellName name)
- {
- return true;
- }
-
- @Override
- public long estimateResultRows() {
- return 0;
- }
- }
-
-
- private class MockColumnIndex extends PerColumnSecondaryIndex
- {
- @Override
- public void init()
- {
- }
-
- @Override
- public void validateOptions() throws ConfigurationException
- {
- }
-
- @Override
- public String getIndexName()
- {
- return null;
- }
-
- @Override
- protected SecondaryIndexSearcher createSecondaryIndexSearcher(Set<ByteBuffer> columns)
- {
- return null;
- }
-
- @Override
- public void forceBlockingFlush()
- {
- }
-
- @Override
- public ColumnFamilyStore getIndexCfs()
- {
- return null;
- }
-
- @Override
- public void removeIndex(ByteBuffer columnName)
- {
- }
-
- @Override
- public void invalidate()
- {
- }
-
- @Override
- public void truncateBlocking(long truncatedAt)
- {
- }
-
- @Override
- public void delete(ByteBuffer rowKey, Cell col, OpOrder.Group opGroup)
- {
- }
-
- @Override
- public void deleteForCleanup(ByteBuffer rowKey, Cell col, OpOrder.Group opGroup) {}
-
- @Override
- public void insert(ByteBuffer rowKey, Cell col, OpOrder.Group opGroup)
- {
- }
-
- @Override
- public void update(ByteBuffer rowKey, Cell oldCol, Cell col, OpOrder.Group opGroup)
- {
- }
-
- @Override
- public void reload()
- {
- }
-
- public boolean indexes(CellName name)
- {
- return true;
- }
-
- @Override
- public long estimateResultRows() {
- return 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/04a2c242/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java b/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
index df0f98c..744fd89 100644
--- a/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
+++ b/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
@@ -31,6 +31,7 @@ import org.apache.cassandra.locator.LocalStrategy;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.utils.ByteBufferUtil;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import static org.junit.Assert.assertNotNull;
@@ -92,7 +93,8 @@ public class ThriftValidationTest extends SchemaLoader
Column column = new Column(ByteBufferUtil.bytes("id"));
column.setValue(ByteBufferUtil.bytes("not a long"));
column.setTimestamp(1234);
- ThriftValidation.validateColumnData(newMetadata, null, column);
+ ByteBuffer key = ByteBufferUtil.bytes("key");
+ ThriftValidation.validateColumnData(newMetadata, key, null, column);
}
@Test
@@ -107,7 +109,8 @@ public class ThriftValidationTest extends SchemaLoader
Column column = new Column(ByteBufferUtil.bytes(CFMetaData.DEFAULT_KEY_ALIAS));
column.setValue(ByteBufferUtil.bytes("not a uuid"));
column.setTimestamp(1234);
- ThriftValidation.validateColumnData(metaData, null, column);
+ ByteBuffer key = ByteBufferUtil.bytes("key");
+ ThriftValidation.validateColumnData(metaData, key, null, column);
IndexExpression expression = new IndexExpression(ByteBufferUtil.bytes(CFMetaData.DEFAULT_KEY_ALIAS), IndexOperator.EQ, ByteBufferUtil.bytes("a"));
ThriftValidation.validateFilterClauses(metaData, Arrays.asList(expression));
@@ -124,7 +127,8 @@ public class ThriftValidationTest extends SchemaLoader
Column column = new Column(ByteBufferUtil.bytes(CFMetaData.DEFAULT_COLUMN_ALIAS + 1));
column.setValue(ByteBufferUtil.bytes("not a long"));
column.setTimestamp(1234);
- ThriftValidation.validateColumnData(metaData, null, column);
+ ByteBuffer key = ByteBufferUtil.bytes("key");
+ ThriftValidation.validateColumnData(metaData, key, null, column);
}
@Test
[2/2] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6b127679
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6b127679
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6b127679
Branch: refs/heads/trunk
Commit: 6b12767912c6b2e54a1abf1773aa68ccd638de81
Parents: 20c5ea0 04a2c24
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed May 6 12:51:14 2015 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed May 6 12:54:44 2015 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cql3/statements/UpdateStatement.java | 21 +-
.../AbstractSimplePerColumnSecondaryIndex.java | 8 +-
.../db/index/PerColumnSecondaryIndex.java | 5 +
.../db/index/PerRowSecondaryIndex.java | 7 +
.../cassandra/db/index/SecondaryIndex.java | 2 +-
.../db/index/SecondaryIndexManager.java | 8 +-
.../cassandra/thrift/CassandraServer.java | 6 +-
.../cassandra/thrift/ThriftValidation.java | 18 +-
.../cql3/IndexedValuesValidationTest.java | 103 +++++++--
.../db/SecondaryIndexCellSizeTest.java | 223 -------------------
.../cassandra/thrift/ThriftValidationTest.java | 20 +-
12 files changed, 145 insertions(+), 277 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6b127679/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index ab92aa0,09bdfb1..dae32db
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,104 -1,5 +1,105 @@@
+3.0
+ * Remove Thrift dependencies in bundled tools (CASSANDRA-8358)
+ * Disable memory mapping of hsperfdata file for JVM statistics (CASSANDRA-9242)
+ * Add pre-startup checks to detect potential incompatibilities (CASSANDRA-8049)
+ * Distinguish between null and unset in protocol v4 (CASSANDRA-7304)
+ * Add user/role permissions for user-defined functions (CASSANDRA-7557)
+ * Allow cassandra config to be updated to restart daemon without unloading classes (CASSANDRA-9046)
+ * Don't initialize compaction writer before checking if iter is empty (CASSANDRA-9117)
+ * Don't execute any functions at prepare-time (CASSANDRA-9037)
+ * Share file handles between all instances of a SegmentedFile (CASSANDRA-8893)
+ * Make it possible to major compact LCS (CASSANDRA-7272)
+ * Make FunctionExecutionException extend RequestExecutionException
+ (CASSANDRA-9055)
+ * Add support for SELECT JSON, INSERT JSON syntax and new toJson(), fromJson()
+ functions (CASSANDRA-7970)
+ * Optimise max purgeable timestamp calculation in compaction (CASSANDRA-8920)
+ * Constrain internode message buffer sizes, and improve IO class hierarchy (CASSANDRA-8670)
+ * New tool added to validate all sstables in a node (CASSANDRA-5791)
+ * Push notification when tracing completes for an operation (CASSANDRA-7807)
+ * Delay "node up" and "node added" notifications until native protocol server is started (CASSANDRA-8236)
+ * Compressed Commit Log (CASSANDRA-6809)
+ * Optimise IntervalTree (CASSANDRA-8988)
+ * Add a key-value payload for third party usage (CASSANDRA-8553, 9212)
+ * Bump metrics-reporter-config dependency for metrics 3.0 (CASSANDRA-8149)
+ * Partition intra-cluster message streams by size, not type (CASSANDRA-8789)
+ * Add WriteFailureException to native protocol, notify coordinator of
+ write failures (CASSANDRA-8592)
+ * Convert SequentialWriter to nio (CASSANDRA-8709)
+ * Add role based access control (CASSANDRA-7653, 8650, 7216, 8760, 8849, 8761, 8850)
+ * Record client ip address in tracing sessions (CASSANDRA-8162)
+ * Indicate partition key columns in response metadata for prepared
+ statements (CASSANDRA-7660)
+ * Merge UUIDType and TimeUUIDType parse logic (CASSANDRA-8759)
+ * Avoid memory allocation when searching index summary (CASSANDRA-8793)
+ * Optimise (Time)?UUIDType Comparisons (CASSANDRA-8730)
+ * Make CRC32Ex into a separate maven dependency (CASSANDRA-8836)
+ * Use preloaded jemalloc w/ Unsafe (CASSANDRA-8714)
+ * Avoid accessing partitioner through StorageProxy (CASSANDRA-8244, 8268)
+ * Upgrade Metrics library and remove depricated metrics (CASSANDRA-5657)
+ * Serializing Row cache alternative, fully off heap (CASSANDRA-7438)
+ * Duplicate rows returned when in clause has repeated values (CASSANDRA-6707)
+ * Make CassandraException unchecked, extend RuntimeException (CASSANDRA-8560)
+ * Support direct buffer decompression for reads (CASSANDRA-8464)
+ * DirectByteBuffer compatible LZ4 methods (CASSANDRA-7039)
+ * Group sstables for anticompaction correctly (CASSANDRA-8578)
+ * Add ReadFailureException to native protocol, respond
+ immediately when replicas encounter errors while handling
+ a read request (CASSANDRA-7886)
+ * Switch CommitLogSegment from RandomAccessFile to nio (CASSANDRA-8308)
+ * Allow mixing token and partition key restrictions (CASSANDRA-7016)
+ * Support index key/value entries on map collections (CASSANDRA-8473)
+ * Modernize schema tables (CASSANDRA-8261)
+ * Support for user-defined aggregation functions (CASSANDRA-8053)
+ * Fix NPE in SelectStatement with empty IN values (CASSANDRA-8419)
+ * Refactor SelectStatement, return IN results in natural order instead
+ of IN value list order and ignore duplicate values in partition key IN restrictions (CASSANDRA-7981)
+ * Support UDTs, tuples, and collections in user-defined
+ functions (CASSANDRA-7563)
+ * Fix aggregate fn results on empty selection, result column name,
+ and cqlsh parsing (CASSANDRA-8229)
+ * Mark sstables as repaired after full repair (CASSANDRA-7586)
+ * Extend Descriptor to include a format value and refactor reader/writer
+ APIs (CASSANDRA-7443)
+ * Integrate JMH for microbenchmarks (CASSANDRA-8151)
+ * Keep sstable levels when bootstrapping (CASSANDRA-7460)
+ * Add Sigar library and perform basic OS settings check on startup (CASSANDRA-7838)
+ * Support for aggregation functions (CASSANDRA-4914)
+ * Remove cassandra-cli (CASSANDRA-7920)
+ * Accept dollar quoted strings in CQL (CASSANDRA-7769)
+ * Make assassinate a first class command (CASSANDRA-7935)
+ * Support IN clause on any partition key column (CASSANDRA-7855)
+ * Support IN clause on any clustering column (CASSANDRA-4762)
+ * Improve compaction logging (CASSANDRA-7818)
+ * Remove YamlFileNetworkTopologySnitch (CASSANDRA-7917)
+ * Do anticompaction in groups (CASSANDRA-6851)
+ * Support user-defined functions (CASSANDRA-7395, 7526, 7562, 7740, 7781, 7929,
+ 7924, 7812, 8063, 7813, 7708)
+ * Permit configurable timestamps with cassandra-stress (CASSANDRA-7416)
+ * Move sstable RandomAccessReader to nio2, which allows using the
+ FILE_SHARE_DELETE flag on Windows (CASSANDRA-4050)
+ * Remove CQL2 (CASSANDRA-5918)
+ * Add Thrift get_multi_slice call (CASSANDRA-6757)
+ * Optimize fetching multiple cells by name (CASSANDRA-6933)
+ * Allow compilation in java 8 (CASSANDRA-7028)
+ * Make incremental repair default (CASSANDRA-7250)
+ * Enable code coverage thru JaCoCo (CASSANDRA-7226)
+ * Switch external naming of 'column families' to 'tables' (CASSANDRA-4369)
+ * Shorten SSTable path (CASSANDRA-6962)
+ * Use unsafe mutations for most unit tests (CASSANDRA-6969)
+ * Fix race condition during calculation of pending ranges (CASSANDRA-7390)
+ * Fail on very large batch sizes (CASSANDRA-8011)
+ * Improve concurrency of repair (CASSANDRA-6455, 8208)
+ * Select optimal CRC32 implementation at runtime (CASSANDRA-8614)
+ * Evaluate MurmurHash of Token once per query (CASSANDRA-7096)
+ * Generalize progress reporting (CASSANDRA-8901)
+ * Resumable bootstrap streaming (CASSANDRA-8838, CASSANDRA-8942)
+ * Allow scrub for secondary index (CASSANDRA-5174)
+ * Save repair data to system table (CASSANDRA-5839)
+ * fix nodetool names that reference column families (CASSANDRA-8872)
+
2.1.6
- * Validate the indexed column rather than the cell's contents for 2i (CASSANDRA-9057)
++ * Validate the indexed column rather than the cell's contents. (CASSANDRA-9057)
* Add support for top-k custom 2i queries (CASSANDRA-8717)
* Fix error when dropping table during compaction (CASSANDRA-9251)
* cassandra-stress supports validation operations over user profiles (CASSANDRA-8773)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6b127679/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
index 67958cf,06282ad..ad46a0f
--- a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
@@@ -118,12 -117,13 +119,13 @@@ public class UpdateStatement extends Mo
}
/**
- * Checks that the value of the indexed columns is valid.
+ * Checks if the values of the indexed columns are valid.
*
+ * @param key row key for the column family
* @param cf the column family
- * @throws InvalidRequestException if one of the values is invalid
+ * @throws InvalidRequestException if one of the values of the indexed columns is not valid
*/
- private void validateIndexedColumns(ColumnFamily cf)
- private void validateIndexedColumns(ByteBuffer key, ColumnFamily cf) throws InvalidRequestException
++ private void validateIndexedColumns(ByteBuffer key, ColumnFamily cf)
{
SecondaryIndexManager indexManager = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfId).indexManager;
if (indexManager.hasIndexes())
@@@ -131,12 -131,15 +133,15 @@@
for (Cell cell : cf)
{
// Indexed values must be validated by any applicable index. See CASSANDRA-3057/4240/8081 for more details
- if (!indexManager.validate(cell))
- throw invalidRequest("Can't index column value of size %d for index %s on %s.%s",
- cell.value().remaining(),
- cfm.getColumnDefinition(cell.name()).getIndexName(),
- cfm.ksName,
- cfm.cfName);
+ SecondaryIndex failedIndex = indexManager.validate(key, cell);
+ if (failedIndex != null)
+ {
- throw new InvalidRequestException(String.format("Can't index column value of size %d for index %s on %s.%s",
- cell.value().remaining(),
- failedIndex.getIndexName(),
- cfm.ksName,
- cfm.cfName));
++ throw invalidRequest(String.format("Can't index column value of size %d for index %s on %s.%s",
++ cell.value().remaining(),
++ failedIndex.getIndexName(),
++ cfm.ksName,
++ cfm.cfName));
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6b127679/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6b127679/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6b127679/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6b127679/src/java/org/apache/cassandra/thrift/CassandraServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6b127679/src/java/org/apache/cassandra/thrift/ThriftValidation.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6b127679/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
index 5df2c47,744fd89..c693f7c
--- a/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
+++ b/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
@@@ -20,46 -20,25 +20,45 @@@ package org.apache.cassandra.thrift
*
*/
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.exceptions.*;
++import java.nio.ByteBuffer;
++import java.util.Arrays;
++
+import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.*;
-import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.*;
- import org.apache.cassandra.exceptions.*;
++import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.LocalStrategy;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
+import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.utils.ByteBufferUtil;
- import org.apache.thrift.TException;
-
- import java.io.IOException;
- import java.util.Arrays;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
--import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertEquals;
++import static org.junit.Assert.assertNotNull;
-public class ThriftValidationTest extends SchemaLoader
+public class ThriftValidationTest
{
+ public static final String KEYSPACE1 = "MultiSliceTest";
+ public static final String CF_STANDARD = "Standard1";
+ public static final String CF_COUNTER = "Counter1";
+ public static final String CF_UUID = "UUIDKeys";
+ public static final String CF_STANDARDLONG3 = "StandardLong3";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_COUNTER).defaultValidator(CounterColumnType.instance),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_UUID).keyValidator(UUIDType.instance),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLONG3, IntegerType.instance));
+ }
+
@Test(expected=org.apache.cassandra.exceptions.InvalidRequestException.class)
public void testValidateCommutativeWithStandard() throws org.apache.cassandra.exceptions.InvalidRequestException
{