You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2011/03/08 19:23:50 UTC
svn commit: r1079467 - in /cassandra/trunk: ./ contrib/
contrib/pig/src/java/org/apache/cassandra/hadoop/pig/
interface/thrift/gen-java/org/apache/cassandra/thrift/
src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/thrift/
test/system/
Author: jbellis
Date: Tue Mar 8 18:23:49 2011
New Revision: 1079467
URL: http://svn.apache.org/viewvc?rev=1079467&view=rev
Log:
merge from 0.7
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/contrib/ (props changed)
cassandra/trunk/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed)
cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
cassandra/trunk/src/java/org/apache/cassandra/thrift/CassandraServer.java
cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java
cassandra/trunk/test/system/test_thrift_server.py
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 18:23:49 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7:1026516-1078859
+/cassandra/branches/cassandra-0.7:1026516-1079465
/cassandra/branches/cassandra-0.7.0:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3:774578-796573
Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1079467&r1=1079466&r2=1079467&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Tue Mar 8 18:23:49 2011
@@ -22,6 +22,9 @@
a key that does not exist (CASSANDRA-2168)
* track and migrate cached pages during compaction (CASSANDRA-1902)
* add incremental_backups option (CASSANDRA-1872)
+ * add configurable row limit to Pig loadfunc (CASSANDRA-2276)
+ * validate column values in batches as well as single-Column inserts
+ (CASSANDRA-2259)
0.7.3
Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 18:23:49 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/contrib:1026516-1078859
+/cassandra/branches/cassandra-0.7/contrib:1026516-1079465
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/contrib:774578-796573
Modified: cassandra/trunk/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java?rev=1079467&r1=1079466&r2=1079467&view=diff
==============================================================================
--- cassandra/trunk/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java (original)
+++ cassandra/trunk/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java Tue Mar 8 18:23:49 2011
@@ -58,14 +58,33 @@ public class CassandraStorage extends Lo
public final static String PIG_PARTITIONER = "PIG_PARTITIONER";
private final static ByteBuffer BOUND = ByteBufferUtil.EMPTY_BYTE_BUFFER;
- private final static int LIMIT = 1024;
private static final Log logger = LogFactory.getLog(CassandraStorage.class);
private Configuration conf;
private RecordReader reader;
private RecordWriter writer;
+ private final int limit;
- @Override
+ public CassandraStorage()
+ {
+ this(1024);
+ }
+
+ /**
+ * @param limit: number of rows to fetch at a time
+ */
+ public CassandraStorage(int limit)
+ {
+ super();
+ this.limit = limit;
+ }
+
+ public int getLimit()
+ {
+ return limit;
+ }
+
+ @Override
public Tuple getNext() throws IOException
{
try
@@ -167,7 +186,7 @@ public class CassandraStorage extends Lo
@Override
public void setLocation(String location, Job job) throws IOException
{
- SliceRange range = new SliceRange(BOUND, BOUND, false, LIMIT);
+ SliceRange range = new SliceRange(BOUND, BOUND, false, limit);
SlicePredicate predicate = new SlicePredicate().setSlice_range(range);
conf = job.getConfiguration();
ConfigHelper.setInputSlicePredicate(conf, predicate);
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 18:23:49 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1078859
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1079465
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 18:23:49 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1078859
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1079465
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 18:23:49 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1078859
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1079465
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 18:23:49 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1078859
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1079465
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 18:23:49 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1078859
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1079465
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java?rev=1079467&r1=1079466&r2=1079467&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java Tue Mar 8 18:23:49 2011
@@ -701,7 +701,7 @@ public class CompactionManager implement
if (Range.isTokenInRanges(row.getKey().token, ranges))
{
writer = maybeCreateWriter(cfs, compactionFileLocation, expectedBloomFilterSize, writer, row.hasRowsInPageCache());
- writer.append(getCompactedRow(row, cfs, sstable.descriptor, true));
+ writer.append(getCompactedRow(row, cfs, sstable.descriptor, false));
totalkeysWritten++;
}
else
Modified: cassandra/trunk/src/java/org/apache/cassandra/thrift/CassandraServer.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/thrift/CassandraServer.java?rev=1079467&r1=1079466&r2=1079467&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/thrift/CassandraServer.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/thrift/CassandraServer.java Tue Mar 8 18:23:49 2011
@@ -349,7 +349,8 @@ public class CassandraServer implements
ThriftValidation.validateKey(key);
ThriftValidation.validateColumnParent(state().getKeyspace(), column_parent);
- ThriftValidation.validateColumn(state().getKeyspace(), column_parent, column);
+ ThriftValidation.validateColumnNames(state().getKeyspace(), column_parent, Arrays.asList(column.name));
+ ThriftValidation.validateColumnData(state().getKeyspace(), column_parent.column_family, column);
RowMutation rm = new RowMutation(state().getKeyspace(), key);
try
Modified: cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java?rev=1079467&r1=1079466&r2=1079467&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java Tue Mar 8 18:23:49 2011
@@ -41,6 +41,14 @@ import org.apache.cassandra.service.Stor
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+/**
+ * This has a lot of building blocks for CassandraServer to call to make sure it has valid input
+ * -- ensuring column names conform to the declared comparator, for instance.
+ *
+ * The methods here mostly try to do just one part of the validation so they can be combined
+ * for different needs -- supercolumns vs regular, range slices vs named, batch vs single-column.
+ * (ValidateColumnPath is the main exception in that it includes table and CF validation.)
+ */
public class ThriftValidation
{
private static final Logger logger = LoggerFactory.getLogger(ThriftValidation.class);
@@ -81,6 +89,9 @@ public class ThriftValidation
return cfType;
}
+ /**
+ * validates the tablename and all parts of the path to the column, including the column name
+ */
static void validateColumnPath(String tablename, ColumnPath column_path) throws InvalidRequestException
{
validateTable(tablename);
@@ -103,11 +114,11 @@ public class ThriftValidation
}
if (column_path.column != null)
{
- validateColumns(tablename, column_path.column_family, column_path.super_column, Arrays.asList(column_path.column));
+ validateColumnNames(tablename, column_path.column_family, column_path.super_column, Arrays.asList(column_path.column));
}
if (column_path.super_column != null)
{
- validateColumns(tablename, column_path.column_family, null, Arrays.asList(column_path.super_column));
+ validateColumnNames(tablename, column_path.column_family, null, Arrays.asList(column_path.super_column));
}
}
@@ -124,7 +135,7 @@ public class ThriftValidation
}
if (column_parent.super_column != null)
{
- validateColumns(tablename, column_parent.column_family, null, Arrays.asList(column_parent.super_column));
+ validateColumnNames(tablename, column_parent.column_family, null, Arrays.asList(column_parent.super_column));
}
}
@@ -149,15 +160,18 @@ public class ThriftValidation
}
if (column_path_or_parent.column != null)
{
- validateColumns(tablename, column_path_or_parent.column_family, column_path_or_parent.super_column, Arrays.asList(column_path_or_parent.column));
+ validateColumnNames(tablename, column_path_or_parent.column_family, column_path_or_parent.super_column, Arrays.asList(column_path_or_parent.column));
}
if (column_path_or_parent.super_column != null)
{
- validateColumns(tablename, column_path_or_parent.column_family, null, Arrays.asList(column_path_or_parent.super_column));
+ validateColumnNames(tablename, column_path_or_parent.column_family, null, Arrays.asList(column_path_or_parent.super_column));
}
}
- private static void validateColumns(String keyspace, String columnFamilyName, ByteBuffer superColumnName, Iterable<ByteBuffer> column_names)
+ /**
+ * Validates the column names but not the parent path or data
+ */
+ private static void validateColumnNames(String keyspace, String columnFamilyName, ByteBuffer superColumnName, Iterable<ByteBuffer> column_names)
throws InvalidRequestException
{
if (superColumnName != null)
@@ -187,9 +201,9 @@ public class ThriftValidation
}
}
- public static void validateColumns(String keyspace, ColumnParent column_parent, Iterable<ByteBuffer> column_names) throws InvalidRequestException
+ public static void validateColumnNames(String keyspace, ColumnParent column_parent, Iterable<ByteBuffer> column_names) throws InvalidRequestException
{
- validateColumns(keyspace, column_parent.column_family, column_parent.super_column, column_names);
+ validateColumnNames(keyspace, column_parent.column_family, column_parent.super_column, column_names);
}
public static void validateRange(String keyspace, ColumnParent column_parent, SliceRange range) throws InvalidRequestException
@@ -224,14 +238,16 @@ public class ThriftValidation
{
validateTtl(cosc.column);
ThriftValidation.validateColumnPath(keyspace, new ColumnPath(cfName).setSuper_column((ByteBuffer)null).setColumn(cosc.column.name));
+ validateColumnData(keyspace, cfName, cosc.column);
}
if (cosc.super_column != null)
{
+ ColumnParent cp = new ColumnParent(cfName).setSuper_column(cosc.super_column.name);
for (Column c : cosc.super_column.columns)
{
- validateTtl(c);
ThriftValidation.validateColumnPath(keyspace, new ColumnPath(cfName).setSuper_column(cosc.super_column.name).setColumn(c.name));
+ validateColumnData(keyspace, cp.column_family, c);
}
}
@@ -298,16 +314,18 @@ public class ThriftValidation
validateRange(keyspace, new ColumnParent(cfName).setSuper_column(scName), predicate.slice_range);
if (predicate.column_names != null)
- validateColumns(keyspace, cfName, scName, predicate.column_names);
+ validateColumnNames(keyspace, cfName, scName, predicate.column_names);
}
- public static void validateColumn(String keyspace, ColumnParent column_parent, Column column) throws InvalidRequestException
+ /**
+ * Validates the data part of the column (everything in the Column object but the name)
+ */
+ public static void validateColumnData(String keyspace, String column_family, Column column) throws InvalidRequestException
{
validateTtl(column);
- validateColumns(keyspace, column_parent, Arrays.asList(column.name));
try
{
- AbstractType validator = DatabaseDescriptor.getValueValidator(keyspace, column_parent.column_family, column.name);
+ AbstractType validator = DatabaseDescriptor.getValueValidator(keyspace, column_family, column.name);
if (validator != null)
validator.validate(column.value);
}
@@ -315,7 +333,7 @@ public class ThriftValidation
{
throw new InvalidRequestException(String.format("[%s][%s][%s] = [%s] failed validation (%s)",
keyspace,
- column_parent.getColumn_family(),
+ column_family,
ByteBufferUtil.bytesToHex(column.name),
ByteBufferUtil.bytesToHex(column.value),
me.getMessage()));
@@ -333,7 +351,7 @@ public class ThriftValidation
if (predicate.getSlice_range() != null)
validateRange(keyspace, column_parent, predicate.slice_range);
else
- validateColumns(keyspace, column_parent, predicate.column_names);
+ validateColumnNames(keyspace, column_parent, predicate.column_names);
}
public static void validateKeyRange(KeyRange range) throws InvalidRequestException
Modified: cassandra/trunk/test/system/test_thrift_server.py
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/system/test_thrift_server.py?rev=1079467&r1=1079466&r2=1079467&view=diff
==============================================================================
--- cassandra/trunk/test/system/test_thrift_server.py (original)
+++ cassandra/trunk/test/system/test_thrift_server.py Tue Mar 8 18:23:49 2011
@@ -583,6 +583,14 @@ class TestMutations(ThriftTester):
ConsistencyLevel.ONE)
_expect_exception(bad_cf, InvalidRequestException)
+ # a column value that does not match the declared validator
+ def send_string_instead_of_long():
+ _set_keyspace('Keyspace1')
+ col = ColumnOrSuperColumn(column=Column('birthdate', 'bar', 0))
+ client.batch_mutate({'key_38': {'Indexed1': [Mutation(col)]}},
+ ConsistencyLevel.ONE)
+ _expect_exception(send_string_instead_of_long, InvalidRequestException)
+
def test_column_name_lengths(self):
_set_keyspace('Keyspace1')
_expect_exception(lambda: client.insert('key1', ColumnParent('Standard1'), Column('', 'value', 0), ConsistencyLevel.ONE), InvalidRequestException)