You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2011/10/19 15:35:59 UTC
svn commit: r1186209 [1/2] - in /cassandra/trunk: ./ contrib/
contrib/pig/src/java/org/apache/cassandra/hadoop/pig/
interface/thrift/gen-java/org/apache/cassandra/thrift/ src/avro/
src/java/org/apache/cassandra/cli/ src/java/org/apache/cassandra/client...
Author: jbellis
Date: Wed Oct 19 13:35:57 2011
New Revision: 1186209
URL: http://svn.apache.org/viewvc?rev=1186209&view=rev
Log:
merge from 1.0
Removed:
cassandra/trunk/src/java/org/apache/cassandra/db/migration/RenameColumnFamily.java
cassandra/trunk/src/java/org/apache/cassandra/db/migration/RenameKeyspace.java
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/contrib/ (props changed)
cassandra/trunk/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed)
cassandra/trunk/src/avro/internode.genavro
cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g
cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java
cassandra/trunk/src/java/org/apache/cassandra/client/RingCache.java
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
cassandra/trunk/src/java/org/apache/cassandra/config/KSMetaData.java
cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g
cassandra/trunk/src/java/org/apache/cassandra/cql/QueryProcessor.java
cassandra/trunk/src/java/org/apache/cassandra/cql/SelectStatement.java
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/trunk/src/java/org/apache/cassandra/db/HintedHandOffManager.java
cassandra/trunk/src/java/org/apache/cassandra/db/RowMutation.java
cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndexBuilder.java
cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
cassandra/trunk/src/java/org/apache/cassandra/db/migration/AddColumnFamily.java
cassandra/trunk/src/java/org/apache/cassandra/db/migration/DropColumnFamily.java
cassandra/trunk/src/java/org/apache/cassandra/db/migration/UpdateKeyspace.java
cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java
cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
cassandra/trunk/src/java/org/apache/cassandra/hadoop/ConfigHelper.java
cassandra/trunk/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/FileUtils.java
cassandra/trunk/src/java/org/apache/cassandra/service/ClientState.java
cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java
cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
cassandra/trunk/src/java/org/apache/cassandra/streaming/StreamInSession.java
cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java
cassandra/trunk/test/distributed/org/apache/cassandra/TestBase.java
cassandra/trunk/test/unit/org/apache/cassandra/SchemaLoader.java
cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java
cassandra/trunk/test/unit/org/apache/cassandra/client/TestRingCache.java
cassandra/trunk/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 19 13:35:57 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7:1026516-1183000
/cassandra/branches/cassandra-0.7.0:1053690-1055654
-/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1183002,1183241
+/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1185965
/cassandra/branches/cassandra-0.8.0:1125021-1130369
/cassandra/branches/cassandra-0.8.1:1101014-1125018
-/cassandra/branches/cassandra-1.0:1167085-1183245,1185665,1185675,1186069
+/cassandra/branches/cassandra-1.0:1167085-1186208
/cassandra/branches/cassandra-1.0.0:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1:1102511-1125020
Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Wed Oct 19 13:35:57 2011
@@ -4,6 +4,8 @@
1.0.1
+ * acquire references during index build to prevent delete problems
+ on Windows (CASSANDRA-3314)
* describe_ring should include datacenter/topology information (CASSANDRA-2882)
* Thrift sockets are not properly buffered (CASSANDRA-3261)
* performance improvement for bytebufferutil compare function (CASSANDRA-3286)
@@ -19,6 +21,23 @@
successfully acquired the compaction lock (CASSANDRA-3344)
* limit nodetool to 32MB of heap (CASSANDRA-3124)
* (CQL) update parser to accept "timestamp" instead of "date" (CASSANDRA-3149)
+ * Fix CLI `show schema` to include "compression_options" (CASSANDRA-3368)
+ * Snapshot to include manifest under LeveledCompactionStrategy (CASSANDRA-3359)
+ * (CQL) SELECT query should allow CF name to be qualified by keyspace (CASSANDRA-3130)
+ * Display CLI version string on startup (CASSANDRA-3196)
+ * (Hadoop) make CFIF try rpc_address or fallback to listen_address
+ (CASSANDRA-3214)
+ * (Hadoop) accept comma delimited lists of initial thrift connections
+ (CASSANDRA-3185)
+ * ColumnFamily min_compaction_threshold should be >= 2 (CASSANDRA-3342)
+ * (Pig) add 0.8+ types and key validation type in schema (CASSANDRA-3280)
+ * Fix completely removing column metadata using CLI (CASSANDRA-3126)
+ * (CQL) Fix internal application error specifying 'using consistency ...'
+ in lower case (CASSANDRA-3366)
+ * CLI `describe cluster;` output should be on separate lines for separate versions
+ (CASSANDRA-3170)
+ * fix changing durable_writes keyspace option during CF creation
+ (CASSANDRA-3292)
1.0.0-final
Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 19 13:35:57 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
/cassandra/branches/cassandra-0.7/contrib:1026516-1183000
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
-/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1183002,1183241
+/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1185965
/cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
/cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
-/cassandra/branches/cassandra-1.0/contrib:1167085-1183245,1185665,1185675,1186069
+/cassandra/branches/cassandra-1.0/contrib:1167085-1186208
/cassandra/branches/cassandra-1.0.0/contrib:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/contrib:1102511-1125020
Modified: cassandra/trunk/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java (original)
+++ cassandra/trunk/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java Wed Oct 19 13:35:57 2011
@@ -107,7 +107,7 @@ public class CassandraStorage extends Lo
return limit;
}
- @Override
+ @Override
public Tuple getNext() throws IOException
{
try
@@ -122,7 +122,7 @@ public class CassandraStorage extends Lo
assert key != null && cf != null;
// and wrap it in a tuple
- Tuple tuple = TupleFactory.getInstance().newTuple(2);
+ Tuple tuple = TupleFactory.getInstance().newTuple(2);
ArrayList<Tuple> columns = new ArrayList<Tuple>();
tuple.set(0, new DataByteArray(key.array(), key.position()+key.arrayOffset(), key.limit()+key.arrayOffset()));
for (Map.Entry<ByteBuffer, IColumn> entry : cf.entrySet())
@@ -187,10 +187,12 @@ public class CassandraStorage extends Lo
ArrayList<AbstractType> marshallers = new ArrayList<AbstractType>();
AbstractType comparator = null;
AbstractType default_validator = null;
+ AbstractType key_validator = null;
try
{
- comparator = TypeParser.parse(cfDef.comparator_type);
- default_validator = TypeParser.parse(cfDef.default_validation_class);
+ comparator = TypeParser.parse(cfDef.getComparator_type());
+ default_validator = TypeParser.parse(cfDef.getDefault_validation_class());
+ key_validator = TypeParser.parse(cfDef.getKey_validation_class());
}
catch (ConfigurationException e)
{
@@ -199,13 +201,14 @@ public class CassandraStorage extends Lo
marshallers.add(comparator);
marshallers.add(default_validator);
+ marshallers.add(key_validator);
return marshallers;
}
- private Map<ByteBuffer,AbstractType> getValidatorMap(CfDef cfDef) throws IOException
+ private Map<ByteBuffer, AbstractType> getValidatorMap(CfDef cfDef) throws IOException
{
Map<ByteBuffer, AbstractType> validators = new HashMap<ByteBuffer, AbstractType>();
- for (ColumnDef cd : cfDef.column_metadata)
+ for (ColumnDef cd : cfDef.getColumn_metadata())
{
if (cd.getValidation_class() != null && !cd.getValidation_class().isEmpty())
{
@@ -236,6 +239,18 @@ public class CassandraStorage extends Lo
this.reader = reader;
}
+ public static Map<String, String> getQueryMap(String query)
+ {
+ String[] params = query.split("&");
+ Map<String, String> map = new HashMap<String, String>();
+ for (String param : params)
+ {
+ String[] keyValue = param.split("=");
+ map.put(keyValue[0], keyValue[1]);
+ }
+ return map;
+ }
+
private void setLocationFromUri(String location) throws IOException
{
// parse uri into keyspace and columnfamily
@@ -247,18 +262,18 @@ public class CassandraStorage extends Lo
String[] urlParts = location.split("\\?");
if (urlParts.length > 1)
{
- for (String param : urlParts[1].split("&"))
- {
- String[] pair = param.split("=");
- if (pair[0].equals("slice_start"))
- slice_start = ByteBufferUtil.bytes(pair[1]);
- else if (pair[0].equals("slice_end"))
- slice_end = ByteBufferUtil.bytes(pair[1]);
- else if (pair[0].equals("reversed"))
- slice_reverse = Boolean.parseBoolean(pair[1]);
- else if (pair[0].equals("limit"))
- limit = Integer.parseInt(pair[1]);
- }
+ Map<String, String> urlQuery = getQueryMap(urlParts[1]);
+ AbstractType comparator = BytesType.instance;
+ if (urlQuery.containsKey("comparator"))
+ comparator = TypeParser.parse(urlQuery.get("comparator"));
+ if (urlQuery.containsKey("slice_start"))
+ slice_start = comparator.fromString(urlQuery.get("slice_start"));
+ if (urlQuery.containsKey("slice_end"))
+ slice_end = comparator.fromString(urlQuery.get("slice_end"));
+ if (urlQuery.containsKey("reversed"))
+ slice_reverse = Boolean.parseBoolean(urlQuery.get("reversed"));
+ if (urlQuery.containsKey("limit"))
+ limit = Integer.parseInt(urlQuery.get("limit"));
}
String[] parts = urlParts[0].split("/+");
keyspace = parts[1];
@@ -312,10 +327,14 @@ public class CassandraStorage extends Lo
// top-level schema, no type
ResourceSchema schema = new ResourceSchema();
+ // get default marshallers and validators
+ List<AbstractType> marshallers = getDefaultMarshallers(cfDef);
+ Map<ByteBuffer,AbstractType> validators = getValidatorMap(cfDef);
+
// add key
ResourceFieldSchema keyFieldSchema = new ResourceFieldSchema();
keyFieldSchema.setName("key");
- keyFieldSchema.setType(DataType.CHARARRAY); //TODO: get key type
+ keyFieldSchema.setType(getPigType(marshallers.get(2)));
// will become the bag of tuples
ResourceFieldSchema bagFieldSchema = new ResourceFieldSchema();
@@ -323,9 +342,6 @@ public class CassandraStorage extends Lo
bagFieldSchema.setType(DataType.BAG);
ResourceSchema bagSchema = new ResourceSchema();
-
- List<AbstractType> marshallers = getDefaultMarshallers(cfDef);
- Map<ByteBuffer,AbstractType> validators = getValidatorMap(cfDef);
List<ResourceFieldSchema> tupleFields = new ArrayList<ResourceFieldSchema>();
// default comparator/validator
@@ -381,6 +397,10 @@ public class CassandraStorage extends Lo
return DataType.CHARARRAY;
else if (type instanceof UTF8Type)
return DataType.CHARARRAY;
+ else if (type instanceof FloatType)
+ return DataType.FLOAT;
+ else if (type instanceof DoubleType)
+ return DataType.DOUBLE;
return DataType.BYTEARRAY;
}
@@ -545,7 +565,7 @@ public class CassandraStorage extends Lo
Cassandra.Client client = null;
try
{
- client = createConnection(ConfigHelper.getInitialAddress(conf), ConfigHelper.getRpcPort(conf), true);
+ client = ConfigHelper.getClientFromAddressList(conf);
CfDef cfDef = null;
client.set_keyspace(keyspace);
KsDef ksDef = client.describe_keyspace(keyspace);
@@ -579,21 +599,6 @@ public class CassandraStorage extends Lo
}
}
- private static Cassandra.Client createConnection(String host, Integer port, boolean framed) throws IOException
- {
- TSocket socket = new TSocket(host, port);
- TTransport trans = framed ? new TFramedTransport(socket) : socket;
- try
- {
- trans.open();
- }
- catch (TTransportException e)
- {
- throw new IOException("unable to connect to server", e);
- }
- return new Cassandra.Client(new TBinaryProtocol(trans));
- }
-
private static String cfdefToString(CfDef cfDef)
{
assert cfDef != null;
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 19 13:35:57 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1183000
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1183002,1183241
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1185965
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1183245,1185665,1185675,1186069
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1186208
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 19 13:35:57 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1183000
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1183002,1183241
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1185965
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1183245,1185665,1185675,1186069
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1186208
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 19 13:35:57 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1183000
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1183002,1183241
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1185965
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1183245,1185665,1185675,1186069
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1186208
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 19 13:35:57 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1183000
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1183002,1183241
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1185965
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1183245,1185665,1185675,1186069
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1186208
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 19 13:35:57 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1183000
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1183002,1183241
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1185965
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1183245,1185665,1185675,1186069
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1186208
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1102511-1125020
Modified: cassandra/trunk/src/avro/internode.genavro
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/avro/internode.genavro?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/avro/internode.genavro (original)
+++ cassandra/trunk/src/avro/internode.genavro Wed Oct 19 13:35:57 2011
@@ -109,20 +109,6 @@ protocol InterNode {
}
@namespace("org.apache.cassandra.db.migration.avro")
- record RenameColumnFamily {
- string ksname;
- int cfid;
- string old_cfname;
- string new_cfname;
- }
-
- @namespace("org.apache.cassandra.db.migration.avro")
- record RenameKeyspace {
- string old_ksname;
- string new_ksname;
- }
-
- @namespace("org.apache.cassandra.db.migration.avro")
record UpdateKeyspace {
KsDef oldKs;
KsDef newKs;
@@ -139,6 +125,6 @@ protocol InterNode {
org.apache.cassandra.utils.avro.UUID new_version;
bytes row_mutation;
string classname;
- union { AddColumnFamily,DropColumnFamily,RenameColumnFamily,AddKeyspace,DropKeyspace,RenameKeyspace,UpdateKeyspace,UpdateColumnFamily } migration;
+ union { AddColumnFamily,DropColumnFamily,AddKeyspace,DropKeyspace,UpdateKeyspace,UpdateColumnFamily } migration;
}
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g Wed Oct 19 13:35:57 2011
@@ -401,8 +401,8 @@ attrValue
arrayConstruct
- : '[' (hashConstruct ','?)+ ']'
- -> ^(ARRAY (hashConstruct)+)
+ : '[' (hashConstruct ','?)* ']'
+ -> ^(ARRAY (hashConstruct)*)
;
hashConstruct
Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java Wed Oct 19 13:35:57 2011
@@ -189,6 +189,7 @@ public class CliClient
public void printBanner()
{
sessionState.out.println(getHelp().banner);
+ sessionState.out.println("Cassandra CLI version " + FBUtilities.getReleaseVersionString());
}
// Execute a CLI Statement
@@ -1688,6 +1689,30 @@ public class CliClient
colSb.append("]");
writeAttrRaw(sb, false, "column_metadata", colSb.toString());
}
+
+ if (cfDef.compression_options != null && !cfDef.compression_options.isEmpty())
+ {
+ StringBuilder compOptions = new StringBuilder();
+
+ compOptions.append("{");
+
+ int i = 0, size = cfDef.compression_options.size();
+
+ for (Map.Entry<String, String> entry : cfDef.compression_options.entrySet())
+ {
+ compOptions.append(CliUtils.quote(entry.getKey())).append(" : ").append(CliUtils.quote(entry.getValue()));
+
+ if (i != size - 1)
+ compOptions.append(", ");
+
+ i++;
+ }
+
+ compOptions.append("}");
+
+ writeAttrRaw(sb, false, "compression_options", compOptions.toString());
+ }
+
sb.append(";");
sb.append(NEWLINE);
sb.append(NEWLINE);
@@ -2097,7 +2122,7 @@ public class CliClient
for (String version : versions.keySet())
{
- sessionState.out.println("\t" + version + ": " + versions.get(version));
+ sessionState.out.println(String.format("\t%s: %s%n", version, versions.get(version)));
}
}
catch (Exception e)
Modified: cassandra/trunk/src/java/org/apache/cassandra/client/RingCache.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/client/RingCache.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/client/RingCache.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/client/RingCache.java Wed Oct 19 13:35:57 2011
@@ -21,25 +21,22 @@ import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Multimap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.hadoop.ConfigHelper;
import org.apache.cassandra.thrift.Cassandra;
import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.thrift.TokenRange;
+import org.apache.hadoop.conf.Configuration;
import org.apache.thrift.TException;
-import org.apache.cassandra.thrift.TBinaryProtocol;
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TSocket;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
/**
* A class for caching the ring map at the client. For usage example, see
@@ -50,42 +47,32 @@ public class RingCache
{
final private static Logger logger_ = LoggerFactory.getLogger(RingCache.class);
- private final Set<String> seeds_ = new HashSet<String>();
- private final int port_;
- private final IPartitioner<?> partitioner_;
- private final String keyspace;
+ private final IPartitioner<?> partitioner;
+ private final Configuration conf;
private Multimap<Range, InetAddress> rangeMap;
- public RingCache(String keyspace, IPartitioner<?> partitioner, String addresses, int port) throws IOException
+ public RingCache(Configuration conf) throws IOException
{
- for (String seed : addresses.split(","))
- seeds_.add(seed);
- this.port_ = port;
- this.keyspace = keyspace;
- this.partitioner_ = partitioner;
+ this.conf = conf;
+ this.partitioner = ConfigHelper.getPartitioner(conf);
refreshEndpointMap();
}
public void refreshEndpointMap()
{
- for (String seed : seeds_)
- {
- try
- {
- TSocket socket = new TSocket(seed, port_);
- TBinaryProtocol binaryProtocol = new TBinaryProtocol(new TFramedTransport(socket));
- Cassandra.Client client = new Cassandra.Client(binaryProtocol);
- socket.open();
+ try {
+
+ Cassandra.Client client = ConfigHelper.getClientFromAddressList(conf);
- List<TokenRange> ring = client.describe_ring(keyspace);
+ List<TokenRange> ring = client.describe_ring(ConfigHelper.getOutputKeyspace(conf));
rangeMap = ArrayListMultimap.create();
for (TokenRange range : ring)
{
- Token<?> left = partitioner_.getTokenFactory().fromString(range.start_token);
- Token<?> right = partitioner_.getTokenFactory().fromString(range.end_token);
- Range r = new Range(left, right, partitioner_);
+ Token<?> left = partitioner.getTokenFactory().fromString(range.start_token);
+ Token<?> right = partitioner.getTokenFactory().fromString(range.end_token);
+ Range r = new Range(left, right, partitioner);
for (String host : range.endpoints)
{
try
@@ -98,19 +85,20 @@ public class RingCache
}
}
}
- break;
}
catch (InvalidRequestException e)
{
throw new RuntimeException(e);
}
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
catch (TException e)
{
- /* let the Exception go and try another seed. log this though */
- logger_.debug("Error contacting seed " + seed + " " + e.getMessage());
+ logger_.debug("Error contacting seed list" + ConfigHelper.getInitialAddress(conf) + " " + e.getMessage());
}
}
- }
/** ListMultimap promises to return a List for get(K) */
public List<InetAddress> getEndpoint(Range range)
@@ -126,7 +114,7 @@ public class RingCache
public Range getRange(ByteBuffer key)
{
// TODO: naive linear search of the token map
- Token<?> t = partitioner_.getToken(key);
+ Token<?> t = partitioner.getToken(key);
for (Range range : rangeMap.keySet())
if (range.contains(t))
return range;
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Wed Oct 19 13:35:57 2011
@@ -18,7 +18,10 @@
package org.apache.cassandra.config;
-import java.io.*;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.io.InputStream;
import java.net.InetAddress;
import java.net.URL;
import java.net.UnknownHostException;
@@ -34,18 +37,18 @@ import org.apache.cassandra.auth.IAuthor
import org.apache.cassandra.config.Config.RequestSchedulerId;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DefsTable;
-import org.apache.cassandra.db.SystemTable;
-import org.apache.cassandra.db.Table;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.MmappedSegmentedFile;
-import org.apache.cassandra.locator.*;
+import org.apache.cassandra.locator.DynamicEndpointSnitch;
+import org.apache.cassandra.locator.EndpointSnitchInfo;
+import org.apache.cassandra.locator.IEndpointSnitch;
+import org.apache.cassandra.locator.SeedProvider;
import org.apache.cassandra.scheduler.IRequestScheduler;
import org.apache.cassandra.scheduler.NoScheduler;
import org.apache.cassandra.thrift.CassandraDaemon;
import org.apache.cassandra.utils.FBUtilities;
-
import org.yaml.snakeyaml.Loader;
import org.yaml.snakeyaml.TypeDescription;
import org.yaml.snakeyaml.Yaml;
@@ -403,16 +406,7 @@ public class DatabaseDescriptor
partitioner.getTokenFactory().validate(conf.initial_token);
// Hardcoded system tables
- KSMetaData systemMeta = new KSMetaData(Table.SYSTEM_TABLE,
- LocalStrategy.class,
- KSMetaData.optsWithRF(1),
- CFMetaData.StatusCf,
- CFMetaData.HintsCf,
- CFMetaData.MigrationsCf,
- CFMetaData.SchemaCf,
- CFMetaData.IndexCf,
- CFMetaData.NodeIdCf,
- CFMetaData.VersionCf);
+ KSMetaData systemMeta = KSMetaData.systemKeyspace();
Schema.instance.load(CFMetaData.StatusCf);
Schema.instance.load(CFMetaData.HintsCf);
Schema.instance.load(CFMetaData.MigrationsCf);
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/KSMetaData.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/KSMetaData.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/KSMetaData.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/KSMetaData.java Wed Oct 19 13:35:57 2011
@@ -21,30 +21,26 @@ package org.apache.cassandra.config;
import java.util.*;
import org.apache.commons.lang.ObjectUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.avro.util.Utf8;
+import org.apache.cassandra.db.Table;
import org.apache.cassandra.io.SerDeUtils;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
+import org.apache.cassandra.locator.LocalStrategy;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.thrift.CfDef;
import org.apache.cassandra.thrift.KsDef;
-import org.apache.commons.lang.StringUtils;
-
public final class KSMetaData
{
public final String name;
public final Class<? extends AbstractReplicationStrategy> strategyClass;
public final Map<String, String> strategyOptions;
private final Map<String, CFMetaData> cfMetaData;
- private boolean durable_writes;
-
- public KSMetaData(String name, Class<? extends AbstractReplicationStrategy> strategyClass, Map<String, String> strategyOptions, CFMetaData... cfDefs)
- {
- this(name, strategyClass, strategyOptions, true, cfDefs);
- }
+ public final boolean durableWrites;
- public KSMetaData(String name, Class<? extends AbstractReplicationStrategy> strategyClass, Map<String, String> strategyOptions, boolean durable_writes, CFMetaData... cfDefs)
+ private KSMetaData(String name, Class<? extends AbstractReplicationStrategy> strategyClass, Map<String, String> strategyOptions, boolean durableWrites, Iterable<CFMetaData> cfDefs)
{
this.name = name;
this.strategyClass = strategyClass == null ? NetworkTopologyStrategy.class : strategyClass;
@@ -53,19 +49,36 @@ public final class KSMetaData
for (CFMetaData cfm : cfDefs)
cfmap.put(cfm.cfName, cfm);
this.cfMetaData = Collections.unmodifiableMap(cfmap);
- this.durable_writes = durable_writes;
+ this.durableWrites = durableWrites;
}
-
- public void setDurableWrites(boolean durable_writes)
+
+ public static KSMetaData cloneWith(KSMetaData ksm, Iterable<CFMetaData> cfDefs)
{
- this.durable_writes = durable_writes;
+ return new KSMetaData(ksm.name, ksm.strategyClass, ksm.strategyOptions, ksm.durableWrites, cfDefs);
}
-
- public boolean isDurableWrites()
+
+ public static KSMetaData systemKeyspace()
{
- return durable_writes;
+ List<CFMetaData> cfDefs = Arrays.asList(CFMetaData.StatusCf,
+ CFMetaData.HintsCf,
+ CFMetaData.MigrationsCf,
+ CFMetaData.SchemaCf,
+ CFMetaData.IndexCf,
+ CFMetaData.NodeIdCf,
+ CFMetaData.VersionCf);
+ return new KSMetaData(Table.SYSTEM_TABLE, LocalStrategy.class, optsWithRF(1), false, cfDefs);
}
-
+
+ public static KSMetaData testMetadata(String name, Class<? extends AbstractReplicationStrategy> strategyClass, Map<String, String> strategyOptions, CFMetaData... cfDefs)
+ {
+ return new KSMetaData(name, strategyClass, strategyOptions, true, Arrays.asList(cfDefs));
+ }
+
+ public static KSMetaData testMetadataNotDurable(String name, Class<? extends AbstractReplicationStrategy> strategyClass, Map<String, String> strategyOptions, CFMetaData... cfDefs)
+ {
+ return new KSMetaData(name, strategyClass, strategyOptions, false, Arrays.asList(cfDefs));
+ }
+
public static Map<String, String> forwardsCompatibleOptions(KsDef ks_def)
{
Map<String, String> options;
@@ -97,7 +110,7 @@ public final class KSMetaData
&& ObjectUtils.equals(other.strategyClass, strategyClass)
&& ObjectUtils.equals(other.strategyOptions, strategyOptions)
&& other.cfMetaData.equals(cfMetaData)
- && other.durable_writes == durable_writes;
+ && other.durableWrites == durableWrites;
}
public Map<String, CFMetaData> cfMetaData()
@@ -122,7 +135,7 @@ public final class KSMetaData
for (CFMetaData cfm : cfMetaData.values())
ks.cf_defs.add(cfm.toAvro());
- ks.durable_writes = durable_writes;
+ ks.durable_writes = durableWrites;
return ks;
}
@@ -137,7 +150,7 @@ public final class KSMetaData
.append("{")
.append(StringUtils.join(cfMetaData.values(), ", "))
.append("}")
- .append(", durable_writes: ").append(durable_writes);
+ .append(", durable_writes: ").append(durableWrites);
return sb.toString();
}
@@ -174,10 +187,10 @@ public final class KSMetaData
maybeAddReplicationFactor(strategyOptions, ks.strategy_class.toString(), ks.replication_factor);
int cfsz = ks.cf_defs.size();
- CFMetaData[] cfMetaData = new CFMetaData[cfsz];
+ List<CFMetaData> cfMetaData = new ArrayList<CFMetaData>(cfsz);
Iterator<org.apache.cassandra.db.migration.avro.CfDef> cfiter = ks.cf_defs.iterator();
for (int i = 0; i < cfsz; i++)
- cfMetaData[i] = CFMetaData.fromAvro(cfiter.next());
+ cfMetaData.add(CFMetaData.fromAvro(cfiter.next()));
return new KSMetaData(ks.name.toString(), repStratClass, strategyOptions, ks.durable_writes, cfMetaData);
}
@@ -201,7 +214,7 @@ public final class KSMetaData
AbstractReplicationStrategy.getClass(ksd.strategy_class),
forwardsCompatibleOptions(ksd),
ksd.durable_writes,
- cfDefs);
+ Arrays.asList(cfDefs));
}
public KsDef toThrift()
@@ -213,7 +226,7 @@ public final class KSMetaData
ksdef.setStrategy_options(strategyOptions);
if (strategyOptions != null && strategyOptions.containsKey("replication_factor"))
ksdef.setReplication_factor(Integer.parseInt(strategyOptions.get("replication_factor")));
- ksdef.setDurable_writes(durable_writes);
+ ksdef.setDurable_writes(durableWrites);
return ksdef;
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cql/Cql.g Wed Oct 19 13:35:57 2011
@@ -147,14 +147,15 @@ selectStatement returns [SelectStatement
( s1=selectExpression { expression = s1; }
| K_COUNT '(' s2=selectExpression ')' { expression = s2; isCountOp = true; }
)
- K_FROM columnFamily=( IDENT | STRING_LITERAL | INTEGER )
- ( K_USING K_CONSISTENCY K_LEVEL { cLevel = ConsistencyLevel.valueOf($K_LEVEL.text); } )?
+ K_FROM (keyspace=(IDENT | STRING_LITERAL | INTEGER) '.')? columnFamily=( IDENT | STRING_LITERAL | INTEGER )
+ ( K_USING K_CONSISTENCY K_LEVEL { cLevel = ConsistencyLevel.valueOf($K_LEVEL.text.toUpperCase()); } )?
( K_WHERE whereClause )?
( K_LIMIT rows=INTEGER { numRecords = Integer.parseInt($rows.text); } )?
endStmnt
{
return new SelectStatement(expression,
isCountOp,
+ $keyspace.text,
$columnFamily.text,
cLevel,
$whereClause.clause,
@@ -230,7 +231,7 @@ usingClauseDelete[Attributes attrs]
;
usingClauseDeleteObjective[Attributes attrs]
- : K_CONSISTENCY K_LEVEL { attrs.setConsistencyLevel(ConsistencyLevel.valueOf($K_LEVEL.text)); }
+ : K_CONSISTENCY K_LEVEL { attrs.setConsistencyLevel(ConsistencyLevel.valueOf($K_LEVEL.text.toUpperCase())); }
| K_TIMESTAMP ts=INTEGER { attrs.setTimestamp(Long.valueOf($ts.text)); }
;
Modified: cassandra/trunk/src/java/org/apache/cassandra/cql/QueryProcessor.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cql/QueryProcessor.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cql/QueryProcessor.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cql/QueryProcessor.java Wed Oct 19 13:35:57 2011
@@ -73,11 +73,10 @@ public class QueryProcessor
public static final String DEFAULT_KEY_NAME = bufferToString(CFMetaData.DEFAULT_KEY_NAME);
- private static List<org.apache.cassandra.db.Row> getSlice(String keyspace, SelectStatement select)
+ private static List<org.apache.cassandra.db.Row> getSlice(CFMetaData metadata, SelectStatement select)
throws InvalidRequestException, TimedOutException, UnavailableException
{
QueryPath queryPath = new QueryPath(select.getColumnFamily());
- CFMetaData metadata = validateColumnFamily(keyspace, select.getColumnFamily());
List<ReadCommand> commands = new ArrayList<ReadCommand>();
// ...of a list of column names
@@ -91,13 +90,13 @@ public class QueryProcessor
ByteBuffer key = rawKey.getByteBuffer(metadata.getKeyValidator());
validateKey(key);
- commands.add(new SliceByNamesReadCommand(keyspace, key, queryPath, columnNames));
+ commands.add(new SliceByNamesReadCommand(metadata.ksName, key, queryPath, columnNames));
}
}
// ...a range (slice) of column names
else
{
- AbstractType<?> comparator = select.getComparator(keyspace);
+ AbstractType<?> comparator = select.getComparator(metadata.ksName);
ByteBuffer start = select.getColumnStart().getByteBuffer(comparator);
ByteBuffer finish = select.getColumnFinish().getByteBuffer(comparator);
@@ -107,7 +106,7 @@ public class QueryProcessor
validateKey(key);
validateSliceRange(metadata, start, finish, select.isColumnsReversed());
- commands.add(new SliceFromReadCommand(keyspace,
+ commands.add(new SliceFromReadCommand(metadata.ksName,
key,
queryPath,
start,
@@ -144,13 +143,13 @@ public class QueryProcessor
return columnNames;
}
- private static List<org.apache.cassandra.db.Row> multiRangeSlice(String keyspace, SelectStatement select)
+ private static List<org.apache.cassandra.db.Row> multiRangeSlice(CFMetaData metadata, SelectStatement select)
throws TimedOutException, UnavailableException, InvalidRequestException
{
List<org.apache.cassandra.db.Row> rows;
IPartitioner<?> p = StorageService.getPartitioner();
- AbstractType<?> keyType = Schema.instance.getCFMetaData(keyspace, select.getColumnFamily()).getKeyValidator();
+ AbstractType<?> keyType = Schema.instance.getCFMetaData(metadata.ksName, select.getColumnFamily()).getKeyValidator();
ByteBuffer startKey = (select.getKeyStart() != null)
? select.getKeyStart().getByteBuffer(keyType)
@@ -170,7 +169,6 @@ public class QueryProcessor
}
AbstractBounds bounds = new Bounds(startToken, finishToken);
- CFMetaData metadata = validateColumnFamily(keyspace, select.getColumnFamily());
// XXX: Our use of Thrift structs internally makes me Sad. :(
SlicePredicate thriftSlicePredicate = slicePredicateFromSelect(select, metadata);
validateSlicePredicate(metadata, thriftSlicePredicate);
@@ -181,7 +179,7 @@ public class QueryProcessor
try
{
- rows = StorageProxy.getRangeSlice(new RangeSliceCommand(keyspace,
+ rows = StorageProxy.getRangeSlice(new RangeSliceCommand(metadata.ksName,
select.getColumnFamily(),
null,
thriftSlicePredicate,
@@ -220,10 +218,9 @@ public class QueryProcessor
return rows.subList(0, select.getNumRecords() < rows.size() ? select.getNumRecords() : rows.size());
}
- private static List<org.apache.cassandra.db.Row> getIndexedSlices(String keyspace, SelectStatement select)
+ private static List<org.apache.cassandra.db.Row> getIndexedSlices(CFMetaData metadata, SelectStatement select)
throws TimedOutException, UnavailableException, InvalidRequestException
{
- CFMetaData metadata = validateColumnFamily(keyspace, select.getColumnFamily());
// XXX: Our use of Thrift structs internally (still) makes me Sad. :~(
SlicePredicate thriftSlicePredicate = slicePredicateFromSelect(select, metadata);
validateSlicePredicate(metadata, thriftSlicePredicate);
@@ -233,21 +230,21 @@ public class QueryProcessor
{
// Left and right side of relational expression encoded according to comparator/validator.
ByteBuffer entity = columnRelation.getEntity().getByteBuffer(metadata.comparator);
- ByteBuffer value = columnRelation.getValue().getByteBuffer(select.getValueValidator(keyspace, entity));
+ ByteBuffer value = columnRelation.getValue().getByteBuffer(select.getValueValidator(metadata.ksName, entity));
expressions.add(new IndexExpression(entity,
IndexOperator.valueOf(columnRelation.operator().toString()),
value));
}
- AbstractType<?> keyType = Schema.instance.getCFMetaData(keyspace, select.getColumnFamily()).getKeyValidator();
+ AbstractType<?> keyType = Schema.instance.getCFMetaData(metadata.ksName, select.getColumnFamily()).getKeyValidator();
ByteBuffer startKey = (!select.isKeyRange()) ? (new Term()).getByteBuffer() : select.getKeyStart().getByteBuffer(keyType);
IndexClause thriftIndexClause = new IndexClause(expressions, startKey, select.getNumRecords());
List<org.apache.cassandra.db.Row> rows;
try
{
- rows = StorageProxy.scan(keyspace,
+ rows = StorageProxy.scan(metadata.ksName,
select.getColumnFamily(),
thriftIndexClause,
thriftSlicePredicate,
@@ -504,7 +501,7 @@ public class QueryProcessor
String keyspace = null;
// Some statements won't have (or don't need) a keyspace (think USE, or CREATE).
- if (StatementType.requiresKeyspace.contains(statement.type))
+ if (statement.type != StatementType.SELECT && StatementType.requiresKeyspace.contains(statement.type))
keyspace = clientState.getKeyspace();
CqlResult result = new CqlResult();
@@ -515,7 +512,20 @@ public class QueryProcessor
{
case SELECT:
SelectStatement select = (SelectStatement)statement.statement;
- clientState.hasColumnFamilyAccess(select.getColumnFamily(), Permission.READ);
+
+ final String oldKeyspace = clientState.getRawKeyspace();
+
+ if (select.isSetKeyspace())
+ {
+ keyspace = CliUtils.unescapeSQLString(select.getKeyspace());
+ ThriftValidation.validateTable(keyspace);
+ }
+ else if (oldKeyspace == null)
+ throw new InvalidRequestException("no keyspace has been specified");
+ else
+ keyspace = oldKeyspace;
+
+ clientState.hasColumnFamilyAccess(keyspace, select.getColumnFamily(), Permission.READ);
metadata = validateColumnFamily(keyspace, select.getColumnFamily());
// need to do this in here because we need a CFMD.getKeyName()
@@ -531,19 +541,19 @@ public class QueryProcessor
// By-key
if (!select.isKeyRange() && (select.getKeys().size() > 0))
{
- rows = getSlice(keyspace, select);
+ rows = getSlice(metadata, select);
}
else
{
// Range query
if ((select.getKeyFinish() != null) || (select.getColumnRelations().size() == 0))
{
- rows = multiRangeSlice(keyspace, select);
+ rows = multiRangeSlice(metadata, select);
}
// Index scan
else
{
- rows = getIndexedSlices(keyspace, select);
+ rows = getIndexedSlices(metadata, select);
}
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/cql/SelectStatement.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cql/SelectStatement.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cql/SelectStatement.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cql/SelectStatement.java Wed Oct 19 13:35:57 2011
@@ -39,15 +39,17 @@ public class SelectStatement
private final SelectExpression expression;
private final boolean isCountOper;
private final String columnFamily;
+ private final String keyspace;
private final ConsistencyLevel cLevel;
private final WhereClause clause;
private final int numRecords;
- public SelectStatement(SelectExpression expression, boolean isCountOper, String columnFamily,
+ public SelectStatement(SelectExpression expression, boolean isCountOper, String keyspace, String columnFamily,
ConsistencyLevel cLevel, WhereClause clause, int numRecords)
{
this.expression = expression;
this.isCountOper = isCountOper;
+ this.keyspace = keyspace;
this.columnFamily = columnFamily;
this.cLevel = cLevel;
this.clause = (clause != null) ? clause : new WhereClause();
@@ -103,7 +105,17 @@ public class SelectStatement
{
return expression.getFinish();
}
-
+
+ public boolean isSetKeyspace()
+ {
+ return keyspace != null;
+ }
+
+ public String getKeyspace()
+ {
+ return keyspace;
+ }
+
public String getColumnFamily()
{
return columnFamily;
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Wed Oct 19 13:35:57 2011
@@ -32,6 +32,7 @@ import javax.management.MBeanServer;
import javax.management.ObjectName;
import com.google.common.collect.Iterables;
+import org.apache.cassandra.db.compaction.LeveledManifest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -256,7 +257,7 @@ public class ColumnFamilyStore implement
catch (Exception e)
{
// this shouldn't block anything.
- logger.warn(e.getMessage(), e);
+ logger.warn("Failed unregistering mbean: " + mbeanName, e);
}
}
@@ -561,8 +562,8 @@ public class ColumnFamilyStore implement
}
logger.info("Loading new SSTable Set for " + table.name + "/" + columnFamily + ": " + sstables);
+ SSTableReader.acquireReferences(sstables);
data.addSSTables(sstables); // this will call updateCacheSizes() for us
-
logger.info("Requesting a full secondary index re-build for " + table.name + "/" + columnFamily);
try
{
@@ -572,6 +573,10 @@ public class ColumnFamilyStore implement
{
throw new IOError(e);
}
+ finally
+ {
+ SSTableReader.releaseReferences(sstables);
+ }
logger.info("Setting up new generation: " + generation);
fileIndexGenerator.set(generation);
@@ -885,11 +890,6 @@ public class ColumnFamilyStore implement
return false;
}
- public boolean isKeyExistenceExpensive(Set<? extends SSTable> sstablesToIgnore)
- {
- return compactionStrategy.isKeyExistenceExpensive(sstablesToIgnore);
- }
-
/*
* Called after a BinaryMemtable flushes its in-memory data, or we add a file
* via bootstrap. This information is cached in the ColumnFamilyStore.
@@ -1394,29 +1394,40 @@ public class ColumnFamilyStore implement
for (ColumnFamilyStore cfs : concatWithIndexes())
{
DataTracker.View currentView = cfs.markCurrentViewReferenced();
+
try
{
for (SSTableReader ssTable : currentView.sstables)
{
- try
- {
- // mkdir
- File dataDirectory = ssTable.descriptor.directory.getParentFile();
- String snapshotDirectoryPath = Table.getSnapshotPath(dataDirectory.getAbsolutePath(), table.name, snapshotName);
- FileUtils.createDirectory(snapshotDirectoryPath);
+ // mkdir
+ File dataDirectory = ssTable.descriptor.directory.getParentFile();
+ String snapshotDirectoryPath = Table.getSnapshotPath(dataDirectory.getAbsolutePath(), table.name, snapshotName);
+ FileUtils.createDirectory(snapshotDirectoryPath);
+
+ // hard links
+ ssTable.createLinks(snapshotDirectoryPath);
+ if (logger.isDebugEnabled())
+ logger.debug("Snapshot for " + table + " keyspace data file " + ssTable.getFilename() +
+ " created in " + snapshotDirectoryPath);
+ }
- // hard links
- ssTable.createLinks(snapshotDirectoryPath);
- if (logger.isDebugEnabled())
- logger.debug("Snapshot for " + table + " keyspace data file " + ssTable.getFilename() +
- " created in " + snapshotDirectoryPath);
- }
- catch (IOException e)
+ if (compactionStrategy instanceof LeveledCompactionStrategy)
+ {
+ File manifest = LeveledManifest.tryGetManifest(cfs);
+
+ if (manifest != null)
{
- throw new IOError(e);
+ File snapshotDirectory = new File(Table.getSnapshotPath(manifest.getParent(), snapshotName));
+ FileUtils.createDirectory(snapshotDirectory);
+
+ CLibrary.createHardLink(manifest, new File(snapshotDirectory, manifest.getName()));
}
}
}
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
finally
{
SSTableReader.releaseReferences(currentView.sstables);
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/HintedHandOffManager.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/HintedHandOffManager.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/HintedHandOffManager.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/HintedHandOffManager.java Wed Oct 19 13:35:57 2011
@@ -236,7 +236,8 @@ public class HintedHandOffManager implem
// sleep a random amount to stagger handoff delivery from different replicas.
// (if we had to wait, then gossiper randomness took care of that for us already.)
if (waited == 0) {
- int sleep = FBUtilities.threadLocalRandom().nextInt(60000);
+ // use a 'rounded' sleep interval because of a strange bug with windows: CASSANDRA-3375
+ int sleep = FBUtilities.threadLocalRandom().nextInt(2000) * 30;
logger_.debug("Sleeping {}ms to stagger hint delivery", sleep);
Thread.sleep(sleep);
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/RowMutation.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/RowMutation.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/RowMutation.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/RowMutation.java Wed Oct 19 13:35:57 2011
@@ -251,8 +251,8 @@ public class RowMutation implements IMut
public void apply() throws IOException
{
KSMetaData ksm = Schema.instance.getTableDefinition(getTable());
-
- Table.open(table_).apply(this, ksm.isDurableWrites());
+
+ Table.open(table_).apply(this, ksm.durableWrites);
}
public void applyUnsafe() throws IOException
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/Table.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Wed Oct 19 13:35:57 2011
@@ -372,16 +372,6 @@ public class Table
columnFamilyStores.put(cfId, ColumnFamilyStore.createColumnFamilyStore(this, cfName));
}
- /** basically a combined drop and add */
- public void renameCf(Integer cfId, String newName) throws IOException
- {
- assert columnFamilyStores.containsKey(cfId);
- ColumnFamilyStore cfs = columnFamilyStores.remove(cfId);
- unloadCf(cfs);
- cfs.renameSSTables(newName);
- initCf(cfId, newName);
- }
-
public Row getRow(QueryFilter filter) throws IOException
{
ColumnFamilyStore cfStore = getColumnFamilyStore(filter.getColumnFamilyName());
@@ -589,7 +579,12 @@ public class Table
public static String getSnapshotPath(String dataDirPath, String tableName, String snapshotName)
{
- return dataDirPath + File.separator + tableName + File.separator + SNAPSHOT_SUBDIR_NAME + File.separator + snapshotName;
+ return getSnapshotPath(dataDirPath + File.separator + tableName, snapshotName);
+ }
+
+ public static String getSnapshotPath(String tableDirectory, String snapshotName)
+ {
+ return tableDirectory + File.separator + SNAPSHOT_SUBDIR_NAME + File.separator + snapshotName;
}
public static Iterable<Table> all()
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionManager.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionManager.java Wed Oct 19 13:35:57 2011
@@ -772,7 +772,7 @@ public class CompactionManager implement
}
// flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
- cfs.indexManager.flushIndexes();
+ cfs.indexManager.flushIndexesBlocking();
cfs.replaceCompactedSSTables(Arrays.asList(sstable), results);
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java Wed Oct 19 13:35:57 2011
@@ -154,7 +154,7 @@ public class LeveledCompactionStrategy e
public long getMaxSSTableSize()
{
- return maxSSTableSizeInMB * 1024 * 1024;
+ return maxSSTableSizeInMB * 1024L * 1024L;
}
public boolean isKeyExistenceExpensive(Set<? extends SSTable> sstablesToIgnore)
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java Wed Oct 19 13:35:57 2011
@@ -65,7 +65,7 @@ public class LeveledCompactionTask exten
@Override
protected boolean newSSTableSegmentThresholdReached(SSTableWriter writer, long position)
{
- return position > sstableSizeInMB * 1024 * 1024;
+ return position > sstableSizeInMB * 1024L * 1024L;
}
@Override
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java Wed Oct 19 13:35:57 2011
@@ -47,6 +47,8 @@ public class LeveledManifest
{
private static final Logger logger = LoggerFactory.getLogger(LeveledCompactionStrategy.class);
+ public static final String EXTENSION = ".json";
+
/**
* limit the number of L0 sstables we do at once, because compaction bloom filter creation
* uses a pessimistic estimate of how many keys overlap (none), so we risk wasting memory
@@ -344,8 +346,8 @@ public class LeveledManifest
File manifestFile = tryGetManifest(cfs);
if (manifestFile == null)
manifestFile = new File(new File(DatabaseDescriptor.getAllDataFileLocations()[0], cfs.table.name), cfs.columnFamily + ".json");
- File oldFile = new File(manifestFile.getPath().replace(".json", "-old.json"));
- File tmpFile = new File(manifestFile.getPath().replace(".json", "-tmp.json"));
+ File oldFile = new File(manifestFile.getPath().replace(EXTENSION, "-old.json"));
+ File tmpFile = new File(manifestFile.getPath().replace(EXTENSION, "-tmp.json"));
JsonFactory f = new JsonFactory();
try
@@ -387,7 +389,7 @@ public class LeveledManifest
{
for (String dir : DatabaseDescriptor.getAllDataFileLocations())
{
- File manifestFile = new File(new File(dir, cfs.table.name), cfs.columnFamily + ".json");
+ File manifestFile = new File(new File(dir, cfs.table.name), cfs.columnFamily + EXTENSION);
if (manifestFile.exists())
{
logger.debug("Found manifest at {}", manifestFile);
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndex.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndex.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndex.java Wed Oct 19 13:35:57 2011
@@ -30,6 +30,7 @@ import org.apache.cassandra.db.SystemTab
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.index.keys.KeysIndex;
import org.apache.cassandra.io.sstable.ReducingKeyIterator;
+import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -149,11 +150,11 @@ public abstract class SecondaryIndex
for (ColumnDefinition cdef : columnDefs)
columnNames.add(cdef.name);
-
+
+ Collection<SSTableReader> sstables = baseCfs.markCurrentSSTablesReferenced();
SecondaryIndexBuilder builder = new SecondaryIndexBuilder(baseCfs,
columnNames,
- new ReducingKeyIterator(baseCfs.getSSTables()));
-
+ new ReducingKeyIterator(sstables));
Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
try
{
@@ -179,6 +180,10 @@ public abstract class SecondaryIndex
{
throw new IOException(e);
}
+ finally
+ {
+ SSTableReader.releaseReferences(sstables);
+ }
logger.info("Index build of " + getIndexName() + " complete");
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndexBuilder.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndexBuilder.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndexBuilder.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndexBuilder.java Wed Oct 19 13:35:57 2011
@@ -29,8 +29,7 @@ import org.apache.cassandra.db.compactio
import org.apache.cassandra.io.sstable.ReducingKeyIterator;
/**
- * Manages building an entire index from column family data
- * Sent to compaction manager
+ * Manages building an entire index from column family data. Runs on to compaction manager.
*/
public class SecondaryIndexBuilder implements CompactionInfo.Holder
{
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java Wed Oct 19 13:35:57 2011
@@ -97,8 +97,11 @@ public class SecondaryIndexManager
/**
- * Does a full rebuild of the indexes specified by columns from the sstables.
+ * Does a full, blocking rebuild of the indexes specified by columns from the sstables.
* Does nothing if columns is empty.
+ *
+ * Caller must acquire and release references to the sstables used here.
+ *
* @param sstables the data to build from
* @param columns the list of columns to index
* @throws IOException
@@ -116,7 +119,7 @@ public class SecondaryIndexManager
try
{
future.get();
- flushIndexes();
+ flushIndexesBlocking();
}
catch (InterruptedException e)
{
@@ -270,7 +273,7 @@ public class SecondaryIndexManager
* @throws ExecutionException
* @throws InterruptedException
*/
- public void flushIndexes() throws IOException
+ public void flushIndexesBlocking() throws IOException
{
for (Map.Entry<ByteBuffer, SecondaryIndex> entry : indexesByColumn.entrySet())
entry.getValue().forceBlockingFlush();
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/migration/AddColumnFamily.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/migration/AddColumnFamily.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/migration/AddColumnFamily.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/migration/AddColumnFamily.java Wed Oct 19 13:35:57 2011
@@ -2,9 +2,9 @@ package org.apache.cassandra.db.migratio
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+
+import com.google.common.collect.Iterables;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.Table;
@@ -67,9 +67,7 @@ public class AddColumnFamily extends Mig
private KSMetaData makeNewKeyspaceDefinition(KSMetaData ksm)
{
- List<CFMetaData> newCfs = new ArrayList<CFMetaData>(ksm.cfMetaData().values());
- newCfs.add(cfm);
- return new KSMetaData(ksm.name, ksm.strategyClass, ksm.strategyOptions, newCfs.toArray(new CFMetaData[newCfs.size()]));
+ return KSMetaData.cloneWith(ksm, Iterables.concat(ksm.cfMetaData().values(), Collections.singleton(cfm)));
}
public void applyModels() throws IOException
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/migration/DropColumnFamily.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/migration/DropColumnFamily.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/migration/DropColumnFamily.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/migration/DropColumnFamily.java Wed Oct 19 13:35:57 2011
@@ -62,7 +62,7 @@ public class DropColumnFamily extends Mi
List<CFMetaData> newCfs = new ArrayList<CFMetaData>(ksm.cfMetaData().values());
newCfs.remove(cfm);
assert newCfs.size() == ksm.cfMetaData().size() - 1;
- return new KSMetaData(ksm.name, ksm.strategyClass, ksm.strategyOptions, newCfs.toArray(new CFMetaData[newCfs.size()]));
+ return KSMetaData.cloneWith(ksm, newCfs);
}
public void applyModels() throws IOException
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/migration/UpdateKeyspace.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/migration/UpdateKeyspace.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/migration/UpdateKeyspace.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/migration/UpdateKeyspace.java Wed Oct 19 13:35:57 2011
@@ -49,7 +49,7 @@ public class UpdateKeyspace extends Migr
if (oldKsm == null)
throw new ConfigurationException(ksm.name + " cannot be updated because it doesn't exist.");
- this.newKsm = new KSMetaData(ksm.name, ksm.strategyClass, ksm.strategyOptions, ksm.isDurableWrites(), oldKsm.cfMetaData().values().toArray(new CFMetaData[]{}));
+ this.newKsm = KSMetaData.cloneWith(ksm, oldKsm.cfMetaData().values());
rm = makeDefinitionMutation(newKsm, oldKsm, newVersion);
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyInputFormat.java Wed Oct 19 13:35:57 2011
@@ -24,16 +24,17 @@ package org.apache.cassandra.hadoop;
import java.io.IOException;
import java.net.InetAddress;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.cassandra.db.IColumn;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
@@ -41,14 +42,16 @@ import org.apache.cassandra.thrift.Cassa
import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.thrift.KeyRange;
import org.apache.cassandra.thrift.TokenRange;
-import org.apache.cassandra.thrift.TBinaryProtocol;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.thrift.TException;
-import org.apache.thrift.transport.TFramedTransport;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransport;
-import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Hadoop InputFormat allowing map/reduce against Cassandra rows within one ColumnFamily.
@@ -188,13 +191,17 @@ public class ColumnFamilyInputFormat ext
{
ArrayList<InputSplit> splits = new ArrayList<InputSplit>();
List<String> tokens = getSubSplits(keyspace, cfName, range, conf);
-
+ assert range.rpc_endpoints.size() == range.endpoints.size() : "rpc_endpoints size must match endpoints size";
// turn the sub-ranges into InputSplits
String[] endpoints = range.endpoints.toArray(new String[range.endpoints.size()]);
// hadoop needs hostname, not ip
- for (int i = 0; i < endpoints.length; i++)
+ int endpointIndex = 0;
+ for (String endpoint: range.rpc_endpoints)
{
- endpoints[i] = InetAddress.getByName(endpoints[i]).getHostName();
+ String endpoint_address = endpoint;
+ if(endpoint_address == null || endpoint_address == "0.0.0.0")
+ endpoint_address = range.endpoints.get(endpointIndex);
+ endpoints[endpointIndex++] = InetAddress.getByName(endpoint_address).getHostName();
}
for (int i = 1; i < tokens.size(); i++)
@@ -210,11 +217,11 @@ public class ColumnFamilyInputFormat ext
private List<String> getSubSplits(String keyspace, String cfName, TokenRange range, Configuration conf) throws IOException
{
int splitsize = ConfigHelper.getInputSplitSize(conf);
- for (String host : range.endpoints)
+ for (String host : range.rpc_endpoints)
{
try
{
- Cassandra.Client client = createConnection(host, ConfigHelper.getRpcPort(conf), true);
+ Cassandra.Client client = ConfigHelper.createConnection(host, ConfigHelper.getRpcPort(conf), true);
client.set_keyspace(keyspace);
return client.describe_splits(cfName, range.start_token, range.end_token, splitsize);
}
@@ -234,47 +241,10 @@ public class ColumnFamilyInputFormat ext
throw new IOException("failed connecting to all endpoints " + StringUtils.join(range.endpoints, ","));
}
- private static Cassandra.Client createConnection(String host, Integer port, boolean framed) throws IOException
- {
- TSocket socket = new TSocket(host, port);
- TTransport trans = framed ? new TFramedTransport(socket) : socket;
- try
- {
- trans.open();
- }
- catch (TTransportException e)
- {
- throw new IOException("unable to connect to server", e);
- }
- return new Cassandra.Client(new TBinaryProtocol(trans));
- }
private List<TokenRange> getRangeMap(Configuration conf) throws IOException
{
- String[] addresses = ConfigHelper.getInitialAddress(conf).split(",");
- Cassandra.Client client = null;
- List<IOException> exceptions = new ArrayList<IOException>();
- for (String address : addresses)
- {
- try
- {
- client = createConnection(address, ConfigHelper.getRpcPort(conf), true);
- break;
- }
- catch (IOException ioe)
- {
- exceptions.add(ioe);
- }
- }
- if (client == null)
- {
- logger.error("failed to connect to any initial addresses");
- for (IOException ioe : exceptions)
- {
- logger.error("", ioe);
- }
- throw exceptions.get(exceptions.size() - 1);
- }
+ Cassandra.Client client = ConfigHelper.getClientFromAddressList(conf);
List<TokenRange> map;
try
@@ -292,6 +262,8 @@ public class ColumnFamilyInputFormat ext
return map;
}
+
+
public RecordReader<ByteBuffer, SortedMap<ByteBuffer, IColumn>> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException
{
return new ColumnFamilyRecordReader();
Modified: cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java?rev=1186209&r1=1186208&r2=1186209&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java Wed Oct 19 13:35:57 2011
@@ -95,10 +95,7 @@ implements org.apache.hadoop.mapred.Reco
ColumnFamilyRecordWriter(Configuration conf) throws IOException
{
this.conf = conf;
- this.ringCache = new RingCache(ConfigHelper.getOutputKeyspace(conf),
- ConfigHelper.getPartitioner(conf),
- ConfigHelper.getInitialAddress(conf),
- ConfigHelper.getRpcPort(conf));
+ this.ringCache = new RingCache(conf);
this.queueSize = conf.getInt(ColumnFamilyOutputFormat.QUEUE_SIZE, 32 * Runtime.getRuntime().availableProcessors());
this.clients = new HashMap<Range,RangeClient>();
batchThreshold = conf.getLong(ColumnFamilyOutputFormat.BATCH_THRESHOLD, 32);