You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2011/03/15 18:31:29 UTC
svn commit: r1081872 - in /cassandra/trunk: ./ conf/ contrib/
contrib/stress/src/org/apache/cassandra/contrib/stress/
contrib/stress/src/org/apache/cassandra/contrib/stress/operations/
contrib/stress/src/org/apache/cassandra/contrib/stress/util/ debian...
Author: jbellis
Date: Tue Mar 15 17:31:28 2011
New Revision: 1081872
URL: http://svn.apache.org/viewvc?rev=1081872&view=rev
Log:
merge from 0.7
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/.rat-excludes
cassandra/trunk/CHANGES.txt
cassandra/trunk/conf/cassandra.yaml
cassandra/trunk/contrib/ (props changed)
cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/Session.java
cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/operations/IndexedRangeSlicer.java
cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/operations/Inserter.java
cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/util/Operation.java
cassandra/trunk/debian/changelog
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed)
cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
cassandra/trunk/src/java/org/apache/cassandra/db/RowIteratorFactory.java
cassandra/trunk/src/java/org/apache/cassandra/db/columniterator/IColumnIterator.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/FileDataInput.java
cassandra/trunk/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java
cassandra/trunk/src/java/org/apache/cassandra/locator/TokenMetadata.java
cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java
cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
cassandra/trunk/src/java/org/apache/cassandra/thrift/CassandraServer.java
cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java
cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
cassandra/trunk/src/java/org/apache/cassandra/utils/ReducingIterator.java
cassandra/trunk/test/system/test_thrift_server.py
cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
cassandra/trunk/test/unit/org/apache/cassandra/db/RowIterationTest.java
cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
cassandra/trunk/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
cassandra/trunk/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java
cassandra/trunk/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 15 17:31:28 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7:1026516-1080684
+/cassandra/branches/cassandra-0.7:1026516-1081840
/cassandra/branches/cassandra-0.7.0:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3:774578-796573
Modified: cassandra/trunk/.rat-excludes
URL: http://svn.apache.org/viewvc/cassandra/trunk/.rat-excludes?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/.rat-excludes (original)
+++ cassandra/trunk/.rat-excludes Tue Mar 15 17:31:28 2011
@@ -23,3 +23,4 @@ redhat/default
.externalToolBuilders/**
test/data/serialization/*/*
**/*.wpr
+conf/schema-sample.txt
Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Tue Mar 15 17:31:28 2011
@@ -1,3 +1,4 @@
+<<<<<<< .working
0.8-dev
* avoid double RowMutation serialization on write path (CASSANDRA-1800)
* adds support for columns that act as incr/decr counters
@@ -11,6 +12,15 @@
* Fix for Cli to support updating replicate_on_write (CASSANDRA-2236)
+=======
+0.7.5
+ * Avoid seeking when sstable2json exports the entire file (CASSANDRA-2318)
+ * fix tombstone handling in repair and sstable2json (CASSANDRA-2279)
+ * clear Built flag in system table when dropping an index (CASSANDRA-2320)
+ * validate index names (CASSANDRA-1761)
+
+
+>>>>>>> .merge-right.r1081840
0.7.4
* add nodetool join command (CASSANDRA-2160)
* fix secondary indexes on pre-existing or streamed data (CASSANDRA-2244)
@@ -32,6 +42,12 @@
* fix fd leak in sstable2json with non-mmap'd i/o (CASSANDRA-2304)
* reduce memory use during streaming of multiple sstables (CASSANDRA-2301)
* purge tombstoned rows from cache after GCGraceSeconds (CASSANDRA-2305)
+ * allow zero replicas in a NTS datacenter (CASSANDRA-1924)
+ * make range queries respect snitch for local replicas (CASSANDRA-2286)
+ * fix HH delivery when column index is larger than 2GB (CASSANDRA-2297)
+ * make 2ary indexes use parent CF flush thresholds during initial build
+ (CASSANDRA-2294)
+ * update memtable_throughput to be a long (CASSANDRA-2158)
0.7.3
@@ -48,7 +64,6 @@
* validate index names for \w+ (CASSANDRA-2196)
* Fix Cassandra cli to respect timeout if schema does not settle
(CASSANDRA-2187)
- * update memtable_throughput to be a long (CASSANDRA-2158)
* fix for compaction and cleanup writing old-format data into new-version
sstable (CASSANDRA-2211, -2216)
* add nodetool scrub (CASSANDRA-2217, -2240)
Modified: cassandra/trunk/conf/cassandra.yaml
URL: http://svn.apache.org/viewvc/cassandra/trunk/conf/cassandra.yaml?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/conf/cassandra.yaml (original)
+++ cassandra/trunk/conf/cassandra.yaml Tue Mar 15 17:31:28 2011
@@ -129,16 +129,6 @@ flush_largest_memtables_at: 0.75
reduce_cache_sizes_at: 0.85
reduce_cache_capacity_to: 0.6
-# Access mode. mmapped i/o is substantially faster, but only practical on
-# a 64bit machine (which notably does not include EC2 "small" instances)
-# or relatively small datasets. "auto", the safe choice, will enable
-# mmapping on a 64bit JVM. Other values are "mmap", "mmap_index_only"
-# (which may allow you to get part of the benefits of mmap on a 32bit
-# machine by mmapping only index files) and "standard".
-# (The buffer size settings that follow only apply to standard,
-# non-mmapped i/o.)
-disk_access_mode: auto
-
# For workloads with more data than can fit in memory, Cassandra's
# bottleneck will be reads that need to fetch data from
# disk. "concurrent_reads" should be set to (16 * number_of_drives) in
@@ -233,10 +223,6 @@ snapshot_before_compaction: false
# lowest priority and that is our default.
# compaction_thread_priority: 1
-# The threshold size in megabytes the binary memtable must grow to,
-# before it's submitted for flushing to disk.
-binary_memtable_throughput_in_mb: 256
-
# Add column indexes to a row after its contents reach this size.
# Increase if your column values are large, or if you have a very large
# number of columns. The competing causes are, Cassandra has to
Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 15 17:31:28 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/contrib:1026516-1080684
+/cassandra/branches/cassandra-0.7/contrib:1026516-1081840
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/contrib:774578-796573
Modified: cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/Session.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/Session.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/Session.java (original)
+++ cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/Session.java Tue Mar 15 17:31:28 2011
@@ -384,17 +384,20 @@ public class Session
public void createKeySpaces()
{
KsDef keyspace = new KsDef();
- ColumnDef standardColumn = new ColumnDef(ByteBuffer.wrap("C1".getBytes()), "UTF8Type");
- ColumnDef superSubColumn = new ColumnDef(ByteBuffer.wrap("S1".getBytes()), "UTF8Type");
+ // column family for standard columns
+ CfDef standardCfDef = new CfDef("Keyspace1", "Standard1");
+ standardCfDef.setComparator_type("AsciiType").setDefault_validation_class("BytesType");
if (indexType != null)
+ {
+ ColumnDef standardColumn = new ColumnDef(ByteBuffer.wrap("C1".getBytes()), "BytesType");
standardColumn.setIndex_type(indexType).setIndex_name("Idx1");
-
- // column family for standard columns
- CfDef standardCfDef = new CfDef("Keyspace1", "Standard1").setColumn_metadata(Arrays.asList(standardColumn));
+ standardCfDef.setColumn_metadata(Arrays.asList(standardColumn));
+ }
// column family with super columns
- CfDef superCfDef = new CfDef("Keyspace1", "Super1").setColumn_metadata(Arrays.asList(superSubColumn)).setColumn_type("Super");
+ CfDef superCfDef = new CfDef("Keyspace1", "Super1").setColumn_type("Super");
+ superCfDef.setComparator_type("AsciiType").setSubcomparator_type("AsciiType").setDefault_validation_class("BytesType");
// column family for standard counters
CfDef counterCfDef = new CfDef("Keyspace1", "Counter1").setDefault_validation_class("CounterColumnType").setReplicate_on_write(replicateOnWrite);
Modified: cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/operations/IndexedRangeSlicer.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/operations/IndexedRangeSlicer.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/operations/IndexedRangeSlicer.java (original)
+++ cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/operations/IndexedRangeSlicer.java Tue Mar 15 17:31:28 2011
@@ -40,7 +40,7 @@ public class IndexedRangeSlicer extends
ByteBuffer.wrap(new byte[] {}),
false, session.getColumnsPerKey()));
- List<String> values = super.generateValues();
+ List<ByteBuffer> values = super.generateValues();
ColumnParent parent = new ColumnParent("Standard1");
int expectedPerValue = session.getNumKeys() / values.size();
@@ -49,7 +49,7 @@ public class IndexedRangeSlicer extends
int received = 0;
String startOffset = "0";
- ByteBuffer value = ByteBufferUtil.bytes(values.get(index % values.size()));
+ ByteBuffer value = values.get(index % values.size());
IndexExpression expression = new IndexExpression(columnName, IndexOperator.EQ, value);
Modified: cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/operations/Inserter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/operations/Inserter.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/operations/Inserter.java (original)
+++ cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/operations/Inserter.java Tue Mar 15 17:31:28 2011
@@ -38,7 +38,7 @@ public class Inserter extends Operation
public void run(Cassandra.Client client) throws IOException
{
- List<String> values = generateValues();
+ List<ByteBuffer> values = generateValues();
List<Column> columns = new ArrayList<Column>();
List<SuperColumn> superColumns = new ArrayList<SuperColumn>();
@@ -48,8 +48,7 @@ public class Inserter extends Operation
for (int i = 0; i < session.getColumnsPerKey(); i++)
{
String columnName = ("C" + Integer.toString(i));
- ByteBuffer columnValue = ByteBufferUtil.bytes(values.get(i % values.size()));
-
+ ByteBuffer columnValue = values.get(i % values.size());
columns.add(new Column(ByteBufferUtil.bytes(columnName), columnValue, System.currentTimeMillis()));
}
Modified: cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/util/Operation.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/util/Operation.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/util/Operation.java (original)
+++ cassandra/trunk/contrib/stress/src/org/apache/cassandra/contrib/stress/util/Operation.java Tue Mar 15 17:31:28 2011
@@ -59,9 +59,9 @@ public abstract class Operation
* Generate values of average size specified by -S, up to cardinality specified by -C
* @return Collection of the values
*/
- protected List<String> generateValues()
+ protected List<ByteBuffer> generateValues()
{
- List<String> values = new ArrayList<String>();
+ List<ByteBuffer> values = new ArrayList<ByteBuffer>();
int limit = 2 * session.getColumnSize();
@@ -69,8 +69,7 @@ public abstract class Operation
{
byte[] value = new byte[Stress.randomizer.nextInt(limit)];
Stress.randomizer.nextBytes(value);
-
- values.add(FBUtilities.bytesToHex(value));
+ values.add(ByteBuffer.wrap(value));
}
return values;
Modified: cassandra/trunk/debian/changelog
URL: http://svn.apache.org/viewvc/cassandra/trunk/debian/changelog?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/debian/changelog (original)
+++ cassandra/trunk/debian/changelog Tue Mar 15 17:31:28 2011
@@ -1,3 +1,9 @@
+cassandra (0.7.4) unstable; urgency=low
+
+ * New stable point release.
+
+ -- Eric Evans <ee...@apache.org> Fri, 11 Mar 2011 17:39:11 -0600
+
cassandra (0.7.3) unstable; urgency=low
* New stable point release.
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 15 17:31:28 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1080684
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1081840
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 15 17:31:28 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1080684
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1081840
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 15 17:31:28 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1080684
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1081840
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 15 17:31:28 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1080684
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1081840
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 15 17:31:28 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1071777,1076891
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1080684
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1081840
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java Tue Mar 15 17:31:28 2011
@@ -285,10 +285,10 @@ public final class CFMetaData
column_metadata);
}
- public static CFMetaData newIndexMetadata(String table, String parentCf, ColumnDefinition info, AbstractType columnComparator)
+ public static CFMetaData newIndexMetadata(CFMetaData parent, ColumnDefinition info, AbstractType columnComparator)
{
- return new CFMetaData(table,
- indexName(parentCf, info),
+ return new CFMetaData(parent.tableName,
+ indexName(parent.cfName, info),
ColumnFamilyType.Standard,
columnComparator,
null,
@@ -297,15 +297,15 @@ public final class CFMetaData
0,
0,
false,
- DEFAULT_GC_GRACE_SECONDS,
+ parent.gcGraceSeconds,
BytesType.instance,
- DEFAULT_MIN_COMPACTION_THRESHOLD,
- DEFAULT_MAX_COMPACTION_THRESHOLD,
- DEFAULT_ROW_CACHE_SAVE_PERIOD_IN_SECONDS,
- DEFAULT_KEY_CACHE_SAVE_PERIOD_IN_SECONDS,
- DEFAULT_MEMTABLE_LIFETIME_IN_MINS,
- DEFAULT_MEMTABLE_THROUGHPUT_IN_MB,
- DEFAULT_MEMTABLE_OPERATIONS_IN_MILLIONS,
+ parent.minCompactionThreshold,
+ parent.maxCompactionThreshold,
+ 0,
+ 0,
+ parent.memtableFlushAfterMins,
+ parent.memtableThroughputInMb,
+ parent.memtableOperationsInMillions,
0,
Collections.<ByteBuffer, ColumnDefinition>emptyMap());
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Tue Mar 15 17:31:28 2011
@@ -26,7 +26,6 @@ import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;
-import javax.management.JMX;
import javax.management.MBeanServer;
import javax.management.ObjectName;
@@ -152,15 +151,20 @@ public class ColumnFamilyStore implement
// only update these runtime-modifiable settings if they have not been modified.
if (!minCompactionThreshold.isModified())
- minCompactionThreshold = new DefaultInteger(metadata.getMinCompactionThreshold());
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ cfs.minCompactionThreshold = new DefaultInteger(metadata.getMinCompactionThreshold());
if (!maxCompactionThreshold.isModified())
- maxCompactionThreshold = new DefaultInteger(metadata.getMaxCompactionThreshold());
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ cfs.maxCompactionThreshold = new DefaultInteger(metadata.getMaxCompactionThreshold());
if (!memtime.isModified())
- memtime = new DefaultInteger(metadata.getMemtableFlushAfterMins());
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ cfs.memtime = new DefaultInteger(metadata.getMemtableFlushAfterMins());
if (!memsize.isModified())
- memsize = new DefaultInteger(metadata.getMemtableThroughputInMb());
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ cfs.memsize = new DefaultInteger(metadata.getMemtableThroughputInMb());
if (!memops.isModified())
- memops = new DefaultDouble(metadata.getMemtableOperationsInMillions());
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ cfs.memops = new DefaultDouble(metadata.getMemtableOperationsInMillions());
if (!rowCacheSaveInSeconds.isModified())
rowCacheSaveInSeconds = new DefaultInteger(metadata.getRowCacheSavePeriodInSeconds());
if (!keyCacheSaveInSeconds.isModified())
@@ -171,20 +175,10 @@ public class ColumnFamilyStore implement
// figure out what needs to be added and dropped.
// future: if/when we have modifiable settings for secondary indexes, they'll need to be handled here.
- for (ByteBuffer indexName : indexedColumns.keySet())
+ for (ByteBuffer indexedColumn : indexedColumns.keySet())
{
- if (!metadata.getColumn_metadata().containsKey(indexName))
- {
- ColumnFamilyStore indexCfs = indexedColumns.remove(indexName);
- if (indexCfs == null)
- {
- logger.debug("index {} already removed; ignoring", ByteBufferUtil.bytesToHex(indexName));
- continue;
- }
- indexCfs.unregisterMBean();
- SystemTable.setIndexRemoved(metadata.tableName, metadata.cfName);
- indexCfs.removeAllSSTables();
- }
+ if (!metadata.getColumn_metadata().containsKey(indexedColumn))
+ removeIndex(indexedColumn);
}
for (ColumnDefinition cdef : metadata.getColumn_metadata().values())
@@ -192,6 +186,19 @@ public class ColumnFamilyStore implement
addIndex(cdef);
}
+ void removeIndex(ByteBuffer indexedColumn)
+ {
+ ColumnFamilyStore indexCfs = indexedColumns.remove(indexedColumn);
+ if (indexCfs == null)
+ {
+ logger.debug("index {} already removed; ignoring", ByteBufferUtil.bytesToHex(indexedColumn));
+ return;
+ }
+ indexCfs.unregisterMBean();
+ SystemTable.setIndexRemoved(metadata.tableName, indexCfs.columnFamily);
+ indexCfs.removeAllSSTables();
+ }
+
private ColumnFamilyStore(Table table, String columnFamilyName, IPartitioner partitioner, int generation, CFMetaData metadata)
{
assert metadata != null : "null metadata for " + table + ":" + columnFamilyName;
@@ -344,7 +351,7 @@ public class ColumnFamilyStore implement
return metadata.getDefaultValidator().isCommutative();
}
- public void addIndex(final ColumnDefinition info)
+ public Future<?> addIndex(final ColumnDefinition info)
{
assert info.getIndexType() != null;
@@ -353,7 +360,7 @@ public class ColumnFamilyStore implement
AbstractType columnComparator = (rowPartitioner instanceof OrderPreservingPartitioner || rowPartitioner instanceof ByteOrderedPartitioner)
? BytesType.instance
: new LocalByPartionerType(StorageService.getPartitioner());
- final CFMetaData indexedCfMetadata = CFMetaData.newIndexMetadata(table.name, columnFamily, info, columnComparator);
+ final CFMetaData indexedCfMetadata = CFMetaData.newIndexMetadata(metadata, info, columnComparator);
ColumnFamilyStore indexedCfs = ColumnFamilyStore.createColumnFamilyStore(table,
indexedCfMetadata.cfName,
new LocalPartitioner(metadata.getColumn_metadata().get(info.name).validator),
@@ -363,11 +370,11 @@ public class ColumnFamilyStore implement
// so we don't have to lock everything while we do the build. it's up to the operator to wait
// until the index is actually built before using in queries.
if (indexedColumns.putIfAbsent(info.name, indexedCfs) != null)
- return;
+ return null;
// if we're just linking in the index to indexedColumns on an already-built index post-restart, we're done
if (indexedCfs.isIndexBuilt())
- return;
+ return null;
// build it asynchronously; addIndex gets called by CFS open and schema update, neither of which
// we want to block for a long period. (actual build is serialized on CompactionManager.)
@@ -391,7 +398,9 @@ public class ColumnFamilyStore implement
SystemTable.setIndexBuilt(table.name, indexedCfMetadata.cfName);
}
};
- new Thread(runnable, "Create index " + indexedCfMetadata.cfName).start();
+ FutureTask<?> f = new FutureTask<Object>(runnable, null);
+ new Thread(f, "Create index " + indexedCfMetadata.cfName).start();
+ return f;
}
public void buildSecondaryIndexes(Collection<SSTableReader> sstables, SortedSet<ByteBuffer> columns)
@@ -738,7 +747,7 @@ public class ColumnFamilyStore implement
// submit the memtable for any indexed sub-cfses, and our own.
List<ColumnFamilyStore> icc = new ArrayList<ColumnFamilyStore>(indexedColumns.size());
// don't assume that this.memtable is dirty; forceFlush can bring us here during index build even if it is not
- for (ColumnFamilyStore cfs : Iterables.concat(Collections.singleton(this), indexedColumns.values()))
+ for (ColumnFamilyStore cfs : concatWithIndexes())
{
if (!cfs.memtable.isClean())
icc.add(cfs);
@@ -810,7 +819,7 @@ public class ColumnFamilyStore implement
// during index build, 2ary index memtables can be dirty even if parent is not. if so,
// we want flushLargestMemtables to flush the 2ary index ones too.
boolean clean = true;
- for (ColumnFamilyStore cfs : Iterables.concat(Collections.singleton(this), getIndexColumnFamilyStores()))
+ for (ColumnFamilyStore cfs : concatWithIndexes())
clean &= cfs.memtable.isClean();
if (clean)
@@ -1900,7 +1909,7 @@ public class ColumnFamilyStore implement
{
// putting markCompacted on the commitlogUpdater thread ensures it will run
// after any compactions that were in progress when truncate was called, are finished
- for (ColumnFamilyStore cfs : Iterables.concat(indexedColumns.values(), Arrays.asList(ColumnFamilyStore.this)))
+ for (ColumnFamilyStore cfs : concatWithIndexes())
{
List<SSTableReader> truncatedSSTables = new ArrayList<SSTableReader>();
for (SSTableReader sstable : cfs.getSSTables())
@@ -2004,11 +2013,6 @@ public class ColumnFamilyStore implement
return indexedColumns.get(column);
}
- public Collection<ColumnFamilyStore> getIndexColumnFamilyStores()
- {
- return indexedColumns.values();
- }
-
public ColumnFamily newIndexedColumnFamily(ByteBuffer column)
{
return ColumnFamily.create(indexedColumns.get(column).metadata);
@@ -2269,4 +2273,9 @@ public class ColumnFamilyStore implement
{
return new SSTableWriter(getTempSSTablePath(location), estimatedRows, metadata, partitioner);
}
+
+ public Iterable<ColumnFamilyStore> concatWithIndexes()
+ {
+ return Iterables.concat(Collections.singleton(this), indexedColumns.values());
+ }
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java Tue Mar 15 17:31:28 2011
@@ -71,7 +71,7 @@ public class Memtable implements Compara
this.cfs = cfs;
creationTime = System.currentTimeMillis();
- THRESHOLD = cfs.getMemtableThroughputInMB() * 1024 * 1024;
+ THRESHOLD = cfs.getMemtableThroughputInMB() * 1024L * 1024L;
THRESHOLD_COUNT = (long) (cfs.getMemtableOperationsInMillions() * 1024 * 1024);
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/RowIteratorFactory.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/RowIteratorFactory.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/RowIteratorFactory.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/RowIteratorFactory.java Tue Mar 15 17:31:28 2011
@@ -99,7 +99,6 @@ public class RowIteratorFactory
}
Iterator<IColumnIterator> collated = IteratorUtils.collatedIterator(COMPARE_BY_KEY, iterators);
- final Memtable firstMemtable = memtables.iterator().next();
// reduce rows from all sources into a single row
ReducingIterator<IColumnIterator, Row> reduced = new ReducingIterator<IColumnIterator, Row>(collated)
@@ -107,11 +106,26 @@ public class RowIteratorFactory
private final int gcBefore = (int) (System.currentTimeMillis() / 1000) - cfs.metadata.getGcGraceSeconds();
private final List<IColumnIterator> colIters = new ArrayList<IColumnIterator>();
private DecoratedKey key;
+ private ColumnFamily returnCF;
+
+ @Override
+ protected void onKeyChange()
+ {
+ this.returnCF = ColumnFamily.create(cfs.metadata);
+ }
public void reduce(IColumnIterator current)
{
this.colIters.add(current);
this.key = current.getKey();
+ try
+ {
+ this.returnCF.delete(current.getColumnFamily());
+ }
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
}
@Override
@@ -125,7 +139,6 @@ public class RowIteratorFactory
Comparator<IColumn> colComparator = filter.filter.getColumnComparator(comparator);
Iterator<IColumn> colCollated = IteratorUtils.collatedIterator(colComparator, colIters);
- ColumnFamily returnCF;
// First check if this row is in the rowCache. If it is we can skip the rest
ColumnFamily cached = cfs.getRawCachedRow(key);
if (cached != null)
@@ -135,33 +148,8 @@ public class RowIteratorFactory
}
else if (colCollated.hasNext())
{
- returnCF = firstMemtable.getColumnFamily(key);
- // TODO this is a little subtle: the Memtable ColumnIterator has to be a shallow clone of the source CF,
- // with deletion times set correctly, so we can use it as the "base" CF to add query results to.
- // (for sstable ColumnIterators we do not care if it is a shallow clone or not.)
- returnCF = returnCF == null ? ColumnFamily.create(firstMemtable.getTableName(), filter.getColumnFamilyName())
- : returnCF.cloneMeShallow();
- long lastDeletedAt = Long.MIN_VALUE;
- for (IColumnIterator columns : colIters)
- {
- columns.hasNext(); // force cf initializtion
- try
- {
- if (columns.getColumnFamily().isMarkedForDelete())
- lastDeletedAt = Math.max(lastDeletedAt, columns.getColumnFamily().getMarkedForDeleteAt());
- }
- catch (IOException e)
- {
- throw new IOError(e);
- }
- }
- returnCF.markedForDeleteAt.set(lastDeletedAt);
filter.collectCollatedColumns(returnCF, colCollated, gcBefore);
}
- else
- {
- returnCF = null;
- }
Row rv = new Row(key, returnCF);
colIters.clear();
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/columniterator/IColumnIterator.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/columniterator/IColumnIterator.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/columniterator/IColumnIterator.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/columniterator/IColumnIterator.java Tue Mar 15 17:31:28 2011
@@ -31,9 +31,11 @@ import org.apache.cassandra.db.IColumn;
public interface IColumnIterator extends Iterator<IColumn>
{
/**
- * returns the CF of the column being iterated. Do not modify the returned CF; clone first.
- * The CF is only guaranteed to be available after a call to next() or hasNext().
- * Guaranteed to be non-null.
+ * returns the CF of the column being iterated.
+ * Do not modify the returned CF; clone first.
+ * This is guaranteed to be non-null and that the returned CF have the correct metadata
+ * (markedForDeleteAt and localDeletionTime). The full CF is however only guaranteed to
+ * be available after a call to next() or hasNext().
* @throws IOException
*/
public abstract ColumnFamily getColumnFamily() throws IOException;
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java Tue Mar 15 17:31:28 2011
@@ -443,14 +443,11 @@ public class BufferedRandomAccessFile ex
seek(markedPointer);
}
- public int bytesPastMark()
+ public long bytesPastMark()
{
long bytes = getFilePointer() - markedPointer;
-
assert bytes >= 0;
- if (bytes > Integer.MAX_VALUE)
- throw new UnsupportedOperationException("Overflow: " + bytes);
- return (int) bytes;
+ return bytes;
}
public FileMark mark()
@@ -465,15 +462,12 @@ public class BufferedRandomAccessFile ex
seek(((BufferedRandomAccessFileMark) mark).pointer);
}
- public int bytesPastMark(FileMark mark)
+ public long bytesPastMark(FileMark mark)
{
assert mark instanceof BufferedRandomAccessFileMark;
long bytes = getFilePointer() - ((BufferedRandomAccessFileMark) mark).pointer;
-
assert bytes >= 0;
- if (bytes > Integer.MAX_VALUE)
- throw new UnsupportedOperationException("Overflow: " + bytes);
- return (int) bytes;
+ return bytes;
}
public static BufferedRandomAccessFile getUncachingReader(String filename) throws IOException
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/util/FileDataInput.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/FileDataInput.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/util/FileDataInput.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/util/FileDataInput.java Tue Mar 15 17:31:28 2011
@@ -38,7 +38,7 @@ public interface FileDataInput extends D
public void reset(FileMark mark) throws IOException;
- public int bytesPastMark(FileMark mark);
+ public long bytesPastMark(FileMark mark);
/**
* Read length bytes from current file position
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java Tue Mar 15 17:31:28 2011
@@ -75,7 +75,7 @@ public class MappedFileDataInput extends
return new MappedFileDataInputMark(position);
}
- public int bytesPastMark(FileMark mark)
+ public long bytesPastMark(FileMark mark)
{
assert mark instanceof MappedFileDataInputMark;
assert position >= ((MappedFileDataInputMark) mark).position;
Modified: cassandra/trunk/src/java/org/apache/cassandra/locator/TokenMetadata.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/locator/TokenMetadata.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/locator/TokenMetadata.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/locator/TokenMetadata.java Tue Mar 15 17:31:28 2011
@@ -529,6 +529,10 @@ public class TokenMetadata
*/
public static Iterator<Token> ringIterator(final ArrayList<Token> ring, Token start, boolean includeMin)
{
+ if (ring.isEmpty())
+ return includeMin ? Iterators.singletonIterator(StorageService.getPartitioner().getMinimumToken())
+ : Iterators.<Token>emptyIterator();
+
final boolean insertMin = (includeMin && !ring.get(0).equals(StorageService.getPartitioner().getMinimumToken())) ? true : false;
final int startIndex = firstTokenIndex(ring, start, insertMin);
return new AbstractIterator<Token>()
Modified: cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/StorageProxy.java Tue Mar 15 17:31:28 2011
@@ -664,8 +664,9 @@ public class StorageProxy implements Sto
for (AbstractBounds range : ranges)
{
List<InetAddress> liveEndpoints = StorageService.instance.getLiveNaturalEndpoints(command.keyspace, range.right);
+ DatabaseDescriptor.getEndpointSnitch().sortByProximity(FBUtilities.getLocalAddress(), liveEndpoints);
- if (consistency_level == ConsistencyLevel.ONE && liveEndpoints.contains(FBUtilities.getLocalAddress()))
+ if (consistency_level == ConsistencyLevel.ONE && !liveEndpoints.isEmpty() && liveEndpoints.get(0).equals(FBUtilities.getLocalAddress()))
{
if (logger.isDebugEnabled())
logger.debug("local range slice");
@@ -688,7 +689,6 @@ public class StorageProxy implements Sto
}
else
{
- DatabaseDescriptor.getEndpointSnitch().sortByProximity(FBUtilities.getLocalAddress(), liveEndpoints);
RangeSliceCommand c2 = new RangeSliceCommand(command.keyspace, command.column_family, command.super_column, command.predicate, range, command.max_keys);
// collect replies and resolve according to consistency level
Modified: cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java Tue Mar 15 17:31:28 2011
@@ -2334,7 +2334,7 @@ public class StorageService implements I
{
long ops = 0;
long throughput = 0;
- for (ColumnFamilyStore subordinate : Iterables.concat(Collections.singleton(cfs), cfs.getIndexColumnFamilyStores()))
+ for (ColumnFamilyStore subordinate : cfs.concatWithIndexes())
{
ops += subordinate.getMemtableColumnsCount();
throughput = subordinate.getMemtableThroughputInMB();
Modified: cassandra/trunk/src/java/org/apache/cassandra/thrift/CassandraServer.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/thrift/CassandraServer.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/thrift/CassandraServer.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/thrift/CassandraServer.java Tue Mar 15 17:31:28 2011
@@ -678,7 +678,7 @@ public class CassandraServer implements
// helper method to apply migration on the migration stage. typical migration failures will throw an
// InvalidRequestException. atypical failures will throw a RuntimeException.
- private static void applyMigrationOnStage(final Migration m) throws InvalidRequestException
+ private static void applyMigrationOnStage(final Migration m)
{
Future f = StageManager.getStage(Stage.MIGRATION).submit(new Callable()
{
@@ -695,23 +695,11 @@ public class CassandraServer implements
}
catch (InterruptedException e)
{
- throw new RuntimeException(e);
+ throw new AssertionError(e);
}
catch (ExecutionException e)
{
- // this means call() threw an exception. deal with it directly.
- if (e.getCause() != null)
- {
- InvalidRequestException ex = new InvalidRequestException(e.getCause().getMessage());
- ex.initCause(e.getCause());
- throw ex;
- }
- else
- {
- InvalidRequestException ex = new InvalidRequestException(e.getMessage());
- ex.initCause(e);
- throw ex;
- }
+ throw new RuntimeException(e);
}
}
@@ -875,6 +863,7 @@ public class CassandraServer implements
{
logger.debug("update_column_family");
state().hasColumnFamilyListAccess(Permission.WRITE);
+ ThriftValidation.validateCfDef(cf_def);
if (cf_def.keyspace == null || cf_def.name == null)
throw new InvalidRequestException("Keyspace and CF name must be set.");
CFMetaData oldCfm = DatabaseDescriptor.getCFMetaData(CFMetaData.getId(cf_def.keyspace, cf_def.name));
Modified: cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java Tue Mar 15 17:31:28 2011
@@ -21,9 +21,7 @@ package org.apache.cassandra.thrift;
*/
import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.Set;
+import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,6 +29,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.MarshalException;
@@ -423,8 +422,16 @@ public class ThriftValidation
AbstractType comparator = cfType == ColumnFamilyType.Standard
? DatabaseDescriptor.getComparator(cf_def.comparator_type)
: DatabaseDescriptor.getComparator(cf_def.subcomparator_type);
+
+ Set<String> indexNames = new HashSet<String>();
for (ColumnDef c : cf_def.column_metadata)
{
+ // Ensure that given idx_names and auto_generated idx_names cannot collide
+ String idxName = CFMetaData.indexName(cf_def.name, ColumnDefinition.fromColumnDef(c));
+ if (indexNames.contains(idxName))
+ throw new InvalidRequestException("Duplicate index names " + idxName);
+ indexNames.add(idxName);
+
DatabaseDescriptor.getComparator(c.validation_class);
try
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java Tue Mar 15 17:31:28 2011
@@ -19,7 +19,6 @@
package org.apache.cassandra.tools;
import java.io.File;
-import java.io.IOError;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.ByteBuffer;
@@ -27,19 +26,17 @@ import java.util.*;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.IColumnIterator;
-import org.apache.cassandra.db.filter.QueryFilter;
-import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.util.BufferedRandomAccessFile;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.ByteBufferUtil;
+
import org.apache.commons.cli.*;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import static org.apache.cassandra.utils.ByteBufferUtil.bytesToHex;
@@ -169,88 +166,25 @@ public class SSTableExport
/**
* Get portion of the columns and serialize in loop while not more columns left in the row
- * @param reader SSTableReader for given SSTable
* @param row SSTableIdentityIterator row representation with Column Family
* @param key Decorated Key for the required row
* @param out output stream
*/
- private static void serializeRow(SSTableReader reader, SSTableIdentityIterator row, DecoratedKey key, PrintStream out)
+ private static void serializeRow(SSTableIdentityIterator row, DecoratedKey key, PrintStream out)
{
ColumnFamily columnFamily = row.getColumnFamily();
boolean isSuperCF = columnFamily.isSuper();
- ByteBuffer startColumn = ByteBufferUtil.EMPTY_BYTE_BUFFER; // initial column name, "blank" for first
+ CFMetaData cfMetaData = columnFamily.metadata();
+ AbstractType comparator = columnFamily.getComparator();
out.print(asKey(bytesToHex(key.key)));
-
out.print(isSuperCF ? "{" : "[");
- while (true)
+ if (isSuperCF)
{
- QueryFilter filter = QueryFilter.getSliceFilter(key,
- new QueryPath(columnFamily.metadata().tableName),
- startColumn,
- ByteBufferUtil.EMPTY_BYTE_BUFFER,
- false,
- PAGE_SIZE);
-
- IColumnIterator columns = filter.getSSTableColumnIterator(reader);
-
- Pair<Integer, ByteBuffer> serialized;
- try
- {
- serialized = serializeRow(columns, isSuperCF, out);
- }
- catch (IOException e)
+ while (row.hasNext())
{
- System.err.println("WARNING: Corrupt row " + key + " (skipping).");
- continue;
- }
- finally
- {
- try
- {
- columns.close();
- }
- catch (IOException e)
- {
- throw new IOError(e);
- }
- }
-
- if (serialized.left < PAGE_SIZE)
- break;
-
- out.print(",");
- }
-
- out.print(isSuperCF ? "}" : "]");
- }
-
- /**
- * Serialize a row with already given column iterator
- *
- * @param columns columns of the row
- * @param isSuper true if wrapping Column Family is Super
- * @param out output stream
- * @return pair of (number of columns serialized, last column serialized)
- *
- * @throws IOException on any I/O error.
- */
- private static Pair<Integer, ByteBuffer> serializeRow(IColumnIterator columns, boolean isSuper, PrintStream out) throws IOException
- {
- ColumnFamily columnFamily = columns.getColumnFamily();
- CFMetaData cfMetaData = columnFamily.metadata();
-
- AbstractType comparator = columnFamily.getComparator();
-
- if (isSuper)
- {
- int n = 0;
- IColumn column = null;
- while (columns.hasNext())
- {
- column = columns.next();
- n++;
+ IColumn column = row.next();
out.print(asKey(comparator.getString(column.name())));
out.print("{");
@@ -263,16 +197,16 @@ public class SSTableExport
out.print("]");
out.print("}");
- if (columns.hasNext())
+ if (row.hasNext())
out.print(", ");
}
-
- return new Pair<Integer, ByteBuffer>(n, column == null ? null : column.name());
}
else
{
- return serializeColumns(columns, out, comparator, cfMetaData);
+ serializeColumns(row, out, comparator, cfMetaData);
}
+
+ out.print(isSuperCF ? "}" : "]");
}
/**
@@ -347,7 +281,7 @@ public class SSTableExport
if (!row.getKey().equals(decoratedKey))
continue;
- serializeRow(reader, row, decoratedKey, outs);
+ serializeRow(row, decoratedKey, outs);
if (i != 0)
outs.println(",");
@@ -390,7 +324,7 @@ public class SSTableExport
else if (i != 0)
outs.println(",");
- serializeRow(reader, row, row.getKey(), outs);
+ serializeRow(row, row.getKey(), outs);
i++;
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/ReducingIterator.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/ReducingIterator.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/utils/ReducingIterator.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/utils/ReducingIterator.java Tue Mar 15 17:31:28 2011
@@ -55,6 +55,7 @@ public abstract class ReducingIterator<T
if (last == null && !source.hasNext())
return endOfData();
+ onKeyChange();
boolean keyChanged = false;
while (!keyChanged)
{
@@ -73,6 +74,12 @@ public abstract class ReducingIterator<T
return getReduced();
}
+ /**
+ * Called at the begining of each new key, before any reduce is called.
+ * To be overriden by implementing classes.
+ */
+ protected void onKeyChange() {}
+
public Iterator<T2> iterator()
{
return this;
Modified: cassandra/trunk/test/system/test_thrift_server.py
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/system/test_thrift_server.py?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/test/system/test_thrift_server.py (original)
+++ cassandra/trunk/test/system/test_thrift_server.py Tue Mar 15 17:31:28 2011
@@ -548,6 +548,15 @@ class TestMutations(ThriftTester):
_assert_columnpath_exists(key, ColumnPath('Super1', super_column='sc1', column=c))
_assert_columnpath_exists(key, ColumnPath('Super2', super_column='sc1', column=c))
+ def test_bad_system_calls(self):
+ def duplicate_index_names():
+ _set_keyspace('Keyspace1')
+ cd1 = ColumnDef('foo', 'BytesType', IndexType.KEYS, 'i')
+ cd2 = ColumnDef('bar', 'BytesType', IndexType.KEYS, 'i')
+ cf = CfDef('Keyspace1', 'BadCF', column_metadata=[cd1, cd2])
+ client.system_add_column_family(cf)
+ _expect_exception(duplicate_index_names, InvalidRequestException)
+
def test_bad_batch_calls(self):
# mutate_does_not_accept_cosc_and_deletion_in_same_mutation
def too_full():
Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java Tue Mar 15 17:31:28 2011
@@ -21,17 +21,11 @@ package org.apache.cassandra.db;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
+import java.nio.charset.CharacterCodingException;
+import java.util.*;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.Future;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.thrift.SlicePredicate;
-import org.apache.cassandra.thrift.SliceRange;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.junit.Test;
@@ -43,14 +37,13 @@ import org.apache.cassandra.config.Confi
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.thrift.IndexClause;
-import org.apache.cassandra.thrift.IndexExpression;
-import org.apache.cassandra.thrift.IndexOperator;
-import org.apache.cassandra.thrift.IndexType;
+import org.apache.cassandra.thrift.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.WrappedRunnable;
import static junit.framework.Assert.assertEquals;
@@ -58,7 +51,6 @@ import static junit.framework.Assert.ass
import static org.apache.cassandra.Util.column;
import static org.apache.cassandra.Util.getBytes;
import static org.junit.Assert.assertNull;
-import org.apache.cassandra.utils.ByteBufferUtil;
public class ColumnFamilyStoreTest extends CleanupHelper
{
@@ -323,13 +315,26 @@ public class ColumnFamilyStoreTest exten
ColumnFamilyStore cfs = table.getColumnFamilyStore("Indexed2");
ColumnDefinition old = cfs.metadata.getColumn_metadata().get(ByteBufferUtil.bytes("birthdate"));
ColumnDefinition cd = new ColumnDefinition(old.name, old.validator.getClass().getName(), IndexType.KEYS, "birthdate_index");
- cfs.addIndex(cd);
- while (!SystemTable.isIndexBuilt("Keyspace1", cfs.getIndexedColumnFamilyStore(ByteBufferUtil.bytes("birthdate")).columnFamily))
- TimeUnit.MILLISECONDS.sleep(100);
-
- // we had a bug (CASSANDRA-2244) where index would get created but not flushed -- check for that
+ Future<?> future = cfs.addIndex(cd);
+ future.get();
+ // we had a bug (CASSANDRA-2244) where index would get created but not flushed -- check for that
assert cfs.getIndexedColumnFamilyStore(cd.name).getSSTables().size() > 0;
+ queryBirthdate(table);
+
+ // validate that drop clears it out & rebuild works (CASSANDRA-2320)
+ ColumnFamilyStore indexedCfs = cfs.getIndexedColumnFamilyStore(ByteBufferUtil.bytes("birthdate"));
+ cfs.removeIndex(ByteBufferUtil.bytes("birthdate"));
+ assert !indexedCfs.isIndexBuilt();
+
+ // rebuild & re-query
+ future = cfs.addIndex(cd);
+ future.get();
+ queryBirthdate(table);
+ }
+
+ private void queryBirthdate(Table table) throws CharacterCodingException
+ {
IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), IndexOperator.EQ, ByteBufferUtil.bytes(1L));
IndexClause clause = new IndexClause(Arrays.asList(expr), ByteBufferUtil.EMPTY_BYTE_BUFFER, 100);
IFilter filter = new IdentityQueryFilter();
@@ -337,9 +342,9 @@ public class ColumnFamilyStoreTest exten
Range range = new Range(p.getMinimumToken(), p.getMinimumToken());
List<Row> rows = table.getColumnFamilyStore("Indexed2").scan(clause, range, filter);
assert rows.size() == 1 : StringUtils.join(rows, ",");
- assertEquals("k1", ByteBufferUtil.string(rows.get(0).key.key));
+ assertEquals("k1", ByteBufferUtil.string(rows.get(0).key.key));
}
-
+
@Test
public void testDeleteSuperRowSticksAfterFlush() throws Throwable
{
Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/RowIterationTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/RowIterationTest.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/RowIterationTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/RowIterationTest.java Tue Mar 15 17:31:28 2011
@@ -62,4 +62,51 @@ public class RowIterationTest extends Cl
store.forceBlockingFlush();
assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(store).size());
}
+
+ @Test
+ public void testRowIterationDeletionTime() throws IOException, ExecutionException, InterruptedException
+ {
+ Table table = Table.open(TABLE1);
+ String CF_NAME = "Standard3";
+ ColumnFamilyStore store = table.getColumnFamilyStore(CF_NAME);
+ DecoratedKey key = Util.dk("key");
+
+ // Delete row in first sstable
+ RowMutation rm = new RowMutation(TABLE1, key.key);
+ rm.delete(new QueryPath(CF_NAME, null, null), 0);
+ rm.add(new QueryPath(CF_NAME, null, ByteBufferUtil.bytes("c")), ByteBufferUtil.bytes("values"), 0L);
+ int tstamp1 = rm.getColumnFamilies().iterator().next().getLocalDeletionTime();
+ rm.apply();
+ store.forceBlockingFlush();
+
+ // Delete row in second sstable with higher timestamp
+ rm = new RowMutation(TABLE1, key.key);
+ rm.delete(new QueryPath(CF_NAME, null, null), 1);
+ rm.add(new QueryPath(CF_NAME, null, ByteBufferUtil.bytes("c")), ByteBufferUtil.bytes("values"), 1L);
+ int tstamp2 = rm.getColumnFamilies().iterator().next().getLocalDeletionTime();
+ rm.apply();
+ store.forceBlockingFlush();
+
+ ColumnFamily cf = Util.getRangeSlice(store).iterator().next().cf;
+ assert cf.getMarkedForDeleteAt() == 1L;
+ assert cf.getLocalDeletionTime() == tstamp2;
+ }
+
+ @Test
+ public void testRowIterationDeletion() throws IOException, ExecutionException, InterruptedException
+ {
+ Table table = Table.open(TABLE1);
+ String CF_NAME = "Standard3";
+ ColumnFamilyStore store = table.getColumnFamilyStore(CF_NAME);
+ DecoratedKey key = Util.dk("key");
+
+ // Delete a row in first sstable
+ RowMutation rm = new RowMutation(TABLE1, key.key);
+ rm.delete(new QueryPath(CF_NAME, null, null), 0);
+ rm.apply();
+ store.forceBlockingFlush();
+
+ ColumnFamily cf = Util.getRangeSlice(store).iterator().next().cf;
+ assert cf != null;
+ }
}
Modified: cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java Tue Mar 15 17:31:28 2011
@@ -1,4 +1,25 @@
package org.apache.cassandra.io.sstable;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
import java.io.File;
Modified: cassandra/trunk/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java Tue Mar 15 17:31:28 2011
@@ -425,18 +425,6 @@ public class BufferedRandomAccessFileTes
// Expect this call to succeed.
rw.bytesPastMark(mark);
-
- // Seek 4gb
- rw.seek(4L*1024L*1024L*1024L*1024L);
-
- // Expect this call to fail -- the distance from mark to current file pointer > 2gb.
- expectException(new Callable<Object>()
- {
- public Object call() throws IOException
- {
- return rw.bytesPastMark(mark);
- }
- }, UnsupportedOperationException.class);
}
@Test
Modified: cassandra/trunk/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java Tue Mar 15 17:31:28 2011
@@ -44,7 +44,7 @@ public class NetworkTopologyStrategyTest
{
IEndpointSnitch snitch = new PropertyFileSnitch();
TokenMetadata metadata = new TokenMetadata();
- createDummyTokens(metadata);
+ createDummyTokens(metadata, true);
Map<String, String> configOptions = new HashMap<String, String>();
configOptions.put("DC1", "3");
@@ -62,7 +62,30 @@ public class NetworkTopologyStrategyTest
assert 6 == new HashSet<InetAddress>(endpoints).size(); // ensure uniqueness
}
- public void createDummyTokens(TokenMetadata metadata) throws UnknownHostException
+ @Test
+ public void testPropertiesWithEmptyDC() throws IOException, ParserConfigurationException, SAXException, ConfigurationException
+ {
+ IEndpointSnitch snitch = new PropertyFileSnitch();
+ TokenMetadata metadata = new TokenMetadata();
+ createDummyTokens(metadata, false);
+
+ Map<String, String> configOptions = new HashMap<String, String>();
+ configOptions.put("DC1", "3");
+ configOptions.put("DC2", "3");
+ configOptions.put("DC3", "0");
+
+ // Set the localhost to the tokenmetadata. Embedded cassandra way?
+ NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(table, metadata, snitch, configOptions);
+ assert strategy.getReplicationFactor("DC1") == 3;
+ assert strategy.getReplicationFactor("DC2") == 3;
+ assert strategy.getReplicationFactor("DC3") == 0;
+ // Query for the natural hosts
+ ArrayList<InetAddress> endpoints = strategy.getNaturalEndpoints(new StringToken("123"));
+ assert 6 == endpoints.size();
+ assert 6 == new HashSet<InetAddress>(endpoints).size(); // ensure uniqueness
+ }
+
+ public void createDummyTokens(TokenMetadata metadata, boolean populateDC3) throws UnknownHostException
{
// DC 1
tokenFactory(metadata, "123", new byte[]{ 10, 0, 0, 10 });
@@ -72,11 +95,15 @@ public class NetworkTopologyStrategyTest
tokenFactory(metadata, "789", new byte[]{ 10, 20, 114, 10 });
tokenFactory(metadata, "890", new byte[]{ 10, 20, 114, 11 });
//tokens for DC3
- tokenFactory(metadata, "456", new byte[]{ 10, 21, 119, 13 });
- tokenFactory(metadata, "567", new byte[]{ 10, 21, 119, 10 });
+ if (populateDC3)
+ {
+ tokenFactory(metadata, "456", new byte[]{ 10, 21, 119, 13 });
+ tokenFactory(metadata, "567", new byte[]{ 10, 21, 119, 10 });
+ }
// Extra Tokens
tokenFactory(metadata, "90A", new byte[]{ 10, 0, 0, 13 });
- tokenFactory(metadata, "0AB", new byte[]{ 10, 21, 119, 14 });
+ if (populateDC3)
+ tokenFactory(metadata, "0AB", new byte[]{ 10, 21, 119, 14 });
tokenFactory(metadata, "ABC", new byte[]{ 10, 20, 114, 15 });
}
Modified: cassandra/trunk/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java?rev=1081872&r1=1081871&r2=1081872&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java Tue Mar 15 17:31:28 2011
@@ -77,4 +77,11 @@ public class TokenMetadataTest
testRingIterator("0", true, "1", "6", "");
testRingIterator("", true, "1", "6", "");
}
+
+ @Test
+ public void testRingIteratorEmptyRing()
+ {
+ RING.clear();
+ testRingIterator("2", false);
+ }
}