You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2011/02/14 17:06:51 UTC
svn commit: r1070541 - in /cassandra/trunk: ./ bin/ conf/ contrib/ debian/
interface/thrift/gen-java/org/apache/cassandra/thrift/
src/java/org/apache/cassandra/cli/ src/java/org/apache/cassandra/config/
src/java/org/apache/cassandra/db/ src/java/org/ap...
Author: jbellis
Date: Mon Feb 14 16:06:50 2011
New Revision: 1070541
URL: http://svn.apache.org/viewvc?rev=1070541&view=rev
Log:
merge from 0.7
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/NEWS.txt
cassandra/trunk/bin/sstablekeys
cassandra/trunk/conf/cassandra.yaml
cassandra/trunk/contrib/ (props changed)
cassandra/trunk/debian/changelog
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed)
cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g
cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java
cassandra/trunk/src/java/org/apache/cassandra/cli/CliMain.java
cassandra/trunk/src/java/org/apache/cassandra/cli/CliOptions.java
cassandra/trunk/src/java/org/apache/cassandra/cli/CliUserHelp.java
cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
cassandra/trunk/src/java/org/apache/cassandra/config/Config.java
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/CacheWriter.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
cassandra/trunk/src/java/org/apache/cassandra/net/MessagingService.java
cassandra/trunk/src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java
cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java
cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java
cassandra/trunk/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
cassandra/trunk/src/java/org/apache/cassandra/utils/StatusLogger.java
cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7:1026516-1068562
+/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7:1026516-1070530
/cassandra/branches/cassandra-0.7.0:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3:774578-796573
Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Mon Feb 14 16:06:50 2011
@@ -11,8 +11,9 @@
0.7.2
- * cache writing moved to CompactionManager to reduce i/o contention and
- updated to use non-cache-polluting writes (CASSANDRA-2053)
+ * copy DecoratedKey.key when inserting into caches to avoid retaining
+ a reference to the underlying buffer (CASSANDRA-2102)
+ * format subcolumn names with subcomparator (CASSANDRA-2136)
0.7.1
@@ -67,6 +68,19 @@
* avoid blocking gossip while deleting handoff hints (CASSANDRA-2073)
* ignore messages from newer versions, keep track of nodes in gossip
regardless of version (CASSANDRA-1970)
+ * cache writing moved to CompactionManager to reduce i/o contention and
+ updated to use non-cache-polluting writes (CASSANDRA-2053)
+ * page through large rows when exporting to JSON (CASSANDRA-2041)
+ * add flush_largest_memtables_at and reduce_cache_sizes_at options
+ (CASSANDRA-2142)
+ * add cli 'describe cluster' command (CASSANDRA-2127)
+ * add cli support for setting username/password at 'connect' command
+ (CASSANDRA-2111)
+ * add -D option to Stress.java to allow reading hosts from a file
+ (CASSANDRA-2149)
+ * bound hints CF throughput between 32M and 256M (CASSANDRA-2148)
+ * continue starting when invalid saved cache entries are encountered
+ (CASSANDRA-2076)
0.7.0-final
@@ -214,6 +228,8 @@
* fix service initialization order deadlock (CASSANDRA-1756)
* multi-line cli commands (CASSANDRA-1742)
* fix race between snapshot and compaction (CASSANDRA-1736)
+ * add listEndpointsPendingHints, deleteHintsForEndpoint JMX methods
+ (CASSANDRA-1551)
0.7.0-beta3
Modified: cassandra/trunk/NEWS.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/NEWS.txt?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/NEWS.txt (original)
+++ cassandra/trunk/NEWS.txt Mon Feb 14 16:06:50 2011
@@ -27,17 +27,33 @@ JMX
0.7.1
=====
-Uprading
---------
+Upgrading
+---------
- 0.7.1 is completely backwards compatible with 0.7.0. Just restart
each node with the new version, one at a time. (The cluster does
not all need to be upgraded simultaneously.)
Features
--------
- - Cassandra can perform writes efficiently across datacenters by
+ - added flush_largest_memtables_at and reduce_cache_sizes_at options
+ to cassandra.yaml as an escape valve for memory pressure
+ - added option to specify -Dcassandra.join_ring=false on startup
+ to allow "warm spare" nodes or performing JMX maintenance before
+ joining the ring
+
+Performance
+-----------
+ - Disk writes and sequential scans avoid polluting page cache
+ (requires JNA to be enabled)
+ - Cassandra performs writes efficiently across datacenters by
sending a single copy of the mutation and having the recipient
forward that to other replicas in its datacenter.
+ - Improved network buffering
+ - Reduced lock contention on memtable flush
+ - Optimized supercolumn deserialization
+ - Zero-copy reads from mmapped sstable files
+ - Explicitly set higher JVM new generation size
+ - Reduced i/o contention during saving of caches
0.7.0
Modified: cassandra/trunk/bin/sstablekeys
URL: http://svn.apache.org/viewvc/cassandra/trunk/bin/sstablekeys?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/bin/sstablekeys (original)
+++ cassandra/trunk/bin/sstablekeys Mon Feb 14 16:06:50 2011
@@ -50,6 +50,6 @@ fi
$JAVA -cp $CLASSPATH -Dstorage-config=$CASSANDRA_CONF \
-Dlog4j.configuration=log4j-tools.properties \
- org.apache.cassandra.tools.SSTableExport "$1" -e
+ org.apache.cassandra.tools.SSTableExport "$@" -e
# vi:ai sw=4 ts=4 tw=0 et
Modified: cassandra/trunk/conf/cassandra.yaml
URL: http://svn.apache.org/viewvc/cassandra/trunk/conf/cassandra.yaml?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/conf/cassandra.yaml (original)
+++ cassandra/trunk/conf/cassandra.yaml Mon Feb 14 16:06:50 2011
@@ -102,6 +102,31 @@ seed_provider:
# seeds is actually a comma-delimited list of addresses.
- seeds: "127.0.0.1"
+# emergency pressure valve: each time heap usage after a full (CMS)
+# garbage collection is above this fraction of the max, Cassandra will
+# flush the largest memtables.
+#
+# Set to 1.0 to disable. Setting this lower than
+# CMSInitiatingOccupancyFraction is not likely to be useful.
+#
+# RELYING ON THIS AS YOUR PRIMARY TUNING MECHANISM WILL WORK POORLY:
+# it is most effective under light to moderate load, or read-heavy
+# workloads; under truly massive write load, it will often be too
+# little, too late.
+flush_largest_memtables_at: 0.75
+
+# emergency pressure valve #2: the first time heap usage after a full
+# (CMS) garbage collection is above this fraction of the max,
+# Cassandra will reduce cache maximum _capacity_ to the given fraction
+# of the current _size_. Should usually be set substantially above
+# flush_largest_memtables_at, since that will have less long-term
+# impact on the system.
+#
+# Set to 1.0 to disable. Setting this lower than
+# CMSInitiatingOccupancyFraction is not likely to be useful.
+reduce_cache_sizes_at: 0.85
+reduce_cache_capacity_to: 0.6
+
# Access mode. mmapped i/o is substantially faster, but only practical on
# a 64bit machine (which notably does not include EC2 "small" instances)
# or relatively small datasets. "auto", the safe choice, will enable
Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/contrib:1026516-1070510
+/cassandra/branches/cassandra-0.7/contrib:1026516-1070530
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/contrib:774578-796573
Modified: cassandra/trunk/debian/changelog
URL: http://svn.apache.org/viewvc/cassandra/trunk/debian/changelog?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/debian/changelog (original)
+++ cassandra/trunk/debian/changelog Mon Feb 14 16:06:50 2011
@@ -2,7 +2,7 @@ cassandra (0.7.1) unstable; urgency=low
* New stable point release.
- -- Eric Evans <ee...@apache.org> Fri, 04 Feb 2011 12:57:52 -0600
+ -- Eric Evans <ee...@apache.org> Thu, 10 Feb 2011 10:34:50 -0600
cassandra (0.7.0~rc4) unstable; urgency=low
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1068562
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1070530
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1068562
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1070530
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1068562
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1070530
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1068562
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1070530
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1068562
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1070530
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g Mon Feb 14 16:06:50 2011
@@ -36,6 +36,7 @@ tokens {
//
NODE_CONNECT;
NODE_DESCRIBE_TABLE;
+ NODE_DESCRIBE_CLUSTER;
NODE_USE_TABLE;
NODE_EXIT;
NODE_HELP;
@@ -138,6 +139,7 @@ statement
| exitStatement
| countStatement
| describeTable
+ | describeCluster
| addKeyspace
| addColumnFamily
| updateKeyspace
@@ -157,10 +159,10 @@ statement
;
connectStatement
- : CONNECT host '/' port
- -> ^(NODE_CONNECT host port)
- | CONNECT ip_address '/' port
- -> ^(NODE_CONNECT ip_address port)
+ : CONNECT host '/' port (username password)?
+ -> ^(NODE_CONNECT host port (username password)?)
+ | CONNECT ip_address '/' port (username password)?
+ -> ^(NODE_CONNECT ip_address port (username password)?)
;
helpStatement
@@ -172,6 +174,8 @@ helpStatement
-> ^(NODE_HELP NODE_USE_TABLE)
| HELP DESCRIBE KEYSPACE
-> ^(NODE_HELP NODE_DESCRIBE_TABLE)
+ | HELP DESCRIBE 'CLUSTER'
+ -> ^(NODE_HELP NODE_DESCRIBE_CLUSTER)
| HELP EXIT
-> ^(NODE_HELP NODE_EXIT)
| HELP QUIT
@@ -325,6 +329,11 @@ describeTable
-> ^(NODE_DESCRIBE_TABLE (keyspace)?)
;
+describeCluster
+ : DESCRIBE 'CLUSTER'
+ -> ^(NODE_DESCRIBE_CLUSTER)
+ ;
+
useKeyspace
: USE keyspace ( username )? ( password )?
-> ^(NODE_USE_TABLE keyspace ( username )? ( password )?)
Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java Mon Feb 14 16:06:50 2011
@@ -173,6 +173,9 @@ public class CliClient extends CliUserHe
case CliParser.NODE_DESCRIBE_TABLE:
executeDescribeKeySpace(tree);
break;
+ case CliParser.NODE_DESCRIBE_CLUSTER:
+ executeDescribeCluster();
+ break;
case CliParser.NODE_USE_TABLE:
executeUseKeySpace(tree);
break;
@@ -314,7 +317,8 @@ public class CliClient extends CliUserHe
AbstractType validator;
CfDef cfDef = getCfDef(columnFamily);
-
+ boolean isSuperCF = cfDef.column_type.equals("Super");
+
// Print out super columns or columns.
for (ColumnOrSuperColumn cosc : columns)
{
@@ -337,9 +341,16 @@ public class CliClient extends CliUserHe
{
Column column = cosc.column;
validator = getValidatorForValue(cfDef, column.getName());
- sessionState.out.printf("=> (column=%s, value=%s, timestamp=%d%s)%n", formatColumnName(keyspace, columnFamily, column),
- validator.getString(column.value), column.timestamp,
- column.isSetTtl() ? String.format(", ttl=%d", column.getTtl()) : "");
+
+ String formattedName = isSuperCF
+ ? formatSubcolumnName(keyspace, columnFamily, column)
+ : formatColumnName(keyspace, columnFamily, column);
+
+ sessionState.out.printf("=> (column=%s, value=%s, timestamp=%d%s)%n",
+ formattedName,
+ validator.getString(column.value),
+ column.timestamp,
+ column.isSetTtl() ? String.format(", ttl=%d", column.getTtl()) : "");
}
}
@@ -461,9 +472,15 @@ public class CliClient extends CliUserHe
valueAsString = (validator == null) ? new String(columnValue, Charsets.UTF_8) : validator.getString(ByteBuffer.wrap(columnValue));
}
+ String formattedColumnName = isSuper
+ ? formatSubcolumnName(keySpace, columnFamily, column)
+ : formatColumnName(keySpace, columnFamily, column);
+
// print results
sessionState.out.printf("=> (column=%s, value=%s, timestamp=%d%s)%n",
- formatColumnName(keySpace, columnFamily, column), valueAsString, column.timestamp,
+ formattedColumnName,
+ valueAsString,
+ column.timestamp,
column.isSetTtl() ? String.format(", ttl=%d", column.getTtl()) : "");
}
@@ -880,7 +897,12 @@ public class CliClient extends CliUserHe
cfDef.setKey_cache_size(Double.parseDouble(mValue));
break;
case READ_REPAIR_CHANCE:
- cfDef.setRead_repair_chance(Double.parseDouble(mValue));
+ double chance = Double.parseDouble(mValue) / 100;
+
+ if (chance > 1)
+ throw new RuntimeException("Error: read_repair_chance / 100 should not be greater than 1.");
+
+ cfDef.setRead_repair_chance(chance);
break;
case GC_GRACE:
cfDef.setGc_grace_seconds(Integer.parseInt(mValue));
@@ -1412,12 +1434,39 @@ public class CliClient extends CliUserHe
describeKeySpace(keySpaceName, null);
}
+ // ^(NODE_DESCRIBE_CLUSTER) or describe: schema_versions, partitioner, snitch
+ private void executeDescribeCluster()
+ {
+ if (!CliMain.isConnected())
+ return;
+
+ sessionState.out.println("Cluster Information:");
+ try
+ {
+ sessionState.out.println(" Snitch: " + thriftClient.describe_snitch());
+ sessionState.out.println(" Partitioner: " + thriftClient.describe_partitioner());
+
+ sessionState.out.println(" Schema versions: ");
+ Map<String,List<String>> versions = thriftClient.describe_schema_versions();
+
+ for (String version : versions.keySet())
+ {
+ sessionState.out.println("\t" + version + ": " + versions.get(version));
+ }
+ }
+ catch (Exception e)
+ {
+ String message = (e instanceof InvalidRequestException) ? ((InvalidRequestException) e).getWhy() : e.getMessage();
+ sessionState.err.println("Error retrieving data: " + message);
+ }
+ }
+
// process a statement of the form: connect hostname/port
private void executeConnect(Tree statement)
{
Tree idList = statement.getChild(0);
int portNumber = Integer.parseInt(statement.getChild(1).getText());
-
+
StringBuilder hostName = new StringBuilder();
int idCount = idList.getChildCount();
for (int idx = 0; idx < idCount; idx++)
@@ -1432,6 +1481,14 @@ public class CliClient extends CliUserHe
// now, connect to the newly specified host name and port
sessionState.hostName = hostName.toString();
sessionState.thriftPort = portNumber;
+
+ // if we have user name and password
+ if (statement.getChildCount() == 4)
+ {
+ sessionState.username = statement.getChild(2).getText();
+ sessionState.password = CliUtils.unescapeSQLString(statement.getChild(3).getText());
+ }
+
CliMain.connect(sessionState.hostName, sessionState.thriftPort);
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/CliMain.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/CliMain.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/CliMain.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/CliMain.java Mon Feb 14 16:06:50 2011
@@ -115,35 +115,38 @@ public class CliMain
return;
}
}
-
+
if ((sessionState.username != null) && (sessionState.password != null))
{
- // Authenticate
+ // Authenticate
Map<String, String> credentials = new HashMap<String, String>();
credentials.put(SimpleAuthenticator.USERNAME_KEY, sessionState.username);
credentials.put(SimpleAuthenticator.PASSWORD_KEY, sessionState.password);
AuthenticationRequest authRequest = new AuthenticationRequest(credentials);
- try
+ try
{
thriftClient.login(authRequest);
cliClient.setUsername(sessionState.username);
- }
- catch (AuthenticationException e)
+ }
+ catch (AuthenticationException e)
{
+ thriftClient = null;
sessionState.err.println("Exception during authentication to the cassandra node, " +
"Verify the keyspace exists, and that you are using the correct credentials.");
return;
- }
- catch (AuthorizationException e)
+ }
+ catch (AuthorizationException e)
{
+ thriftClient = null;
sessionState.err.println("You are not authorized to use keyspace: " + sessionState.keyspace);
return;
}
- catch (TException e)
+ catch (TException e)
{
+ thriftClient = null;
sessionState.err.println("Login failure. Did you specify 'keyspace', 'username' and 'password'?");
return;
- }
+ }
}
// Lookup the cluster name, this is to make it clear which cluster the user is connected to
@@ -155,7 +158,6 @@ public class CliMain
}
catch (Exception e)
{
-
sessionState.err.println("Exception retrieving information about the cassandra node, check you have connected to the thrift port.");
if (sessionState.debug)
Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/CliOptions.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/CliOptions.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/CliOptions.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/CliOptions.java Mon Feb 14 16:06:50 2011
@@ -124,11 +124,19 @@ public class CliOptions
{
css.username = cmd.getOptionValue(USERNAME_OPTION);
}
+ else
+ {
+ css.username = "default";
+ }
if (cmd.hasOption(PASSWORD_OPTION))
{
css.password = cmd.getOptionValue(PASSWORD_OPTION);
}
+ else
+ {
+ css.password = "";
+ }
// Look for keyspace
if (cmd.hasOption(KEYSPACE_OPTION))
Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/CliUserHelp.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/CliUserHelp.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/CliUserHelp.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/CliUserHelp.java Mon Feb 14 16:06:50 2011
@@ -83,10 +83,12 @@ public class CliUserHelp {
state.out.println("Display the general help page with a list of available commands.");
break;
case CliParser.NODE_CONNECT:
- state.out.println("connect <hostname>/<port>;\n");
- state.out.println("Connect to the specified host on the specified port.\n");
+ state.out.println("connect <hostname>/<port> (<username> '<password>')?;\n");
+ state.out.println("Connect to the specified host on the specified port (using specified username and password).\n");
state.out.println("example:");
state.out.println("connect localhost/9160;");
+ state.out.println("connect localhost/9160 user 'badpasswd';");
+ state.out.println("connect 127.0.0.1/9160 user 'badpasswd';");
break;
case CliParser.NODE_USE_TABLE:
@@ -104,6 +106,11 @@ public class CliUserHelp {
state.out.println("describe keyspace system;");
break;
+ case CliParser.NODE_DESCRIBE_CLUSTER:
+ state.out.println("describe cluster;\n");
+ state.out.println("Display information about cluster: snitch, partitioner, schema versions.");
+ break;
+
case CliParser.NODE_EXIT:
state.out.println("exit;");
state.out.println("quit;\n");
@@ -330,11 +337,12 @@ public class CliUserHelp {
state.out.println("? Display this message.");
state.out.println("help; Display this help.");
state.out.println("help <command>; Display detailed, command-specific help.");
- state.out.println("connect <hostname>/<port>; Connect to thrift service.");
+ state.out.println("connect <hostname>/<port> (<username> '<password>')?; Connect to thrift service.");
state.out.println("use <keyspace> [<username> 'password']; Switch to a keyspace.");
state.out.println("describe keyspace (<keyspacename>)?; Describe keyspace.");
state.out.println("exit; Exit CLI.");
state.out.println("quit; Exit CLI.");
+ state.out.println("describe cluster; Display information about cluster.");
state.out.println("show cluster name; Display cluster name.");
state.out.println("show keyspaces; Show list of keyspaces.");
state.out.println("show api version; Show server API version.");
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java Mon Feb 14 16:06:50 2011
@@ -46,10 +46,12 @@ import org.apache.cassandra.utils.Pair;
public final class CFMetaData
{
+
public final static double DEFAULT_ROW_CACHE_SIZE = 0.0;
public final static double DEFAULT_KEY_CACHE_SIZE = 200000;
public final static double DEFAULT_READ_REPAIR_CHANCE = 1.0;
public final static boolean DEFAULT_REPLICATE_ON_WRITE = false;
+ public final static int DEFAULT_SYSTEM_MEMTABLE_THROUGHPUT_IN_MB = 8;
public final static int DEFAULT_ROW_CACHE_SAVE_PERIOD_IN_SECONDS = 0;
public final static int DEFAULT_KEY_CACHE_SAVE_PERIOD_IN_SECONDS = 4 * 3600;
public final static int DEFAULT_GC_GRACE_SECONDS = 864000;
@@ -65,13 +67,13 @@ public final class CFMetaData
private static final BiMap<Pair<String, String>, Integer> cfIdMap = HashBiMap.create();
- public static final CFMetaData StatusCf = newSystemTable(SystemTable.STATUS_CF, 0, "persistent metadata for the local node", BytesType.instance, null);
- public static final CFMetaData HintsCf = newSystemTable(HintedHandOffManager.HINTS_CF, 1, "hinted handoff data", BytesType.instance, BytesType.instance);
- public static final CFMetaData MigrationsCf = newSystemTable(Migration.MIGRATIONS_CF, 2, "individual schema mutations", TimeUUIDType.instance, null);
- public static final CFMetaData SchemaCf = newSystemTable(Migration.SCHEMA_CF, 3, "current state of the schema", UTF8Type.instance, null);
- public static final CFMetaData IndexCf = newSystemTable(SystemTable.INDEX_CF, 5, "indexes that have been completed", UTF8Type.instance, null);
+ public static final CFMetaData StatusCf = newSystemTable(SystemTable.STATUS_CF, 0, "persistent metadata for the local node", BytesType.instance, null, DEFAULT_SYSTEM_MEMTABLE_THROUGHPUT_IN_MB);
+ public static final CFMetaData HintsCf = newSystemTable(HintedHandOffManager.HINTS_CF, 1, "hinted handoff data", BytesType.instance, BytesType.instance, Math.min(256, Math.max(32, DEFAULT_MEMTABLE_THROUGHPUT_IN_MB / 2)));
+ public static final CFMetaData MigrationsCf = newSystemTable(Migration.MIGRATIONS_CF, 2, "individual schema mutations", TimeUUIDType.instance, null, DEFAULT_SYSTEM_MEMTABLE_THROUGHPUT_IN_MB);
+ public static final CFMetaData SchemaCf = newSystemTable(Migration.SCHEMA_CF, 3, "current state of the schema", UTF8Type.instance, null, DEFAULT_SYSTEM_MEMTABLE_THROUGHPUT_IN_MB);
+ public static final CFMetaData IndexCf = newSystemTable(SystemTable.INDEX_CF, 5, "indexes that have been completed", UTF8Type.instance, null, DEFAULT_SYSTEM_MEMTABLE_THROUGHPUT_IN_MB);
- private static CFMetaData newSystemTable(String cfName, int cfId, String comment, AbstractType comparator, AbstractType subComparator)
+ private static CFMetaData newSystemTable(String cfName, int cfId, String comment, AbstractType comparator, AbstractType subComparator, int memtableThroughPutInMB)
{
return new CFMetaData(Table.SYSTEM_TABLE,
cfName,
@@ -90,8 +92,8 @@ public final class CFMetaData
DEFAULT_ROW_CACHE_SAVE_PERIOD_IN_SECONDS,
DEFAULT_KEY_CACHE_SAVE_PERIOD_IN_SECONDS,
DEFAULT_MEMTABLE_LIFETIME_IN_MINS,
- DEFAULT_MEMTABLE_THROUGHPUT_IN_MB,
- DEFAULT_MEMTABLE_OPERATIONS_IN_MILLIONS,
+ memtableThroughPutInMB,
+ sizeMemtableOperations(memtableThroughPutInMB),
cfId,
Collections.<ByteBuffer, ColumnDefinition>emptyMap());
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/Config.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/Config.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/Config.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/Config.java Mon Feb 14 16:06:50 2011
@@ -106,7 +106,10 @@ public class Config
public Integer index_interval = 128;
public List<RawKeyspace> keyspaces;
-
+ public Double flush_largest_memtables_at = 1.0;
+ public Double reduce_cache_sizes_at = 1.0;
+ public double reduce_cache_capacity_to = 0.6;
+
public static enum CommitLogSync {
periodic,
batch
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Mon Feb 14 16:06:50 2011
@@ -20,7 +20,6 @@ package org.apache.cassandra.config;
import java.io.*;
import java.net.InetAddress;
-import java.net.MalformedURLException;
import java.net.URL;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
@@ -1177,4 +1176,19 @@ public class DatabaseDescriptor
{
return conf.encryption_options;
}
+
+ public static double getFlushLargestMemtablesAt()
+ {
+ return conf.flush_largest_memtables_at;
+ }
+
+ public static double getReduceCacheSizesAt()
+ {
+ return conf.reduce_cache_sizes_at;
+ }
+
+ public static double getReduceCacheCapacityTo()
+ {
+ return conf.reduce_cache_capacity_to;
+ }
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon Feb 14 16:06:50 2011
@@ -258,7 +258,19 @@ public class ColumnFamilyStore implement
int size = in.readInt();
byte[] bytes = new byte[size];
in.readFully(bytes);
- keys.add(StorageService.getPartitioner().decorateKey(ByteBuffer.wrap(bytes)));
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ DecoratedKey key;
+ try
+ {
+ key = StorageService.getPartitioner().decorateKey(buffer);
+ }
+ catch (Exception e)
+ {
+ logger.info(String.format("unable to read entry #%s from saved cache %s; skipping remaining entries",
+ keys.size(), path.getAbsolutePath()), e);
+ break;
+ }
+ keys.add(key);
}
if (logger.isDebugEnabled())
logger.debug(String.format("completed reading (%d ms; %d keys) saved cache %s",
@@ -465,7 +477,8 @@ public class ColumnFamilyStore implement
generations.add(desc.generation);
if (desc.isFromTheFuture())
{
- throw new RuntimeException("you can't open sstables from the future!");
+ throw new RuntimeException(String.format("Can't open sstables from the future! Current version %s, found file: %s",
+ Descriptor.CURRENT_VERSION, desc));
}
}
Collections.sort(generations);
@@ -1182,7 +1195,8 @@ public class ColumnFamilyStore implement
}
}
- ssTables.getRowCache().put(key, cached);
+ // avoid keeping a permanent reference to the original key buffer
+ ssTables.getRowCache().put(new DecoratedKey(key.token, ByteBufferUtil.clone(key.key)), cached);
}
return cached;
}
@@ -1973,7 +1987,7 @@ public class ColumnFamilyStore implement
public String toString()
{
return "ColumnFamilyStore(" +
- "table='" + table + '\'' +
+ "table='" + table.name + '\'' +
", columnFamily='" + columnFamily + '\'' +
')';
}
@@ -2114,4 +2128,26 @@ public class ColumnFamilyStore implement
{
return partitioner instanceof LocalPartitioner;
}
+
+ /**
+ * sets each cache's maximum capacity to 75% of its current size
+ */
+ public void reduceCacheSizes()
+ {
+ if (ssTables.getRowCache().getCapacity() > 0)
+ {
+ int newCapacity = (int) (DatabaseDescriptor.getReduceCacheCapacityTo() * ssTables.getRowCache().getSize());
+ logger.warn(String.format("Reducing %s row cache capacity from %d to %s to reduce memory pressure",
+ columnFamily, ssTables.getRowCache().getCapacity(), newCapacity));
+ ssTables.getRowCache().setCapacity(newCapacity);
+ }
+
+ if (ssTables.getKeyCache().getCapacity() > 0)
+ {
+ int newCapacity = (int) (DatabaseDescriptor.getReduceCacheCapacityTo() * ssTables.getKeyCache().getSize());
+ logger.warn(String.format("Reducing %s key cache capacity from %d to %s to reduce memory pressure",
+ columnFamily, ssTables.getKeyCache().getCapacity(), newCapacity));
+ ssTables.getKeyCache().setCapacity(newCapacity);
+ }
+ }
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java Mon Feb 14 16:06:50 2011
@@ -491,7 +491,7 @@ public class CompactionManager implement
long expectedRangeFileSize = cfs.getExpectedCompactedFileSize(Arrays.asList(sstable)) / 2;
String compactionFileLocation = table.getDataFileLocation(expectedRangeFileSize);
if (compactionFileLocation == null)
- throw new UnsupportedOperationException("disk full");
+ throw new IOException("disk full");
long startTime = System.currentTimeMillis();
long totalkeysWritten = 0;
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java Mon Feb 14 16:06:50 2011
@@ -211,16 +211,21 @@ public class CommitLog
logger.debug("Reading mutation at " + reader.getFilePointer());
long claimedCRC32;
-
Checksum checksum = new CRC32();
int serializedSize;
try
{
// any of the reads may hit EOF
serializedSize = reader.readInt();
+ // RowMutation must be at LEAST 10 bytes:
+ // 3 each for a non-empty Table and Key (including the 2-byte length from
+ // writeUTF/writeWithShortLength) and 4 bytes for column count.
+ // This prevents CRC by being fooled by special-case garbage in the file; see CASSANDRA-2128
+ if (serializedSize < 10)
+ break;
long claimedSizeChecksum = reader.readLong();
checksum.update(serializedSize);
- if (checksum.getValue() != claimedSizeChecksum || serializedSize <= 0)
+ if (checksum.getValue() != claimedSizeChecksum)
break; // entry wasn't synced correctly/fully. that's ok.
if (serializedSize > bytes.length)
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/CacheWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/CacheWriter.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/CacheWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/CacheWriter.java Mon Feb 14 16:06:50 2011
@@ -1,4 +1,25 @@
package org.apache.cassandra.io.sstable;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
import java.io.File;
import java.io.IOException;
@@ -6,7 +27,6 @@ import java.nio.ByteBuffer;
import java.util.Set;
import com.google.common.base.Function;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -23,7 +43,7 @@ public class CacheWriter<K, V> implement
private final Function<K, ByteBuffer> converter;
private final Set<K> keys;
private final String columnFamily;
- private final long totalBytes;
+ private final long estimatedTotalBytes;
private long bytesWritten;
public CacheWriter(String columnFamily, JMXInstrumentedCache<K, V> cache, File path, Function<K, ByteBuffer> converter)
@@ -36,7 +56,9 @@ public class CacheWriter<K, V> implement
long bytes = 0;
for (K key : keys)
bytes += converter.apply(key).remaining();
- totalBytes = bytes;
+
+ // an approximation -- the keyset can change while saving
+ estimatedTotalBytes = bytes;
}
public void saveCache() throws IOException
@@ -67,7 +89,8 @@ public class CacheWriter<K, V> implement
public long getTotalBytes()
{
- return totalBytes;
+ // keyset can change in size, thus totalBytes can too
+ return Math.max(estimatedTotalBytes, getBytesComplete());
}
public long getBytesComplete()
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Mon Feb 14 16:06:50 2011
@@ -307,7 +307,7 @@ public class SSTableReader extends SSTab
if (shouldAddEntry)
indexSummary.addEntry(decoratedKey, indexPosition);
if (cacheLoading && keysToLoadInCache.contains(decoratedKey))
- keyCache.put(new Pair<Descriptor, DecoratedKey>(descriptor, decoratedKey), dataPosition);
+ cacheKey(decoratedKey, dataPosition);
}
indexSummary.incrementRowid();
@@ -417,7 +417,9 @@ public class SSTableReader extends SSTab
public void cacheKey(DecoratedKey key, Long info)
{
- keyCache.put(new Pair<Descriptor, DecoratedKey>(descriptor, key), info);
+ // avoid keeping a permanent reference to the original key buffer
+ DecoratedKey copiedKey = new DecoratedKey(key.token, key.key == null ? null : ByteBufferUtil.clone(key.key));
+ keyCache.put(new Pair<Descriptor, DecoratedKey>(descriptor, copiedKey), info);
}
public Long getCachedPosition(DecoratedKey key)
@@ -440,8 +442,12 @@ public class SSTableReader extends SSTab
public long getPosition(DecoratedKey decoratedKey, Operator op)
{
// first, check bloom filter
- if (op == Operator.EQ && !bf.isPresent(decoratedKey.key))
- return -1;
+ if (op == Operator.EQ)
+ {
+ assert decoratedKey.key != null; // null is ok for GE scans
+ if (!bf.isPresent(decoratedKey.key))
+ return -1;
+ }
// next, the key cache
Pair<Descriptor, DecoratedKey> unifiedKey = new Pair<Descriptor, DecoratedKey>(descriptor, decoratedKey);
@@ -481,7 +487,7 @@ public class SSTableReader extends SSTab
if (op == Operator.EQ)
bloomFilterTracker.addTruePositive();
// store exact match for the key
- keyCache.put(unifiedKey, dataPosition);
+ cacheKey(decoratedKey, dataPosition);
}
return dataPosition;
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/net/MessagingService.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/net/MessagingService.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/net/MessagingService.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/net/MessagingService.java Mon Feb 14 16:06:50 2011
@@ -339,8 +339,8 @@ public final class MessagingService impl
*/
private void sendOneWay(Message message, String id, InetAddress to)
{
- if (logger_.isDebugEnabled())
- logger_.debug(FBUtilities.getLocalAddress() + " sending " + message.getVerb() + " to " + id + "@" + to);
+ if (logger_.isTraceEnabled())
+ logger_.trace(FBUtilities.getLocalAddress() + " sending " + message.getVerb() + " to " + id + "@" + to);
// do local deliveries
if ( message.getFrom().equals(to) )
Modified: cassandra/trunk/src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java Mon Feb 14 16:06:50 2011
@@ -161,6 +161,15 @@ public abstract class AbstractCassandraD
Table.open(table);
}
+ try
+ {
+ GCInspector.instance.start();
+ }
+ catch (Throwable t)
+ {
+ logger.warn("Unable to start GCInspector (currently only supported on the Sun JVM)");
+ }
+
// replay the log if necessary and check for compaction candidates
CommitLog.recover();
CompactionManager.instance.checkAllColumnFamilies();
Modified: cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java Mon Feb 14 16:06:50 2011
@@ -32,6 +32,7 @@ import javax.management.ObjectName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.utils.StatusLogger;
public class GCInspector
@@ -46,6 +47,7 @@ public class GCInspector
private HashMap<String, Long> gctimes = new HashMap<String, Long>();
List<Object> beans = new ArrayList<Object>(); // these are instances of com.sun.management.GarbageCollectorMXBean
+ private volatile boolean cacheSizesReduced;
public GCInspector()
{
@@ -87,13 +89,13 @@ public class GCInspector
{
public void run()
{
- logIntervalGCStats();
+ logGCResults();
}
};
StorageService.scheduledTasks.scheduleWithFixedDelay(t, INTERVAL_IN_MS, INTERVAL_IN_MS, TimeUnit.MILLISECONDS);
}
- private void logIntervalGCStats()
+ private void logGCResults()
{
for (Object gc : beans)
{
@@ -121,7 +123,7 @@ public class GCInspector
}
String st = String.format("GC for %s: %s ms, %s reclaimed leaving %s used; max is %s",
- gcw.getName(), gcw.getDuration(), previousMemoryUsed - memoryUsed, memoryUsed, memoryMax);
+ gcw.getName(), gcw.getDuration(), previousMemoryUsed - memoryUsed, memoryUsed, memoryMax);
if (gcw.getDuration() > MIN_DURATION)
logger.info(st);
else if (logger.isDebugEnabled())
@@ -129,6 +131,25 @@ public class GCInspector
if (gcw.getDuration() > MIN_DURATION_TPSTATS)
StatusLogger.log();
+
+ // if we just finished a full collection and we're still using a lot of memory, try to reduce the pressure
+ if (gcw.getName().equals("ConcurrentMarkSweep"))
+ {
+ double usage = (double) memoryUsed / memoryMax;
+
+ if (memoryUsed > DatabaseDescriptor.getReduceCacheSizesAt() * memoryMax && !cacheSizesReduced)
+ {
+ cacheSizesReduced = true;
+ logger.warn("Heap is " + usage + " full. You may need to reduce memtable and/or cache sizes. Cassandra is now reducing cache sizes to free up memory. Adjust reduce_cache_sizes_at threshold in cassandra.yaml if you don't want Cassandra to do this automatically");
+ StorageService.instance.reduceCacheSizes();
+ }
+
+ if (memoryUsed > DatabaseDescriptor.getFlushLargestMemtablesAt() * memoryMax)
+ {
+ logger.warn("Heap is " + usage + " full. You may need to reduce memtable and/or cache sizes. Cassandra will now flush up to the two largest memtables to free up memory. Adjust flush_largest_memtables_at threshold in cassandra.yaml if you don't want Cassandra to do this automatically");
+ StorageService.instance.flushLargestMemtables();
+ }
+ }
}
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java Mon Feb 14 16:06:50 2011
@@ -366,15 +366,6 @@ public class StorageService implements I
initialized = true;
isClientMode = false;
- try
- {
- GCInspector.instance.start();
- }
- catch (Throwable t)
- {
- logger_.warn("Unable to start GCInspector (currently only supported on the Sun JVM)");
- }
-
if (Boolean.parseBoolean(System.getProperty("cassandra.load_ring_state", "true")))
{
logger_.info("Loading persisted ring state");
@@ -2177,4 +2168,39 @@ public class StorageService implements I
{
return efficientCrossDCWrites;
}
+
+ /**
+ * Flushes the two largest memtables by ops and by throughput
+ */
+ public void flushLargestMemtables()
+ {
+ ColumnFamilyStore largestByOps = null;
+ ColumnFamilyStore largestByThroughput = null;
+ for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
+ {
+ if (largestByOps == null || cfs.getMemtableColumnsCount() > largestByOps.getMemtableColumnsCount())
+ largestByOps = cfs;
+ if (largestByThroughput == null || cfs.getMemtableThroughputInMB() > largestByThroughput.getMemtableThroughputInMB())
+ largestByThroughput = cfs;
+ }
+ if (largestByOps == null)
+ {
+ logger_.error("Unable to reduce heap usage since there are no column families defined");
+ return;
+ }
+
+ logger_.warn("Flushing " + largestByOps + " to relieve memory pressure");
+ largestByOps.forceFlush();
+ if (largestByThroughput != largestByOps)
+ {
+ logger_.warn("Flushing " + largestByThroughput + " to relieve memory pressure");
+ largestByThroughput.forceFlush();
+ }
+ }
+
+ public void reduceCacheSizes()
+ {
+ for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
+ cfs.reduceCacheSizes();
+ }
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java Mon Feb 14 16:06:50 2011
@@ -458,8 +458,8 @@ public class NodeCmd {
output.println(String.format("%-10d%10s%18s%18s%18s%18s",
offsets[i],
(i < sprh.length ? sprh[i] : ""),
- (i < rrlh.length ? rrlh[i] : ""),
- (i < rwlh.length ? rwlh[i] : ""),
+ (i < rwlh.length ? rrlh[i] : ""),
+ (i < rrlh.length ? rwlh[i] : ""),
(i < ersh.length ? ersh[i] : ""),
(i < ecch.length ? ecch[i] : "")));
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/ByteBufferUtil.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/utils/ByteBufferUtil.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/utils/ByteBufferUtil.java Mon Feb 14 16:06:50 2011
@@ -116,7 +116,7 @@ public class ByteBufferUtil
ByteBuffer copy = buffer.duplicate();
copy.position(position);
copy.limit(copy.position() + length);
- return string(buffer, charset);
+ return string(copy, charset);
}
public static String string(ByteBuffer buffer, Charset charset) throws CharacterCodingException
@@ -153,22 +153,19 @@ public class ByteBufferUtil
}
/**
- * ByteBuffer adoption of org.apache.commons.lang.ArrayUtils.lastIndexOf method
+ * ByteBuffer adaptation of org.apache.commons.lang.ArrayUtils.lastIndexOf method
*
* @param buffer the array to traverse for looking for the object, may be <code>null</code>
* @param valueToFind the value to find
* @param startIndex the start index to travers backwards from
- * @return the last index of the value within the array,
- * <code>-1</code> if not found or <code>null</code> array input
+ * @return the last index of the value within the array, relative to buffer's arrayOffset
+ * [that is, between buffer.position() and buffer.limit()]; <code>-1</code> if not found.
*/
public static int lastIndexOf(ByteBuffer buffer, byte valueToFind, int startIndex)
{
- if (buffer == null)
- {
- return -1;
- }
+ assert buffer != null;
- if (startIndex < 0)
+ if (startIndex < buffer.position())
{
return -1;
}
@@ -177,12 +174,10 @@ public class ByteBufferUtil
startIndex = buffer.limit() - 1;
}
- for (int i = startIndex; i >= 0; i--)
+ for (int i = startIndex; i >= buffer.position(); i--)
{
if (valueToFind == buffer.get(i))
- {
return i;
- }
}
return -1;
Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/StatusLogger.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/StatusLogger.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/utils/StatusLogger.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/utils/StatusLogger.java Mon Feb 14 16:06:50 2011
@@ -1,4 +1,25 @@
package org.apache.cassandra.utils;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
import java.lang.management.ManagementFactory;
import java.util.Set;
Modified: cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java Mon Feb 14 16:06:50 2011
@@ -115,6 +115,8 @@ public class CliTest extends CleanupHelp
"set myCF['key']['scName']['firstname'] = 'John';",
"get myCF['key']['scName']",
"use TestKEYSpace;",
+ "describe cluster;",
+ "help describe cluster;",
"show cluster name",
"show api version",
"help help",