You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yu...@apache.org on 2012/10/22 20:55:18 UTC
[1/7] git commit: Merge branch 'cassandra-1.1' into trunk
Updated Branches:
refs/heads/cassandra-1.1 f22e2c459 -> 95fb613bf
refs/heads/trunk a58b87020 -> dd8a3c450
Merge branch 'cassandra-1.1' into trunk
Conflicts:
src/java/org/apache/cassandra/service/StorageService.java
src/java/org/apache/cassandra/thrift/CassandraServer.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/dd8a3c45
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/dd8a3c45
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/dd8a3c45
Branch: refs/heads/trunk
Commit: dd8a3c45072efbc0fea14e98552b4b5a9dab9de3
Parents: a58b870 95fb613
Author: Yuki Morishita <yu...@apache.org>
Authored: Mon Oct 22 13:53:11 2012 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Mon Oct 22 13:53:11 2012 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/thrift/CassandraServer.java | 12 ++-
.../cassandra/service/CassandraServerTest.java | 91 ++++++++++-----
3 files changed, 70 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dd8a3c45/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 4026428,f309ef1..0f7caba
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -40,76 -5,8 +40,77 @@@ Merged from 1.1
(CASSANDRA-4765)
* fix wrong leveled compaction progress calculation (CASSANDRA-4807)
* add a close() method to CRAR to prevent leaking file descriptors (CASSANDRA-4820)
+ * fix potential infinite loop in get_count (CASSANDRA-4833)
+
+1.2-beta1
+ * add atomic_batch_mutate (CASSANDRA-4542, -4635)
+ * increase default max_hint_window_in_ms to 3h (CASSANDRA-4632)
+ * include message initiation time to replicas so they can more
+ accurately drop timed-out requests (CASSANDRA-2858)
+ * fix clientutil.jar dependencies (CASSANDRA-4566)
+ * optimize WriteResponse (CASSANDRA-4548)
+ * new metrics (CASSANDRA-4009)
+ * redesign KEYS indexes to avoid read-before-write (CASSANDRA-2897)
+ * debug tracing (CASSANDRA-1123)
+ * parallelize row cache loading (CASSANDRA-4282)
+ * Make compaction, flush JBOD-aware (CASSANDRA-4292)
+ * run local range scans on the read stage (CASSANDRA-3687)
+ * clean up ioexceptions (CASSANDRA-2116)
+ * add disk_failure_policy (CASSANDRA-2118)
+ * Introduce new json format with row level deletion (CASSANDRA-4054)
+ * remove redundant "name" column from schema_keyspaces (CASSANDRA-4433)
+ * improve "nodetool ring" handling of multi-dc clusters (CASSANDRA-3047)
+ * update NTS calculateNaturalEndpoints to be O(N log N) (CASSANDRA-3881)
+ * add UseCondCardMark XX jvm settings on jdk 1.7 (CASSANDRA-4366)
+ * split up rpc timeout by operation type (CASSANDRA-2819)
+ * rewrite key cache save/load to use only sequential i/o (CASSANDRA-3762)
+ * update MS protocol with a version handshake + broadcast address id
+ (CASSANDRA-4311)
+ * multithreaded hint replay (CASSANDRA-4189)
+ * add inter-node message compression (CASSANDRA-3127)
+ * remove COPP (CASSANDRA-2479)
+ * Track tombstone expiration and compact when tombstone content is
+ higher than a configurable threshold, default 20% (CASSANDRA-3442, 4234)
+ * update MurmurHash to version 3 (CASSANDRA-2975)
+ * (CLI) track elapsed time for `delete' operation (CASSANDRA-4060)
+ * (CLI) jline version is bumped to 1.0 to properly support
+ 'delete' key function (CASSANDRA-4132)
+ * Save IndexSummary into new SSTable 'Summary' component (CASSANDRA-2392, 4289)
+ * Add support for range tombstones (CASSANDRA-3708)
+ * Improve MessagingService efficiency (CASSANDRA-3617)
+ * Avoid ID conflicts from concurrent schema changes (CASSANDRA-3794)
+ * Set thrift HSHA server thread limit to unlimited by default (CASSANDRA-4277)
+ * Avoids double serialization of CF id in RowMutation messages
+ (CASSANDRA-4293)
+ * stream compressed sstables directly with java nio (CASSANDRA-4297)
+ * Support multiple ranges in SliceQueryFilter (CASSANDRA-3885)
+ * Add column metadata to system column families (CASSANDRA-4018)
+ * (cql3) Always use composite types by default (CASSANDRA-4329)
+ * (cql3) Add support for set, map and list (CASSANDRA-3647)
+ * Validate date type correctly (CASSANDRA-4441)
+ * (cql3) Allow definitions with only a PK (CASSANDRA-4361)
+ * (cql3) Add support for row key composites (CASSANDRA-4179)
+ * improve DynamicEndpointSnitch by using reservoir sampling (CASSANDRA-4038)
+ * (cql3) Add support for 2ndary indexes (CASSANDRA-3680)
+ * (cql3) fix defining more than one PK to be invalid (CASSANDRA-4477)
+ * remove schema agreement checking from all external APIs (Thrift, CQL and CQL3) (CASSANDRA-4487)
+ * add Murmur3Partitioner and make it default for new installations (CASSANDRA-3772, 4621)
+ * (cql3) update pseudo-map syntax to use map syntax (CASSANDRA-4497)
+ * Finer grained exceptions hierarchy and provides error code with exceptions (CASSANDRA-3979)
+ * Adds events push to binary protocol (CASSANDRA-4480)
+ * Rewrite nodetool help (CASSANDRA-2293)
+ * Make CQL3 the default for CQL (CASSANDRA-4640)
+ * update stress tool to be able to use CQL3 (CASSANDRA-4406)
+ * Accept all thrift update on CQL3 cf but don't expose their metadata (CASSANDRA-4377)
+ * Replace Throttle with Guava's RateLimiter for HintedHandOff (CASSANDRA-4541)
+ * fix counter add/get using CQL2 and CQL3 in stress tool (CASSANDRA-4633)
+ * Add sstable count per level to cfstats (CASSANDRA-4537)
+ * (cql3) Add ALTER KEYSPACE statement (CASSANDRA-4611)
+ * (cql3) Allow defining default consistency levels (CASSANDRA-4448)
+ * (cql3) Fix queries using LIMIT missing results (CASSANDRA-4579)
+ * fix cross-version gossip messaging (CASSANDRA-4576)
+
1.1.6
* Wait for writes on synchronous read digest mismatch (CASSANDRA-4792)
* fix commitlog replay for nanotime-infected sstables (CASSANDRA-4782)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dd8a3c45/src/java/org/apache/cassandra/thrift/CassandraServer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/thrift/CassandraServer.java
index 2c37ba6,39b57f3..d21d9a2
--- a/src/java/org/apache/cassandra/thrift/CassandraServer.java
+++ b/src/java/org/apache/cassandra/thrift/CassandraServer.java
@@@ -464,80 -414,48 +464,84 @@@ public class CassandraServer implement
}
else
{
- pageSize = COUNT_PAGE_SIZE;
+ logger.debug("get_count");
}
- int totalCount = 0;
- List<ColumnOrSuperColumn> columns;
-
- if (predicate.slice_range == null)
+ try
{
- predicate.slice_range = new SliceRange(ByteBufferUtil.EMPTY_BYTE_BUFFER,
- ByteBufferUtil.EMPTY_BYTE_BUFFER,
- false,
- Integer.MAX_VALUE);
- }
+ ClientState cState = state();
+ cState.hasColumnFamilyAccess(column_parent.column_family, Permission.SELECT);
+ Table table = Table.open(cState.getKeyspace());
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(column_parent.column_family);
- final int requestedCount = predicate.slice_range.count;
- int remaining = requestedCount;
- int pages = 0;
- while (true)
- {
- predicate.slice_range.count = Math.min(pageSize, Math.max(2, remaining)); // fetch at least two columns
- columns = get_slice(key, column_parent, predicate, consistency_level);
- if (columns.isEmpty())
- break;
+ if (predicate.column_names != null)
+ return get_slice(key, column_parent, predicate, consistency_level).size();
- ByteBuffer firstName = getName(columns.get(0));
- int newColumns = pages == 0 || !firstName.equals(predicate.slice_range.start) ? columns.size() : columns.size() - 1;
- totalCount += newColumns;
- // if we over-counted, just return original limit
- if (totalCount > requestedCount)
- return requestedCount;
- remaining -= newColumns;
- pages++;
- // We're done if either:
- // - We've querying the number of columns requested by the user
- // - The last page wasn't full
- if (remaining == 0 || columns.size() < predicate.slice_range.count)
- break;
+ int pageSize;
+ // request by page if this is a large row
+ if (cfs.getMeanColumns() > 0)
+ {
+ int averageColumnSize = (int) (cfs.getMeanRowSize() / cfs.getMeanColumns());
+ pageSize = Math.min(COUNT_PAGE_SIZE,
+ DatabaseDescriptor.getInMemoryCompactionLimit() / averageColumnSize);
+ pageSize = Math.max(2, pageSize);
+ logger.debug("average row column size is {}; using pageSize of {}", averageColumnSize, pageSize);
+ }
else
- predicate.slice_range.start = getName(columns.get(columns.size() - 1));
- }
+ {
+ pageSize = COUNT_PAGE_SIZE;
+ }
+
+ int totalCount = 0;
+ List<ColumnOrSuperColumn> columns;
- return totalCount;
+ if (predicate.slice_range == null)
+ {
+ predicate.slice_range = new SliceRange(ByteBufferUtil.EMPTY_BYTE_BUFFER,
+ ByteBufferUtil.EMPTY_BYTE_BUFFER,
+ false,
+ Integer.MAX_VALUE);
+ }
+
- int requestedCount = predicate.slice_range.count;
++ final int requestedCount = predicate.slice_range.count;
++ int remaining = requestedCount;
+ int pages = 0;
+ while (true)
+ {
- predicate.slice_range.count = Math.min(pageSize, requestedCount);
++ predicate.slice_range.count = Math.min(pageSize, Math.max(2, remaining)); // fetch at least two columns
+ columns = get_slice(key, column_parent, predicate, consistency_level);
+ if (columns.isEmpty())
+ break;
+
+ ByteBuffer firstName = getName(columns.get(0));
+ int newColumns = pages == 0 || !firstName.equals(predicate.slice_range.start) ? columns.size()
+ : columns.size() - 1;
+
+ totalCount += newColumns;
- requestedCount -= newColumns;
++ // if we over-counted, just return original limit
++ if (totalCount > requestedCount)
++ return requestedCount;
++ remaining -= newColumns;
+ pages++;
+ // We're done if either:
+ // - We've querying the number of columns requested by the user
+ // - The last page wasn't full
- if (requestedCount == 0 || columns.size() < predicate.slice_range.count)
++ if (remaining == 0 || columns.size() < predicate.slice_range.count)
+ break;
+ else
+ predicate.slice_range.start = getName(columns.get(columns.size() - 1));
+ }
+
+ return totalCount;
+ }
+ catch (RequestValidationException e)
+ {
+ throw ThriftConversion.toThrift(e);
+ }
+ finally
+ {
+ Tracing.instance().stopSession();
+ }
}
private static ByteBuffer getName(ColumnOrSuperColumn cosc)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dd8a3c45/test/unit/org/apache/cassandra/service/CassandraServerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/service/CassandraServerTest.java
index 495d697,48701de..af2d1c7
--- a/test/unit/org/apache/cassandra/service/CassandraServerTest.java
+++ b/test/unit/org/apache/cassandra/service/CassandraServerTest.java
@@@ -18,6 -18,6 +18,8 @@@
*/
package org.apache.cassandra.service;
++import java.net.InetSocketAddress;
++
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
@@@ -24,37 -31,58 +33,59 @@@ import org.apache.cassandra.utils.ByteB
public class CassandraServerTest extends SchemaLoader
{
+ /**
+ * test get_count() to work correctly with 'count' settings around page size.
+ * (CASSANDRA-4833)
+ */
@Test
- public void test_get_column() throws Throwable {
- /*
- CassandraServer server = new CassandraServer();
- server.start();
-
- try {
- Column c1 = column("c1", "0", 0L);
- Column c2 = column("c2", "0", 0L);
- List<Column> columns = new ArrayList<Column>();
- columns.add(c1);
- columns.add(c2);
- Map<String, List<Column>> cfmap = new HashMap<String, List<Column>>();
- cfmap.put("Standard1", columns);
- cfmap.put("Standard2", columns);
-
- BatchMutation m = new BatchMutation("Keyspace1", "key1", cfmap);
- server.batch_insert(m, 1);
-
- Column column;
- column = server.get_column("Keyspace1", "key1", "Standard1:c2");
- assert column.value.equals("0");
-
- column = server.get_column("Keyspace1", "key1", "Standard2:c2");
- assert column.value.equals("0");
-
- ArrayList<Column> Columns = server.get_slice_strong("Keyspace1", "key1", "Standard1", -1, -1);
- assert Columns.size() == 2;
- } finally {
- server.shutdown();
+ public void test_get_count() throws Exception
+ {
+ Schema.instance.clear(); // Schema are now written on disk and will be reloaded
+ new EmbeddedCassandraService().start();
++ ThriftSessionManager.instance.setCurrentSocket(new InetSocketAddress(9160));
+
+ DecoratedKey key = Util.dk("testkey");
+ for (int i = 0; i < 3050; i++)
+ {
+ RowMutation rm = new RowMutation("Keyspace1", key.key);
+ rm.add(new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i))),
+ ByteBufferUtil.EMPTY_BYTE_BUFFER,
+ System.currentTimeMillis());
+ rm.apply();
}
- */
+
+ CassandraServer server = new CassandraServer();
+ server.set_keyspace("Keyspace1");
+
+ // same as page size
+ int count = server.get_count(key.key, new ColumnParent("Standard1"), predicateWithCount(1024), ConsistencyLevel.ONE);
+ assert count == 1024 : "expected 1024 but was " + count;
+
+ // 1 above page size
+ count = server.get_count(key.key, new ColumnParent("Standard1"), predicateWithCount(1025), ConsistencyLevel.ONE);
+ assert count == 1025 : "expected 1025 but was " + count;
+
+ // above number of columns
+ count = server.get_count(key.key, new ColumnParent("Standard1"), predicateWithCount(4000), ConsistencyLevel.ONE);
+ assert count == 3050 : "expected 3050 but was " + count;
+
+ // same as number of columns
+ count = server.get_count(key.key, new ColumnParent("Standard1"), predicateWithCount(3050), ConsistencyLevel.ONE);
+ assert count == 3050 : "expected 3050 but was " + count;
+
+ // 1 above number of columns
+ count = server.get_count(key.key, new ColumnParent("Standard1"), predicateWithCount(3051), ConsistencyLevel.ONE);
+ assert count == 3050 : "expected 3050 but was " + count;
+ }
+
+ private SlicePredicate predicateWithCount(int count)
+ {
+ SliceRange range = new SliceRange();
+ range.setStart("".getBytes());
+ range.setFinish("".getBytes());
+ range.setCount(count);
+ SlicePredicate predicate = new SlicePredicate();
+ predicate.setSlice_range(range);
+ return predicate;
}
}