You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2014/12/19 18:08:38 UTC
[1/3] cassandra git commit: Fix paging with multi-partition IN queries
Repository: cassandra
Updated Branches:
refs/heads/trunk 82a97f52e -> 9616a88f7
Fix paging with multi-partition IN queries
patch by thobbs; reviewed by slebresne for CASSANDRA-8408
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/62189935
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/62189935
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/62189935
Branch: refs/heads/trunk
Commit: 621899355a23eca6c503aa2ced2e944ff6a5fe66
Parents: d8bce50
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Dec 19 17:53:46 2014 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Dec 19 17:53:46 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/cql3/statements/SelectStatement.java | 2 +-
.../cassandra/service/pager/MultiPartitionPager.java | 15 ++++-----------
.../org/apache/cassandra/service/pager/Pageable.java | 5 ++++-
.../apache/cassandra/service/pager/QueryPagers.java | 7 +------
.../org/apache/cassandra/service/QueryPagerTest.java | 2 +-
6 files changed, 12 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/62189935/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bad24e7..516b4a2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.12:
+ * Fix paging for multi-partition IN queries (CASSANDRA-8408)
* Fix MOVED_NODE topology event never being emitted when a node
moves its token (CASSANDRA-8373)
* Fix validation of indexes in COMPACT tables (CASSANDRA-8156)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/62189935/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
index db25716..f08f6b8 100644
--- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@ -212,7 +212,7 @@ public class SelectStatement implements CQLStatement, MeasurableForPreparedCache
else
{
List<ReadCommand> commands = getSliceCommands(variables, limitForQuery, now);
- command = commands == null ? null : new Pageable.ReadCommands(commands);
+ command = commands == null ? null : new Pageable.ReadCommands(commands, limitForQuery);
}
int pageSize = options.getPageSize();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/62189935/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
index 35d6752..e478d3a 100644
--- a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
+++ b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
@@ -46,7 +46,7 @@ class MultiPartitionPager implements QueryPager
private int remaining;
private int current;
- MultiPartitionPager(List<ReadCommand> commands, ConsistencyLevel consistencyLevel, boolean localQuery, PagingState state)
+ MultiPartitionPager(List<ReadCommand> commands, ConsistencyLevel consistencyLevel, boolean localQuery, PagingState state, int limitForQuery)
{
int i = 0;
// If it's not the beginning (state != null), we need to find where we were and skip previous commands
@@ -76,7 +76,8 @@ class MultiPartitionPager implements QueryPager
throw new IllegalArgumentException("All commands must have the same timestamp or weird results may happen.");
pagers[j - i] = makePager(command, consistencyLevel, localQuery, null);
}
- remaining = state == null ? computeRemaining(pagers) : state.remaining;
+
+ remaining = state == null ? limitForQuery : state.remaining;
}
private static SinglePartitionPager makePager(ReadCommand command, ConsistencyLevel consistencyLevel, boolean localQuery, PagingState state)
@@ -86,14 +87,6 @@ class MultiPartitionPager implements QueryPager
: new NamesQueryPager((SliceByNamesReadCommand)command, consistencyLevel, localQuery);
}
- private static int computeRemaining(SinglePartitionPager[] pagers)
- {
- long remaining = 0;
- for (SinglePartitionPager pager : pagers)
- remaining += pager.maxRemaining();
- return remaining > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int)remaining;
- }
-
public PagingState state()
{
// Sets current to the first non-exhausted pager
@@ -123,7 +116,7 @@ class MultiPartitionPager implements QueryPager
{
List<Row> result = new ArrayList<Row>();
- int remainingThisQuery = pageSize;
+ int remainingThisQuery = Math.min(remaining, pageSize);
while (remainingThisQuery > 0 && !isExhausted())
{
// isExhausted has set us on the first non-exhausted pager
http://git-wip-us.apache.org/repos/asf/cassandra/blob/62189935/src/java/org/apache/cassandra/service/pager/Pageable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/pager/Pageable.java b/src/java/org/apache/cassandra/service/pager/Pageable.java
index 3a69bf4..d4986f7 100644
--- a/src/java/org/apache/cassandra/service/pager/Pageable.java
+++ b/src/java/org/apache/cassandra/service/pager/Pageable.java
@@ -30,9 +30,12 @@ public interface Pageable
{
public final List<ReadCommand> commands;
- public ReadCommands(List<ReadCommand> commands)
+ public final int limitForQuery;
+
+ public ReadCommands(List<ReadCommand> commands, int limitForQuery)
{
this.commands = commands;
+ this.limitForQuery = limitForQuery;
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/62189935/src/java/org/apache/cassandra/service/pager/QueryPagers.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/pager/QueryPagers.java b/src/java/org/apache/cassandra/service/pager/QueryPagers.java
index 65112aa..72d76fe 100644
--- a/src/java/org/apache/cassandra/service/pager/QueryPagers.java
+++ b/src/java/org/apache/cassandra/service/pager/QueryPagers.java
@@ -98,7 +98,7 @@ public class QueryPagers
if (commands.size() == 1)
return pager(commands.get(0), consistencyLevel, local, state);
- return new MultiPartitionPager(commands, consistencyLevel, local, state);
+ return new MultiPartitionPager(commands, consistencyLevel, local, state, ((Pageable.ReadCommands) command).limitForQuery);
}
else if (command instanceof ReadCommand)
{
@@ -115,11 +115,6 @@ public class QueryPagers
}
}
- public static QueryPager pager(Pageable command, ConsistencyLevel consistencyLevel)
- {
- return pager(command, consistencyLevel, false, null);
- }
-
public static QueryPager pager(Pageable command, ConsistencyLevel consistencyLevel, PagingState state)
{
return pager(command, consistencyLevel, false, state);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/62189935/test/unit/org/apache/cassandra/service/QueryPagerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/service/QueryPagerTest.java b/test/unit/org/apache/cassandra/service/QueryPagerTest.java
index 0645433..7dbd7b9 100644
--- a/test/unit/org/apache/cassandra/service/QueryPagerTest.java
+++ b/test/unit/org/apache/cassandra/service/QueryPagerTest.java
@@ -236,7 +236,7 @@ public class QueryPagerTest extends SchemaLoader
QueryPager pager = QueryPagers.localPager(new Pageable.ReadCommands(new ArrayList<ReadCommand>() {{
add(sliceQuery("k1", "c2", "c6", 10));
add(sliceQuery("k4", "c3", "c5", 10));
- }}));
+ }}, 10));
List<Row> page;
[3/3] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by sl...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9616a88f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9616a88f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9616a88f
Branch: refs/heads/trunk
Commit: 9616a88f7e358865d0d05ce0575a392dc075c5a5
Parents: 82a97f5 7c6993f
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Dec 19 18:07:35 2014 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Dec 19 18:07:35 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/cql3/statements/SelectStatement.java | 2 +-
.../cassandra/service/pager/MultiPartitionPager.java | 15 ++++-----------
.../org/apache/cassandra/service/pager/Pageable.java | 5 ++++-
.../apache/cassandra/service/pager/QueryPagers.java | 2 +-
.../org/apache/cassandra/service/QueryPagerTest.java | 2 +-
6 files changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9616a88f/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9616a88f/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9616a88f/test/unit/org/apache/cassandra/service/QueryPagerTest.java
----------------------------------------------------------------------
[2/3] cassandra git commit: Merge branch 'cassandra-2.0' into
cassandra-2.1
Posted by sl...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
CHANGES.txt
src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
src/java/org/apache/cassandra/service/pager/QueryPagers.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7c6993f8
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7c6993f8
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7c6993f8
Branch: refs/heads/trunk
Commit: 7c6993f82bfa1c8f294f3de669b2b0219b2cd306
Parents: 58529c7 6218993
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Dec 19 18:02:01 2014 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Dec 19 18:02:01 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/cql3/statements/SelectStatement.java | 2 +-
.../cassandra/service/pager/MultiPartitionPager.java | 15 ++++-----------
.../org/apache/cassandra/service/pager/Pageable.java | 5 ++++-
.../apache/cassandra/service/pager/QueryPagers.java | 2 +-
.../org/apache/cassandra/service/QueryPagerTest.java | 2 +-
6 files changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7c6993f8/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index ac28d78,516b4a2..4d88aa9
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,46 -1,5 +1,47 @@@
-2.0.12:
+2.1.3
+ * Fix regression in SSTableRewriter causing some rows to become unreadable
+ during compaction (CASSANDRA-8429)
+ * Run major compactions for repaired/unrepaired in parallel (CASSANDRA-8510)
+ * (cqlsh) Fix compression options in DESCRIBE TABLE output when compression
+ is disabled (CASSANDRA-8288)
+ * (cqlsh) Fix DESCRIBE output after keyspaces are altered (CASSANDRA-7623)
+ * Make sure we set lastCompactedKey correctly (CASSANDRA-8463)
+ * (cqlsh) Fix output of CONSISTENCY command (CASSANDRA-8507)
+ * (cqlsh) Fixed the handling of LIST statements (CASSANDRA-8370)
+ * Make sstablescrub check leveled manifest again (CASSANDRA-8432)
+ * Check first/last keys in sstable when giving out positions (CASSANDRA-8458)
+ * Disable mmap on Windows (CASSANDRA-6993)
+ * Add missing ConsistencyLevels to cassandra-stress (CASSANDRA-8253)
+ * Add auth support to cassandra-stress (CASSANDRA-7985)
+ * Fix ArrayIndexOutOfBoundsException when generating error message
+ for some CQL syntax errors (CASSANDRA-8455)
+ * Scale memtable slab allocation logarithmically (CASSANDRA-7882)
+ * cassandra-stress simultaneous inserts over same seed (CASSANDRA-7964)
+ * Reduce cassandra-stress sampling memory requirements (CASSANDRA-7926)
+ * Ensure memtable flush cannot expire commit log entries from its future (CASSANDRA-8383)
+ * Make read "defrag" async to reclaim memtables (CASSANDRA-8459)
+ * Remove tmplink files for offline compactions (CASSANDRA-8321)
+ * Reduce maxHintsInProgress (CASSANDRA-8415)
+ * BTree updates may call provided update function twice (CASSANDRA-8018)
+ * Release sstable references after anticompaction (CASSANDRA-8386)
+ * Handle abort() in SSTableRewriter properly (CASSANDRA-8320)
+ * Fix high size calculations for prepared statements (CASSANDRA-8231)
+ * Centralize shared executors (CASSANDRA-8055)
+ * Fix filtering for CONTAINS (KEY) relations on frozen collection
+ clustering columns when the query is restricted to a single
+ partition (CASSANDRA-8203)
+ * Do more aggressive entire-sstable TTL expiry checks (CASSANDRA-8243)
+ * Add more log info if readMeter is null (CASSANDRA-8238)
+ * add check of the system wall clock time at startup (CASSANDRA-8305)
+ * Support for frozen collections (CASSANDRA-7859)
+ * Fix overflow on histogram computation (CASSANDRA-8028)
+ * Have paxos reuse the timestamp generation of normal queries (CASSANDRA-7801)
+ * Fix incremental repair not remove parent session on remote (CASSANDRA-8291)
+ * Improve JBOD disk utilization (CASSANDRA-7386)
+ * Log failed host when preparing incremental repair (CASSANDRA-8228)
+ * Force config client mode in CQLSSTableWriter (CASSANDRA-8281)
+Merged from 2.0:
+ * Fix paging for multi-partition IN queries (CASSANDRA-8408)
* Fix MOVED_NODE topology event never being emitted when a node
moves its token (CASSANDRA-8373)
* Fix validation of indexes in COMPACT tables (CASSANDRA-8156)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7c6993f8/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
index 6d7bdbb,f08f6b8..4163315
--- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@@ -219,22 -245,7 +219,22 @@@ public class SelectStatement implement
}
}
- private ResultMessage.Rows execute(Pageable command, ConsistencyLevel cl, List<ByteBuffer> variables, int limit, long now) throws RequestValidationException, RequestExecutionException
+ private Pageable getPageableCommand(QueryOptions options, int limit, long now) throws RequestValidationException
+ {
+ int limitForQuery = updateLimitForQuery(limit);
+ if (isKeyRange || usesSecondaryIndexing)
+ return getRangeCommand(options, limitForQuery, now);
+
+ List<ReadCommand> commands = getSliceCommands(options, limitForQuery, now);
- return commands == null ? null : new Pageable.ReadCommands(commands);
++ return commands == null ? null : new Pageable.ReadCommands(commands, limitForQuery);
+ }
+
+ public Pageable getPageableCommand(QueryOptions options) throws RequestValidationException
+ {
+ return getPageableCommand(options, getLimit(options), System.currentTimeMillis());
+ }
+
+ private ResultMessage.Rows execute(Pageable command, QueryOptions options, int limit, long now, QueryState state) throws RequestValidationException, RequestExecutionException
{
List<Row> rows;
if (command == null)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7c6993f8/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
index 6ed635f,e478d3a..35d0971
--- a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
+++ b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
@@@ -47,7 -46,7 +47,7 @@@ class MultiPartitionPager implements Qu
private int remaining;
private int current;
- MultiPartitionPager(List<ReadCommand> commands, ConsistencyLevel consistencyLevel, ClientState cState, boolean localQuery, PagingState state)
- MultiPartitionPager(List<ReadCommand> commands, ConsistencyLevel consistencyLevel, boolean localQuery, PagingState state, int limitForQuery)
++ MultiPartitionPager(List<ReadCommand> commands, ConsistencyLevel consistencyLevel, ClientState cState, boolean localQuery, PagingState state, int limitForQuery)
{
int i = 0;
// If it's not the beginning (state != null), we need to find where we were and skip previous commands
@@@ -75,26 -74,19 +75,19 @@@
ReadCommand command = commands.get(j);
if (command.timestamp != timestamp)
throw new IllegalArgumentException("All commands must have the same timestamp or weird results may happen.");
- pagers[j - i] = makePager(command, consistencyLevel, localQuery, null);
+ pagers[j - i] = makePager(command, consistencyLevel, cState, localQuery, null);
}
- remaining = state == null ? computeRemaining(pagers) : state.remaining;
+
+ remaining = state == null ? limitForQuery : state.remaining;
}
- private static SinglePartitionPager makePager(ReadCommand command, ConsistencyLevel consistencyLevel, boolean localQuery, PagingState state)
+ private static SinglePartitionPager makePager(ReadCommand command, ConsistencyLevel consistencyLevel, ClientState cState, boolean localQuery, PagingState state)
{
return command instanceof SliceFromReadCommand
- ? new SliceQueryPager((SliceFromReadCommand)command, consistencyLevel, localQuery, state)
- : new NamesQueryPager((SliceByNamesReadCommand)command, consistencyLevel, localQuery);
+ ? new SliceQueryPager((SliceFromReadCommand)command, consistencyLevel, cState, localQuery, state)
+ : new NamesQueryPager((SliceByNamesReadCommand)command, consistencyLevel, cState, localQuery);
}
- private static int computeRemaining(SinglePartitionPager[] pagers)
- {
- long remaining = 0;
- for (SinglePartitionPager pager : pagers)
- remaining += pager.maxRemaining();
- return remaining > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int)remaining;
- }
-
public PagingState state()
{
// Sets current to the first non-exhausted pager
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7c6993f8/src/java/org/apache/cassandra/service/pager/QueryPagers.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/pager/QueryPagers.java
index c03e8ec,72d76fe..f933ccb
--- a/src/java/org/apache/cassandra/service/pager/QueryPagers.java
+++ b/src/java/org/apache/cassandra/service/pager/QueryPagers.java
@@@ -97,9 -96,9 +97,9 @@@ public class QueryPager
{
List<ReadCommand> commands = ((Pageable.ReadCommands)command).commands;
if (commands.size() == 1)
- return pager(commands.get(0), consistencyLevel, local, state);
+ return pager(commands.get(0), consistencyLevel, cState, local, state);
- return new MultiPartitionPager(commands, consistencyLevel, cState, local, state);
- return new MultiPartitionPager(commands, consistencyLevel, local, state, ((Pageable.ReadCommands) command).limitForQuery);
++ return new MultiPartitionPager(commands, consistencyLevel, cState, local, state, ((Pageable.ReadCommands) command).limitForQuery);
}
else if (command instanceof ReadCommand)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7c6993f8/test/unit/org/apache/cassandra/service/QueryPagerTest.java
----------------------------------------------------------------------