You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by bl...@apache.org on 2017/07/14 15:31:01 UTC
[1/7] cassandra git commit: Fix queries with LIMIT and filtering on
clustering columns
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.11 bd89f5623 -> 7aa89a64e
Fix queries with LIMIT and filtering on clustering columns
patch by Benjamin Lerer; reviewed by Stefania Alborghetti for CASSANDRA-11223
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b08843de
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b08843de
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b08843de
Branch: refs/heads/cassandra-3.11
Commit: b08843de67b3c63fa9c0efe10bb9eda07c007f6c
Parents: 5b982d7
Author: Benjamin Lerer <b....@gmail.com>
Authored: Fri Jul 14 17:11:15 2017 +0200
Committer: Benjamin Lerer <b....@gmail.com>
Committed: Fri Jul 14 17:11:15 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/ColumnFamily.java | 2 +-
.../cassandra/db/filter/ColumnCounter.java | 21 +-
.../cassandra/db/filter/NamesQueryFilter.java | 2 +-
.../cassandra/db/filter/SliceQueryFilter.java | 17 +-
.../validation/operations/SelectLimitTest.java | 209 +++++++++++++++++++
6 files changed, 238 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b08843de/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 122ba54..bda510f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.11
+ * Fix queries with LIMIT and filtering on clustering columns (CASSANDRA-11223)
* Fix potential NPE when resume bootstrap fails (CASSANDRA-13272)
* Fix toJSONString for the UDT, tuple and collection types (CASSANDRA-13592)
* Fix nested Tuples/UDTs validation (CASSANDRA-13646)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b08843de/src/java/org/apache/cassandra/db/ColumnFamily.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamily.java b/src/java/org/apache/cassandra/db/ColumnFamily.java
index a7243a2..1532439 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamily.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamily.java
@@ -92,7 +92,7 @@ public abstract class ColumnFamily implements Iterable<Cell>, IRowCacheEntry
{
ColumnCounter counter = getComparator().isDense()
? new ColumnCounter(now)
- : new ColumnCounter.GroupByPrefix(now, getComparator(), metadata.clusteringColumns().size());
+ : new ColumnCounter.GroupByPrefix(now, getComparator(), metadata.clusteringColumns().size(), true);
return counter.countAll(this).live();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b08843de/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/ColumnCounter.java b/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
index 594fde8..a00d588 100644
--- a/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
+++ b/src/java/org/apache/cassandra/db/filter/ColumnCounter.java
@@ -90,6 +90,7 @@ public class ColumnCounter
{
protected final CellNameType type;
protected final int toGroup;
+ protected final boolean countPartitionsWithOnlyStaticData;
protected CellName previous;
/**
@@ -101,12 +102,15 @@ public class ColumnCounter
* @param toGroup the number of composite components on which to group
* column. If 0, all columns are grouped, otherwise we group
* those for which the {@code toGroup} first component are equals.
+ * @param countPartitionsWithOnlyStaticData if {@code true} the partitions with only static data should be
+ * counted as 1 valid row.
*/
- public GroupByPrefix(long timestamp, CellNameType type, int toGroup)
+ public GroupByPrefix(long timestamp, CellNameType type, int toGroup, boolean countPartitionsWithOnlyStaticData)
{
super(timestamp);
this.type = type;
this.toGroup = toGroup;
+ this.countPartitionsWithOnlyStaticData = countPartitionsWithOnlyStaticData;
assert toGroup == 0 || type != null;
}
@@ -153,14 +157,16 @@ public class ColumnCounter
// We want to count the static group as 1 (CQL) row only if it's the only
// group in the partition. So, since we have already counted it at this point,
// just don't count the 2nd group if there is one and the first one was static
- if (previous.isStatic())
+ if (previous.isStatic() && countPartitionsWithOnlyStaticData)
{
previous = current;
return true;
}
}
- live++;
+ if (!current.isStatic() || countPartitionsWithOnlyStaticData)
+ live++;
+
previous = current;
return true;
@@ -172,9 +178,14 @@ public class ColumnCounter
*/
public static class GroupByPrefixReversed extends GroupByPrefix
{
- public GroupByPrefixReversed(long timestamp, CellNameType type, int toGroup)
+ public GroupByPrefixReversed(long timestamp, CellNameType type, int toGroup, boolean countPartitionsWithOnlyStaticData)
{
- super(timestamp, type, toGroup);
+ // GroupByPrefixReversed ignores countPartitionsWithOnlyStaticData because the original problem (CASSANDRA-11223)
+ // only affect range queries and multi-partition queries. Range queries do not accept an ORDER BY clause.
+ // Multi-partition queries only accept an ORDER BY clause when paging is off. The limit in this case is used
+ // only when the rows with only static data have already been discarded. So, in practice
+ // changing GroupByPrefixReversed.count() has no effect.
+ super(timestamp, type, toGroup, countPartitionsWithOnlyStaticData);
}
@Override
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b08843de/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java b/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
index c8f63bb..74ca1fd 100644
--- a/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
@@ -183,7 +183,7 @@ public class NamesQueryFilter implements IDiskAtomFilter
public ColumnCounter columnCounter(CellNameType comparator, long now)
{
return countCQL3Rows
- ? new ColumnCounter.GroupByPrefix(now, null, 0)
+ ? new ColumnCounter.GroupByPrefix(now, null, 0, false)
: new ColumnCounter(now);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b08843de/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index 822d838..95f67e6 100644
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@ -26,7 +26,6 @@ import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.composites.*;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -344,12 +343,16 @@ public class SliceQueryFilter implements IDiskAtomFilter
{
if (compositesToGroup < 0)
return new ColumnCounter(now);
- else if (compositesToGroup == 0)
- return new ColumnCounter.GroupByPrefix(now, null, 0);
- else if (reversed)
- return new ColumnCounter.GroupByPrefixReversed(now, comparator, compositesToGroup);
- else
- return new ColumnCounter.GroupByPrefix(now, comparator, compositesToGroup);
+
+ boolean countPartitionsWithOnlyStaticData = Arrays.equals(slices, ColumnSlice.ALL_COLUMNS_ARRAY);
+
+ if (compositesToGroup == 0)
+ return new ColumnCounter.GroupByPrefix(now, null, 0, countPartitionsWithOnlyStaticData);
+
+ if (reversed)
+ return new ColumnCounter.GroupByPrefixReversed(now, comparator, compositesToGroup, countPartitionsWithOnlyStaticData);
+
+ return new ColumnCounter.GroupByPrefix(now, comparator, compositesToGroup, countPartitionsWithOnlyStaticData);
}
public ColumnFamily trim(ColumnFamily cf, int trimTo, long now)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b08843de/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
index 5fbf36d..0ffb799 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
@@ -129,4 +129,213 @@ public class SelectLimitTest extends CQLTester
// strict bound (v > 1) over a range of partitions is not supported for compact storage if limit is provided
assertInvalidThrow(InvalidRequestException.class, "SELECT * FROM %s WHERE v > 1 AND v <= 3 LIMIT 6 ALLOW FILTERING");
}
+
+ @Test
+ public void testFilteringOnClusteringColumnsWithLimitAndStaticColumns() throws Throwable
+ {
+ // With only one clustering column
+ createTable("CREATE TABLE %s (a int, b int, s int static, c int, primary key (a, b))"
+ + " WITH caching = {'keys': 'ALL', 'rows_per_partition' : 'ALL'}");
+
+ for (int i = 0; i < 4; i++)
+ {
+ execute("INSERT INTO %s (a, s) VALUES (?, ?)", i, i);
+ for (int j = 0; j < 3; j++)
+ if (!((i == 0 || i == 3) && j == 1))
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", i, j, i + j);
+ }
+
+ for (boolean forceFlush : new boolean[]{false, true})
+ {
+ if (forceFlush)
+ flush();
+
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, 0, 0, 0),
+ row(0, 2, 0, 2),
+ row(1, 0, 1, 1),
+ row(1, 1, 1, 2),
+ row(1, 2, 1, 3),
+ row(2, 0, 2, 2),
+ row(2, 1, 2, 3),
+ row(2, 2, 2, 4),
+ row(3, 0, 3, 3),
+ row(3, 2, 3, 5));
+
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ // The problem was that the static row of the partition 0 used to be only filtered in SelectStatement and was
+ // by consequence counted as a row. In which case the query was returning one row less.
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+ }
+ }
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b = 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b = 1 ORDER BY b DESC LIMIT 2 ALLOW FILTERING"),
+ row(2, 1, 2, 3),
+ row(1, 1, 1, 2));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b >= 1 AND b <= 1 ORDER BY b DESC LIMIT 2 ALLOW FILTERING"),
+ row(2, 1, 2, 3),
+ row(1, 1, 1, 2));
+
+ execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3)"); // Load all data in the row cache
+
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+ }
+
+ // With multiple clustering columns
+ createTable("CREATE TABLE %s (a int, b int, c int, s int static, d int, primary key (a, b, c))"
+ + " WITH caching = {'keys': 'ALL', 'rows_per_partition' : 'ALL'}");
+
+ for (int i = 0; i < 3; i++)
+ {
+ execute("INSERT INTO %s (a, s) VALUES (?, ?)", i, i);
+ for (int j = 0; j < 3; j++)
+ if (!(i == 0 && j == 1))
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", i, j, j, i + j);
+ }
+
+ for (boolean forceFlush : new boolean[]{false, true})
+ {
+ if (forceFlush)
+ flush();
+
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, 0, 0, 0, 0),
+ row(0, 2, 2, 0, 2),
+ row(1, 0, 0, 1, 1),
+ row(1, 1, 1, 1, 2),
+ row(1, 2, 2, 1, 3),
+ row(2, 0, 0, 2, 2),
+ row(2, 1, 1, 2, 3),
+ row(2, 2, 2, 2, 4));
+
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+ }
+ }
+
+ execute("SELECT * FROM %s WHERE a IN (0, 1, 2)"); // Load data in the row cache
+
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+ }
+ }
+
+ @Test
+ public void testIndexOnRegularColumnWithPartitionWithoutRows() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, c int, s int static, v int, PRIMARY KEY(pk, c))");
+ createIndex("CREATE INDEX ON %s (v)");
+
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 1, 9, 1);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 2, 9, 2);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 3, 1, 9, 1);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 4, 1, 9, 1);
+ flush();
+
+ assertRows(execute("SELECT * FROM %s WHERE v = ?", 1),
+ row(1, 1, 9, 1),
+ row(3, 1, 9, 1),
+ row(4, 1, 9, 1));
+
+ execute("DELETE FROM %s WHERE pk = ? AND c = ?", 3, 1);
+
+ // Test without paging
+ assertRows(execute("SELECT * FROM %s WHERE v = ?", 1),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+
+ assertRows(execute("SELECT * FROM %s WHERE v = ? LIMIT 2", 1),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE v = 1", pageSize),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE v = 1 LIMIT 2", pageSize),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[5/7] cassandra git commit: Merge branch cassandra-2.2 into
cassandra-3.0
Posted by bl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
index b4211bb,0000000..319eeb4
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
@@@ -1,1089 -1,0 +1,1106 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+import org.apache.cassandra.cache.IRowCacheEntry;
+import org.apache.cassandra.cache.RowCacheKey;
+import org.apache.cassandra.cache.RowCacheSentinel;
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.concurrent.StageManager;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.lifecycle.*;
+import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.service.pager.*;
+import org.apache.cassandra.thrift.ThriftResultsMerger;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.SearchIterator;
+import org.apache.cassandra.utils.btree.BTreeSet;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.HeapAllocator;
+
+
+/**
+ * A read command that selects a (part of a) single partition.
+ */
+public class SinglePartitionReadCommand extends ReadCommand
+{
+ protected static final SelectionDeserializer selectionDeserializer = new Deserializer();
+
+ private final DecoratedKey partitionKey;
+ private final ClusteringIndexFilter clusteringIndexFilter;
+
+ private int oldestUnrepairedTombstone = Integer.MAX_VALUE;
+
+ public SinglePartitionReadCommand(boolean isDigest,
+ int digestVersion,
+ boolean isForThrift,
+ CFMetaData metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DecoratedKey partitionKey,
+ ClusteringIndexFilter clusteringIndexFilter)
+ {
+ super(Kind.SINGLE_PARTITION, isDigest, digestVersion, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits);
+ assert partitionKey.getPartitioner() == metadata.partitioner;
+ this.partitionKey = partitionKey;
+ this.clusteringIndexFilter = clusteringIndexFilter;
+ }
+
+ /**
+ * Creates a new read command on a single partition.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param columnFilter the column filter to use for the query.
+ * @param rowFilter the row filter to use for the query.
+ * @param limits the limits to use for the query.
+ * @param partitionKey the partition key for the partition to query.
+ * @param clusteringIndexFilter the clustering index filter to use for the query.
+ *
+ * @return a newly created read command.
+ */
+ public static SinglePartitionReadCommand create(CFMetaData metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DecoratedKey partitionKey,
+ ClusteringIndexFilter clusteringIndexFilter)
+ {
+ return create(false, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter);
+ }
+
+ /**
+ * Creates a new read command on a single partition for thrift.
+ *
+ * @param isForThrift whether the query is for thrift or not.
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param columnFilter the column filter to use for the query.
+ * @param rowFilter the row filter to use for the query.
+ * @param limits the limits to use for the query.
+ * @param partitionKey the partition key for the partition to query.
+ * @param clusteringIndexFilter the clustering index filter to use for the query.
+ *
+ * @return a newly created read command.
+ */
+ public static SinglePartitionReadCommand create(boolean isForThrift,
+ CFMetaData metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DecoratedKey partitionKey,
+ ClusteringIndexFilter clusteringIndexFilter)
+ {
+ return new SinglePartitionReadCommand(false, 0, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter);
+ }
+
+ /**
+ * Creates a new read command on a single partition.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param key the partition key for the partition to query.
+ * @param columnFilter the column filter to use for the query.
+ * @param filter the clustering index filter to use for the query.
+ *
+ * @return a newly created read command. The returned command will use no row filter and have no limits.
+ */
+ public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, ColumnFilter columnFilter, ClusteringIndexFilter filter)
+ {
+ return create(metadata, nowInSec, columnFilter, RowFilter.NONE, DataLimits.NONE, key, filter);
+ }
+
+ /**
+ * Creates a new read command that queries a single partition in its entirety.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param key the partition key for the partition to query.
+ *
+ * @return a newly created read command that queries all the rows of {@code key}.
+ */
+ public static SinglePartitionReadCommand fullPartitionRead(CFMetaData metadata, int nowInSec, DecoratedKey key)
+ {
+ return SinglePartitionReadCommand.create(metadata, nowInSec, key, Slices.ALL);
+ }
+
+ /**
+ * Creates a new read command that queries a single partition in its entirety.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param key the partition key for the partition to query.
+ *
+ * @return a newly created read command that queries all the rows of {@code key}.
+ */
+ public static SinglePartitionReadCommand fullPartitionRead(CFMetaData metadata, int nowInSec, ByteBuffer key)
+ {
+ return SinglePartitionReadCommand.create(metadata, nowInSec, metadata.decorateKey(key), Slices.ALL);
+ }
+
+ /**
+ * Creates a new single partition slice command for the provided single slice.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param key the partition key for the partition to query.
+ * @param slice the slice of rows to query.
+ *
+ * @return a newly created read command that queries {@code slice} in {@code key}. The returned query will
+ * query every columns for the table (without limit or row filtering) and be in forward order.
+ */
+ public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, Slice slice)
+ {
+ return create(metadata, nowInSec, key, Slices.with(metadata.comparator, slice));
+ }
+
+ /**
+ * Creates a new single partition slice command for the provided slices.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param key the partition key for the partition to query.
+ * @param slices the slices of rows to query.
+ *
+ * @return a newly created read command that queries the {@code slices} in {@code key}. The returned query will
+ * query every columns for the table (without limit or row filtering) and be in forward order.
+ */
+ public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, Slices slices)
+ {
+ ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, false);
+ return SinglePartitionReadCommand.create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, key, filter);
+ }
+
+ /**
+ * Creates a new single partition slice command for the provided slices.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param key the partition key for the partition to query.
+ * @param slices the slices of rows to query.
+ *
+ * @return a newly created read command that queries the {@code slices} in {@code key}. The returned query will
+ * query every columns for the table (without limit or row filtering) and be in forward order.
+ */
+ public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, ByteBuffer key, Slices slices)
+ {
+ return create(metadata, nowInSec, metadata.decorateKey(key), slices);
+ }
+
+ /**
+ * Creates a new single partition name command for the provided rows.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param key the partition key for the partition to query.
+ * @param names the clustering for the rows to query.
+ *
+ * @return a newly created read command that queries the {@code names} in {@code key}. The returned query will
+ * query every columns (without limit or row filtering) and be in forward order.
+ */
+ public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, NavigableSet<Clustering> names)
+ {
+ ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(names, false);
+ return SinglePartitionReadCommand.create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, key, filter);
+ }
+
+ /**
+ * Creates a new single partition name command for the provided row.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ * @param key the partition key for the partition to query.
+ * @param name the clustering for the row to query.
+ *
+ * @return a newly created read command that queries {@code name} in {@code key}. The returned query will
+ * query every columns (without limit or row filtering).
+ */
+ public static SinglePartitionReadCommand create(CFMetaData metadata, int nowInSec, DecoratedKey key, Clustering name)
+ {
+ return create(metadata, nowInSec, key, FBUtilities.singleton(name, metadata.comparator));
+ }
+
+ public SinglePartitionReadCommand copy()
+ {
+ return new SinglePartitionReadCommand(isDigestQuery(), digestVersion(), isForThrift(), metadata(), nowInSec(), columnFilter(), rowFilter(), limits(), partitionKey(), clusteringIndexFilter());
+ }
+
+ public DecoratedKey partitionKey()
+ {
+ return partitionKey;
+ }
+
+ public ClusteringIndexFilter clusteringIndexFilter()
+ {
+ return clusteringIndexFilter;
+ }
+
+ public ClusteringIndexFilter clusteringIndexFilter(DecoratedKey key)
+ {
+ return clusteringIndexFilter;
+ }
+
+ public long getTimeout()
+ {
+ return DatabaseDescriptor.getReadRpcTimeout();
+ }
+
+ public boolean selectsKey(DecoratedKey key)
+ {
+ if (!this.partitionKey().equals(key))
+ return false;
+
+ return rowFilter().partitionKeyRestrictionsAreSatisfiedBy(key, metadata().getKeyValidator());
+ }
+
+ public boolean selectsClustering(DecoratedKey key, Clustering clustering)
+ {
+ if (clustering == Clustering.STATIC_CLUSTERING)
+ return !columnFilter().fetchedColumns().statics.isEmpty();
+
+ if (!clusteringIndexFilter().selects(clustering))
+ return false;
+
+ return rowFilter().clusteringKeyRestrictionsAreSatisfiedBy(clustering);
+ }
+
+ /**
+ * Returns a new command suitable to paging from the last returned row.
+ *
+ * @param lastReturned the last row returned by the previous page. The newly created command
+ * will only query row that comes after this (in query order). This can be {@code null} if this
+ * is the first page.
+ * @param pageSize the size to use for the page to query.
+ *
+ * @return the newly create command.
+ */
+ public SinglePartitionReadCommand forPaging(Clustering lastReturned, int pageSize)
+ {
+ // We shouldn't have set digest yet when reaching that point
+ assert !isDigestQuery();
+ return create(isForThrift(),
+ metadata(),
+ nowInSec(),
+ columnFilter(),
+ rowFilter(),
+ limits().forPaging(pageSize),
+ partitionKey(),
+ lastReturned == null ? clusteringIndexFilter() : clusteringIndexFilter.forPaging(metadata().comparator, lastReturned, false));
+ }
+
+ public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState) throws RequestExecutionException
+ {
+ return StorageProxy.read(Group.one(this), consistency, clientState);
+ }
+
+ public SinglePartitionPager getPager(PagingState pagingState, int protocolVersion)
+ {
+ return getPager(this, pagingState, protocolVersion);
+ }
+
+ private static SinglePartitionPager getPager(SinglePartitionReadCommand command, PagingState pagingState, int protocolVersion)
+ {
+ return new SinglePartitionPager(command, pagingState, protocolVersion);
+ }
+
+ protected void recordLatency(TableMetrics metric, long latencyNanos)
+ {
+ metric.readLatency.addNano(latencyNanos);
+ }
+
+ @SuppressWarnings("resource") // we close the created iterator through closing the result of this method (and SingletonUnfilteredPartitionIterator ctor cannot fail)
+ protected UnfilteredPartitionIterator queryStorage(final ColumnFamilyStore cfs, ReadOrderGroup orderGroup)
+ {
+ UnfilteredRowIterator partition = cfs.isRowCacheEnabled()
+ ? getThroughCache(cfs, orderGroup.baseReadOpOrderGroup())
+ : queryMemtableAndDisk(cfs, orderGroup.baseReadOpOrderGroup());
+ return new SingletonUnfilteredPartitionIterator(partition, isForThrift());
+ }
+
+ /**
+ * Fetch the rows requested if in cache; if not, read it from disk and cache it.
+ * <p>
+ * If the partition is cached, and the filter given is within its bounds, we return
+ * from cache, otherwise from disk.
+ * <p>
+ * If the partition is is not cached, we figure out what filter is "biggest", read
+ * that from disk, then filter the result and either cache that or return it.
+ */
+ private UnfilteredRowIterator getThroughCache(ColumnFamilyStore cfs, OpOrder.Group readOp)
+ {
+ assert !cfs.isIndex(); // CASSANDRA-5732
+ assert cfs.isRowCacheEnabled() : String.format("Row cache is not enabled on table [%s]", cfs.name);
+
+ RowCacheKey key = new RowCacheKey(metadata().ksAndCFName, partitionKey());
+
+ // Attempt a sentinel-read-cache sequence. if a write invalidates our sentinel, we'll return our
+ // (now potentially obsolete) data, but won't cache it. see CASSANDRA-3862
+ // TODO: don't evict entire partitions on writes (#2864)
+ IRowCacheEntry cached = CacheService.instance.rowCache.get(key);
+ if (cached != null)
+ {
+ if (cached instanceof RowCacheSentinel)
+ {
+ // Some other read is trying to cache the value, just do a normal non-caching read
+ Tracing.trace("Row cache miss (race)");
+ cfs.metric.rowCacheMiss.inc();
+ return queryMemtableAndDisk(cfs, readOp);
+ }
+
+ CachedPartition cachedPartition = (CachedPartition)cached;
+ if (cfs.isFilterFullyCoveredBy(clusteringIndexFilter(), limits(), cachedPartition, nowInSec()))
+ {
+ cfs.metric.rowCacheHit.inc();
+ Tracing.trace("Row cache hit");
+ UnfilteredRowIterator unfilteredRowIterator = clusteringIndexFilter().getUnfilteredRowIterator(columnFilter(), cachedPartition);
+ cfs.metric.updateSSTableIterated(0);
+ return unfilteredRowIterator;
+ }
+
+ cfs.metric.rowCacheHitOutOfRange.inc();
+ Tracing.trace("Ignoring row cache as cached value could not satisfy query");
+ return queryMemtableAndDisk(cfs, readOp);
+ }
+
+ cfs.metric.rowCacheMiss.inc();
+ Tracing.trace("Row cache miss");
+
+ // Note that on tables with no clustering keys, any positive value of
+ // rowsToCache implies caching the full partition
+ boolean cacheFullPartitions = metadata().clusteringColumns().size() > 0 ?
+ metadata().params.caching.cacheAllRows() :
+ metadata().params.caching.cacheRows();
+
+ // To be able to cache what we read, what we read must at least covers what the cache holds, that
+ // is the 'rowsToCache' first rows of the partition. We could read those 'rowsToCache' first rows
+ // systematically, but we'd have to "extend" that to whatever is needed for the user query that the
+ // 'rowsToCache' first rows don't cover and it's not trivial with our existing filters. So currently
+ // we settle for caching what we read only if the user query does query the head of the partition since
+ // that's the common case of when we'll be able to use the cache anyway. One exception is if we cache
+ // full partitions, in which case we just always read it all and cache.
+ if (cacheFullPartitions || clusteringIndexFilter().isHeadFilter())
+ {
+ RowCacheSentinel sentinel = new RowCacheSentinel();
+ boolean sentinelSuccess = CacheService.instance.rowCache.putIfAbsent(key, sentinel);
+ boolean sentinelReplaced = false;
+
+ try
+ {
+ final int rowsToCache = metadata().params.caching.rowsPerPartitionToCache();
+
+ @SuppressWarnings("resource") // we close on exception or upon closing the result of this method
+ UnfilteredRowIterator iter = SinglePartitionReadCommand.fullPartitionRead(metadata(), nowInSec(), partitionKey()).queryMemtableAndDisk(cfs, readOp);
+ try
+ {
+ // Use a custom iterator instead of DataLimits to avoid stopping the original iterator
+ UnfilteredRowIterator toCacheIterator = new WrappingUnfilteredRowIterator(iter)
+ {
+ private int rowsCounted = 0;
+
+ @Override
+ public boolean hasNext()
+ {
+ return rowsCounted < rowsToCache && super.hasNext();
+ }
+
+ @Override
+ public Unfiltered next()
+ {
+ Unfiltered unfiltered = super.next();
+ if (unfiltered.isRow())
+ {
+ Row row = (Row) unfiltered;
+ if (row.hasLiveData(nowInSec()))
+ rowsCounted++;
+ }
+ return unfiltered;
+ }
+ };
+
+ // We want to cache only rowsToCache rows
+ CachedPartition toCache = CachedBTreePartition.create(toCacheIterator, nowInSec());
+ if (sentinelSuccess && !toCache.isEmpty())
+ {
+ Tracing.trace("Caching {} rows", toCache.rowCount());
+ CacheService.instance.rowCache.replace(key, sentinel, toCache);
+ // Whether or not the previous replace has worked, our sentinel is not in the cache anymore
+ sentinelReplaced = true;
+ }
+
+ // We then re-filter out what this query wants.
+ // Note that in the case where we don't cache full partitions, it's possible that the current query is interested in more
+ // than what we've cached, so we can't just use toCache.
+ UnfilteredRowIterator cacheIterator = clusteringIndexFilter().getUnfilteredRowIterator(columnFilter(), toCache);
+ if (cacheFullPartitions)
+ {
+ // Everything is guaranteed to be in 'toCache', we're done with 'iter'
+ assert !iter.hasNext();
+ iter.close();
+ return cacheIterator;
+ }
+ return UnfilteredRowIterators.concat(cacheIterator, clusteringIndexFilter().filterNotIndexed(columnFilter(), iter));
+ }
+ catch (RuntimeException | Error e)
+ {
+ iter.close();
+ throw e;
+ }
+ }
+ finally
+ {
+ if (sentinelSuccess && !sentinelReplaced)
+ cfs.invalidateCachedPartition(key);
+ }
+ }
+
+ Tracing.trace("Fetching data but not populating cache as query does not query from the start of the partition");
+ return queryMemtableAndDisk(cfs, readOp);
+ }
+
+ /**
+ * Queries both memtable and sstables to fetch the result of this query.
+ * <p>
+ * Please note that this method:
+ * 1) does not check the row cache.
+ * 2) does not apply the query limit, nor the row filter (and so ignore 2ndary indexes).
+ * Those are applied in {@link ReadCommand#executeLocally}.
+ * 3) does not record some of the read metrics (latency, scanned cells histograms) nor
+ * throws TombstoneOverwhelmingException.
+ * It is publicly exposed because there is a few places where that is exactly what we want,
+ * but it should be used only where you know you don't need thoses things.
+ * <p>
+ * Also note that one must have "started" a {@code OpOrder.Group} on the queried table, and that is
+ * to enforce that that it is required as parameter, even though it's not explicitlly used by the method.
+ */
+ public UnfilteredRowIterator queryMemtableAndDisk(ColumnFamilyStore cfs, OpOrder.Group readOp)
+ {
+ Tracing.trace("Executing single-partition query on {}", cfs.name);
+
+ boolean copyOnHeap = Memtable.MEMORY_POOL.needToCopyOnHeap();
+ return queryMemtableAndDiskInternal(cfs, copyOnHeap);
+ }
+
+ @Override
+ protected int oldestUnrepairedTombstone()
+ {
+ return oldestUnrepairedTombstone;
+ }
+
+ private UnfilteredRowIterator queryMemtableAndDiskInternal(ColumnFamilyStore cfs, boolean copyOnHeap)
+ {
+ /*
+ * We have 2 main strategies:
+ * 1) We query memtables and sstables simulateneously. This is our most generic strategy and the one we use
+ * unless we have a names filter that we know we can optimize futher.
+ * 2) If we have a name filter (so we query specific rows), we can make a bet: that all column for all queried row
+ * will have data in the most recent sstable(s), thus saving us from reading older ones. This does imply we
+ * have a way to guarantee we have all the data for what is queried, which is only possible for name queries
+ * and if we have neither non-frozen collections/UDTs nor counters (indeed, for a non-frozen collection or UDT,
+ * we can't guarantee an older sstable won't have some elements that weren't in the most recent sstables,
+ * and counters are intrinsically a collection of shards and so have the same problem).
+ */
+ if (clusteringIndexFilter() instanceof ClusteringIndexNamesFilter && !queriesMulticellType())
+ return queryMemtableAndSSTablesInTimestampOrder(cfs, copyOnHeap, (ClusteringIndexNamesFilter)clusteringIndexFilter());
+
+ Tracing.trace("Acquiring sstable references");
+ ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, partitionKey()));
+
+ List<UnfilteredRowIterator> iterators = new ArrayList<>(Iterables.size(view.memtables) + view.sstables.size());
+ ClusteringIndexFilter filter = clusteringIndexFilter();
+
+ try
+ {
+ for (Memtable memtable : view.memtables)
+ {
+ Partition partition = memtable.getPartition(partitionKey());
+ if (partition == null)
+ continue;
+
+ @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
+ UnfilteredRowIterator iter = filter.getUnfilteredRowIterator(columnFilter(), partition);
+ @SuppressWarnings("resource") // same as above
+ UnfilteredRowIterator maybeCopied = copyOnHeap ? UnfilteredRowIterators.cloningIterator(iter, HeapAllocator.instance) : iter;
+ oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, partition.stats().minLocalDeletionTime);
+ iterators.add(isForThrift() ? ThriftResultsMerger.maybeWrap(maybeCopied, nowInSec()) : maybeCopied);
+ }
+ /*
+ * We can't eliminate full sstables based on the timestamp of what we've already read like
+ * in collectTimeOrderedData, but we still want to eliminate sstable whose maxTimestamp < mostRecentTombstone
+ * we've read. We still rely on the sstable ordering by maxTimestamp since if
+ * maxTimestamp_s1 > maxTimestamp_s0,
+ * we're guaranteed that s1 cannot have a row tombstone such that
+ * timestamp(tombstone) > maxTimestamp_s0
+ * since we necessarily have
+ * timestamp(tombstone) <= maxTimestamp_s1
+ * In other words, iterating in maxTimestamp order allow to do our mostRecentPartitionTombstone elimination
+ * in one pass, and minimize the number of sstables for which we read a partition tombstone.
+ */
+ Collections.sort(view.sstables, SSTableReader.maxTimestampComparator);
+ List<SSTableReader> skippedSSTables = null;
+ long mostRecentPartitionTombstone = Long.MIN_VALUE;
+ long minTimestamp = Long.MAX_VALUE;
+ int nonIntersectingSSTables = 0;
+ SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector();
+
+ for (SSTableReader sstable : view.sstables)
+ {
+ minTimestamp = Math.min(minTimestamp, sstable.getMinTimestamp());
+ // if we've already seen a partition tombstone with a timestamp greater
+ // than the most recent update to this sstable, we can skip it
+ if (sstable.getMaxTimestamp() < mostRecentPartitionTombstone)
+ break;
+
+ if (!shouldInclude(sstable))
+ {
+ nonIntersectingSSTables++;
+ // sstable contains no tombstone if maxLocalDeletionTime == Integer.MAX_VALUE, so we can safely skip those entirely
+ if (sstable.hasTombstones())
+ {
+ if (skippedSSTables == null)
+ skippedSSTables = new ArrayList<>();
+ skippedSSTables.add(sstable);
+ }
+ continue;
+ }
+
+ @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator
+ UnfilteredRowIterator iter = filter.filter(sstable.iterator(partitionKey(),
+ columnFilter(),
+ filter.isReversed(),
+ isForThrift(),
+ metricsCollector));
+ if (!sstable.isRepaired())
+ oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, sstable.getMinLocalDeletionTime());
+
+ iterators.add(isForThrift() ? ThriftResultsMerger.maybeWrap(iter, nowInSec()) : iter);
+ mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone, iter.partitionLevelDeletion().markedForDeleteAt());
+ }
+
+ int includedDueToTombstones = 0;
+ // Check for partition tombstones in the skipped sstables
+ if (skippedSSTables != null)
+ {
+ for (SSTableReader sstable : skippedSSTables)
+ {
+ if (sstable.getMaxTimestamp() <= minTimestamp)
+ continue;
+
+ @SuppressWarnings("resource") // 'iter' is either closed right away, or added to iterators which is close on exception, or through the closing of the final merged iterator
+ UnfilteredRowIterator iter = filter.filter(sstable.iterator(partitionKey(),
+ columnFilter(),
+ filter.isReversed(),
+ isForThrift(),
+ metricsCollector));
+
+ if (iter.partitionLevelDeletion().markedForDeleteAt() > minTimestamp)
+ {
+ iterators.add(iter);
+ if (!sstable.isRepaired())
+ oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, sstable.getMinLocalDeletionTime());
+ includedDueToTombstones++;
+ }
+ else
+ {
+ iter.close();
+ }
+ }
+ }
+ if (Tracing.isTracing())
+ Tracing.trace("Skipped {}/{} non-slice-intersecting sstables, included {} due to tombstones",
+ nonIntersectingSSTables, view.sstables.size(), includedDueToTombstones);
+
+ cfs.metric.updateSSTableIterated(metricsCollector.getMergedSSTables());
+
+ if (iterators.isEmpty())
+ return EmptyIterators.unfilteredRow(cfs.metadata, partitionKey(), filter.isReversed());
+
+ Tracing.trace("Merging data from memtables and {} sstables", metricsCollector.getMergedSSTables());
+
+ @SuppressWarnings("resource") // Closed through the closing of the result of that method.
+ UnfilteredRowIterator merged = UnfilteredRowIterators.merge(iterators, nowInSec());
+ if (!merged.isEmpty())
+ {
+ DecoratedKey key = merged.partitionKey();
+ cfs.metric.samplers.get(TableMetrics.Sampler.READS).addSample(key.getKey(), key.hashCode(), 1);
+ }
+
+ return merged;
+ }
+ catch (RuntimeException | Error e)
+ {
+ try
+ {
+ FBUtilities.closeAll(iterators);
+ }
+ catch (Exception suppressed)
+ {
+ e.addSuppressed(suppressed);
+ }
+ throw e;
+ }
+ }
+
+ private boolean shouldInclude(SSTableReader sstable)
+ {
+ // If some static columns are queried, we should always include the sstable: the clustering values stats of the sstable
+ // don't tell us if the sstable contains static values in particular.
+ // TODO: we could record if a sstable contains any static value at all.
+ if (!columnFilter().fetchedColumns().statics.isEmpty())
+ return true;
+
+ return clusteringIndexFilter().shouldInclude(sstable);
+ }
+
+ private boolean queriesMulticellType()
+ {
+ for (ColumnDefinition column : columnFilter().fetchedColumns())
+ {
+ if (column.type.isMultiCell() || column.type.isCounter())
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Do a read by querying the memtable(s) first, and then each relevant sstables sequentially by order of the sstable
+ * max timestamp.
+ *
+ * This is used for names query in the hope of only having to query the 1 or 2 most recent query and then knowing nothing
+ * more recent could be in the older sstables (which we can only guarantee if we know exactly which row we queries, and if
+ * no collection or counters are included).
+ * This method assumes the filter is a {@code ClusteringIndexNamesFilter}.
+ */
+ private UnfilteredRowIterator queryMemtableAndSSTablesInTimestampOrder(ColumnFamilyStore cfs, boolean copyOnHeap, ClusteringIndexNamesFilter filter)
+ {
+ Tracing.trace("Acquiring sstable references");
+ ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, partitionKey()));
+
+ ImmutableBTreePartition result = null;
+
+ Tracing.trace("Merging memtable contents");
+ for (Memtable memtable : view.memtables)
+ {
+ Partition partition = memtable.getPartition(partitionKey());
+ if (partition == null)
+ continue;
+
+ try (UnfilteredRowIterator iter = filter.getUnfilteredRowIterator(columnFilter(), partition))
+ {
+ if (iter.isEmpty())
+ continue;
+
+ UnfilteredRowIterator clonedFilter = copyOnHeap
+ ? UnfilteredRowIterators.cloningIterator(iter, HeapAllocator.instance)
+ : iter;
+ result = add(isForThrift() ? ThriftResultsMerger.maybeWrap(clonedFilter, nowInSec()) : clonedFilter, result, filter, false);
+ }
+ }
+
+ /* add the SSTables on disk */
+ Collections.sort(view.sstables, SSTableReader.maxTimestampComparator);
+ boolean onlyUnrepaired = true;
+ // read sorted sstables
+ SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector();
+ for (SSTableReader sstable : view.sstables)
+ {
+ // if we've already seen a partition tombstone with a timestamp greater
+ // than the most recent update to this sstable, we're done, since the rest of the sstables
+ // will also be older
+ if (result != null && sstable.getMaxTimestamp() < result.partitionLevelDeletion().markedForDeleteAt())
+ break;
+
+ long currentMaxTs = sstable.getMaxTimestamp();
+ filter = reduceFilter(filter, result, currentMaxTs);
+ if (filter == null)
+ break;
+
+ if (!shouldInclude(sstable))
+ {
+ // This mean that nothing queried by the filter can be in the sstable. One exception is the top-level partition deletion
+ // however: if it is set, it impacts everything and must be included. Getting that top-level partition deletion costs us
+ // some seek in general however (unless the partition is indexed and is in the key cache), so we first check if the sstable
+ // has any tombstone at all as a shortcut.
+ if (!sstable.hasTombstones())
+ continue; // Means no tombstone at all, we can skip that sstable
+
+ // We need to get the partition deletion and include it if it's live. In any case though, we're done with that sstable.
+ try (UnfilteredRowIterator iter = filter.filter(sstable.iterator(partitionKey(),
+ columnFilter(),
+ filter.isReversed(),
+ isForThrift(),
+ metricsCollector)))
+ {
+ if (!iter.partitionLevelDeletion().isLive())
+ result = add(UnfilteredRowIterators.noRowsIterator(iter.metadata(), iter.partitionKey(), Rows.EMPTY_STATIC_ROW, iter.partitionLevelDeletion(), filter.isReversed()), result, filter, sstable.isRepaired());
+ else
+ result = add(iter, result, filter, sstable.isRepaired());
+ }
+ continue;
+ }
+
+ Tracing.trace("Merging data from sstable {}", sstable.descriptor.generation);
+ try (UnfilteredRowIterator iter = filter.filter(sstable.iterator(partitionKey(),
+ columnFilter(),
+ filter.isReversed(),
+ isForThrift(),
+ metricsCollector)))
+ {
+ if (iter.isEmpty())
+ continue;
+
+ if (sstable.isRepaired())
+ onlyUnrepaired = false;
+ result = add(isForThrift() ? ThriftResultsMerger.maybeWrap(iter, nowInSec()) : iter, result, filter, sstable.isRepaired());
+ }
+ }
+
+ cfs.metric.updateSSTableIterated(metricsCollector.getMergedSSTables());
+
+ if (result == null || result.isEmpty())
+ return EmptyIterators.unfilteredRow(metadata(), partitionKey(), false);
+
+ DecoratedKey key = result.partitionKey();
+ cfs.metric.samplers.get(TableMetrics.Sampler.READS).addSample(key.getKey(), key.hashCode(), 1);
+
+ // "hoist up" the requested data into a more recent sstable
+ if (metricsCollector.getMergedSSTables() > cfs.getMinimumCompactionThreshold()
+ && onlyUnrepaired
+ && !cfs.isAutoCompactionDisabled()
+ && cfs.getCompactionStrategyManager().shouldDefragment())
+ {
+ // !!WARNING!! if we stop copying our data to a heap-managed object,
+ // we will need to track the lifetime of this mutation as well
+ Tracing.trace("Defragmenting requested data");
+
+ try (UnfilteredRowIterator iter = result.unfilteredIterator(columnFilter(), Slices.ALL, false))
+ {
+ final Mutation mutation = new Mutation(PartitionUpdate.fromIterator(iter));
+ StageManager.getStage(Stage.MUTATION).execute(new Runnable()
+ {
+ public void run()
+ {
+ // skipping commitlog and index updates is fine since we're just de-fragmenting existing data
+ Keyspace.open(mutation.getKeyspaceName()).apply(mutation, false, false);
+ }
+ });
+ }
+ }
+
+ return result.unfilteredIterator(columnFilter(), Slices.ALL, clusteringIndexFilter().isReversed());
+ }
+
+ private ImmutableBTreePartition add(UnfilteredRowIterator iter, ImmutableBTreePartition result, ClusteringIndexNamesFilter filter, boolean isRepaired)
+ {
+ if (!isRepaired)
+ oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, iter.stats().minLocalDeletionTime);
+
+ int maxRows = Math.max(filter.requestedRows().size(), 1);
+ if (result == null)
+ return ImmutableBTreePartition.create(iter, maxRows);
+
+ try (UnfilteredRowIterator merged = UnfilteredRowIterators.merge(Arrays.asList(iter, result.unfilteredIterator(columnFilter(), Slices.ALL, filter.isReversed())), nowInSec()))
+ {
+ return ImmutableBTreePartition.create(merged, maxRows);
+ }
+ }
+
+ private ClusteringIndexNamesFilter reduceFilter(ClusteringIndexNamesFilter filter, Partition result, long sstableTimestamp)
+ {
+ if (result == null)
+ return filter;
+
+ SearchIterator<Clustering, Row> searchIter = result.searchIterator(columnFilter(), false);
+
+ PartitionColumns columns = columnFilter().fetchedColumns();
+ NavigableSet<Clustering> clusterings = filter.requestedRows();
+
+ // We want to remove rows for which we have values for all requested columns. We have to deal with both static and regular rows.
+ // TODO: we could also remove a selected column if we've found values for every requested row but we'll leave
+ // that for later.
+
+ boolean removeStatic = false;
+ if (!columns.statics.isEmpty())
+ {
+ Row staticRow = searchIter.next(Clustering.STATIC_CLUSTERING);
+ removeStatic = staticRow != null && canRemoveRow(staticRow, columns.statics, sstableTimestamp);
+ }
+
+ NavigableSet<Clustering> toRemove = null;
+ for (Clustering clustering : clusterings)
+ {
+ Row row = searchIter.next(clustering);
+ if (row == null || !canRemoveRow(row, columns.regulars, sstableTimestamp))
+ continue;
+
+ if (toRemove == null)
+ toRemove = new TreeSet<>(result.metadata().comparator);
+ toRemove.add(clustering);
+ }
+
+ if (!removeStatic && toRemove == null)
+ return filter;
+
+ // Check if we have everything we need
+ boolean hasNoMoreStatic = columns.statics.isEmpty() || removeStatic;
+ boolean hasNoMoreClusterings = clusterings.isEmpty() || (toRemove != null && toRemove.size() == clusterings.size());
+ if (hasNoMoreStatic && hasNoMoreClusterings)
+ return null;
+
+ if (toRemove != null)
+ {
+ BTreeSet.Builder<Clustering> newClusterings = BTreeSet.builder(result.metadata().comparator);
+ newClusterings.addAll(Sets.difference(clusterings, toRemove));
+ clusterings = newClusterings.build();
+ }
+ return new ClusteringIndexNamesFilter(clusterings, filter.isReversed());
+ }
+
+ private boolean canRemoveRow(Row row, Columns requestedColumns, long sstableTimestamp)
+ {
+ // We can remove a row if it has data that is more recent that the next sstable to consider for the data that the query
+ // cares about. And the data we care about is 1) the row timestamp (since every query cares if the row exists or not)
+ // and 2) the requested columns.
+ if (row.primaryKeyLivenessInfo().isEmpty() || row.primaryKeyLivenessInfo().timestamp() <= sstableTimestamp)
+ return false;
+
+ for (ColumnDefinition column : requestedColumns)
+ {
+ Cell cell = row.getCell(column);
+ if (cell == null || cell.timestamp() <= sstableTimestamp)
+ return false;
+ }
+ return true;
+ }
+
+ @Override
++ public boolean selectsFullPartition()
++ {
++ return clusteringIndexFilter.selectsAllPartition() && !rowFilter().hasExpressionOnClusteringOrRegularColumns();
++ }
++
++ @Override
+ public String toString()
+ {
+ return String.format("Read(%s.%s columns=%s rowFilter=%s limits=%s key=%s filter=%s, nowInSec=%d)",
+ metadata().ksName,
+ metadata().cfName,
+ columnFilter(),
+ rowFilter(),
+ limits(),
+ metadata().getKeyValidator().getString(partitionKey().getKey()),
+ clusteringIndexFilter.toString(metadata()),
+ nowInSec());
+ }
+
+ public MessageOut<ReadCommand> createMessage(int version)
+ {
+ return new MessageOut<>(MessagingService.Verb.READ, this, readSerializer);
+ }
+
+ protected void appendCQLWhereClause(StringBuilder sb)
+ {
+ sb.append(" WHERE ");
+
+ sb.append(ColumnDefinition.toCQLString(metadata().partitionKeyColumns())).append(" = ");
+ DataRange.appendKeyString(sb, metadata().getKeyValidator(), partitionKey().getKey());
+
+ // We put the row filter first because the clustering index filter can end by "ORDER BY"
+ if (!rowFilter().isEmpty())
+ sb.append(" AND ").append(rowFilter());
+
+ String filterString = clusteringIndexFilter().toCQLString(metadata());
+ if (!filterString.isEmpty())
+ sb.append(" AND ").append(filterString);
+ }
+
+ protected void serializeSelection(DataOutputPlus out, int version) throws IOException
+ {
+ metadata().getKeyValidator().writeValue(partitionKey().getKey(), out);
+ ClusteringIndexFilter.serializer.serialize(clusteringIndexFilter(), out, version);
+ }
+
+ protected long selectionSerializedSize(int version)
+ {
+ return metadata().getKeyValidator().writtenLength(partitionKey().getKey())
+ + ClusteringIndexFilter.serializer.serializedSize(clusteringIndexFilter(), version);
+ }
+
+ /**
+ * Groups multiple single partition read commands.
+ */
+ public static class Group implements ReadQuery
+ {
+ public final List<SinglePartitionReadCommand> commands;
+ private final DataLimits limits;
+ private final int nowInSec;
++ private final boolean selectsFullPartitions;
+
+ public Group(List<SinglePartitionReadCommand> commands, DataLimits limits)
+ {
+ assert !commands.isEmpty();
+ this.commands = commands;
+ this.limits = limits;
- this.nowInSec = commands.get(0).nowInSec();
++ SinglePartitionReadCommand firstCommand = commands.get(0);
++ this.nowInSec = firstCommand.nowInSec();
++ this.selectsFullPartitions = firstCommand.selectsFullPartition();
+ for (int i = 1; i < commands.size(); i++)
+ assert commands.get(i).nowInSec() == nowInSec;
+ }
+
+ public static Group one(SinglePartitionReadCommand command)
+ {
+ return new Group(Collections.<SinglePartitionReadCommand>singletonList(command), command.limits());
+ }
+
+ public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState) throws RequestExecutionException
+ {
+ return StorageProxy.read(this, consistency, clientState);
+ }
+
+ public int nowInSec()
+ {
+ return nowInSec;
+ }
+
+ public DataLimits limits()
+ {
+ return limits;
+ }
+
+ public CFMetaData metadata()
+ {
+ return commands.get(0).metadata();
+ }
+
++ @Override
++ public boolean selectsFullPartition()
++ {
++ return selectsFullPartitions;
++ }
++
+ public ReadOrderGroup startOrderGroup()
+ {
+ // Note that the only difference between the command in a group must be the partition key on which
+ // they applied. So as far as ReadOrderGroup is concerned, we can use any of the commands to start one.
+ return commands.get(0).startOrderGroup();
+ }
+
+ public PartitionIterator executeInternal(ReadOrderGroup orderGroup)
+ {
+ List<PartitionIterator> partitions = new ArrayList<>(commands.size());
+ for (SinglePartitionReadCommand cmd : commands)
+ partitions.add(cmd.executeInternal(orderGroup));
+
+ // Because we only have enforce the limit per command, we need to enforce it globally.
- return limits.filter(PartitionIterators.concat(partitions), nowInSec);
++ return limits.filter(PartitionIterators.concat(partitions),
++ nowInSec,
++ selectsFullPartitions);
+ }
+
+ public QueryPager getPager(PagingState pagingState, int protocolVersion)
+ {
+ if (commands.size() == 1)
+ return SinglePartitionReadCommand.getPager(commands.get(0), pagingState, protocolVersion);
+
+ return new MultiPartitionPager(this, pagingState, protocolVersion);
+ }
+
+ public boolean selectsKey(DecoratedKey key)
+ {
+ return Iterables.any(commands, c -> c.selectsKey(key));
+ }
+
+ public boolean selectsClustering(DecoratedKey key, Clustering clustering)
+ {
+ return Iterables.any(commands, c -> c.selectsClustering(key, clustering));
+ }
+
+ @Override
+ public String toString()
+ {
+ return commands.toString();
+ }
+ }
+
+ private static class Deserializer extends SelectionDeserializer
+ {
+ public ReadCommand deserialize(DataInputPlus in, int version, boolean isDigest, int digestVersion, boolean isForThrift, CFMetaData metadata, int nowInSec, ColumnFilter columnFilter, RowFilter rowFilter, DataLimits limits, Optional<IndexMetadata> index)
+ throws IOException
+ {
+ DecoratedKey key = metadata.decorateKey(metadata.getKeyValidator().readValue(in, DatabaseDescriptor.getMaxValueSize()));
+ ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata);
+ return new SinglePartitionReadCommand(isDigest, digestVersion, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter);
+ }
+ }
+
+ /**
+ * {@code SSTableReaderListener} used to collect metrics about SSTable read access.
+ */
+ private static final class SSTableReadMetricsCollector implements SSTableReadsListener
+ {
+ /**
+ * The number of SSTables that need to be merged. This counter is only updated for single partition queries
+ * since this has been the behavior so far.
+ */
+ private int mergedSSTables;
+
+ @Override
+ public void onSSTableSelected(SSTableReader sstable, RowIndexEntry<?> indexEntry, SelectionReason reason)
+ {
+ sstable.incrementReadCount();
+ mergedSSTables++;
+ }
+
+ /**
+ * Returns the number of SSTables that need to be merged.
+ * @return the number of SSTables that need to be merged.
+ */
+ public int getMergedSSTables()
+ {
+ return mergedSSTables;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[4/7] cassandra git commit: Merge branch cassandra-2.2 into
cassandra-3.0
Posted by bl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/db/filter/DataLimits.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/filter/DataLimits.java
index 94f43dc,0000000..48ec06a
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/filter/DataLimits.java
+++ b/src/java/org/apache/cassandra/db/filter/DataLimits.java
@@@ -1,814 -1,0 +1,827 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.filter;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.transform.BasePartitions;
+import org.apache.cassandra.db.transform.BaseRows;
+import org.apache.cassandra.db.transform.StoppingTransformation;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * Object in charge of tracking if we have fetch enough data for a given query.
+ *
+ * The reason this is not just a simple integer is that Thrift and CQL3 count
+ * stuffs in different ways. This is what abstract those differences.
+ */
+public abstract class DataLimits
+{
+ public static final Serializer serializer = new Serializer();
+
+ public static final int NO_LIMIT = Integer.MAX_VALUE;
+
+ public static final DataLimits NONE = new CQLLimits(NO_LIMIT)
+ {
+ @Override
- public boolean hasEnoughLiveData(CachedPartition cached, int nowInSec)
++ public boolean hasEnoughLiveData(CachedPartition cached, int nowInSec, boolean countPartitionsWithOnlyStaticData)
+ {
+ return false;
+ }
+
+ @Override
- public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, int nowInSec)
++ public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter,
++ int nowInSec,
++ boolean countPartitionsWithOnlyStaticData)
+ {
+ return iter;
+ }
+
+ @Override
- public UnfilteredRowIterator filter(UnfilteredRowIterator iter, int nowInSec)
++ public UnfilteredRowIterator filter(UnfilteredRowIterator iter,
++ int nowInSec,
++ boolean countPartitionsWithOnlyStaticData)
+ {
+ return iter;
+ }
+ };
+
+ // We currently deal with distinct queries by querying full partitions but limiting the result at 1 row per
+ // partition (see SelectStatement.makeFilter). So an "unbounded" distinct is still actually doing some filtering.
+ public static final DataLimits DISTINCT_NONE = new CQLLimits(NO_LIMIT, 1, true);
+
+ public enum Kind { CQL_LIMIT, CQL_PAGING_LIMIT, THRIFT_LIMIT, SUPER_COLUMN_COUNTING_LIMIT }
+
+ public static DataLimits cqlLimits(int cqlRowLimit)
+ {
+ return new CQLLimits(cqlRowLimit);
+ }
+
+ public static DataLimits cqlLimits(int cqlRowLimit, int perPartitionLimit)
+ {
+ return new CQLLimits(cqlRowLimit, perPartitionLimit);
+ }
+
+ public static DataLimits distinctLimits(int cqlRowLimit)
+ {
+ return CQLLimits.distinct(cqlRowLimit);
+ }
+
+ public static DataLimits thriftLimits(int partitionLimit, int cellPerPartitionLimit)
+ {
+ return new ThriftLimits(partitionLimit, cellPerPartitionLimit);
+ }
+
+ public static DataLimits superColumnCountingLimits(int partitionLimit, int cellPerPartitionLimit)
+ {
+ return new SuperColumnCountingLimits(partitionLimit, cellPerPartitionLimit);
+ }
+
+ public abstract Kind kind();
+
+ public abstract boolean isUnlimited();
+ public abstract boolean isDistinct();
+
+ public abstract DataLimits forPaging(int pageSize);
+ public abstract DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining);
+
+ public abstract DataLimits forShortReadRetry(int toFetch);
+
- public abstract boolean hasEnoughLiveData(CachedPartition cached, int nowInSec);
++ public abstract boolean hasEnoughLiveData(CachedPartition cached, int nowInSec, boolean countPartitionsWithOnlyStaticData);
+
+ /**
+ * Returns a new {@code Counter} for this limits.
+ *
+ * @param nowInSec the current time in second (to decide what is expired or not).
+ * @param assumeLiveData if true, the counter will assume that every row passed is live and won't
+ * thus check for liveness, otherwise it will. This should be {@code true} when used on a
+ * {@code RowIterator} (since it only returns live rows), false otherwise.
++ * @param countPartitionsWithOnlyStaticData if {@code true} the partitions with only static data should be counted
++ * as 1 valid row.
+ * @return a new {@code Counter} for this limits.
+ */
- public abstract Counter newCounter(int nowInSec, boolean assumeLiveData);
++ public abstract Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData);
+
+ /**
+ * The max number of results this limits enforces.
+ * <p>
+ * Note that the actual definition of "results" depends a bit: for CQL, it's always rows, but for
+ * thrift, it means cells.
+ *
+ * @return the maximum number of results this limits enforces.
+ */
+ public abstract int count();
+
+ public abstract int perPartitionCount();
+
- public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, int nowInSec)
++ public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter,
++ int nowInSec,
++ boolean countPartitionsWithOnlyStaticData)
+ {
- return this.newCounter(nowInSec, false).applyTo(iter);
++ return this.newCounter(nowInSec, false, countPartitionsWithOnlyStaticData).applyTo(iter);
+ }
+
- public UnfilteredRowIterator filter(UnfilteredRowIterator iter, int nowInSec)
++ public UnfilteredRowIterator filter(UnfilteredRowIterator iter,
++ int nowInSec,
++ boolean countPartitionsWithOnlyStaticData)
+ {
- return this.newCounter(nowInSec, false).applyTo(iter);
++ return this.newCounter(nowInSec, false, countPartitionsWithOnlyStaticData).applyTo(iter);
+ }
+
- public PartitionIterator filter(PartitionIterator iter, int nowInSec)
++ public PartitionIterator filter(PartitionIterator iter, int nowInSec, boolean countPartitionsWithOnlyStaticData)
+ {
- return this.newCounter(nowInSec, true).applyTo(iter);
++ return this.newCounter(nowInSec, true, countPartitionsWithOnlyStaticData).applyTo(iter);
+ }
+
+ /**
+ * Estimate the number of results (the definition of "results" will be rows for CQL queries
+ * and partitions for thrift ones) that a full scan of the provided cfs would yield.
+ */
+ public abstract float estimateTotalResults(ColumnFamilyStore cfs);
+
+ public static abstract class Counter extends StoppingTransformation<BaseRowIterator<?>>
+ {
+ // false means we do not propagate our stop signals onto the iterator, we only count
+ private boolean enforceLimits = true;
+
+ public Counter onlyCount()
+ {
+ this.enforceLimits = false;
+ return this;
+ }
+
+ public PartitionIterator applyTo(PartitionIterator partitions)
+ {
+ return Transformation.apply(partitions, this);
+ }
+
+ public UnfilteredPartitionIterator applyTo(UnfilteredPartitionIterator partitions)
+ {
+ return Transformation.apply(partitions, this);
+ }
+
+ public UnfilteredRowIterator applyTo(UnfilteredRowIterator partition)
+ {
+ return (UnfilteredRowIterator) applyToPartition(partition);
+ }
+
+ public RowIterator applyTo(RowIterator partition)
+ {
+ return (RowIterator) applyToPartition(partition);
+ }
+
+ /**
+ * The number of results counted.
+ * <p>
+ * Note that the definition of "results" should be the same that for {@link #count}.
+ *
+ * @return the number of results counted.
+ */
+ public abstract int counted();
+ public abstract int countedInCurrentPartition();
+
+ public abstract boolean isDone();
+ public abstract boolean isDoneForPartition();
+
+ @Override
+ protected BaseRowIterator<?> applyToPartition(BaseRowIterator<?> partition)
+ {
+ return partition instanceof UnfilteredRowIterator ? Transformation.apply((UnfilteredRowIterator) partition, this)
+ : Transformation.apply((RowIterator) partition, this);
+ }
+
+ // called before we process a given partition
+ protected abstract void applyToPartition(DecoratedKey partitionKey, Row staticRow);
+
+ @Override
+ protected void attachTo(BasePartitions partitions)
+ {
+ if (enforceLimits)
+ super.attachTo(partitions);
+ if (isDone())
+ stop();
+ }
+
+ @Override
+ protected void attachTo(BaseRows rows)
+ {
+ if (enforceLimits)
+ super.attachTo(rows);
+ applyToPartition(rows.partitionKey(), rows.staticRow());
+ if (isDoneForPartition())
+ stopInPartition();
+ }
+ }
+
+ /**
+ * Limits used by CQL; this counts rows.
+ */
+ private static class CQLLimits extends DataLimits
+ {
+ protected final int rowLimit;
+ protected final int perPartitionLimit;
+
+ // Whether the query is a distinct query or not.
+ protected final boolean isDistinct;
+
+ private CQLLimits(int rowLimit)
+ {
+ this(rowLimit, NO_LIMIT);
+ }
+
+ private CQLLimits(int rowLimit, int perPartitionLimit)
+ {
+ this(rowLimit, perPartitionLimit, false);
+ }
+
+ private CQLLimits(int rowLimit, int perPartitionLimit, boolean isDistinct)
+ {
+ this.rowLimit = rowLimit;
+ this.perPartitionLimit = perPartitionLimit;
+ this.isDistinct = isDistinct;
+ }
+
+ private static CQLLimits distinct(int rowLimit)
+ {
+ return new CQLLimits(rowLimit, 1, true);
+ }
+
+ public Kind kind()
+ {
+ return Kind.CQL_LIMIT;
+ }
+
+ public boolean isUnlimited()
+ {
+ return rowLimit == NO_LIMIT && perPartitionLimit == NO_LIMIT;
+ }
+
+ public boolean isDistinct()
+ {
+ return isDistinct;
+ }
+
+ public DataLimits forPaging(int pageSize)
+ {
+ return new CQLLimits(pageSize, perPartitionLimit, isDistinct);
+ }
+
+ public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+ {
+ return new CQLPagingLimits(pageSize, perPartitionLimit, isDistinct, lastReturnedKey, lastReturnedKeyRemaining);
+ }
+
+ public DataLimits forShortReadRetry(int toFetch)
+ {
+ // When we do a short read retry, we're only ever querying the single partition on which we have a short read. So
+ // we use toFetch as the row limit and use no perPartitionLimit (it would be equivalent in practice to use toFetch
+ // for both argument or just for perPartitionLimit with no limit on rowLimit).
+ return new CQLLimits(toFetch, NO_LIMIT, isDistinct);
+ }
+
- public boolean hasEnoughLiveData(CachedPartition cached, int nowInSec)
++ public boolean hasEnoughLiveData(CachedPartition cached, int nowInSec, boolean countPartitionsWithOnlyStaticData)
+ {
+ // We want the number of row that are currently live. Getting that precise number forces
+ // us to iterate the cached partition in general, but we can avoid that if:
+ // - The number of rows with at least one non-expiring cell is greater than what we ask,
+ // in which case we know we have enough live.
+ // - The number of rows is less than requested, in which case we know we won't have enough.
+ if (cached.rowsWithNonExpiringCells() >= rowLimit)
+ return true;
+
+ if (cached.rowCount() < rowLimit)
+ return false;
+
+ // Otherwise, we need to re-count
+
- DataLimits.Counter counter = newCounter(nowInSec, false);
++ DataLimits.Counter counter = newCounter(nowInSec, false, countPartitionsWithOnlyStaticData);
+ try (UnfilteredRowIterator cacheIter = cached.unfilteredIterator(ColumnFilter.selection(cached.columns()), Slices.ALL, false);
+ UnfilteredRowIterator iter = counter.applyTo(cacheIter))
+ {
+ // Consume the iterator until we've counted enough
+ while (iter.hasNext())
+ iter.next();
+ return counter.isDone();
+ }
+ }
+
- public Counter newCounter(int nowInSec, boolean assumeLiveData)
++ public Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
+ {
- return new CQLCounter(nowInSec, assumeLiveData);
++ return new CQLCounter(nowInSec, assumeLiveData, countPartitionsWithOnlyStaticData);
+ }
+
+ public int count()
+ {
+ return rowLimit;
+ }
+
+ public int perPartitionCount()
+ {
+ return perPartitionLimit;
+ }
+
+ public float estimateTotalResults(ColumnFamilyStore cfs)
+ {
+ // TODO: we should start storing stats on the number of rows (instead of the number of cells, which
+ // is what getMeanColumns returns)
+ float rowsPerPartition = ((float) cfs.getMeanColumns()) / cfs.metadata.partitionColumns().regulars.size();
+ return rowsPerPartition * (cfs.estimateKeys());
+ }
+
+ protected class CQLCounter extends Counter
+ {
+ protected final int nowInSec;
+ protected final boolean assumeLiveData;
++ protected final boolean countPartitionsWithOnlyStaticData;
+
+ protected int rowCounted;
+ protected int rowInCurrentPartition;
+
+ protected boolean hasLiveStaticRow;
+
- public CQLCounter(int nowInSec, boolean assumeLiveData)
++ public CQLCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
+ {
+ this.nowInSec = nowInSec;
+ this.assumeLiveData = assumeLiveData;
++ this.countPartitionsWithOnlyStaticData = countPartitionsWithOnlyStaticData;
+ }
+
+ @Override
+ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
+ {
+ rowInCurrentPartition = 0;
+ hasLiveStaticRow = !staticRow.isEmpty() && (assumeLiveData || staticRow.hasLiveData(nowInSec));
+ }
+
+ @Override
+ public Row applyToRow(Row row)
+ {
+ if (assumeLiveData || row.hasLiveData(nowInSec))
+ incrementRowCount();
+ return row;
+ }
+
+ @Override
+ public void onPartitionClose()
+ {
+ // Normally, we don't count static rows as from a CQL point of view, it will be merge with other
+ // rows in the partition. However, if we only have the static row, it will be returned as one row
+ // so count it.
- if (hasLiveStaticRow && rowInCurrentPartition == 0)
++ if (countPartitionsWithOnlyStaticData && hasLiveStaticRow && rowInCurrentPartition == 0)
+ incrementRowCount();
+ super.onPartitionClose();
+ }
+
+ private void incrementRowCount()
+ {
+ if (++rowCounted >= rowLimit)
+ stop();
+ if (++rowInCurrentPartition >= perPartitionLimit)
+ stopInPartition();
+ }
+
+ public int counted()
+ {
+ return rowCounted;
+ }
+
+ public int countedInCurrentPartition()
+ {
+ return rowInCurrentPartition;
+ }
+
+ public boolean isDone()
+ {
+ return rowCounted >= rowLimit;
+ }
+
+ public boolean isDoneForPartition()
+ {
+ return isDone() || rowInCurrentPartition >= perPartitionLimit;
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+
+ if (rowLimit != NO_LIMIT)
+ {
+ sb.append("LIMIT ").append(rowLimit);
+ if (perPartitionLimit != NO_LIMIT)
+ sb.append(' ');
+ }
+
+ if (perPartitionLimit != NO_LIMIT)
+ sb.append("PER PARTITION LIMIT ").append(perPartitionLimit);
+
+ return sb.toString();
+ }
+ }
+
+ private static class CQLPagingLimits extends CQLLimits
+ {
+ private final ByteBuffer lastReturnedKey;
+ private final int lastReturnedKeyRemaining;
+
+ public CQLPagingLimits(int rowLimit, int perPartitionLimit, boolean isDistinct, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+ {
+ super(rowLimit, perPartitionLimit, isDistinct);
+ this.lastReturnedKey = lastReturnedKey;
+ this.lastReturnedKeyRemaining = lastReturnedKeyRemaining;
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.CQL_PAGING_LIMIT;
+ }
+
+ @Override
+ public DataLimits forPaging(int pageSize)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
- public Counter newCounter(int nowInSec, boolean assumeLiveData)
++ public Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
+ {
- return new PagingAwareCounter(nowInSec, assumeLiveData);
++ return new PagingAwareCounter(nowInSec, assumeLiveData, countPartitionsWithOnlyStaticData);
+ }
+
+ private class PagingAwareCounter extends CQLCounter
+ {
- private PagingAwareCounter(int nowInSec, boolean assumeLiveData)
++ private PagingAwareCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
+ {
- super(nowInSec, assumeLiveData);
++ super(nowInSec, assumeLiveData, countPartitionsWithOnlyStaticData);
+ }
+
+ @Override
+ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
+ {
+ if (partitionKey.getKey().equals(lastReturnedKey))
+ {
+ rowInCurrentPartition = perPartitionLimit - lastReturnedKeyRemaining;
+ // lastReturnedKey is the last key for which we're returned rows in the first page.
+ // So, since we know we have returned rows, we know we have accounted for the static row
+ // if any already, so force hasLiveStaticRow to false so we make sure to not count it
+ // once more.
+ hasLiveStaticRow = false;
+ }
+ else
+ {
+ super.applyToPartition(partitionKey, staticRow);
+ }
+ }
+ }
+ }
+
+ /**
+ * Limits used by thrift; this count partition and cells.
+ */
+ private static class ThriftLimits extends DataLimits
+ {
+ protected final int partitionLimit;
+ protected final int cellPerPartitionLimit;
+
+ private ThriftLimits(int partitionLimit, int cellPerPartitionLimit)
+ {
+ this.partitionLimit = partitionLimit;
+ this.cellPerPartitionLimit = cellPerPartitionLimit;
+ }
+
+ public Kind kind()
+ {
+ return Kind.THRIFT_LIMIT;
+ }
+
+ public boolean isUnlimited()
+ {
+ return partitionLimit == NO_LIMIT && cellPerPartitionLimit == NO_LIMIT;
+ }
+
+ public boolean isDistinct()
+ {
+ return false;
+ }
+
+ public DataLimits forPaging(int pageSize)
+ {
+ // We don't support paging on thrift in general but do use paging under the hood for get_count. For
+ // that case, we only care about limiting cellPerPartitionLimit (since it's paging over a single
+ // partition). We do check that the partition limit is 1 however to make sure this is not misused
+ // (as this wouldn't work properly for range queries).
+ assert partitionLimit == 1;
+ return new ThriftLimits(partitionLimit, pageSize);
+ }
+
+ public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public DataLimits forShortReadRetry(int toFetch)
+ {
+ // Short read retries are always done for a single partition at a time, so it's ok to ignore the
+ // partition limit for those
+ return new ThriftLimits(1, toFetch);
+ }
+
- public boolean hasEnoughLiveData(CachedPartition cached, int nowInSec)
++ public boolean hasEnoughLiveData(CachedPartition cached, int nowInSec, boolean countPartitionsWithOnlyStaticData)
+ {
+ // We want the number of cells that are currently live. Getting that precise number forces
+ // us to iterate the cached partition in general, but we can avoid that if:
+ // - The number of non-expiring live cells is greater than the number of cells asked (we then
+ // know we have enough live cells).
+ // - The number of cells cached is less than requested, in which case we know we won't have enough.
+ if (cached.nonExpiringLiveCells() >= cellPerPartitionLimit)
+ return true;
+
+ if (cached.nonTombstoneCellCount() < cellPerPartitionLimit)
+ return false;
+
+ // Otherwise, we need to re-count
- DataLimits.Counter counter = newCounter(nowInSec, false);
++ DataLimits.Counter counter = newCounter(nowInSec, false, countPartitionsWithOnlyStaticData);
+ try (UnfilteredRowIterator cacheIter = cached.unfilteredIterator(ColumnFilter.selection(cached.columns()), Slices.ALL, false);
+ UnfilteredRowIterator iter = counter.applyTo(cacheIter))
+ {
+ // Consume the iterator until we've counted enough
+ while (iter.hasNext())
+ iter.next();
+ return counter.isDone();
+ }
+ }
+
- public Counter newCounter(int nowInSec, boolean assumeLiveData)
++ public Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
+ {
+ return new ThriftCounter(nowInSec, assumeLiveData);
+ }
+
+ public int count()
+ {
+ return partitionLimit * cellPerPartitionLimit;
+ }
+
+ public int perPartitionCount()
+ {
+ return cellPerPartitionLimit;
+ }
+
+ public float estimateTotalResults(ColumnFamilyStore cfs)
+ {
+ // remember that getMeansColumns returns a number of cells: we should clean nomenclature
+ float cellsPerPartition = ((float) cfs.getMeanColumns()) / cfs.metadata.partitionColumns().regulars.size();
+ return cellsPerPartition * cfs.estimateKeys();
+ }
+
+ protected class ThriftCounter extends Counter
+ {
+ protected final int nowInSec;
+ protected final boolean assumeLiveData;
+
+ protected int partitionsCounted;
+ protected int cellsCounted;
+ protected int cellsInCurrentPartition;
+
+ public ThriftCounter(int nowInSec, boolean assumeLiveData)
+ {
+ this.nowInSec = nowInSec;
+ this.assumeLiveData = assumeLiveData;
+ }
+
+ @Override
+ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
+ {
+ cellsInCurrentPartition = 0;
+ if (!staticRow.isEmpty())
+ applyToRow(staticRow);
+ }
+
+ @Override
+ public Row applyToRow(Row row)
+ {
+ for (Cell cell : row.cells())
+ {
+ if (assumeLiveData || cell.isLive(nowInSec))
+ {
+ ++cellsCounted;
+ if (++cellsInCurrentPartition >= cellPerPartitionLimit)
+ stopInPartition();
+ }
+ }
+ return row;
+ }
+
+ @Override
+ public void onPartitionClose()
+ {
+ if (++partitionsCounted >= partitionLimit)
+ stop();
+ super.onPartitionClose();
+ }
+
+ public int counted()
+ {
+ return cellsCounted;
+ }
+
+ public int countedInCurrentPartition()
+ {
+ return cellsInCurrentPartition;
+ }
+
+ public boolean isDone()
+ {
+ return partitionsCounted >= partitionLimit;
+ }
+
+ public boolean isDoneForPartition()
+ {
+ return isDone() || cellsInCurrentPartition >= cellPerPartitionLimit;
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ // This is not valid CQL, but that's ok since it's not used for CQL queries.
+ return String.format("THRIFT LIMIT (partitions=%d, cells_per_partition=%d)", partitionLimit, cellPerPartitionLimit);
+ }
+ }
+
+ /**
+ * Limits used for thrift get_count when we only want to count super columns.
+ */
+ private static class SuperColumnCountingLimits extends ThriftLimits
+ {
+ private SuperColumnCountingLimits(int partitionLimit, int cellPerPartitionLimit)
+ {
+ super(partitionLimit, cellPerPartitionLimit);
+ }
+
+ public Kind kind()
+ {
+ return Kind.SUPER_COLUMN_COUNTING_LIMIT;
+ }
+
+ public DataLimits forPaging(int pageSize)
+ {
+ // We don't support paging on thrift in general but do use paging under the hood for get_count. For
+ // that case, we only care about limiting cellPerPartitionLimit (since it's paging over a single
+ // partition). We do check that the partition limit is 1 however to make sure this is not misused
+ // (as this wouldn't work properly for range queries).
+ assert partitionLimit == 1;
+ return new SuperColumnCountingLimits(partitionLimit, pageSize);
+ }
+
+ public DataLimits forShortReadRetry(int toFetch)
+ {
+ // Short read retries are always done for a single partition at a time, so it's ok to ignore the
+ // partition limit for those
+ return new SuperColumnCountingLimits(1, toFetch);
+ }
+
- public Counter newCounter(int nowInSec, boolean assumeLiveData)
++ @Override
++ public Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
+ {
+ return new SuperColumnCountingCounter(nowInSec, assumeLiveData);
+ }
+
+ protected class SuperColumnCountingCounter extends ThriftCounter
+ {
+ public SuperColumnCountingCounter(int nowInSec, boolean assumeLiveData)
+ {
+ super(nowInSec, assumeLiveData);
+ }
+
+ @Override
+ public Row applyToRow(Row row)
+ {
+ // In the internal format, a row == a super column, so that's what we want to count.
+ if (assumeLiveData || row.hasLiveData(nowInSec))
+ {
+ ++cellsCounted;
+ if (++cellsInCurrentPartition >= cellPerPartitionLimit)
+ stopInPartition();
+ }
+ return row;
+ }
+ }
+ }
+
+ public static class Serializer
+ {
+ public void serialize(DataLimits limits, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeByte(limits.kind().ordinal());
+ switch (limits.kind())
+ {
+ case CQL_LIMIT:
+ case CQL_PAGING_LIMIT:
+ CQLLimits cqlLimits = (CQLLimits)limits;
+ out.writeUnsignedVInt(cqlLimits.rowLimit);
+ out.writeUnsignedVInt(cqlLimits.perPartitionLimit);
+ out.writeBoolean(cqlLimits.isDistinct);
+ if (limits.kind() == Kind.CQL_PAGING_LIMIT)
+ {
+ CQLPagingLimits pagingLimits = (CQLPagingLimits)cqlLimits;
+ ByteBufferUtil.writeWithVIntLength(pagingLimits.lastReturnedKey, out);
+ out.writeUnsignedVInt(pagingLimits.lastReturnedKeyRemaining);
+ }
+ break;
+ case THRIFT_LIMIT:
+ case SUPER_COLUMN_COUNTING_LIMIT:
+ ThriftLimits thriftLimits = (ThriftLimits)limits;
+ out.writeUnsignedVInt(thriftLimits.partitionLimit);
+ out.writeUnsignedVInt(thriftLimits.cellPerPartitionLimit);
+ break;
+ }
+ }
+
+ public DataLimits deserialize(DataInputPlus in, int version) throws IOException
+ {
+ Kind kind = Kind.values()[in.readUnsignedByte()];
+ switch (kind)
+ {
+ case CQL_LIMIT:
+ case CQL_PAGING_LIMIT:
+ int rowLimit = (int)in.readUnsignedVInt();
+ int perPartitionLimit = (int)in.readUnsignedVInt();
+ boolean isDistinct = in.readBoolean();
+ if (kind == Kind.CQL_LIMIT)
+ return new CQLLimits(rowLimit, perPartitionLimit, isDistinct);
+
+ ByteBuffer lastKey = ByteBufferUtil.readWithVIntLength(in);
+ int lastRemaining = (int)in.readUnsignedVInt();
+ return new CQLPagingLimits(rowLimit, perPartitionLimit, isDistinct, lastKey, lastRemaining);
+ case THRIFT_LIMIT:
+ case SUPER_COLUMN_COUNTING_LIMIT:
+ int partitionLimit = (int)in.readUnsignedVInt();
+ int cellPerPartitionLimit = (int)in.readUnsignedVInt();
+ return kind == Kind.THRIFT_LIMIT
+ ? new ThriftLimits(partitionLimit, cellPerPartitionLimit)
+ : new SuperColumnCountingLimits(partitionLimit, cellPerPartitionLimit);
+ }
+ throw new AssertionError();
+ }
+
+ public long serializedSize(DataLimits limits, int version)
+ {
+ long size = TypeSizes.sizeof((byte)limits.kind().ordinal());
+ switch (limits.kind())
+ {
+ case CQL_LIMIT:
+ case CQL_PAGING_LIMIT:
+ CQLLimits cqlLimits = (CQLLimits)limits;
+ size += TypeSizes.sizeofUnsignedVInt(cqlLimits.rowLimit);
+ size += TypeSizes.sizeofUnsignedVInt(cqlLimits.perPartitionLimit);
+ size += TypeSizes.sizeof(cqlLimits.isDistinct);
+ if (limits.kind() == Kind.CQL_PAGING_LIMIT)
+ {
+ CQLPagingLimits pagingLimits = (CQLPagingLimits)cqlLimits;
+ size += ByteBufferUtil.serializedSizeWithVIntLength(pagingLimits.lastReturnedKey);
+ size += TypeSizes.sizeofUnsignedVInt(pagingLimits.lastReturnedKeyRemaining);
+ }
+ break;
+ case THRIFT_LIMIT:
+ case SUPER_COLUMN_COUNTING_LIMIT:
+ ThriftLimits thriftLimits = (ThriftLimits)limits;
+ size += TypeSizes.sizeofUnsignedVInt(thriftLimits.partitionLimit);
+ size += TypeSizes.sizeofUnsignedVInt(thriftLimits.cellPerPartitionLimit);
+ break;
+ default:
+ throw new AssertionError();
+ }
+ return size;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/db/filter/RowFilter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/filter/RowFilter.java
index 5ffe2ab,0000000..8d11038
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/filter/RowFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/RowFilter.java
@@@ -1,994 -1,0 +1,1009 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.filter;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+import com.google.common.base.Objects;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.context.*;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.cql3.statements.RequestValidations.checkBindValueSet;
+import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse;
+import static org.apache.cassandra.cql3.statements.RequestValidations.checkNotNull;
+
+/**
+ * A filter on which rows a given query should include or exclude.
+ * <p>
+ * This corresponds to the restrictions on rows that are not handled by the query
+ * {@link ClusteringIndexFilter}. Some of the expressions of this filter may
+ * be handled by a 2ndary index, and the rest is simply filtered out from the
+ * result set (the later can only happen if the query was using ALLOW FILTERING).
+ */
+public abstract class RowFilter implements Iterable<RowFilter.Expression>
+{
+ public static final Serializer serializer = new Serializer();
+ public static final RowFilter NONE = new CQLFilter(Collections.emptyList());
+
+ protected final List<Expression> expressions;
+
+ protected RowFilter(List<Expression> expressions)
+ {
+ this.expressions = expressions;
+ }
+
+ public static RowFilter create()
+ {
+ return new CQLFilter(new ArrayList<>());
+ }
+
+ public static RowFilter create(int capacity)
+ {
+ return new CQLFilter(new ArrayList<>(capacity));
+ }
+
+ public static RowFilter forThrift(int capacity)
+ {
+ return new ThriftFilter(new ArrayList<>(capacity));
+ }
+
+ public void add(ColumnDefinition def, Operator op, ByteBuffer value)
+ {
+ add(new SimpleExpression(def, op, value));
+ }
+
+ public void addMapEquality(ColumnDefinition def, ByteBuffer key, Operator op, ByteBuffer value)
+ {
+ add(new MapEqualityExpression(def, key, op, value));
+ }
+
+ public void addThriftExpression(CFMetaData metadata, ByteBuffer name, Operator op, ByteBuffer value)
+ {
+ assert (this instanceof ThriftFilter);
+ add(new ThriftExpression(metadata, name, op, value));
+ }
+
+ public void addCustomIndexExpression(CFMetaData cfm, IndexMetadata targetIndex, ByteBuffer value)
+ {
+ add(new CustomExpression(cfm, targetIndex, value));
+ }
+
+ private void add(Expression expression)
+ {
+ expression.validate();
+ expressions.add(expression);
+ }
+
+ public List<Expression> getExpressions()
+ {
+ return expressions;
+ }
+
+ /**
++ * Checks if some of the expressions apply to clustering or regular columns.
++ * @return {@code true} if some of the expressions apply to clustering or regular columns, {@code false} otherwise.
++ */
++ public boolean hasExpressionOnClusteringOrRegularColumns()
++ {
++ for (Expression expression : expressions)
++ {
++ ColumnDefinition column = expression.column();
++ if (column.isClusteringColumn() || column.isRegular())
++ return true;
++ }
++ return false;
++ }
++
++ /**
+ * Filters the provided iterator so that only the row satisfying the expression of this filter
+ * are included in the resulting iterator.
+ *
+ * @param iter the iterator to filter
+ * @param nowInSec the time of query in seconds.
+ * @return the filtered iterator.
+ */
+ public abstract UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, int nowInSec);
+
+ /**
+ * Whether the provided row in the provided partition satisfies this filter.
+ *
+ * @param metadata the table metadata.
+ * @param partitionKey the partition key for partition to test.
+ * @param row the row to test.
+ * @param nowInSec the current time in seconds (to know what is live and what isn't).
+ * @return {@code true} if {@code row} in partition {@code partitionKey} satisfies this row filter.
+ */
+ public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row, int nowInSec)
+ {
+ // We purge all tombstones as the expressions isSatisfiedBy methods expects it
+ Row purged = row.purge(DeletionPurger.PURGE_ALL, nowInSec);
+ if (purged == null)
+ return expressions.isEmpty();
+
+ for (Expression e : expressions)
+ {
+ if (!e.isSatisfiedBy(metadata, partitionKey, purged))
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Returns true if all of the expressions within this filter that apply to the partition key are satisfied by
+ * the given key, false otherwise.
+ */
+ public boolean partitionKeyRestrictionsAreSatisfiedBy(DecoratedKey key, AbstractType<?> keyValidator)
+ {
+ for (Expression e : expressions)
+ {
+ if (!e.column.isPartitionKey())
+ continue;
+
+ ByteBuffer value = keyValidator instanceof CompositeType
+ ? ((CompositeType) keyValidator).split(key.getKey())[e.column.position()]
+ : key.getKey();
+ if (!e.operator().isSatisfiedBy(e.column.type, value, e.value))
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Returns true if all of the expressions within this filter that apply to the clustering key are satisfied by
+ * the given Clustering, false otherwise.
+ */
+ public boolean clusteringKeyRestrictionsAreSatisfiedBy(Clustering clustering)
+ {
+ for (Expression e : expressions)
+ {
+ if (!e.column.isClusteringColumn())
+ continue;
+
+ if (!e.operator().isSatisfiedBy(e.column.type, clustering.get(e.column.position()), e.value))
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Returns this filter but without the provided expression. This method
+ * *assumes* that the filter contains the provided expression.
+ */
+ public RowFilter without(Expression expression)
+ {
+ assert expressions.contains(expression);
+ if (expressions.size() == 1)
+ return RowFilter.NONE;
+
+ List<Expression> newExpressions = new ArrayList<>(expressions.size() - 1);
+ for (Expression e : expressions)
+ if (!e.equals(expression))
+ newExpressions.add(e);
+
+ return withNewExpressions(newExpressions);
+ }
+
+ protected abstract RowFilter withNewExpressions(List<Expression> expressions);
+
+ public boolean isEmpty()
+ {
+ return expressions.isEmpty();
+ }
+
+ public Iterator<Expression> iterator()
+ {
+ return expressions.iterator();
+ }
+
+ private static Clustering makeCompactClustering(CFMetaData metadata, ByteBuffer name)
+ {
+ assert metadata.isCompactTable();
+ if (metadata.isCompound())
+ {
+ List<ByteBuffer> values = CompositeType.splitName(name);
+ return new Clustering(values.toArray(new ByteBuffer[metadata.comparator.size()]));
+ }
+ else
+ {
+ return new Clustering(name);
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < expressions.size(); i++)
+ {
+ if (i > 0)
+ sb.append(" AND ");
+ sb.append(expressions.get(i));
+ }
+ return sb.toString();
+ }
+
+ private static class CQLFilter extends RowFilter
+ {
+ private CQLFilter(List<Expression> expressions)
+ {
+ super(expressions);
+ }
+
+ public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, int nowInSec)
+ {
+ if (expressions.isEmpty())
+ return iter;
+
+ final CFMetaData metadata = iter.metadata();
+ long numberOfStaticColumnExpressions = expressions.stream().filter(e -> e.column.isStatic()).count();
+ final boolean filterStaticColumns = numberOfStaticColumnExpressions != 0;
+ final boolean filterNonStaticColumns = (expressions.size() - numberOfStaticColumnExpressions) > 0;
+
+ class IsSatisfiedFilter extends Transformation<UnfilteredRowIterator>
+ {
+ DecoratedKey pk;
+ public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
+ {
+ // The filter might be on static columns, so need to check static row first.
+ if (filterStaticColumns && applyToRow(partition.staticRow()) == null)
+ return null;
+
+ pk = partition.partitionKey();
+ UnfilteredRowIterator iterator = Transformation.apply(partition, this);
+
+ return (filterNonStaticColumns && !iterator.hasNext()) ? null : iterator;
+ }
+
+ public Row applyToRow(Row row)
+ {
+ Row purged = row.purge(DeletionPurger.PURGE_ALL, nowInSec);
+ if (purged == null)
+ return null;
+
+ for (Expression e : expressions)
+ if (!e.isSatisfiedBy(metadata, pk, purged))
+ return null;
+ return row;
+ }
+ }
+
+ return Transformation.apply(iter, new IsSatisfiedFilter());
+ }
+
+ protected RowFilter withNewExpressions(List<Expression> expressions)
+ {
+ return new CQLFilter(expressions);
+ }
+ }
+
+ private static class ThriftFilter extends RowFilter
+ {
+ private ThriftFilter(List<Expression> expressions)
+ {
+ super(expressions);
+ }
+
+ public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, final int nowInSec)
+ {
+ if (expressions.isEmpty())
+ return iter;
+
+ class IsSatisfiedThriftFilter extends Transformation<UnfilteredRowIterator>
+ {
+ @Override
+ public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator iter)
+ {
+ // Thrift does not filter rows, it filters entire partition if any of the expression is not
+ // satisfied, which forces us to materialize the result (in theory we could materialize only
+ // what we need which might or might not be everything, but we keep it simple since in practice
+ // it's not worth that it has ever been).
+ ImmutableBTreePartition result = ImmutableBTreePartition.create(iter);
+ iter.close();
+
+ // The partition needs to have a row for every expression, and the expression needs to be valid.
+ for (Expression expr : expressions)
+ {
+ assert expr instanceof ThriftExpression;
+ Row row = result.getRow(makeCompactClustering(iter.metadata(), expr.column().name.bytes));
+ if (row == null || !expr.isSatisfiedBy(iter.metadata(), iter.partitionKey(), row))
+ return null;
+ }
+ // If we get there, it means all expressions where satisfied, so return the original result
+ return result.unfilteredIterator();
+ }
+ }
+ return Transformation.apply(iter, new IsSatisfiedThriftFilter());
+ }
+
+ protected RowFilter withNewExpressions(List<Expression> expressions)
+ {
+ return new ThriftFilter(expressions);
+ }
+ }
+
+ public static abstract class Expression
+ {
+ private static final Serializer serializer = new Serializer();
+
+ // Note: the order of this enum matter, it's used for serialization
+ protected enum Kind { SIMPLE, MAP_EQUALITY, THRIFT_DYN_EXPR, CUSTOM }
+
+ abstract Kind kind();
+ protected final ColumnDefinition column;
+ protected final Operator operator;
+ protected final ByteBuffer value;
+
+ protected Expression(ColumnDefinition column, Operator operator, ByteBuffer value)
+ {
+ this.column = column;
+ this.operator = operator;
+ this.value = value;
+ }
+
+ public boolean isCustom()
+ {
+ return kind() == Kind.CUSTOM;
+ }
+
+ public ColumnDefinition column()
+ {
+ return column;
+ }
+
+ public Operator operator()
+ {
+ return operator;
+ }
+
+ /**
+ * Checks if the operator of this <code>IndexExpression</code> is a <code>CONTAINS</code> operator.
+ *
+ * @return <code>true</code> if the operator of this <code>IndexExpression</code> is a <code>CONTAINS</code>
+ * operator, <code>false</code> otherwise.
+ */
+ public boolean isContains()
+ {
+ return Operator.CONTAINS == operator;
+ }
+
+ /**
+ * Checks if the operator of this <code>IndexExpression</code> is a <code>CONTAINS_KEY</code> operator.
+ *
+ * @return <code>true</code> if the operator of this <code>IndexExpression</code> is a <code>CONTAINS_KEY</code>
+ * operator, <code>false</code> otherwise.
+ */
+ public boolean isContainsKey()
+ {
+ return Operator.CONTAINS_KEY == operator;
+ }
+
+ /**
+ * If this expression is used to query an index, the value to use as
+ * partition key for that index query.
+ */
+ public ByteBuffer getIndexValue()
+ {
+ return value;
+ }
+
+ public void validate()
+ {
+ checkNotNull(value, "Unsupported null value for column %s", column.name);
+ checkBindValueSet(value, "Unsupported unset value for column %s", column.name);
+ }
+
+ @Deprecated
+ public void validateForIndexing()
+ {
+ checkFalse(value.remaining() > FBUtilities.MAX_UNSIGNED_SHORT,
+ "Index expression values may not be larger than 64K");
+ }
+
+ /**
+ * Returns whether the provided row satisfied this expression or not.
+ *
+ * @param partitionKey the partition key for row to check.
+ * @param row the row to check. It should *not* contain deleted cells
+ * (i.e. it should come from a RowIterator).
+ * @return whether the row is satisfied by this expression.
+ */
+ public abstract boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row);
+
+ protected ByteBuffer getValue(CFMetaData metadata, DecoratedKey partitionKey, Row row)
+ {
+ switch (column.kind)
+ {
+ case PARTITION_KEY:
+ return metadata.getKeyValidator() instanceof CompositeType
+ ? CompositeType.extractComponent(partitionKey.getKey(), column.position())
+ : partitionKey.getKey();
+ case CLUSTERING:
+ return row.clustering().get(column.position());
+ default:
+ Cell cell = row.getCell(column);
+ return cell == null ? null : cell.value();
+ }
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof Expression))
+ return false;
+
+ Expression that = (Expression)o;
+
+ return Objects.equal(this.kind(), that.kind())
+ && Objects.equal(this.column.name, that.column.name)
+ && Objects.equal(this.operator, that.operator)
+ && Objects.equal(this.value, that.value);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(column.name, operator, value);
+ }
+
+ private static class Serializer
+ {
+ public void serialize(Expression expression, DataOutputPlus out, int version) throws IOException
+ {
+ if (version >= MessagingService.VERSION_30)
+ out.writeByte(expression.kind().ordinal());
+
+ // Custom expressions include neither a column or operator, but all
+ // other expressions do. Also, custom expressions are 3.0+ only, so
+ // the column & operator will always be the first things written for
+ // any pre-3.0 version
+ if (expression.kind() == Kind.CUSTOM)
+ {
+ assert version >= MessagingService.VERSION_30;
+ IndexMetadata.serializer.serialize(((CustomExpression)expression).targetIndex, out, version);
+ ByteBufferUtil.writeWithShortLength(expression.value, out);
+ return;
+ }
+
+ ByteBufferUtil.writeWithShortLength(expression.column.name.bytes, out);
+ expression.operator.writeTo(out);
+
+ switch (expression.kind())
+ {
+ case SIMPLE:
+ ByteBufferUtil.writeWithShortLength(((SimpleExpression)expression).value, out);
+ break;
+ case MAP_EQUALITY:
+ MapEqualityExpression mexpr = (MapEqualityExpression)expression;
+ if (version < MessagingService.VERSION_30)
+ {
+ ByteBufferUtil.writeWithShortLength(mexpr.getIndexValue(), out);
+ }
+ else
+ {
+ ByteBufferUtil.writeWithShortLength(mexpr.key, out);
+ ByteBufferUtil.writeWithShortLength(mexpr.value, out);
+ }
+ break;
+ case THRIFT_DYN_EXPR:
+ ByteBufferUtil.writeWithShortLength(((ThriftExpression)expression).value, out);
+ break;
+ }
+ }
+
+ public Expression deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
+ {
+ Kind kind = null;
+ ByteBuffer name;
+ Operator operator;
+ ColumnDefinition column;
+
+ if (version >= MessagingService.VERSION_30)
+ {
+ kind = Kind.values()[in.readByte()];
+ // custom expressions (3.0+ only) do not contain a column or operator, only a value
+ if (kind == Kind.CUSTOM)
+ {
+ return new CustomExpression(metadata,
+ IndexMetadata.serializer.deserialize(in, version, metadata),
+ ByteBufferUtil.readWithShortLength(in));
+ }
+ }
+
+ name = ByteBufferUtil.readWithShortLength(in);
+ operator = Operator.readFrom(in);
+ column = metadata.getColumnDefinition(name);
+ if (!metadata.isCompactTable() && column == null)
+ throw new RuntimeException("Unknown (or dropped) column " + UTF8Type.instance.getString(name) + " during deserialization");
+
+ if (version < MessagingService.VERSION_30)
+ {
+ if (column == null)
+ kind = Kind.THRIFT_DYN_EXPR;
+ else if (column.type instanceof MapType && operator == Operator.EQ)
+ kind = Kind.MAP_EQUALITY;
+ else
+ kind = Kind.SIMPLE;
+ }
+
+ assert kind != null;
+ switch (kind)
+ {
+ case SIMPLE:
+ return new SimpleExpression(column, operator, ByteBufferUtil.readWithShortLength(in));
+ case MAP_EQUALITY:
+ ByteBuffer key, value;
+ if (version < MessagingService.VERSION_30)
+ {
+ ByteBuffer composite = ByteBufferUtil.readWithShortLength(in);
+ key = CompositeType.extractComponent(composite, 0);
+ value = CompositeType.extractComponent(composite, 0);
+ }
+ else
+ {
+ key = ByteBufferUtil.readWithShortLength(in);
+ value = ByteBufferUtil.readWithShortLength(in);
+ }
+ return new MapEqualityExpression(column, key, operator, value);
+ case THRIFT_DYN_EXPR:
+ return new ThriftExpression(metadata, name, operator, ByteBufferUtil.readWithShortLength(in));
+ }
+ throw new AssertionError();
+ }
+
+
+ public long serializedSize(Expression expression, int version)
+ {
+ // version 3.0+ includes a byte for Kind
+ long size = version >= MessagingService.VERSION_30 ? 1 : 0;
+
+ // custom expressions don't include a column or operator, all other expressions do
+ if (expression.kind() != Kind.CUSTOM)
+ size += ByteBufferUtil.serializedSizeWithShortLength(expression.column().name.bytes)
+ + expression.operator.serializedSize();
+
+ switch (expression.kind())
+ {
+ case SIMPLE:
+ size += ByteBufferUtil.serializedSizeWithShortLength(((SimpleExpression)expression).value);
+ break;
+ case MAP_EQUALITY:
+ MapEqualityExpression mexpr = (MapEqualityExpression)expression;
+ if (version < MessagingService.VERSION_30)
+ size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.getIndexValue());
+ else
+ size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.key)
+ + ByteBufferUtil.serializedSizeWithShortLength(mexpr.value);
+ break;
+ case THRIFT_DYN_EXPR:
+ size += ByteBufferUtil.serializedSizeWithShortLength(((ThriftExpression)expression).value);
+ break;
+ case CUSTOM:
+ if (version >= MessagingService.VERSION_30)
+ size += IndexMetadata.serializer.serializedSize(((CustomExpression)expression).targetIndex, version)
+ + ByteBufferUtil.serializedSizeWithShortLength(expression.value);
+ break;
+ }
+ return size;
+ }
+ }
+ }
+
+ /**
+ * An expression of the form 'column' 'op' 'value'.
+ */
+ private static class SimpleExpression extends Expression
+ {
+ public SimpleExpression(ColumnDefinition column, Operator operator, ByteBuffer value)
+ {
+ super(column, operator, value);
+ }
+
+ public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row)
+ {
+ // We support null conditions for LWT (in ColumnCondition) but not for RowFilter.
+ // TODO: we should try to merge both code someday.
+ assert value != null;
+
+ if (row.isStatic() != column.isStatic())
+ return true;
+
+ switch (operator)
+ {
+ case EQ:
+ case LT:
+ case LTE:
+ case GTE:
+ case GT:
+ {
+ assert !column.isComplex() : "Only CONTAINS and CONTAINS_KEY are supported for 'complex' types";
+
+ // In order to support operators on Counter types, their value has to be extracted from internal
+ // representation. See CASSANDRA-11629
+ if (column.type.isCounter())
+ {
+ ByteBuffer foundValue = getValue(metadata, partitionKey, row);
+ if (foundValue == null)
+ return false;
+
+ ByteBuffer counterValue = LongType.instance.decompose(CounterContext.instance().total(foundValue));
+ return operator.isSatisfiedBy(LongType.instance, counterValue, value);
+ }
+ else
+ {
+ // Note that CQL expression are always of the form 'x < 4', i.e. the tested value is on the left.
+ ByteBuffer foundValue = getValue(metadata, partitionKey, row);
+ return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value);
+ }
+ }
+ case NEQ:
+ {
+ assert !column.isComplex() : "Only CONTAINS and CONTAINS_KEY are supported for 'complex' types";
+ ByteBuffer foundValue = getValue(metadata, partitionKey, row);
+ // Note that CQL expression are always of the form 'x < 4', i.e. the tested value is on the left.
+ return foundValue != null && operator.isSatisfiedBy(column.type, foundValue, value);
+ }
+ case CONTAINS:
+ assert column.type.isCollection();
+ CollectionType<?> type = (CollectionType<?>)column.type;
+ if (column.isComplex())
+ {
+ ComplexColumnData complexData = row.getComplexColumnData(column);
+ if (complexData != null)
+ {
+ for (Cell cell : complexData)
+ {
+ if (type.kind == CollectionType.Kind.SET)
+ {
+ if (type.nameComparator().compare(cell.path().get(0), value) == 0)
+ return true;
+ }
+ else
+ {
+ if (type.valueComparator().compare(cell.value(), value) == 0)
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+ else
+ {
+ ByteBuffer foundValue = getValue(metadata, partitionKey, row);
+ if (foundValue == null)
+ return false;
+
+ switch (type.kind)
+ {
+ case LIST:
+ ListType<?> listType = (ListType<?>)type;
+ return listType.compose(foundValue).contains(listType.getElementsType().compose(value));
+ case SET:
+ SetType<?> setType = (SetType<?>)type;
+ return setType.compose(foundValue).contains(setType.getElementsType().compose(value));
+ case MAP:
+ MapType<?,?> mapType = (MapType<?, ?>)type;
+ return mapType.compose(foundValue).containsValue(mapType.getValuesType().compose(value));
+ }
+ throw new AssertionError();
+ }
+ case CONTAINS_KEY:
+ assert column.type.isCollection() && column.type instanceof MapType;
+ MapType<?, ?> mapType = (MapType<?, ?>)column.type;
+ if (column.isComplex())
+ {
+ return row.getCell(column, CellPath.create(value)) != null;
+ }
+ else
+ {
+ ByteBuffer foundValue = getValue(metadata, partitionKey, row);
+ return foundValue != null && mapType.getSerializer().getSerializedValue(foundValue, value, mapType.getKeysType()) != null;
+ }
+
+ case IN:
+ // It wouldn't be terribly hard to support this (though doing so would imply supporting
+ // IN for 2ndary index) but currently we don't.
+ throw new AssertionError();
+ }
+ throw new AssertionError();
+ }
+
+ @Override
+ public String toString()
+ {
+ AbstractType<?> type = column.type;
+ switch (operator)
+ {
+ case CONTAINS:
+ assert type instanceof CollectionType;
+ CollectionType<?> ct = (CollectionType<?>)type;
+ type = ct.kind == CollectionType.Kind.SET ? ct.nameComparator() : ct.valueComparator();
+ break;
+ case CONTAINS_KEY:
+ assert type instanceof MapType;
+ type = ((MapType<?, ?>)type).nameComparator();
+ break;
+ case IN:
+ type = ListType.getInstance(type, false);
+ break;
+ default:
+ break;
+ }
+ return String.format("%s %s %s", column.name, operator, type.getString(value));
+ }
+
+ @Override
+ Kind kind()
+ {
+ return Kind.SIMPLE;
+ }
+ }
+
+ /**
+ * An expression of the form 'column' ['key'] = 'value' (which is only
+ * supported when 'column' is a map).
+ */
+ private static class MapEqualityExpression extends Expression
+ {
+ private final ByteBuffer key;
+
+ public MapEqualityExpression(ColumnDefinition column, ByteBuffer key, Operator operator, ByteBuffer value)
+ {
+ super(column, operator, value);
+ assert column.type instanceof MapType && operator == Operator.EQ;
+ this.key = key;
+ }
+
+ @Override
+ public void validate() throws InvalidRequestException
+ {
+ checkNotNull(key, "Unsupported null map key for column %s", column.name);
+ checkBindValueSet(key, "Unsupported unset map key for column %s", column.name);
+ checkNotNull(value, "Unsupported null map value for column %s", column.name);
+ checkBindValueSet(value, "Unsupported unset map value for column %s", column.name);
+ }
+
+ @Override
+ public ByteBuffer getIndexValue()
+ {
+ return CompositeType.build(key, value);
+ }
+
+ public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row)
+ {
+ assert key != null;
+ // We support null conditions for LWT (in ColumnCondition) but not for RowFilter.
+ // TODO: we should try to merge both code someday.
+ assert value != null;
+
+ if (row.isStatic() != column.isStatic())
+ return true;
+
+ MapType<?, ?> mt = (MapType<?, ?>)column.type;
+ if (column.isComplex())
+ {
+ Cell cell = row.getCell(column, CellPath.create(key));
+ return cell != null && mt.valueComparator().compare(cell.value(), value) == 0;
+ }
+ else
+ {
+ ByteBuffer serializedMap = getValue(metadata, partitionKey, row);
+ if (serializedMap == null)
+ return false;
+
+ ByteBuffer foundValue = mt.getSerializer().getSerializedValue(serializedMap, key, mt.getKeysType());
+ return foundValue != null && mt.valueComparator().compare(foundValue, value) == 0;
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ MapType<?, ?> mt = (MapType<?, ?>)column.type;
+ return String.format("%s[%s] = %s", column.name, mt.nameComparator().getString(key), mt.valueComparator().getString(value));
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof MapEqualityExpression))
+ return false;
+
+ MapEqualityExpression that = (MapEqualityExpression)o;
+
+ return Objects.equal(this.column.name, that.column.name)
+ && Objects.equal(this.operator, that.operator)
+ && Objects.equal(this.key, that.key)
+ && Objects.equal(this.value, that.value);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hashCode(column.name, operator, key, value);
+ }
+
+ @Override
+ Kind kind()
+ {
+ return Kind.MAP_EQUALITY;
+ }
+ }
+
+ /**
+ * An expression of the form 'name' = 'value', but where 'name' is actually the
+ * clustering value for a compact table. This is only for thrift.
+ */
+ private static class ThriftExpression extends Expression
+ {
+ public ThriftExpression(CFMetaData metadata, ByteBuffer name, Operator operator, ByteBuffer value)
+ {
+ super(makeDefinition(metadata, name), operator, value);
+ assert metadata.isCompactTable();
+ }
+
+ private static ColumnDefinition makeDefinition(CFMetaData metadata, ByteBuffer name)
+ {
+ ColumnDefinition def = metadata.getColumnDefinition(name);
+ if (def != null)
+ return def;
+
+ // In thrift, we actually allow expression on non-defined columns for the sake of filtering. To accomodate
+ // this we create a "fake" definition. This is messy but it works so is probably good enough.
+ return ColumnDefinition.regularDef(metadata, name, metadata.compactValueColumn().type);
+ }
+
+ public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row)
+ {
+ assert value != null;
+
+ // On thrift queries, even if the column expression is a "static" one, we'll have convert it as a "dynamic"
+ // one in ThriftResultsMerger, so we always expect it to be a dynamic one. Further, we expect this is only
+ // called when the row clustering does match the column (see ThriftFilter above).
+ assert row.clustering().equals(makeCompactClustering(metadata, column.name.bytes));
+ Cell cell = row.getCell(metadata.compactValueColumn());
+ return cell != null && operator.isSatisfiedBy(column.type, cell.value(), value);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("%s %s %s", column.name, operator, column.type.getString(value));
+ }
+
+ @Override
+ Kind kind()
+ {
+ return Kind.THRIFT_DYN_EXPR;
+ }
+ }
+
+ /**
+ * A custom index expression for use with 2i implementations which support custom syntax and which are not
+ * necessarily linked to a single column in the base table.
+ */
+ public static final class CustomExpression extends Expression
+ {
+ private final IndexMetadata targetIndex;
+ private final CFMetaData cfm;
+
+ public CustomExpression(CFMetaData cfm, IndexMetadata targetIndex, ByteBuffer value)
+ {
+ // The operator is not relevant, but Expression requires it so for now we just hardcode EQ
+ super(makeDefinition(cfm, targetIndex), Operator.EQ, value);
+ this.targetIndex = targetIndex;
+ this.cfm = cfm;
+ }
+
+ private static ColumnDefinition makeDefinition(CFMetaData cfm, IndexMetadata index)
+ {
+ // Similarly to how we handle non-defined columns in thift, we create a fake column definition to
+ // represent the target index. This is definitely something that can be improved though.
+ return ColumnDefinition.regularDef(cfm, ByteBuffer.wrap(index.name.getBytes()), BytesType.instance);
+ }
+
+ public IndexMetadata getTargetIndex()
+ {
+ return targetIndex;
+ }
+
+ public ByteBuffer getValue()
+ {
+ return value;
+ }
+
+ public String toString()
+ {
+ return String.format("expr(%s, %s)",
+ targetIndex.name,
+ Keyspace.openAndGetStore(cfm)
+ .indexManager
+ .getIndex(targetIndex)
+ .customExpressionValueType());
+ }
+
+ Kind kind()
+ {
+ return Kind.CUSTOM;
+ }
+
+ // Filtering by custom expressions isn't supported yet, so just accept any row
+ public boolean isSatisfiedBy(CFMetaData metadata, DecoratedKey partitionKey, Row row)
+ {
+ return true;
+ }
+ }
+
+ public static class Serializer
+ {
+ public void serialize(RowFilter filter, DataOutputPlus out, int version) throws IOException
+ {
+ out.writeBoolean(filter instanceof ThriftFilter);
+ out.writeUnsignedVInt(filter.expressions.size());
+ for (Expression expr : filter.expressions)
+ Expression.serializer.serialize(expr, out, version);
+
+ }
+
+ public RowFilter deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
+ {
+ boolean forThrift = in.readBoolean();
+ int size = (int)in.readUnsignedVInt();
+ List<Expression> expressions = new ArrayList<>(size);
+ for (int i = 0; i < size; i++)
+ expressions.add(Expression.serializer.deserialize(in, version, metadata));
+
+ return forThrift
+ ? new ThriftFilter(expressions)
+ : new CQLFilter(expressions);
+ }
+
+ public long serializedSize(RowFilter filter, int version)
+ {
+ long size = 1 // forThrift
+ + TypeSizes.sizeofUnsignedVInt(filter.expressions.size());
+ for (Expression expr : filter.expressions)
+ size += Expression.serializer.serializedSize(expr, version);
+ return size;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/service/CacheService.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/CacheService.java
index c51a5d1,a13a52d..d23bdb0
--- a/src/java/org/apache/cassandra/service/CacheService.java
+++ b/src/java/org/apache/cassandra/service/CacheService.java
@@@ -439,13 -414,10 +439,13 @@@ public class CacheService implements Ca
{
public Pair<RowCacheKey, IRowCacheEntry> call() throws Exception
{
- DecoratedKey key = cfs.partitioner.decorateKey(buffer);
- QueryFilter cacheFilter = new QueryFilter(key, cfs.getColumnFamilyName(), cfs.readFilterForCache(), Integer.MIN_VALUE);
- ColumnFamily data = cfs.getTopLevelColumns(cacheFilter, Integer.MIN_VALUE);
- return Pair.create(new RowCacheKey(cfs.metadata.ksAndCFName, key), (IRowCacheEntry) data);
+ DecoratedKey key = cfs.decorateKey(buffer);
+ int nowInSec = FBUtilities.nowInSeconds();
+ try (OpOrder.Group op = cfs.readOrdering.start(); UnfilteredRowIterator iter = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata, nowInSec, key).queryMemtableAndDisk(cfs, op))
+ {
- CachedPartition toCache = CachedBTreePartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec), nowInSec);
++ CachedPartition toCache = CachedBTreePartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec, true), nowInSec);
+ return Pair.create(new RowCacheKey(cfs.metadata.ksAndCFName, key), (IRowCacheEntry)toCache);
+ }
}
});
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[2/7] cassandra git commit: Merge branch cassandra-2.2 into
cassandra-3.0
Posted by bl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
index aeb3d56,0ffb799..7e90c0a
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
@@@ -26,14 -26,14 +26,15 @@@ import org.junit.Test
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
--import org.apache.cassandra.exceptions.InvalidRequestException;
++import org.apache.cassandra.service.StorageService;
public class SelectLimitTest extends CQLTester
{
@BeforeClass
public static void setUp()
{
- DatabaseDescriptor.setPartitioner(ByteOrderedPartitioner.instance);
++ StorageService.instance.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
+ DatabaseDescriptor.setPartitionerUnsafe(ByteOrderedPartitioner.instance);
}
/**
@@@ -125,43 -125,217 +126,296 @@@
row(1, 1),
row(1, 2),
row(1, 3));
+ assertRows(execute("SELECT * FROM %s WHERE v > 1 AND v <= 3 LIMIT 6 ALLOW FILTERING"),
+ row(0, 2),
+ row(0, 3),
+ row(1, 2),
+ row(1, 3),
+ row(2, 2),
+ row(2, 3));
+ }
- // strict bound (v > 1) over a range of partitions is not supported for compact storage if limit is provided
- assertInvalidThrow(InvalidRequestException.class, "SELECT * FROM %s WHERE v > 1 AND v <= 3 LIMIT 6 ALLOW FILTERING");
+ @Test
+ public void testLimitWithDeletedRowsAndStaticColumns() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, c int, v int, s int static, PRIMARY KEY (pk, c))");
+
+ execute("INSERT INTO %s (pk, c, v, s) VALUES (1, -1, 1, 1)");
+ execute("INSERT INTO %s (pk, c, v, s) VALUES (2, -1, 1, 1)");
+ execute("INSERT INTO %s (pk, c, v, s) VALUES (3, -1, 1, 1)");
+ execute("INSERT INTO %s (pk, c, v, s) VALUES (4, -1, 1, 1)");
+ execute("INSERT INTO %s (pk, c, v, s) VALUES (5, -1, 1, 1)");
+
+ assertRows(execute("SELECT * FROM %s"),
+ row(1, -1, 1, 1),
+ row(2, -1, 1, 1),
+ row(3, -1, 1, 1),
+ row(4, -1, 1, 1),
+ row(5, -1, 1, 1));
+
+ execute("DELETE FROM %s WHERE pk = 2");
+
+ assertRows(execute("SELECT * FROM %s"),
+ row(1, -1, 1, 1),
+ row(3, -1, 1, 1),
+ row(4, -1, 1, 1),
+ row(5, -1, 1, 1));
+
+ assertRows(execute("SELECT * FROM %s LIMIT 2"),
+ row(1, -1, 1, 1),
+ row(3, -1, 1, 1));
}
+
+ @Test
+ public void testFilteringOnClusteringColumnsWithLimitAndStaticColumns() throws Throwable
+ {
+ // With only one clustering column
+ createTable("CREATE TABLE %s (a int, b int, s int static, c int, primary key (a, b))"
+ + " WITH caching = {'keys': 'ALL', 'rows_per_partition' : 'ALL'}");
+
+ for (int i = 0; i < 4; i++)
+ {
+ execute("INSERT INTO %s (a, s) VALUES (?, ?)", i, i);
+ for (int j = 0; j < 3; j++)
+ if (!((i == 0 || i == 3) && j == 1))
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", i, j, i + j);
+ }
+
- for (boolean forceFlush : new boolean[]{false, true})
++ beforeAndAfterFlush(() ->
+ {
- if (forceFlush)
- flush();
-
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, 0, 0, 0),
+ row(0, 2, 0, 2),
+ row(1, 0, 1, 1),
+ row(1, 1, 1, 2),
+ row(1, 2, 1, 3),
+ row(2, 0, 2, 2),
+ row(2, 1, 2, 3),
+ row(2, 2, 2, 4),
+ row(3, 0, 3, 3),
+ row(3, 2, 3, 5));
+
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ // The problem was that the static row of the partition 0 used to be only filtered in SelectStatement and was
+ // by consequence counted as a row. In which case the query was returning one row less.
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+ }
- }
++ });
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b = 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b = 1 ORDER BY b DESC LIMIT 2 ALLOW FILTERING"),
- row(2, 1, 2, 3),
- row(1, 1, 1, 2));
++ row(1, 1, 1, 2),
++ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b >= 1 AND b <= 1 ORDER BY b DESC LIMIT 2 ALLOW FILTERING"),
- row(2, 1, 2, 3),
- row(1, 1, 1, 2));
++ row(1, 1, 1, 2),
++ row(2, 1, 2, 3));
+
+ execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3)"); // Load all data in the row cache
+
++ // Partition range queries
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
++ // Multiple partitions queries
++ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2) AND b = 1 LIMIT 2 ALLOW FILTERING"),
++ row(1, 1, 1, 2),
++ row(2, 1, 2, 3));
++
++ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2) AND b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
++ row(1, 1, 1, 2),
++ row(2, 1, 2, 3));
++
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+ }
+
+ // With multiple clustering columns
+ createTable("CREATE TABLE %s (a int, b int, c int, s int static, d int, primary key (a, b, c))"
- + " WITH caching = {'keys': 'ALL', 'rows_per_partition' : 'ALL'}");
++ + " WITH caching = {'keys': 'ALL', 'rows_per_partition' : 'ALL'}");
+
+ for (int i = 0; i < 3; i++)
+ {
+ execute("INSERT INTO %s (a, s) VALUES (?, ?)", i, i);
- for (int j = 0; j < 3; j++)
- if (!(i == 0 && j == 1))
- execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", i, j, j, i + j);
++ for (int j = 0; j < 3; j++)
++ if (!(i == 0 && j == 1))
++ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", i, j, j, i + j);
+ }
+
- for (boolean forceFlush : new boolean[]{false, true})
++ beforeAndAfterFlush(() ->
+ {
- if (forceFlush)
- flush();
-
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, 0, 0, 0, 0),
+ row(0, 2, 2, 0, 2),
+ row(1, 0, 0, 1, 1),
+ row(1, 1, 1, 1, 2),
+ row(1, 2, 2, 1, 3),
+ row(2, 0, 0, 2, 2),
+ row(2, 1, 1, 2, 3),
+ row(2, 2, 2, 2, 4));
+
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+ }
- }
++ });
+
+ execute("SELECT * FROM %s WHERE a IN (0, 1, 2)"); // Load data in the row cache
+
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
++ assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 3, 4) AND b = 1 ALLOW FILTERING"),
++ row(1, 1, 1, 1, 2),
++ row(2, 1, 1, 2, 3));
++
++ assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 3, 4) AND b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING"),
++ row(1, 1, 1, 1, 2),
++ row(2, 1, 1, 2, 3));
++
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+ }
+ }
+
+ @Test
+ public void testIndexOnRegularColumnWithPartitionWithoutRows() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, c int, s int static, v int, PRIMARY KEY(pk, c))");
+ createIndex("CREATE INDEX ON %s (v)");
+
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 1, 9, 1);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 2, 9, 2);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 3, 1, 9, 1);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 4, 1, 9, 1);
+ flush();
+
+ assertRows(execute("SELECT * FROM %s WHERE v = ?", 1),
+ row(1, 1, 9, 1),
+ row(3, 1, 9, 1),
+ row(4, 1, 9, 1));
+
- execute("DELETE FROM %s WHERE pk = ? AND c = ?", 3, 1);
++ execute("DELETE FROM %s WHERE pk = ? and c = ?", 3, 1);
+
+ // Test without paging
- assertRows(execute("SELECT * FROM %s WHERE v = ?", 1),
++ assertRows(execute("SELECT * FROM %s WHERE v = ? LIMIT 2", 1),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+
- assertRows(execute("SELECT * FROM %s WHERE v = ? LIMIT 2", 1),
++ // Test with paging
++ for (int pageSize = 1; pageSize < 4; pageSize++)
++ {
++ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE v = 1 LIMIT 2", pageSize),
++ row(1, 1, 9, 1),
++ row(4, 1, 9, 1));
++ }
++ }
++
++ @Test
++ public void testFilteringWithPartitionWithoutRows() throws Throwable
++ {
++ createTable("CREATE TABLE %s (pk int, c int, s int static, v int, PRIMARY KEY(pk, c))");
++
++ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 1, 9, 1);
++ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 2, 9, 2);
++ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 3, 1, 9, 1);
++ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 4, 1, 9, 1);
++ flush();
++
++ assertRows(execute("SELECT * FROM %s WHERE v = ? ALLOW FILTERING", 1),
++ row(1, 1, 9, 1),
++ row(3, 1, 9, 1),
++ row(4, 1, 9, 1));
++
++ execute("DELETE FROM %s WHERE pk = ? and c = ?", 3, 1);
++
++ // Test without paging
++ assertRows(execute("SELECT * FROM %s WHERE v = ? LIMIT 2 ALLOW FILTERING", 1),
++ row(1, 1, 9, 1),
++ row(4, 1, 9, 1));
++
++ assertRows(execute("SELECT * FROM %s WHERE pk IN ? AND v = ? LIMIT 2 ALLOW FILTERING", list(1, 3, 4), 1),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE v = 1", pageSize),
++ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE v = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+
- assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE v = 1 LIMIT 2", pageSize),
++ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE pk IN (1, 3, 4) AND v = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
index 43b9549,0000000..0387ba2
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
@@@ -1,185 -1,0 +1,185 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.rows;
+
+import java.util.Arrays;
+import java.util.Iterator;
+
+import org.junit.Test;
+
+import junit.framework.Assert;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.EmptyIterators;
+import org.apache.cassandra.db.PartitionColumns;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class UnfilteredRowIteratorsTest
+{
+ static final CFMetaData metadata;
+ static final ColumnDefinition v1Metadata;
+ static final ColumnDefinition v2Metadata;
+
+ static
+ {
+ metadata = CFMetaData.Builder.create("", "")
+ .addPartitionKey("pk", Int32Type.instance)
+ .addClusteringColumn("ck", Int32Type.instance)
+ .addRegularColumn("v1", Int32Type.instance)
+ .addRegularColumn("v2", Int32Type.instance)
+ .build();
+ v1Metadata = metadata.partitionColumns().columns(false).getSimple(0);
+ v2Metadata = metadata.partitionColumns().columns(false).getSimple(1);
+ }
+
+
+ @Test
+ public void concatTest()
+ {
+ UnfilteredRowIterator iter1, iter2, iter3, concat;
+ // simple concatenation
+ iter1 = rows(metadata.partitionColumns(), 1,
+ row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+ row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
+ iter2 = rows(metadata.partitionColumns(), 1,
+ row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
+ row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
+ concat = UnfilteredRowIterators.concat(iter1, iter2);
+ Assert.assertEquals(concat.columns(), metadata.partitionColumns());
+ assertRows(concat,
+ row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+ row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)),
+ row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
+ row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
+
+ // concat with RHS empty iterator
+ iter1 = rows(metadata.partitionColumns(), 1,
+ row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+ row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
+ Assert.assertEquals(concat.columns(), metadata.partitionColumns());
+ assertRows(UnfilteredRowIterators.concat(iter1, EmptyIterators.unfilteredRow(metadata, dk(1), false, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE)),
+ row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+ row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
+
+ // concat with LHS empty iterator
+ iter1 = rows(metadata.partitionColumns(), 1,
+ row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+ row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
+ Assert.assertEquals(concat.columns(), metadata.partitionColumns());
+ assertRows(UnfilteredRowIterators.concat(EmptyIterators.unfilteredRow(metadata, dk(1), false, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE), iter1),
+ row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+ row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
+
+ // concat with different columns
+ iter1 = rows(metadata.partitionColumns().without(v1Metadata), 1,
+ row(1, cell(v2Metadata, 1)), row(2, cell(v2Metadata, 2)));
+ iter2 = rows(metadata.partitionColumns().without(v2Metadata), 1,
+ row(3, cell(v1Metadata, 3)), row(4, cell(v1Metadata, 4)));
+ concat = UnfilteredRowIterators.concat(iter1, iter2);
+ Assert.assertEquals(concat.columns(), PartitionColumns.of(v1Metadata).mergeTo(PartitionColumns.of(v2Metadata)));
+ assertRows(concat,
+ row(1, cell(v2Metadata, 1)), row(2, cell(v2Metadata, 2)),
+ row(3, cell(v1Metadata, 3)), row(4, cell(v1Metadata, 4)));
+
+ // concat with CQL limits
+ iter1 = rows(metadata.partitionColumns(), 1,
+ row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+ row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
+ iter2 = rows(metadata.partitionColumns(), 1,
+ row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
+ row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
- concat = UnfilteredRowIterators.concat(DataLimits.cqlLimits(1).filter(iter1, FBUtilities.nowInSeconds()),
- DataLimits.cqlLimits(1).filter(iter2, FBUtilities.nowInSeconds()));
++ concat = UnfilteredRowIterators.concat(DataLimits.cqlLimits(1).filter(iter1, FBUtilities.nowInSeconds(), true),
++ DataLimits.cqlLimits(1).filter(iter2, FBUtilities.nowInSeconds(), true));
+ Assert.assertEquals(concat.columns(), metadata.partitionColumns());
+ assertRows(concat,
+ row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+ row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)));
+
+ // concat concatenated iterators
+ iter1 = rows(metadata.partitionColumns(), 1,
+ row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+ row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
+ iter2 = rows(metadata.partitionColumns(), 1,
+ row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
+ row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
+
- concat = UnfilteredRowIterators.concat(DataLimits.cqlLimits(1).filter(iter1, FBUtilities.nowInSeconds()),
- DataLimits.cqlLimits(1).filter(iter2, FBUtilities.nowInSeconds()));
++ concat = UnfilteredRowIterators.concat(DataLimits.cqlLimits(1).filter(iter1, FBUtilities.nowInSeconds(), true),
++ DataLimits.cqlLimits(1).filter(iter2, FBUtilities.nowInSeconds(), true));
+
+ iter3 = rows(metadata.partitionColumns(), 1,
+ row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)),
+ row(5, cell(v1Metadata, 5), cell(v2Metadata, 5)));
- concat = UnfilteredRowIterators.concat(concat, DataLimits.cqlLimits(1).filter(iter3, FBUtilities.nowInSeconds()));
++ concat = UnfilteredRowIterators.concat(concat, DataLimits.cqlLimits(1).filter(iter3, FBUtilities.nowInSeconds(), true));
+
+ Assert.assertEquals(concat.columns(), metadata.partitionColumns());
+ assertRows(concat,
+ row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+ row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
+ row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
+ }
+
+ public static void assertRows(UnfilteredRowIterator iterator, Row... rows)
+ {
+ Iterator<Row> rowsIterator = Arrays.asList(rows).iterator();
+
+ while (iterator.hasNext() && rowsIterator.hasNext())
+ Assert.assertEquals(iterator.next(), rowsIterator.next());
+
+ Assert.assertTrue(iterator.hasNext() == rowsIterator.hasNext());
+ }
+
+ public static DecoratedKey dk(int pk)
+ {
+ return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(pk), ByteBufferUtil.bytes(pk));
+ }
+
+ public static UnfilteredRowIterator rows(PartitionColumns columns, int pk, Row... rows)
+ {
+ Iterator<Row> rowsIterator = Arrays.asList(rows).iterator();
+ return new AbstractUnfilteredRowIterator(metadata, dk(pk), DeletionTime.LIVE, columns, Rows.EMPTY_STATIC_ROW, false, EncodingStats.NO_STATS) {
+ protected Unfiltered computeNext()
+ {
+ return rowsIterator.hasNext() ? rowsIterator.next() : endOfData();
+ }
+ };
+ }
+
+ public Row row(int ck, Cell... columns)
+ {
+ BTreeRow.Builder builder = new BTreeRow.Builder(true);
+ builder.newRow(Util.clustering(metadata.comparator, ck));
+ for (Cell cell : columns)
+ builder.addCell(cell);
+ return builder.build();
+ }
+
+ public Cell cell(ColumnDefinition metadata, int v)
+ {
+ return new BufferCell(metadata,
+ 1L, BufferCell.NO_TTL, BufferCell.NO_DELETION_TIME, ByteBufferUtil.bytes(v), null);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[6/7] cassandra git commit: Merge branch cassandra-2.2 into
cassandra-3.0
Posted by bl...@apache.org.
Merge branch cassandra-2.2 into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e2445cfb
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e2445cfb
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e2445cfb
Branch: refs/heads/cassandra-3.11
Commit: e2445cfb18f8b8c05acc0199df37560abfe936e6
Parents: 7de853b b08843d
Author: Benjamin Lerer <b....@gmail.com>
Authored: Fri Jul 14 17:14:38 2017 +0200
Committer: Benjamin Lerer <b....@gmail.com>
Committed: Fri Jul 14 17:15:48 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 3 +-
src/java/org/apache/cassandra/db/DataRange.java | 5 +
.../cassandra/db/PartitionRangeReadCommand.java | 7 +-
.../org/apache/cassandra/db/ReadCommand.java | 2 +-
src/java/org/apache/cassandra/db/ReadQuery.java | 12 +
.../db/SinglePartitionReadCommand.java | 21 +-
.../apache/cassandra/db/filter/DataLimits.java | 63 +++--
.../apache/cassandra/db/filter/RowFilter.java | 15 ++
.../apache/cassandra/service/CacheService.java | 2 +-
.../apache/cassandra/service/DataResolver.java | 4 +-
.../apache/cassandra/service/StorageProxy.java | 8 +-
.../service/pager/AbstractQueryPager.java | 2 +-
.../service/pager/MultiPartitionPager.java | 9 +-
.../cassandra/service/pager/QueryPagers.java | 2 +-
.../org/apache/cassandra/cql3/CQLTester.java | 8 +-
.../validation/operations/SelectLimitTest.java | 256 ++++++++++++++++++-
.../db/rows/UnfilteredRowIteratorsTest.java | 10 +-
18 files changed, 379 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fffda7f,bda510f..c916452
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,65 -1,18 +1,66 @@@
-2.2.11
- * Fix queries with LIMIT and filtering on clustering columns (CASSANDRA-11223)
- * Fix potential NPE when resume bootstrap fails (CASSANDRA-13272)
- * Fix toJSONString for the UDT, tuple and collection types (CASSANDRA-13592)
- * Fix nested Tuples/UDTs validation (CASSANDRA-13646)
- * Remove unused max_value_size_in_mb config setting from yaml (CASSANDRA-13625
+3.0.15
+ * Make concat work with iterators that have different subsets of columns (CASSANDRA-13482)
+ * Set test.runners based on cores and memory size (CASSANDRA-13078)
+ * Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557)
+ * Allow native function calls in CQLSSTableWriter (CASSANDRA-12606)
+ * Fix secondary index queries on COMPACT tables (CASSANDRA-13627)
+ * Nodetool listsnapshots output is missing a newline, if there are no snapshots (CASSANDRA-13568)
+ Merged from 2.2:
++ * Fix queries with LIMIT and filtering on clustering columns (CASSANDRA-11223)
+ * Fix potential NPE when resume bootstrap fails (CASSANDRA-13272)
+ * Fix toJSONString for the UDT, tuple and collection types (CASSANDRA-13592)
+ * Fix nested Tuples/UDTs validation (CASSANDRA-13646)
-
-2.2.10
+3.0.14
+ * Ensure int overflow doesn't occur when calculating large partition warning size (CASSANDRA-13172)
+ * Ensure consistent view of partition columns between coordinator and replica in ColumnFilter (CASSANDRA-13004)
+ * Failed unregistering mbean during drop keyspace (CASSANDRA-13346)
+ * nodetool scrub/cleanup/upgradesstables exit code is wrong (CASSANDRA-13542)
+ * Fix the reported number of sstable data files accessed per read (CASSANDRA-13120)
+ * Fix schema digest mismatch during rolling upgrades from versions before 3.0.12 (CASSANDRA-13559)
+ * Upgrade JNA version to 4.4.0 (CASSANDRA-13072)
+ * Interned ColumnIdentifiers should use minimal ByteBuffers (CASSANDRA-13533)
+ * ReverseIndexedReader may drop rows during 2.1 to 3.0 upgrade (CASSANDRA-13525)
+ * Fix repair process violating start/end token limits for small ranges (CASSANDRA-13052)
+ * Add storage port options to sstableloader (CASSANDRA-13518)
+ * Properly handle quoted index names in cqlsh DESCRIBE output (CASSANDRA-12847)
+ * Avoid reading static row twice from old format sstables (CASSANDRA-13236)
+ * Fix NPE in StorageService.excise() (CASSANDRA-13163)
+ * Expire OutboundTcpConnection messages by a single Thread (CASSANDRA-13265)
+ * Fail repair if insufficient responses received (CASSANDRA-13397)
+ * Fix SSTableLoader fail when the loaded table contains dropped columns (CASSANDRA-13276)
+ * Avoid name clashes in CassandraIndexTest (CASSANDRA-13427)
+ * Handling partially written hint files (CASSANDRA-12728)
+ * Interrupt replaying hints on decommission (CASSANDRA-13308)
+ * Fix schema version calculation for rolling upgrades (CASSANDRA-13441)
+Merged from 2.2:
* Nodes started with join_ring=False should be able to serve requests when authentication is enabled (CASSANDRA-11381)
* cqlsh COPY FROM: increment error count only for failures, not for attempts (CASSANDRA-13209)
- * nodetool upgradesstables should upgrade system tables (CASSANDRA-13119)
+
+3.0.13
+ * Make reading of range tombstones more reliable (CASSANDRA-12811)
+ * Fix startup problems due to schema tables not completely flushed (CASSANDRA-12213)
+ * Fix view builder bug that can filter out data on restart (CASSANDRA-13405)
+ * Fix 2i page size calculation when there are no regular columns (CASSANDRA-13400)
+ * Fix the conversion of 2.X expired rows without regular column data (CASSANDRA-13395)
+ * Fix hint delivery when using ext+internal IPs with prefer_local enabled (CASSANDRA-13020)
+ * Fix possible NPE on upgrade to 3.0/3.X in case of IO errors (CASSANDRA-13389)
+ * Legacy deserializer can create empty range tombstones (CASSANDRA-13341)
+ * Use the Kernel32 library to retrieve the PID on Windows and fix startup checks (CASSANDRA-13333)
+ * Fix code to not exchange schema across major versions (CASSANDRA-13274)
+ * Dropping column results in "corrupt" SSTable (CASSANDRA-13337)
+ * Bugs handling range tombstones in the sstable iterators (CASSANDRA-13340)
+ * Fix CONTAINS filtering for null collections (CASSANDRA-13246)
+ * Applying: Use a unique metric reservoir per test run when using Cassandra-wide metrics residing in MBeans (CASSANDRA-13216)
+ * Propagate row deletions in 2i tables on upgrade (CASSANDRA-13320)
+ * Slice.isEmpty() returns false for some empty slices (CASSANDRA-13305)
+ * Add formatted row output to assertEmpty in CQL Tester (CASSANDRA-13238)
+ * Legacy caching options can prevent 3.0 upgrade (CASSANDRA-13384)
+ * Nodetool upgradesstables/scrub/compact ignores system tables (CASSANDRA-13410)
+ * Fix NPE issue in StorageService (CASSANDRA-13060)
+Merged from 2.2:
* Avoid starting gossiper in RemoveTest (CASSANDRA-13407)
* Fix weightedSize() for row-cache reported by JMX and NodeTool (CASSANDRA-13393)
- * Fix JVM metric paths (CASSANDRA-13103)
* Honor truststore-password parameter in cassandra-stress (CASSANDRA-12773)
* Discard in-flight shadow round responses (CASSANDRA-12653)
* Don't anti-compact repaired data to avoid inconsistencies (CASSANDRA-13153)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index e82f1d3,2e52eb2..f720330
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -1461,37 -1712,284 +1461,38 @@@ public class ColumnFamilyStore implemen
return data.getUncompacting();
}
- public ColumnFamily getColumnFamily(DecoratedKey key,
- Composite start,
- Composite finish,
- boolean reversed,
- int limit,
- long timestamp)
+ public boolean isFilterFullyCoveredBy(ClusteringIndexFilter filter, DataLimits limits, CachedPartition cached, int nowInSec)
{
- return getColumnFamily(QueryFilter.getSliceFilter(key, name, start, finish, reversed, limit, timestamp));
+ // We can use the cached value only if we know that no data it doesn't contain could be covered
+ // by the query filter, that is if:
+ // 1) either the whole partition is cached
+ // 2) or we can ensure than any data the filter selects is in the cached partition
+
+ // We can guarantee that a partition is fully cached if the number of rows it contains is less than
+ // what we're caching. Wen doing that, we should be careful about expiring cells: we should count
+ // something expired that wasn't when the partition was cached, or we could decide that the whole
+ // partition is cached when it's not. This is why we use CachedPartition#cachedLiveRows.
+ if (cached.cachedLiveRows() < metadata.params.caching.rowsPerPartitionToCache())
+ return true;
+
+ // If the whole partition isn't cached, then we must guarantee that the filter cannot select data that
+ // is not in the cache. We can guarantee that if either the filter is a "head filter" and the cached
+ // partition has more live rows that queried (where live rows refers to the rows that are live now),
+ // or if we can prove that everything the filter selects is in the cached partition based on its content.
- return (filter.isHeadFilter() && limits.hasEnoughLiveData(cached, nowInSec)) || filter.isFullyCoveredBy(cached);
++ return (filter.isHeadFilter() && limits.hasEnoughLiveData(cached, nowInSec, filter.selectsAllPartition()))
++ || filter.isFullyCoveredBy(cached);
}
- /**
- * Fetch the row and columns given by filter.key if it is in the cache; if not, read it from disk and cache it
- *
- * If row is cached, and the filter given is within its bounds, we return from cache, otherwise from disk
- *
- * If row is not cached, we figure out what filter is "biggest", read that from disk, then
- * filter the result and either cache that or return it.
- *
- * @param cfId the column family to read the row from
- * @param filter the columns being queried.
- * @return the requested data for the filter provided
- */
- private ColumnFamily getThroughCache(UUID cfId, QueryFilter filter)
+ public int gcBefore(int nowInSec)
{
- assert isRowCacheEnabled()
- : String.format("Row cache is not enabled on table [" + name + "]");
-
- RowCacheKey key = new RowCacheKey(metadata.ksAndCFName, filter.key);
+ return nowInSec - metadata.params.gcGraceSeconds;
+ }
- // attempt a sentinel-read-cache sequence. if a write invalidates our sentinel, we'll return our
- // (now potentially obsolete) data, but won't cache it. see CASSANDRA-3862
- // TODO: don't evict entire rows on writes (#2864)
- IRowCacheEntry cached = CacheService.instance.rowCache.get(key);
- if (cached != null)
- {
- if (cached instanceof RowCacheSentinel)
- {
- // Some other read is trying to cache the value, just do a normal non-caching read
- Tracing.trace("Row cache miss (race)");
- metric.rowCacheMiss.inc();
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
-
- ColumnFamily cachedCf = (ColumnFamily)cached;
- if (isFilterFullyCoveredBy(filter.filter, cachedCf, filter.timestamp))
- {
- metric.rowCacheHit.inc();
- Tracing.trace("Row cache hit");
- ColumnFamily result = filterColumnFamily(cachedCf, filter);
- metric.updateSSTableIterated(0);
- return result;
- }
-
- metric.rowCacheHitOutOfRange.inc();
- Tracing.trace("Ignoring row cache as cached value could not satisfy query");
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
-
- metric.rowCacheMiss.inc();
- Tracing.trace("Row cache miss");
- RowCacheSentinel sentinel = new RowCacheSentinel();
- boolean sentinelSuccess = CacheService.instance.rowCache.putIfAbsent(key, sentinel);
- ColumnFamily data = null;
- ColumnFamily toCache = null;
- try
- {
- // If we are explicitely asked to fill the cache with full partitions, we go ahead and query the whole thing
- if (metadata.getCaching().rowCache.cacheFullPartitions())
- {
- data = getTopLevelColumns(QueryFilter.getIdentityFilter(filter.key, name, filter.timestamp), Integer.MIN_VALUE);
- toCache = data;
- Tracing.trace("Populating row cache with the whole partition");
- if (sentinelSuccess && toCache != null)
- CacheService.instance.rowCache.replace(key, sentinel, toCache);
- return filterColumnFamily(data, filter);
- }
-
- // Otherwise, if we want to cache the result of the query we're about to do, we must make sure this query
- // covers what needs to be cached. And if the user filter does not satisfy that, we sometimes extend said
- // filter so we can populate the cache but only if:
- // 1) we can guarantee it is a strict extension, i.e. that we will still fetch the data asked by the user.
- // 2) the extension does not make us query more than getRowsPerPartitionToCache() (as a mean to limit the
- // amount of extra work we'll do on a user query for the purpose of populating the cache).
- //
- // In practice, we can only guarantee those 2 points if the filter is one that queries the head of the
- // partition (and if that filter actually counts CQL3 rows since that's what we cache and it would be
- // bogus to compare the filter count to the 'rows to cache' otherwise).
- if (filter.filter.isHeadFilter() && filter.filter.countCQL3Rows(metadata.comparator))
- {
- SliceQueryFilter sliceFilter = (SliceQueryFilter)filter.filter;
- int rowsToCache = metadata.getCaching().rowCache.rowsToCache;
-
- SliceQueryFilter cacheSlice = readFilterForCache();
- QueryFilter cacheFilter = new QueryFilter(filter.key, name, cacheSlice, filter.timestamp);
-
- // If the filter count is less than the number of rows cached, we simply extend it to make sure we do cover the
- // number of rows to cache, and if that count is greater than the number of rows to cache, we simply filter what
- // needs to be cached afterwards.
- if (sliceFilter.count < rowsToCache)
- {
- toCache = getTopLevelColumns(cacheFilter, Integer.MIN_VALUE);
- if (toCache != null)
- {
- Tracing.trace("Populating row cache ({} rows cached)", cacheSlice.lastCounted());
- data = filterColumnFamily(toCache, filter);
- }
- }
- else
- {
- data = getTopLevelColumns(filter, Integer.MIN_VALUE);
- if (data != null)
- {
- // The filter limit was greater than the number of rows to cache. But, if the filter had a non-empty
- // finish bound, we may have gotten less than what needs to be cached, in which case we shouldn't cache it
- // (otherwise a cache hit would assume the whole partition is cached which is not the case).
- if (sliceFilter.finish().isEmpty() || sliceFilter.lastCounted() >= rowsToCache)
- {
- toCache = filterColumnFamily(data, cacheFilter);
- Tracing.trace("Caching {} rows (out of {} requested)", cacheSlice.lastCounted(), sliceFilter.count);
- }
- else
- {
- Tracing.trace("Not populating row cache, not enough rows fetched ({} fetched but {} required for the cache)", sliceFilter.lastCounted(), rowsToCache);
- }
- }
- }
-
- if (sentinelSuccess && toCache != null)
- CacheService.instance.rowCache.replace(key, sentinel, toCache);
- return data;
- }
- else
- {
- Tracing.trace("Fetching data but not populating cache as query does not query from the start of the partition");
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
- }
- finally
- {
- if (sentinelSuccess && toCache == null)
- invalidateCachedRow(key);
- }
- }
-
- public SliceQueryFilter readFilterForCache()
- {
- // We create a new filter everytime before for now SliceQueryFilter is unfortunatly mutable.
- return new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, false, metadata.getCaching().rowCache.rowsToCache, metadata.clusteringColumns().size());
- }
-
- public boolean isFilterFullyCoveredBy(IDiskAtomFilter filter, ColumnFamily cachedCf, long now)
- {
- // We can use the cached value only if we know that no data it doesn't contain could be covered
- // by the query filter, that is if:
- // 1) either the whole partition is cached
- // 2) or we can ensure than any data the filter selects are in the cached partition
-
- // When counting rows to decide if the whole row is cached, we should be careful with expiring
- // columns: if we use a timestamp newer than the one that was used when populating the cache, we might
- // end up deciding the whole partition is cached when it's really not (just some rows expired since the
- // cf was cached). This is the reason for Integer.MIN_VALUE below.
- boolean wholePartitionCached = cachedCf.liveCQL3RowCount(Integer.MIN_VALUE) < metadata.getCaching().rowCache.rowsToCache;
-
- // Contrarily to the "wholePartitionCached" check above, we do want isFullyCoveredBy to take the
- // timestamp of the query into account when dealing with expired columns. Otherwise, we could think
- // the cached partition has enough live rows to satisfy the filter when it doesn't because some
- // are now expired.
- return wholePartitionCached || filter.isFullyCoveredBy(cachedCf, now);
- }
-
- public int gcBefore(long now)
- {
- return (int) (now / 1000) - metadata.getGcGraceSeconds();
- }
-
- /**
- * get a list of columns starting from a given column, in a specified order.
- * only the latest version of a column is returned.
- * @return null if there is no data and no tombstones; otherwise a ColumnFamily
- */
- public ColumnFamily getColumnFamily(QueryFilter filter)
- {
- assert name.equals(filter.getColumnFamilyName()) : filter.getColumnFamilyName();
-
- ColumnFamily result = null;
-
- long start = System.nanoTime();
- try
- {
- int gcBefore = gcBefore(filter.timestamp);
- if (isRowCacheEnabled())
- {
- assert !isIndex(); // CASSANDRA-5732
- UUID cfId = metadata.cfId;
-
- ColumnFamily cached = getThroughCache(cfId, filter);
- if (cached == null)
- {
- logger.trace("cached row is empty");
- return null;
- }
-
- result = cached;
- }
- else
- {
- ColumnFamily cf = getTopLevelColumns(filter, gcBefore);
-
- if (cf == null)
- return null;
-
- result = removeDeletedCF(cf, gcBefore);
- }
-
- removeDroppedColumns(result);
-
- if (filter.filter instanceof SliceQueryFilter)
- {
- // Log the number of tombstones scanned on single key queries
- metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastTombstones());
- metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive());
- }
- }
- finally
- {
- metric.readLatency.addNano(System.nanoTime() - start);
- }
-
- return result;
- }
-
- /**
- * Filter a cached row, which will not be modified by the filter, but may be modified by throwing out
- * tombstones that are no longer relevant.
- * The returned column family won't be thread safe.
- */
- ColumnFamily filterColumnFamily(ColumnFamily cached, QueryFilter filter)
- {
- if (cached == null)
- return null;
-
- ColumnFamily cf = cached.cloneMeShallow(ArrayBackedSortedColumns.factory, filter.filter.isReversed());
- int gcBefore = gcBefore(filter.timestamp);
- filter.collateOnDiskAtom(cf, filter.getIterator(cached), gcBefore);
- return removeDeletedCF(cf, gcBefore);
- }
-
- public Set<SSTableReader> getUnrepairedSSTables()
- {
- Set<SSTableReader> unRepairedSSTables = new HashSet<>(getSSTables());
- Iterator<SSTableReader> sstableIterator = unRepairedSSTables.iterator();
- while(sstableIterator.hasNext())
- {
- SSTableReader sstable = sstableIterator.next();
- if (sstable.isRepaired())
- sstableIterator.remove();
- }
- return unRepairedSSTables;
- }
-
- public Set<SSTableReader> getRepairedSSTables()
- {
- Set<SSTableReader> repairedSSTables = new HashSet<>(getSSTables());
- Iterator<SSTableReader> sstableIterator = repairedSSTables.iterator();
- while(sstableIterator.hasNext())
- {
- SSTableReader sstable = sstableIterator.next();
- if (!sstable.isRepaired())
- sstableIterator.remove();
- }
- return repairedSSTables;
- }
-
- @SuppressWarnings("resource")
- public RefViewFragment selectAndReference(Function<View, List<SSTableReader>> filter)
- {
- long failingSince = -1L;
- while (true)
+ @SuppressWarnings("resource")
+ public RefViewFragment selectAndReference(Function<View, Iterable<SSTableReader>> filter)
+ {
+ long failingSince = -1L;
+ while (true)
{
ViewFragment view = select(filter);
Refs<SSTableReader> refs = Refs.tryRef(view.sstables);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/db/DataRange.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/DataRange.java
index ffe041e,1e6f8c8..d2f9c76
--- a/src/java/org/apache/cassandra/db/DataRange.java
+++ b/src/java/org/apache/cassandra/db/DataRange.java
@@@ -179,30 -119,16 +179,35 @@@ public class DataRang
return keyRange.contains(pos);
}
- public int getLiveCount(ColumnFamily data, long now)
+ /**
+ * Whether this {@code DataRange} queries everything (has no restriction neither on the
+ * partition queried, nor within the queried partition).
+ *
+ * @return Whether this {@code DataRange} queries everything.
+ */
+ public boolean isUnrestricted()
+ {
+ return startKey().isMinimum() && stopKey().isMinimum() && clusteringIndexFilter.selectsAllPartition();
++ }
++
++ public boolean selectsAllPartition()
+ {
- return columnFilter instanceof SliceQueryFilter
- ? ((SliceQueryFilter)columnFilter).lastCounted()
- : columnFilter.getLiveCount(data, now);
++ return clusteringIndexFilter.selectsAllPartition();
}
- public boolean selectsFullRowFor(ByteBuffer rowKey)
+ /**
+ * The clustering index filter to use for the provided key.
+ * <p>
+ * This may or may not be the same filter for all keys (that is, paging range
+ * use a different filter for their start key).
+ *
+ * @param key the partition key for which we want the clustering index filter.
+ *
+ * @return the clustering filter to use for {@code key}.
+ */
+ public ClusteringIndexFilter clusteringIndexFilter(DecoratedKey key)
{
- return selectFullRow;
+ return clusteringIndexFilter;
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
index 1cf332d,0000000..617e2f5
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
+++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
@@@ -1,341 -1,0 +1,346 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import com.google.common.collect.Iterables;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.filter.*;
- import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.db.lifecycle.View;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.rows.BaseRowIterator;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReadsListener;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.metrics.TableMetrics;
+import org.apache.cassandra.net.MessageOut;
+import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.service.pager.*;
+import org.apache.cassandra.thrift.ThriftResultsMerger;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * A read command that selects a (part of a) range of partitions.
+ */
+public class PartitionRangeReadCommand extends ReadCommand
+{
+ protected static final SelectionDeserializer selectionDeserializer = new Deserializer();
+
+ private final DataRange dataRange;
+ private int oldestUnrepairedTombstone = Integer.MAX_VALUE;
+
+ public PartitionRangeReadCommand(boolean isDigest,
+ int digestVersion,
+ boolean isForThrift,
+ CFMetaData metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DataRange dataRange,
+ Optional<IndexMetadata> index)
+ {
+ super(Kind.PARTITION_RANGE, isDigest, digestVersion, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits);
+ this.dataRange = dataRange;
+ this.index = index;
+ }
+
+ public PartitionRangeReadCommand(CFMetaData metadata,
+ int nowInSec,
+ ColumnFilter columnFilter,
+ RowFilter rowFilter,
+ DataLimits limits,
+ DataRange dataRange,
+ Optional<IndexMetadata> index)
+ {
+ this(false, 0, false, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, index);
+ }
+
+ /**
+ * Creates a new read command that query all the data in the table.
+ *
+ * @param metadata the table to query.
+ * @param nowInSec the time in seconds to use are "now" for this query.
+ *
+ * @return a newly created read command that queries everything in the table.
+ */
+ public static PartitionRangeReadCommand allDataRead(CFMetaData metadata, int nowInSec)
+ {
+ return new PartitionRangeReadCommand(metadata,
+ nowInSec,
+ ColumnFilter.all(metadata),
+ RowFilter.NONE,
+ DataLimits.NONE,
+ DataRange.allData(metadata.partitioner),
+ Optional.empty());
+ }
+
+ public DataRange dataRange()
+ {
+ return dataRange;
+ }
+
+ public ClusteringIndexFilter clusteringIndexFilter(DecoratedKey key)
+ {
+ return dataRange.clusteringIndexFilter(key);
+ }
+
+ public boolean isNamesQuery()
+ {
+ return dataRange.isNamesQuery();
+ }
+
+ public PartitionRangeReadCommand forSubRange(AbstractBounds<PartitionPosition> range)
+ {
+ return new PartitionRangeReadCommand(isDigestQuery(), digestVersion(), isForThrift(), metadata(), nowInSec(), columnFilter(), rowFilter(), limits(), dataRange().forSubRange(range), index);
+ }
+
+ public PartitionRangeReadCommand copy()
+ {
+ return new PartitionRangeReadCommand(isDigestQuery(), digestVersion(), isForThrift(), metadata(), nowInSec(), columnFilter(), rowFilter(), limits(), dataRange(), index);
+ }
+
+ public PartitionRangeReadCommand withUpdatedLimit(DataLimits newLimits)
+ {
+ return new PartitionRangeReadCommand(metadata(), nowInSec(), columnFilter(), rowFilter(), newLimits, dataRange(), index);
+ }
+
+ public long getTimeout()
+ {
+ return DatabaseDescriptor.getRangeRpcTimeout();
+ }
+
+ public boolean selectsKey(DecoratedKey key)
+ {
+ if (!dataRange().contains(key))
+ return false;
+
+ return rowFilter().partitionKeyRestrictionsAreSatisfiedBy(key, metadata().getKeyValidator());
+ }
+
+ public boolean selectsClustering(DecoratedKey key, Clustering clustering)
+ {
+ if (clustering == Clustering.STATIC_CLUSTERING)
+ return !columnFilter().fetchedColumns().statics.isEmpty();
+
+ if (!dataRange().clusteringIndexFilter(key).selects(clustering))
+ return false;
+ return rowFilter().clusteringKeyRestrictionsAreSatisfiedBy(clustering);
+ }
+
+ public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState) throws RequestExecutionException
+ {
+ return StorageProxy.getRangeSlice(this, consistency);
+ }
+
+ public QueryPager getPager(PagingState pagingState, int protocolVersion)
+ {
+ return new PartitionRangeQueryPager(this, pagingState, protocolVersion);
+ }
+
+ protected void recordLatency(TableMetrics metric, long latencyNanos)
+ {
+ metric.rangeLatency.addNano(latencyNanos);
+ }
+
+ protected UnfilteredPartitionIterator queryStorage(final ColumnFamilyStore cfs, ReadOrderGroup orderGroup)
+ {
+ ColumnFamilyStore.ViewFragment view = cfs.select(View.selectLive(dataRange().keyRange()));
+ Tracing.trace("Executing seq scan across {} sstables for {}", view.sstables.size(), dataRange().keyRange().getString(metadata().getKeyValidator()));
+
+ // fetch data from current memtable, historical memtables, and SSTables in the correct order.
+ final List<UnfilteredPartitionIterator> iterators = new ArrayList<>(Iterables.size(view.memtables) + view.sstables.size());
+
+ try
+ {
+ for (Memtable memtable : view.memtables)
+ {
+ @SuppressWarnings("resource") // We close on exception and on closing the result returned by this method
+ Memtable.MemtableUnfilteredPartitionIterator iter = memtable.makePartitionIterator(columnFilter(), dataRange(), isForThrift());
+ oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, iter.getMinLocalDeletionTime());
+ iterators.add(isForThrift() ? ThriftResultsMerger.maybeWrap(iter, metadata(), nowInSec()) : iter);
+ }
+
+ SSTableReadsListener readCountUpdater = newReadCountUpdater();
+ for (SSTableReader sstable : view.sstables)
+ {
+ @SuppressWarnings("resource") // We close on exception and on closing the result returned by this method
+ UnfilteredPartitionIterator iter = sstable.getScanner(columnFilter(), dataRange(), isForThrift(), readCountUpdater);
+ iterators.add(isForThrift() ? ThriftResultsMerger.maybeWrap(iter, metadata(), nowInSec()) : iter);
+ if (!sstable.isRepaired())
+ oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, sstable.getMinLocalDeletionTime());
+ }
+ return iterators.isEmpty() ? EmptyIterators.unfilteredPartition(metadata(), isForThrift())
+ : checkCacheFilter(UnfilteredPartitionIterators.mergeLazily(iterators, nowInSec()), cfs);
+ }
+ catch (RuntimeException | Error e)
+ {
+ try
+ {
+ FBUtilities.closeAll(iterators);
+ }
+ catch (Exception suppressed)
+ {
+ e.addSuppressed(suppressed);
+ }
+
+ throw e;
+ }
+ }
+
+ /**
+ * Creates a new {@code SSTableReadsListener} to update the SSTables read counts.
+ * @return a new {@code SSTableReadsListener} to update the SSTables read counts.
+ */
+ private static SSTableReadsListener newReadCountUpdater()
+ {
+ return new SSTableReadsListener()
+ {
+ @Override
+ public void onScanningStarted(SSTableReader sstable)
+ {
+ sstable.incrementReadCount();
+ }
+ };
+ }
+
+ @Override
+ protected int oldestUnrepairedTombstone()
+ {
+ return oldestUnrepairedTombstone;
+ }
+
+ private UnfilteredPartitionIterator checkCacheFilter(UnfilteredPartitionIterator iter, final ColumnFamilyStore cfs)
+ {
+ class CacheFilter extends Transformation
+ {
+ @Override
+ public BaseRowIterator applyToPartition(BaseRowIterator iter)
+ {
+ // Note that we rely on the fact that until we actually advance 'iter', no really costly operation is actually done
+ // (except for reading the partition key from the index file) due to the call to mergeLazily in queryStorage.
+ DecoratedKey dk = iter.partitionKey();
+
+ // Check if this partition is in the rowCache and if it is, if it covers our filter
+ CachedPartition cached = cfs.getRawCachedPartition(dk);
+ ClusteringIndexFilter filter = dataRange().clusteringIndexFilter(dk);
+
+ if (cached != null && cfs.isFilterFullyCoveredBy(filter, limits(), cached, nowInSec()))
+ {
+ // We won't use 'iter' so close it now.
+ iter.close();
+
+ return filter.getUnfilteredRowIterator(columnFilter(), cached);
+ }
+
+ return iter;
+ }
+ }
+ return Transformation.apply(iter, new CacheFilter());
+ }
+
+ public MessageOut<ReadCommand> createMessage(int version)
+ {
+ return dataRange().isPaging()
+ ? new MessageOut<>(MessagingService.Verb.PAGED_RANGE, this, pagedRangeSerializer)
+ : new MessageOut<>(MessagingService.Verb.RANGE_SLICE, this, rangeSliceSerializer);
+ }
+
+ protected void appendCQLWhereClause(StringBuilder sb)
+ {
+ if (dataRange.isUnrestricted() && rowFilter().isEmpty())
+ return;
+
+ sb.append(" WHERE ");
+ // We put the row filter first because the data range can end by "ORDER BY"
+ if (!rowFilter().isEmpty())
+ {
+ sb.append(rowFilter());
+ if (!dataRange.isUnrestricted())
+ sb.append(" AND ");
+ }
+ if (!dataRange.isUnrestricted())
+ sb.append(dataRange.toCQLString(metadata()));
+ }
+
+ /**
+ * Allow to post-process the result of the query after it has been reconciled on the coordinator
+ * but before it is passed to the CQL layer to return the ResultSet.
+ *
+ * See CASSANDRA-8717 for why this exists.
+ */
+ public PartitionIterator postReconciliationProcessing(PartitionIterator result)
+ {
+ ColumnFamilyStore cfs = Keyspace.open(metadata().ksName).getColumnFamilyStore(metadata().cfName);
+ Index index = getIndex(cfs);
+ return index == null ? result : index.postProcessorFor(this).apply(result, this);
+ }
+
+ @Override
++ public boolean selectsFullPartition()
++ {
++ return dataRange.selectsAllPartition() && !rowFilter().hasExpressionOnClusteringOrRegularColumns();
++ }
++
++ @Override
+ public String toString()
+ {
+ return String.format("Read(%s.%s columns=%s rowfilter=%s limits=%s %s)",
+ metadata().ksName,
+ metadata().cfName,
+ columnFilter(),
+ rowFilter(),
+ limits(),
+ dataRange().toString(metadata()));
+ }
+
+ protected void serializeSelection(DataOutputPlus out, int version) throws IOException
+ {
+ DataRange.serializer.serialize(dataRange(), out, version, metadata());
+ }
+
+ protected long selectionSerializedSize(int version)
+ {
+ return DataRange.serializer.serializedSize(dataRange(), version, metadata());
+ }
+
+ private static class Deserializer extends SelectionDeserializer
+ {
+ public ReadCommand deserialize(DataInputPlus in, int version, boolean isDigest, int digestVersion, boolean isForThrift, CFMetaData metadata, int nowInSec, ColumnFilter columnFilter, RowFilter rowFilter, DataLimits limits, Optional<IndexMetadata> index)
+ throws IOException
+ {
+ DataRange range = DataRange.serializer.deserialize(in, version, metadata);
+ return new PartitionRangeReadCommand(isDigest, digestVersion, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, range, index);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/db/ReadCommand.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ReadCommand.java
index 64da428,cd86336..76180cc
--- a/src/java/org/apache/cassandra/db/ReadCommand.java
+++ b/src/java/org/apache/cassandra/db/ReadCommand.java
@@@ -264,699 -95,55 +264,699 @@@ public abstract class ReadCommand imple
return this;
}
- public String getColumnFamilyName()
+ /**
+ * Sets the digest version, for when digest for that command is requested.
+ * <p>
+ * Note that we allow setting this independently of setting the command as a digest query as
+ * this allows us to use the command as a carrier of the digest version even if we only call
+ * setIsDigestQuery on some copy of it.
+ *
+ * @param digestVersion the version for the digest is this command is used for digest query..
+ * @return this read command.
+ */
+ public ReadCommand setDigestVersion(int digestVersion)
{
- return cfName;
+ this.digestVersion = digestVersion;
+ return this;
}
+ /**
+ * Whether this query is for thrift or not.
+ *
+ * @return whether this query is for thrift.
+ */
+ public boolean isForThrift()
+ {
+ return isForThrift;
+ }
+
+ /**
+ * The clustering index filter this command to use for the provided key.
+ * <p>
+ * Note that that method should only be called on a key actually queried by this command
+ * and in practice, this will almost always return the same filter, but for the sake of
+ * paging, the filter on the first key of a range command might be slightly different.
+ *
+ * @param key a partition key queried by this command.
+ *
+ * @return the {@code ClusteringIndexFilter} to use for the partition of key {@code key}.
+ */
+ public abstract ClusteringIndexFilter clusteringIndexFilter(DecoratedKey key);
+
+ /**
+ * Returns a copy of this command.
+ *
+ * @return a copy of this command.
+ */
public abstract ReadCommand copy();
- public abstract Row getRow(Keyspace keyspace);
+ protected abstract UnfilteredPartitionIterator queryStorage(ColumnFamilyStore cfs, ReadOrderGroup orderGroup);
+
+ protected abstract int oldestUnrepairedTombstone();
- public abstract IDiskAtomFilter filter();
+ public ReadResponse createResponse(UnfilteredPartitionIterator iterator)
+ {
+ return isDigestQuery()
+ ? ReadResponse.createDigestResponse(iterator, this)
+ : ReadResponse.createDataResponse(iterator, this);
+ }
- public String getKeyspace()
+ public long indexSerializedSize(int version)
{
- return ksName;
+ if (index.isPresent())
+ return IndexMetadata.serializer.serializedSize(index.get(), version);
+ else
+ return 0;
}
- // maybeGenerateRetryCommand is used to generate a retry for short reads
- public ReadCommand maybeGenerateRetryCommand(RowDataResolver resolver, Row row)
+ public Index getIndex(ColumnFamilyStore cfs)
{
- return null;
+ // if we've already consulted the index manager, and it returned a valid index
+ // the result should be cached here.
+ if(index.isPresent())
+ return cfs.indexManager.getIndex(index.get());
+
+ // if no cached index is present, but we've already consulted the index manager
+ // then no registered index is suitable for this command, so just return null.
+ if (indexManagerQueried)
+ return null;
+
+ // do the lookup, set the flag to indicate so and cache the result if not null
+ Index selected = cfs.indexManager.getBestIndexFor(this);
+ indexManagerQueried = true;
+
+ if (selected == null)
+ return null;
+
+ index = Optional.of(selected.getIndexMetadata());
+ return selected;
}
- // maybeTrim removes columns from a response that is too long
- public Row maybeTrim(Row row)
+ /**
+ * If the index manager for the CFS determines that there's an applicable
+ * 2i that can be used to execute this command, call its (optional)
+ * validation method to check that nothing in this command's parameters
+ * violates the implementation specific validation rules.
+ */
+ public void maybeValidateIndex()
{
- return row;
+ Index index = getIndex(Keyspace.openAndGetStore(metadata));
+ if (null != index)
+ index.validate(this);
}
- public long getTimeout()
+ /**
+ * Executes this command on the local host.
+ *
+ * @param orderGroup the operation group spanning this command
+ *
+ * @return an iterator over the result of executing this command locally.
+ */
+ @SuppressWarnings("resource") // The result iterator is closed upon exceptions (we know it's fine to potentially not close the intermediary
+ // iterators created inside the try as long as we do close the original resultIterator), or by closing the result.
+ public UnfilteredPartitionIterator executeLocally(ReadOrderGroup orderGroup)
{
- return DatabaseDescriptor.getReadRpcTimeout();
+ long startTimeNanos = System.nanoTime();
+
+ ColumnFamilyStore cfs = Keyspace.openAndGetStore(metadata());
+ Index index = getIndex(cfs);
+
+ Index.Searcher searcher = null;
+ if (index != null)
+ {
+ if (!cfs.indexManager.isIndexQueryable(index))
+ throw new IndexNotAvailableException(index);
+
+ searcher = index.searcherFor(this);
+ Tracing.trace("Executing read on {}.{} using index {}", cfs.metadata.ksName, cfs.metadata.cfName, index.getIndexMetadata().name);
+ }
+
+ UnfilteredPartitionIterator resultIterator = searcher == null
+ ? queryStorage(cfs, orderGroup)
+ : searcher.search(orderGroup);
+
+ try
+ {
+ resultIterator = withMetricsRecording(withoutPurgeableTombstones(resultIterator, cfs), cfs.metric, startTimeNanos);
+
+ // If we've used a 2ndary index, we know the result already satisfy the primary expression used, so
+ // no point in checking it again.
+ RowFilter updatedFilter = searcher == null
+ ? rowFilter()
+ : index.getPostIndexQueryFilter(rowFilter());
+
+ // TODO: We'll currently do filtering by the rowFilter here because it's convenient. However,
+ // we'll probably want to optimize by pushing it down the layer (like for dropped columns) as it
+ // would be more efficient (the sooner we discard stuff we know we don't care, the less useless
+ // processing we do on it).
- return limits().filter(updatedFilter.filter(resultIterator, nowInSec()), nowInSec());
++ return limits().filter(updatedFilter.filter(resultIterator, nowInSec()), nowInSec(), selectsFullPartition());
+ }
+ catch (RuntimeException | Error e)
+ {
+ resultIterator.close();
+ throw e;
+ }
}
-}
-class ReadCommandSerializer implements IVersionedSerializer<ReadCommand>
-{
- public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException
+ protected abstract void recordLatency(TableMetrics metric, long latencyNanos);
+
+ public PartitionIterator executeInternal(ReadOrderGroup orderGroup)
+ {
+ return UnfilteredPartitionIterators.filter(executeLocally(orderGroup), nowInSec());
+ }
+
+ public ReadOrderGroup startOrderGroup()
+ {
+ return ReadOrderGroup.forCommand(this);
+ }
+
+ /**
+ * Wraps the provided iterator so that metrics on what is scanned by the command are recorded.
+ * This also log warning/trow TombstoneOverwhelmingException if appropriate.
+ */
+ private UnfilteredPartitionIterator withMetricsRecording(UnfilteredPartitionIterator iter, final TableMetrics metric, final long startTimeNanos)
+ {
+ class MetricRecording extends Transformation<UnfilteredRowIterator>
+ {
+ private final int failureThreshold = DatabaseDescriptor.getTombstoneFailureThreshold();
+ private final int warningThreshold = DatabaseDescriptor.getTombstoneWarnThreshold();
+
+ private final boolean respectTombstoneThresholds = !Schema.isSystemKeyspace(ReadCommand.this.metadata().ksName);
+
+ private int liveRows = 0;
+ private int tombstones = 0;
+
+ private DecoratedKey currentKey;
+
+ @Override
+ public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator iter)
+ {
+ currentKey = iter.partitionKey();
+ return Transformation.apply(iter, this);
+ }
+
+ @Override
+ public Row applyToStatic(Row row)
+ {
+ return applyToRow(row);
+ }
+
+ @Override
+ public Row applyToRow(Row row)
+ {
+ if (row.hasLiveData(ReadCommand.this.nowInSec()))
+ ++liveRows;
+
+ for (Cell cell : row.cells())
+ {
+ if (!cell.isLive(ReadCommand.this.nowInSec()))
+ countTombstone(row.clustering());
+ }
+ return row;
+ }
+
+ @Override
+ public RangeTombstoneMarker applyToMarker(RangeTombstoneMarker marker)
+ {
+ countTombstone(marker.clustering());
+ return marker;
+ }
+
+ private void countTombstone(ClusteringPrefix clustering)
+ {
+ ++tombstones;
+ if (tombstones > failureThreshold && respectTombstoneThresholds)
+ {
+ String query = ReadCommand.this.toCQLString();
+ Tracing.trace("Scanned over {} tombstones for query {}; query aborted (see tombstone_failure_threshold)", failureThreshold, query);
+ throw new TombstoneOverwhelmingException(tombstones, query, ReadCommand.this.metadata(), currentKey, clustering);
+ }
+ }
+
+ @Override
+ public void onClose()
+ {
+ recordLatency(metric, System.nanoTime() - startTimeNanos);
+
+ metric.tombstoneScannedHistogram.update(tombstones);
+ metric.liveScannedHistogram.update(liveRows);
+
+ boolean warnTombstones = tombstones > warningThreshold && respectTombstoneThresholds;
+ if (warnTombstones)
+ {
+ String msg = String.format("Read %d live rows and %d tombstone cells for query %1.512s (see tombstone_warn_threshold)", liveRows, tombstones, ReadCommand.this.toCQLString());
+ ClientWarn.instance.warn(msg);
+ logger.warn(msg);
+ }
+
+ Tracing.trace("Read {} live and {} tombstone cells{}", liveRows, tombstones, (warnTombstones ? " (see tombstone_warn_threshold)" : ""));
+ }
+ };
+
+ return Transformation.apply(iter, new MetricRecording());
+ }
+
+ /**
+ * Creates a message for this command.
+ */
+ public abstract MessageOut<ReadCommand> createMessage(int version);
+
+ protected abstract void appendCQLWhereClause(StringBuilder sb);
+
+ // Skip purgeable tombstones. We do this because it's safe to do (post-merge of the memtable and sstable at least), it
+ // can save us some bandwith, and avoid making us throw a TombstoneOverwhelmingException for purgeable tombstones (which
+ // are to some extend an artefact of compaction lagging behind and hence counting them is somewhat unintuitive).
+ protected UnfilteredPartitionIterator withoutPurgeableTombstones(UnfilteredPartitionIterator iterator, ColumnFamilyStore cfs)
+ {
+ final boolean isForThrift = iterator.isForThrift();
+ class WithoutPurgeableTombstones extends PurgeFunction
+ {
+ public WithoutPurgeableTombstones()
+ {
+ super(isForThrift, nowInSec(), cfs.gcBefore(nowInSec()), oldestUnrepairedTombstone(), cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones());
+ }
+
+ protected Predicate<Long> getPurgeEvaluator()
+ {
+ return time -> true;
+ }
+ }
+ return Transformation.apply(iterator, new WithoutPurgeableTombstones());
+ }
+
+ /**
+ * Recreate the CQL string corresponding to this query.
+ * <p>
+ * Note that in general the returned string will not be exactly the original user string, first
+ * because there isn't always a single syntax for a given query, but also because we don't have
+ * all the information needed (we know the non-PK columns queried but not the PK ones as internally
+ * we query them all). So this shouldn't be relied too strongly, but this should be good enough for
+ * debugging purpose which is what this is for.
+ */
+ public String toCQLString()
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.append("SELECT ").append(columnFilter());
+ sb.append(" FROM ").append(metadata().ksName).append('.').append(metadata.cfName);
+ appendCQLWhereClause(sb);
+
+ if (limits() != DataLimits.NONE)
+ sb.append(' ').append(limits());
+ return sb.toString();
+ }
+
+ private static class Serializer implements IVersionedSerializer<ReadCommand>
+ {
+ private static int digestFlag(boolean isDigest)
+ {
+ return isDigest ? 0x01 : 0;
+ }
+
+ private static boolean isDigest(int flags)
+ {
+ return (flags & 0x01) != 0;
+ }
+
+ private static int thriftFlag(boolean isForThrift)
+ {
+ return isForThrift ? 0x02 : 0;
+ }
+
+ private static boolean isForThrift(int flags)
+ {
+ return (flags & 0x02) != 0;
+ }
+
+ private static int indexFlag(boolean hasIndex)
+ {
+ return hasIndex ? 0x04 : 0;
+ }
+
+ private static boolean hasIndex(int flags)
+ {
+ return (flags & 0x04) != 0;
+ }
+
+ public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException
+ {
+ assert version >= MessagingService.VERSION_30;
+
+ out.writeByte(command.kind.ordinal());
+ out.writeByte(digestFlag(command.isDigestQuery()) | thriftFlag(command.isForThrift()) | indexFlag(command.index.isPresent()));
+ if (command.isDigestQuery())
+ out.writeUnsignedVInt(command.digestVersion());
+ CFMetaData.serializer.serialize(command.metadata(), out, version);
+ out.writeInt(command.nowInSec());
+ ColumnFilter.serializer.serialize(command.columnFilter(), out, version);
+ RowFilter.serializer.serialize(command.rowFilter(), out, version);
+ DataLimits.serializer.serialize(command.limits(), out, version);
+ if (command.index.isPresent())
+ IndexMetadata.serializer.serialize(command.index.get(), out, version);
+
+ command.serializeSelection(out, version);
+ }
+
+ public ReadCommand deserialize(DataInputPlus in, int version) throws IOException
+ {
+ assert version >= MessagingService.VERSION_30;
+
+ Kind kind = Kind.values()[in.readByte()];
+ int flags = in.readByte();
+ boolean isDigest = isDigest(flags);
+ boolean isForThrift = isForThrift(flags);
+ boolean hasIndex = hasIndex(flags);
+ int digestVersion = isDigest ? (int)in.readUnsignedVInt() : 0;
+ CFMetaData metadata = CFMetaData.serializer.deserialize(in, version);
+ int nowInSec = in.readInt();
+ ColumnFilter columnFilter = ColumnFilter.serializer.deserialize(in, version, metadata);
+ RowFilter rowFilter = RowFilter.serializer.deserialize(in, version, metadata);
+ DataLimits limits = DataLimits.serializer.deserialize(in, version);
+ Optional<IndexMetadata> index = hasIndex
+ ? deserializeIndexMetadata(in, version, metadata)
+ : Optional.empty();
+
+ return kind.selectionDeserializer.deserialize(in, version, isDigest, digestVersion, isForThrift, metadata, nowInSec, columnFilter, rowFilter, limits, index);
+ }
+
+ private Optional<IndexMetadata> deserializeIndexMetadata(DataInputPlus in, int version, CFMetaData cfm) throws IOException
+ {
+ try
+ {
+ return Optional.of(IndexMetadata.serializer.deserialize(in, version, cfm));
+ }
+ catch (UnknownIndexException e)
+ {
+ String message = String.format("Couldn't find a defined index on %s.%s with the id %s. " +
+ "If an index was just created, this is likely due to the schema not " +
+ "being fully propagated. Local read will proceed without using the " +
+ "index. Please wait for schema agreement after index creation.",
+ cfm.ksName, cfm.cfName, e.indexId.toString());
+ logger.info(message);
+ return Optional.empty();
+ }
+ }
+
+ public long serializedSize(ReadCommand command, int version)
+ {
+ assert version >= MessagingService.VERSION_30;
+
+ return 2 // kind + flags
+ + (command.isDigestQuery() ? TypeSizes.sizeofUnsignedVInt(command.digestVersion()) : 0)
+ + CFMetaData.serializer.serializedSize(command.metadata(), version)
+ + TypeSizes.sizeof(command.nowInSec())
+ + ColumnFilter.serializer.serializedSize(command.columnFilter(), version)
+ + RowFilter.serializer.serializedSize(command.rowFilter(), version)
+ + DataLimits.serializer.serializedSize(command.limits(), version)
+ + command.selectionSerializedSize(version)
+ + command.indexSerializedSize(version);
+ }
+ }
+
+ private enum LegacyType
+ {
+ GET_BY_NAMES((byte)1),
+ GET_SLICES((byte)2);
+
+ public final byte serializedValue;
+
+ LegacyType(byte b)
+ {
+ this.serializedValue = b;
+ }
+
+ public static LegacyType fromPartitionFilterKind(ClusteringIndexFilter.Kind kind)
+ {
+ return kind == ClusteringIndexFilter.Kind.SLICE
+ ? GET_SLICES
+ : GET_BY_NAMES;
+ }
+
+ public static LegacyType fromSerializedValue(byte b)
+ {
+ return b == 1 ? GET_BY_NAMES : GET_SLICES;
+ }
+ }
+
+ /**
+ * Serializer for pre-3.0 RangeSliceCommands.
+ */
+ private static class LegacyRangeSliceCommandSerializer implements IVersionedSerializer<ReadCommand>
{
- out.writeByte(command.commandType.serializedValue);
- switch (command.commandType)
+ public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException
+ {
+ assert version < MessagingService.VERSION_30;
+
+ PartitionRangeReadCommand rangeCommand = (PartitionRangeReadCommand) command;
+ assert !rangeCommand.dataRange().isPaging();
+
+ // convert pre-3.0 incompatible names filters to slice filters
+ rangeCommand = maybeConvertNamesToSlice(rangeCommand);
+
+ CFMetaData metadata = rangeCommand.metadata();
+
+ out.writeUTF(metadata.ksName);
+ out.writeUTF(metadata.cfName);
+ out.writeLong(rangeCommand.nowInSec() * 1000L); // convert from seconds to millis
+
+ // begin DiskAtomFilterSerializer.serialize()
+ if (rangeCommand.isNamesQuery())
+ {
+ out.writeByte(1); // 0 for slices, 1 for names
+ ClusteringIndexNamesFilter filter = (ClusteringIndexNamesFilter) rangeCommand.dataRange().clusteringIndexFilter;
+ LegacyReadCommandSerializer.serializeNamesFilter(rangeCommand, filter, out);
+ }
+ else
+ {
+ out.writeByte(0); // 0 for slices, 1 for names
+
+ // slice filter serialization
+ ClusteringIndexSliceFilter filter = (ClusteringIndexSliceFilter) rangeCommand.dataRange().clusteringIndexFilter;
+
+ boolean makeStaticSlice = !rangeCommand.columnFilter().fetchedColumns().statics.isEmpty() && !filter.requestedSlices().selects(Clustering.STATIC_CLUSTERING);
+ LegacyReadCommandSerializer.serializeSlices(out, filter.requestedSlices(), filter.isReversed(), makeStaticSlice, metadata);
+
+ out.writeBoolean(filter.isReversed());
+
+ // limit
+ DataLimits limits = rangeCommand.limits();
+ if (limits.isDistinct())
+ out.writeInt(1);
+ else
+ out.writeInt(LegacyReadCommandSerializer.updateLimitForQuery(rangeCommand.limits().count(), filter.requestedSlices()));
+
+ int compositesToGroup;
+ boolean selectsStatics = !rangeCommand.columnFilter().fetchedColumns().statics.isEmpty() && filter.requestedSlices().selects(Clustering.STATIC_CLUSTERING);
+ if (limits.kind() == DataLimits.Kind.THRIFT_LIMIT)
+ compositesToGroup = -1;
+ else if (limits.isDistinct() && !selectsStatics)
+ compositesToGroup = -2; // for DISTINCT queries (CASSANDRA-8490)
+ else
+ compositesToGroup = metadata.isDense() ? -1 : metadata.clusteringColumns().size();
+
+ out.writeInt(compositesToGroup);
+ }
+
+ serializeRowFilter(out, rangeCommand.rowFilter());
+ AbstractBounds.rowPositionSerializer.serialize(rangeCommand.dataRange().keyRange(), out, version);
+
+ // maxResults
+ out.writeInt(rangeCommand.limits().count());
+
+ // countCQL3Rows
+ if (rangeCommand.isForThrift() || rangeCommand.limits().perPartitionCount() == 1) // if for Thrift or DISTINCT
+ out.writeBoolean(false);
+ else
+ out.writeBoolean(true);
+
+ // isPaging
+ out.writeBoolean(false);
+ }
+
+ public ReadCommand deserialize(DataInputPlus in, int version) throws IOException
+ {
+ assert version < MessagingService.VERSION_30;
+
+ String keyspace = in.readUTF();
+ String columnFamily = in.readUTF();
+
+ CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnFamily);
+ if (metadata == null)
+ {
+ String message = String.format("Got legacy range command for nonexistent table %s.%s.", keyspace, columnFamily);
+ throw new UnknownColumnFamilyException(message, null);
+ }
+
+ int nowInSec = (int) (in.readLong() / 1000); // convert from millis to seconds
+
+ ClusteringIndexFilter filter;
+ ColumnFilter selection;
+ int compositesToGroup = 0;
+ int perPartitionLimit = -1;
+ byte readType = in.readByte(); // 0 for slices, 1 for names
+ if (readType == 1)
+ {
+ Pair<ColumnFilter, ClusteringIndexNamesFilter> selectionAndFilter = LegacyReadCommandSerializer.deserializeNamesSelectionAndFilter(in, metadata);
+ selection = selectionAndFilter.left;
+ filter = selectionAndFilter.right;
+ }
+ else
+ {
+ Pair<ClusteringIndexSliceFilter, Boolean> p = LegacyReadCommandSerializer.deserializeSlicePartitionFilter(in, metadata);
+ filter = p.left;
+ perPartitionLimit = in.readInt();
+ compositesToGroup = in.readInt();
+ selection = getColumnSelectionForSlice(p.right, compositesToGroup, metadata);
+ }
+
+ RowFilter rowFilter = deserializeRowFilter(in, metadata);
+
+ AbstractBounds<PartitionPosition> keyRange = AbstractBounds.rowPositionSerializer.deserialize(in, metadata.partitioner, version);
+ int maxResults = in.readInt();
+
+ boolean countCQL3Rows = in.readBoolean(); // countCQL3Rows (not needed)
+ in.readBoolean(); // isPaging (not needed)
+
+ boolean selectsStatics = (!selection.fetchedColumns().statics.isEmpty() || filter.selects(Clustering.STATIC_CLUSTERING));
+ // We have 2 types of DISTINCT queries: ones on only the partition key, and ones on the partition key and static columns. For the former,
+ // we can easily detect the case because compositeToGroup is -2 and that's the only case it can be that. The latter one is slightly less
+ // direct, but we know that on 2.1/2.2 queries, DISTINCT queries are the only CQL queries that have countCQL3Rows to false so we use
+ // that fact.
+ boolean isDistinct = compositesToGroup == -2 || (compositesToGroup != -1 && !countCQL3Rows);
+ DataLimits limits;
+ if (isDistinct)
+ limits = DataLimits.distinctLimits(maxResults);
+ else if (compositesToGroup == -1)
+ limits = DataLimits.thriftLimits(maxResults, perPartitionLimit);
+ else
+ limits = DataLimits.cqlLimits(maxResults);
+
+ return new PartitionRangeReadCommand(false, 0, true, metadata, nowInSec, selection, rowFilter, limits, new DataRange(keyRange, filter), Optional.empty());
+ }
+
+ static void serializeRowFilter(DataOutputPlus out, RowFilter rowFilter) throws IOException
+ {
+ ArrayList<RowFilter.Expression> indexExpressions = Lists.newArrayList(rowFilter.iterator());
+ out.writeInt(indexExpressions.size());
+ for (RowFilter.Expression expression : indexExpressions)
+ {
+ ByteBufferUtil.writeWithShortLength(expression.column().name.bytes, out);
+ expression.operator().writeTo(out);
+ ByteBufferUtil.writeWithShortLength(expression.getIndexValue(), out);
+ }
+ }
+
+ static RowFilter deserializeRowFilter(DataInputPlus in, CFMetaData metadata) throws IOException
+ {
+ int numRowFilters = in.readInt();
+ if (numRowFilters == 0)
+ return RowFilter.NONE;
+
+ RowFilter rowFilter = RowFilter.create(numRowFilters);
+ for (int i = 0; i < numRowFilters; i++)
+ {
+ ByteBuffer columnName = ByteBufferUtil.readWithShortLength(in);
+ ColumnDefinition column = metadata.getColumnDefinition(columnName);
+ Operator op = Operator.readFrom(in);
+ ByteBuffer indexValue = ByteBufferUtil.readWithShortLength(in);
+ rowFilter.add(column, op, indexValue);
+ }
+ return rowFilter;
+ }
+
+ static long serializedRowFilterSize(RowFilter rowFilter)
+ {
+ long size = TypeSizes.sizeof(0); // rowFilterCount
+ for (RowFilter.Expression expression : rowFilter)
+ {
+ size += ByteBufferUtil.serializedSizeWithShortLength(expression.column().name.bytes);
+ size += TypeSizes.sizeof(0); // operator int value
+ size += ByteBufferUtil.serializedSizeWithShortLength(expression.getIndexValue());
+ }
+ return size;
+ }
+
+ public long serializedSize(ReadCommand command, int version)
+ {
+ assert version < MessagingService.VERSION_30;
+ assert command.kind == Kind.PARTITION_RANGE;
+
+ PartitionRangeReadCommand rangeCommand = (PartitionRangeReadCommand) command;
+ rangeCommand = maybeConvertNamesToSlice(rangeCommand);
+ CFMetaData metadata = rangeCommand.metadata();
+
+ long size = TypeSizes.sizeof(metadata.ksName);
+ size += TypeSizes.sizeof(metadata.cfName);
+ size += TypeSizes.sizeof((long) rangeCommand.nowInSec());
+
+ size += 1; // single byte flag: 0 for slices, 1 for names
+ if (rangeCommand.isNamesQuery())
+ {
+ PartitionColumns columns = rangeCommand.columnFilter().fetchedColumns();
+ ClusteringIndexNamesFilter filter = (ClusteringIndexNamesFilter) rangeCommand.dataRange().clusteringIndexFilter;
+ size += LegacyReadCommandSerializer.serializedNamesFilterSize(filter, metadata, columns);
+ }
+ else
+ {
+ ClusteringIndexSliceFilter filter = (ClusteringIndexSliceFilter) rangeCommand.dataRange().clusteringIndexFilter;
+ boolean makeStaticSlice = !rangeCommand.columnFilter().fetchedColumns().statics.isEmpty() && !filter.requestedSlices().selects(Clustering.STATIC_CLUSTERING);
+ size += LegacyReadCommandSerializer.serializedSlicesSize(filter.requestedSlices(), makeStaticSlice, metadata);
+ size += TypeSizes.sizeof(filter.isReversed());
+ size += TypeSizes.sizeof(rangeCommand.limits().perPartitionCount());
+ size += TypeSizes.sizeof(0); // compositesToGroup
+ }
+
+ if (rangeCommand.rowFilter().equals(RowFilter.NONE))
+ {
+ size += TypeSizes.sizeof(0);
+ }
+ else
+ {
+ ArrayList<RowFilter.Expression> indexExpressions = Lists.newArrayList(rangeCommand.rowFilter().iterator());
+ size += TypeSizes.sizeof(indexExpressions.size());
+ for (RowFilter.Expression expression : indexExpressions)
+ {
+ size += ByteBufferUtil.serializedSizeWithShortLength(expression.column().name.bytes);
+ size += TypeSizes.sizeof(expression.operator().ordinal());
+ size += ByteBufferUtil.serializedSizeWithShortLength(expression.getIndexValue());
+ }
+ }
+
+ size += AbstractBounds.rowPositionSerializer.serializedSize(rangeCommand.dataRange().keyRange(), version);
+ size += TypeSizes.sizeof(rangeCommand.limits().count());
+ size += TypeSizes.sizeof(!rangeCommand.isForThrift());
+ return size + TypeSizes.sizeof(rangeCommand.dataRange().isPaging());
+ }
+
+ static PartitionRangeReadCommand maybeConvertNamesToSlice(PartitionRangeReadCommand command)
{
- case GET_BY_NAMES:
- SliceByNamesReadCommand.serializer.serialize(command, out, version);
- break;
- case GET_SLICES:
- SliceFromReadCommand.serializer.serialize(command, out, version);
- break;
- default:
- throw new AssertionError();
+ if (!command.dataRange().isNamesQuery())
+ return command;
+
+ CFMetaData metadata = command.metadata();
+ if (!LegacyReadCommandSerializer.shouldConvertNamesToSlice(metadata, command.columnFilter().fetchedColumns()))
+ return command;
+
+ ClusteringIndexNamesFilter filter = (ClusteringIndexNamesFilter) command.dataRange().clusteringIndexFilter;
+ ClusteringIndexSliceFilter sliceFilter = LegacyReadCommandSerializer.convertNamesFilterToSliceFilter(filter, metadata);
+ DataRange newRange = new DataRange(command.dataRange().keyRange(), sliceFilter);
+ return new PartitionRangeReadCommand(
+ command.isDigestQuery(), command.digestVersion(), command.isForThrift(), metadata, command.nowInSec(),
+ command.columnFilter(), command.rowFilter(), command.limits(), newRange, Optional.empty());
+ }
+
+ static ColumnFilter getColumnSelectionForSlice(boolean selectsStatics, int compositesToGroup, CFMetaData metadata)
+ {
+ // A value of -2 indicates this is a DISTINCT query that doesn't select static columns, only partition keys.
+ // In that case, we'll basically be querying the first row of the partition, but we must make sure we include
+ // all columns so we get at least one cell if there is a live row as it would confuse pre-3.0 nodes otherwise.
+ if (compositesToGroup == -2)
+ return ColumnFilter.all(metadata);
+
+ // if a slice query from a pre-3.0 node doesn't cover statics, we shouldn't select them at all
+ PartitionColumns columns = selectsStatics
+ ? metadata.partitionColumns()
+ : metadata.partitionColumns().withoutStatics();
+ return ColumnFilter.selectionBuilder().addAll(columns).build();
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/db/ReadQuery.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ReadQuery.java
index 178ca7c,0000000..75ba8f5
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/ReadQuery.java
+++ b/src/java/org/apache/cassandra/db/ReadQuery.java
@@@ -1,141 -1,0 +1,153 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.service.pager.QueryPager;
+import org.apache.cassandra.service.pager.PagingState;
+
+/**
+ * Generic abstraction for read queries.
+ * <p>
+ * The main implementation of this is {@link ReadCommand} but we have this interface because
+ * {@link SinglePartitionReadCommand.Group} is also consider as a "read query" but is not a
+ * {@code ReadCommand}.
+ */
+public interface ReadQuery
+{
+ ReadQuery EMPTY = new ReadQuery()
+ {
+ public ReadOrderGroup startOrderGroup()
+ {
+ return ReadOrderGroup.emptyGroup();
+ }
+
+ public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState) throws RequestExecutionException
+ {
+ return EmptyIterators.partition();
+ }
+
+ public PartitionIterator executeInternal(ReadOrderGroup orderGroup)
+ {
+ return EmptyIterators.partition();
+ }
+
+ public DataLimits limits()
+ {
+ // What we return here doesn't matter much in practice. However, returning DataLimits.NONE means
+ // "no particular limit", which makes SelectStatement.execute() take the slightly more complex "paging"
+ // path. Not a big deal but it's easy enough to return a limit of 0 rows which avoids this.
+ return DataLimits.cqlLimits(0);
+ }
+
+ public QueryPager getPager(PagingState state, int protocolVersion)
+ {
+ return QueryPager.EMPTY;
+ }
+
+ public QueryPager getLocalPager()
+ {
+ return QueryPager.EMPTY;
+ }
+
+ public boolean selectsKey(DecoratedKey key)
+ {
+ return false;
+ }
+
+ public boolean selectsClustering(DecoratedKey key, Clustering clustering)
+ {
+ return false;
+ }
++
++ @Override
++ public boolean selectsFullPartition()
++ {
++ return false;
++ }
+ };
+
+ /**
+ * Starts a new read operation.
+ * <p>
+ * This must be called before {@link executeInternal} and passed to it to protect the read.
+ * The returned object <b>must</b> be closed on all path and it is thus strongly advised to
+ * use it in a try-with-ressource construction.
+ *
+ * @return a newly started order group for this {@code ReadQuery}.
+ */
+ public ReadOrderGroup startOrderGroup();
+
+ /**
+ * Executes the query at the provided consistency level.
+ *
+ * @param consistency the consistency level to achieve for the query.
+ * @param clientState the {@code ClientState} for the query. In practice, this can be null unless
+ * {@code consistency} is a serial consistency.
+ *
+ * @return the result of the query.
+ */
+ public PartitionIterator execute(ConsistencyLevel consistency, ClientState clientState) throws RequestExecutionException;
+
+ /**
+ * Execute the query for internal queries (that is, it basically executes the query locally).
+ *
+ * @param orderGroup the {@code ReadOrderGroup} protecting the read.
+ * @return the result of the query.
+ */
+ public PartitionIterator executeInternal(ReadOrderGroup orderGroup);
+
+ /**
+ * Returns a pager for the query.
+ *
+ * @param pagingState the {@code PagingState} to start from if this is a paging continuation. This can be
+ * {@code null} if this is the start of paging.
+ * @param protocolVersion the protocol version to use for the paging state of that pager.
+ *
+ * @return a pager for the query.
+ */
+ public QueryPager getPager(PagingState pagingState, int protocolVersion);
+
+ /**
+ * The limits for the query.
+ *
+ * @return The limits for the query.
+ */
+ public DataLimits limits();
+
+ /**
+ * @return true if the read query would select the given key, including checks against the row filter, if
+ * checkRowFilter is true
+ */
+ public boolean selectsKey(DecoratedKey key);
+
+ /**
+ * @return true if the read query would select the given clustering, including checks against the row filter, if
+ * checkRowFilter is true
+ */
+ public boolean selectsClustering(DecoratedKey key, Clustering clustering);
++
++ /**
++ * Checks if this {@code ReadQuery} selects full partitions, that is it has no filtering on clustering or regular columns.
++ * @return {@code true} if this {@code ReadQuery} selects full partitions, {@code false} otherwise.
++ */
++ public boolean selectsFullPartition();
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[7/7] cassandra git commit: Merge branch cassandra-3.0 into
cassandra-3.11
Posted by bl...@apache.org.
Merge branch cassandra-3.0 into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7aa89a64
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7aa89a64
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7aa89a64
Branch: refs/heads/cassandra-3.11
Commit: 7aa89a64e09c57061418c1d83c03ae7cfd0cd745
Parents: bd89f56 e2445cf
Author: Benjamin Lerer <b....@gmail.com>
Authored: Fri Jul 14 17:18:19 2017 +0200
Committer: Benjamin Lerer <b....@gmail.com>
Committed: Fri Jul 14 17:19:38 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 6 +-
.../apache/cassandra/db/ColumnFamilyStore.java | 3 +-
src/java/org/apache/cassandra/db/DataRange.java | 5 +
.../cassandra/db/PartitionRangeReadCommand.java | 6 +
.../org/apache/cassandra/db/ReadCommand.java | 2 +-
src/java/org/apache/cassandra/db/ReadQuery.java | 12 +
.../db/SinglePartitionReadCommand.java | 22 +-
.../apache/cassandra/db/filter/DataLimits.java | 82 +++---
.../apache/cassandra/db/filter/RowFilter.java | 15 +
.../apache/cassandra/service/CacheService.java | 2 +-
.../apache/cassandra/service/DataResolver.java | 4 +-
.../apache/cassandra/service/StorageProxy.java | 8 +-
.../service/pager/AbstractQueryPager.java | 2 +-
.../service/pager/MultiPartitionPager.java | 9 +-
.../cassandra/service/pager/QueryPagers.java | 2 +-
.../org/apache/cassandra/cql3/CQLTester.java | 8 +-
.../validation/operations/SelectLimitTest.java | 279 +++++++++++++++++++
.../db/rows/UnfilteredRowIteratorsTest.java | 10 +-
18 files changed, 418 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index e7ad6fb,c916452..edd66e2
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -6,53 -2,16 +6,53 @@@ Merged from 3.0
* Make concat work with iterators that have different subsets of columns (CASSANDRA-13482)
* Set test.runners based on cores and memory size (CASSANDRA-13078)
* Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557)
- * Allow native function calls in CQLSSTableWriter (CASSANDRA-12606)
* Fix secondary index queries on COMPACT tables (CASSANDRA-13627)
* Nodetool listsnapshots output is missing a newline, if there are no snapshots (CASSANDRA-13568)
- Merged from 2.2:
- * Fix queries with LIMIT and filtering on clustering columns (CASSANDRA-11223)
- * Fix potential NPE when resume bootstrap fails (CASSANDRA-13272)
- * Fix toJSONString for the UDT, tuple and collection types (CASSANDRA-13592)
- * Fix nested Tuples/UDTs validation (CASSANDRA-13646)
+Merged from 2.2:
- * Fix potential NPE when resume bootstrap fails (CASSANDRA-13272)
- * Fix toJSONString for the UDT, tuple and collection types (CASSANDRA-13592)
- * Fix nested Tuples/UDTs validation (CASSANDRA-13646)
++ * Fix queries with LIMIT and filtering on clustering columns (CASSANDRA-11223)
++ * Fix potential NPE when resume bootstrap fails (CASSANDRA-13272)
++ * Fix toJSONString for the UDT, tuple and collection types (CASSANDRA-13592) * Fix nested Tuples/UDTs validation (CASSANDRA-13646)
-3.0.14
+3.11.0
+ * Allow native function calls in CQLSSTableWriter (CASSANDRA-12606)
+ * Replace string comparison with regex/number checks in MessagingService test (CASSANDRA-13216)
+ * Fix formatting of duration columns in CQLSH (CASSANDRA-13549)
+ * Fix the problem with duplicated rows when using paging with SASI (CASSANDRA-13302)
+ * Allow CONTAINS statements filtering on the partition key and it’s parts (CASSANDRA-13275)
+ * Fall back to even ranges calculation in clusters with vnodes when tokens are distributed unevenly (CASSANDRA-13229)
+ * Fix duration type validation to prevent overflow (CASSANDRA-13218)
+ * Forbid unsupported creation of SASI indexes over partition key columns (CASSANDRA-13228)
+ * Reject multiple values for a key in CQL grammar. (CASSANDRA-13369)
+ * UDA fails without input rows (CASSANDRA-13399)
+ * Fix compaction-stress by using daemonInitialization (CASSANDRA-13188)
+ * V5 protocol flags decoding broken (CASSANDRA-13443)
+ * Use write lock not read lock for removing sstables from compaction strategies. (CASSANDRA-13422)
+ * Use corePoolSize equal to maxPoolSize in JMXEnabledThreadPoolExecutors (CASSANDRA-13329)
+ * Avoid rebuilding SASI indexes containing no values (CASSANDRA-12962)
+ * Add charset to Analyser input stream (CASSANDRA-13151)
+ * Fix testLimitSSTables flake caused by concurrent flush (CASSANDRA-12820)
+ * cdc column addition strikes again (CASSANDRA-13382)
+ * Fix static column indexes (CASSANDRA-13277)
+ * DataOutputBuffer.asNewBuffer broken (CASSANDRA-13298)
+ * unittest CipherFactoryTest failed on MacOS (CASSANDRA-13370)
+ * Forbid SELECT restrictions and CREATE INDEX over non-frozen UDT columns (CASSANDRA-13247)
+ * Default logging we ship will incorrectly print "?:?" for "%F:%L" pattern (CASSANDRA-13317)
+ * Possible AssertionError in UnfilteredRowIteratorWithLowerBound (CASSANDRA-13366)
+ * Support unaligned memory access for AArch64 (CASSANDRA-13326)
+ * Improve SASI range iterator efficiency on intersection with an empty range (CASSANDRA-12915).
+ * Fix equality comparisons of columns using the duration type (CASSANDRA-13174)
+ * Obfuscate password in stress-graphs (CASSANDRA-12233)
+ * Move to FastThreadLocalThread and FastThreadLocal (CASSANDRA-13034)
+ * nodetool stopdaemon errors out (CASSANDRA-13030)
+ * Tables in system_distributed should not use gcgs of 0 (CASSANDRA-12954)
+ * Fix primary index calculation for SASI (CASSANDRA-12910)
+ * More fixes to the TokenAllocator (CASSANDRA-12990)
+ * NoReplicationTokenAllocator should work with zero replication factor (CASSANDRA-12983)
+ * Address message coalescing regression (CASSANDRA-12676)
+ * Delete illegal character from StandardTokenizerImpl.jflex (CASSANDRA-13417)
+ * Fix cqlsh automatic protocol downgrade regression (CASSANDRA-13307)
+ * Tracing payload not passed from QueryMessage to tracing session (CASSANDRA-12835)
+Merged from 3.0:
* Ensure int overflow doesn't occur when calculating large partition warning size (CASSANDRA-13172)
* Ensure consistent view of partition columns between coordinator and replica in ColumnFilter (CASSANDRA-13004)
* Failed unregistering mbean during drop keyspace (CASSANDRA-13346)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/db/ReadCommand.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/db/ReadQuery.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
index 7d72212,319eeb4..11444db
--- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
@@@ -479,6 -462,6 +479,7 @@@ public class SinglePartitionReadComman
// We want to cache only rowsToCache rows
CachedPartition toCache = CachedBTreePartition.create(toCacheIterator, nowInSec());
++
if (sentinelSuccess && !toCache.isEmpty())
{
Tracing.trace("Caching {} rows", toCache.rowCount());
@@@ -1047,47 -1015,32 +1057,55 @@@
return commands.get(0).metadata();
}
+ @Override
+ public boolean selectsFullPartition()
+ {
+ return selectsFullPartitions;
+ }
+
- public ReadOrderGroup startOrderGroup()
+ public ReadExecutionController executionController()
{
// Note that the only difference between the command in a group must be the partition key on which
// they applied. So as far as ReadOrderGroup is concerned, we can use any of the commands to start one.
- return commands.get(0).startOrderGroup();
+ return commands.get(0).executionController();
}
- public PartitionIterator executeInternal(ReadOrderGroup orderGroup)
+ public PartitionIterator executeInternal(ReadExecutionController controller)
{
- return limits.filter(UnfilteredPartitionIterators.filter(executeLocally(controller, false), nowInSec), nowInSec);
- List<PartitionIterator> partitions = new ArrayList<>(commands.size());
- for (SinglePartitionReadCommand cmd : commands)
- partitions.add(cmd.executeInternal(orderGroup));
-
- // Because we only have enforce the limit per command, we need to enforce it globally.
- return limits.filter(PartitionIterators.concat(partitions),
++ return limits.filter(UnfilteredPartitionIterators.filter(executeLocally(controller, false), nowInSec),
+ nowInSec,
+ selectsFullPartitions);
}
- public QueryPager getPager(PagingState pagingState, int protocolVersion)
+ public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController)
+ {
+ return executeLocally(executionController, true);
+ }
+
+ /**
+ * Implementation of {@link ReadQuery#executeLocally(ReadExecutionController)}.
+ *
+ * @param executionController - the {@code ReadExecutionController} protecting the read.
+ * @param sort - whether to sort the inner commands by partition key, required for merging the iterator
+ * later on. This will be false when called by {@link ReadQuery#executeInternal(ReadExecutionController)}
+ * because in this case it is safe to do so as there is no merging involved and we don't want to
+ * change the old behavior which was to not sort by partition.
+ *
+ * @return - the iterator that can be used to retrieve the query result.
+ */
+ private UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController, boolean sort)
+ {
+ List<Pair<DecoratedKey, UnfilteredPartitionIterator>> partitions = new ArrayList<>(commands.size());
+ for (SinglePartitionReadCommand cmd : commands)
+ partitions.add(Pair.of(cmd.partitionKey, cmd.executeLocally(executionController)));
+
+ if (sort)
+ Collections.sort(partitions, (p1, p2) -> p1.getLeft().compareTo(p2.getLeft()));
+
+ return UnfilteredPartitionIterators.concat(partitions.stream().map(p -> p.getRight()).collect(Collectors.toList()));
+ }
+
+ public QueryPager getPager(PagingState pagingState, ProtocolVersion protocolVersion)
{
if (commands.size() == 1)
return SinglePartitionReadCommand.getPager(commands.get(0), pagingState, protocolVersion);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/db/filter/DataLimits.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/filter/DataLimits.java
index abe029b,48ec06a..0c8cd37
--- a/src/java/org/apache/cassandra/db/filter/DataLimits.java
+++ b/src/java/org/apache/cassandra/db/filter/DataLimits.java
@@@ -135,18 -109,7 +139,18 @@@ public abstract class DataLimit
public abstract DataLimits forShortReadRetry(int toFetch);
+ /**
+ * Creates a <code>DataLimits</code> instance to be used for paginating internally GROUP BY queries.
+ *
+ * @param state the <code>GroupMaker</code> state
+ * @return a <code>DataLimits</code> instance to be used for paginating internally GROUP BY queries
+ */
+ public DataLimits forGroupByInternalPaging(GroupingState state)
+ {
+ throw new UnsupportedOperationException();
+ }
+
- public abstract boolean hasEnoughLiveData(CachedPartition cached, int nowInSec);
+ public abstract boolean hasEnoughLiveData(CachedPartition cached, int nowInSec, boolean countPartitionsWithOnlyStaticData);
/**
* Returns a new {@code Counter} for this limits.
@@@ -171,25 -136,23 +177,29 @@@
public abstract int perPartitionCount();
+ /**
+ * Returns equivalent limits but where any internal state kept to track where we are of paging and/or grouping is
+ * discarded.
+ */
+ public abstract DataLimits withoutState();
+
- public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter, int nowInSec)
+ public UnfilteredPartitionIterator filter(UnfilteredPartitionIterator iter,
+ int nowInSec,
+ boolean countPartitionsWithOnlyStaticData)
{
- return this.newCounter(nowInSec, false).applyTo(iter);
+ return this.newCounter(nowInSec, false, countPartitionsWithOnlyStaticData).applyTo(iter);
}
- public UnfilteredRowIterator filter(UnfilteredRowIterator iter, int nowInSec)
+ public UnfilteredRowIterator filter(UnfilteredRowIterator iter,
+ int nowInSec,
+ boolean countPartitionsWithOnlyStaticData)
{
- return this.newCounter(nowInSec, false).applyTo(iter);
+ return this.newCounter(nowInSec, false, countPartitionsWithOnlyStaticData).applyTo(iter);
}
- public PartitionIterator filter(PartitionIterator iter, int nowInSec)
+ public PartitionIterator filter(PartitionIterator iter, int nowInSec, boolean countPartitionsWithOnlyStaticData)
{
- return this.newCounter(nowInSec, true).applyTo(iter);
+ return this.newCounter(nowInSec, true, countPartitionsWithOnlyStaticData).applyTo(iter);
}
/**
@@@ -429,14 -352,20 +439,16 @@@
protected class CQLCounter extends Counter
{
- protected final int nowInSec;
- protected final boolean assumeLiveData;
- protected final boolean countPartitionsWithOnlyStaticData;
-
protected int rowCounted;
protected int rowInCurrentPartition;
++ protected final boolean countPartitionsWithOnlyStaticData;
protected boolean hasLiveStaticRow;
- public CQLCounter(int nowInSec, boolean assumeLiveData)
+ public CQLCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
{
- this.nowInSec = nowInSec;
- this.assumeLiveData = assumeLiveData;
+ super(nowInSec, assumeLiveData);
+ this.countPartitionsWithOnlyStaticData = countPartitionsWithOnlyStaticData;
}
@Override
@@@ -554,15 -473,9 +566,15 @@@
}
@Override
+ public DataLimits withoutState()
+ {
+ return new CQLLimits(rowLimit, perPartitionLimit, isDistinct);
+ }
+
+ @Override
- public Counter newCounter(int nowInSec, boolean assumeLiveData)
+ public Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
{
- return new PagingAwareCounter(nowInSec, assumeLiveData);
+ return new PagingAwareCounter(nowInSec, assumeLiveData, countPartitionsWithOnlyStaticData);
}
private class PagingAwareCounter extends CQLCounter
@@@ -593,499 -506,6 +605,502 @@@
}
/**
+ * <code>CQLLimits</code> used for GROUP BY queries or queries with aggregates.
+ * <p>Internally, GROUP BY queries are always paginated by number of rows to avoid OOMExceptions. By consequence,
+ * the limits keep track of the number of rows as well as the number of groups.</p>
+ * <p>A group can only be counted if the next group or the end of the data is reached.</p>
+ */
+ private static class CQLGroupByLimits extends CQLLimits
+ {
+ /**
+ * The <code>GroupMaker</code> state
+ */
+ protected final GroupingState state;
+
+ /**
+ * The GROUP BY specification
+ */
+ protected final AggregationSpecification groupBySpec;
+
+ /**
+ * The limit on the number of groups
+ */
+ protected final int groupLimit;
+
+ /**
+ * The limit on the number of groups per partition
+ */
+ protected final int groupPerPartitionLimit;
+
+ public CQLGroupByLimits(int groupLimit,
+ int groupPerPartitionLimit,
+ int rowLimit,
+ AggregationSpecification groupBySpec)
+ {
+ this(groupLimit, groupPerPartitionLimit, rowLimit, groupBySpec, GroupingState.EMPTY_STATE);
+ }
+
+ private CQLGroupByLimits(int groupLimit,
+ int groupPerPartitionLimit,
+ int rowLimit,
+ AggregationSpecification groupBySpec,
+ GroupingState state)
+ {
+ super(rowLimit, NO_LIMIT, false);
+ this.groupLimit = groupLimit;
+ this.groupPerPartitionLimit = groupPerPartitionLimit;
+ this.groupBySpec = groupBySpec;
+ this.state = state;
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.CQL_GROUP_BY_LIMIT;
+ }
+
+ @Override
+ public boolean isGroupByLimit()
+ {
+ return true;
+ }
+
+ public boolean isUnlimited()
+ {
+ return groupLimit == NO_LIMIT && groupPerPartitionLimit == NO_LIMIT && rowLimit == NO_LIMIT;
+ }
+
+ public DataLimits forShortReadRetry(int toFetch)
+ {
+ return new CQLLimits(toFetch);
+ }
+
+ @Override
+ public float estimateTotalResults(ColumnFamilyStore cfs)
+ {
+ // For the moment, we return the estimated number of rows as we have no good way of estimating
+ // the number of groups that will be returned. Hopefully, we should be able to fix
+ // that problem at some point.
+ return super.estimateTotalResults(cfs);
+ }
+
+ @Override
+ public DataLimits forPaging(int pageSize)
+ {
+ return new CQLGroupByLimits(pageSize,
+ groupPerPartitionLimit,
+ rowLimit,
+ groupBySpec,
+ state);
+ }
+
+ @Override
+ public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+ {
+ return new CQLGroupByPagingLimits(pageSize,
+ groupPerPartitionLimit,
+ rowLimit,
+ groupBySpec,
+ state,
+ lastReturnedKey,
+ lastReturnedKeyRemaining);
+ }
+
+ @Override
+ public DataLimits forGroupByInternalPaging(GroupingState state)
+ {
+ return new CQLGroupByLimits(rowLimit,
+ groupPerPartitionLimit,
+ rowLimit,
+ groupBySpec,
+ state);
+ }
+
+ @Override
- public Counter newCounter(int nowInSec, boolean assumeLiveData)
++ public Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
+ {
- return new GroupByAwareCounter(nowInSec, assumeLiveData);
++ return new GroupByAwareCounter(nowInSec, assumeLiveData, countPartitionsWithOnlyStaticData);
+ }
+
+ @Override
+ public int count()
+ {
+ return groupLimit;
+ }
+
+ @Override
+ public int perPartitionCount()
+ {
+ return groupPerPartitionLimit;
+ }
+
+ @Override
+ public DataLimits withoutState()
+ {
+ return state == GroupingState.EMPTY_STATE
+ ? this
+ : new CQLGroupByLimits(groupLimit, groupPerPartitionLimit, rowLimit, groupBySpec);
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+
+ if (groupLimit != NO_LIMIT)
+ {
+ sb.append("GROUP LIMIT ").append(groupLimit);
+ if (groupPerPartitionLimit != NO_LIMIT || rowLimit != NO_LIMIT)
+ sb.append(' ');
+ }
+
+ if (groupPerPartitionLimit != NO_LIMIT)
+ {
+ sb.append("GROUP PER PARTITION LIMIT ").append(groupPerPartitionLimit);
+ if (rowLimit != NO_LIMIT)
+ sb.append(' ');
+ }
+
+ if (rowLimit != NO_LIMIT)
+ {
+ sb.append("LIMIT ").append(rowLimit);
+ }
+
+ return sb.toString();
+ }
+
+ @Override
+ public boolean isExhausted(Counter counter)
+ {
+ return ((GroupByAwareCounter) counter).rowCounted < rowLimit
+ && counter.counted() < groupLimit;
+ }
+
+ protected class GroupByAwareCounter extends Counter
+ {
+ private final GroupMaker groupMaker;
+
++ protected final boolean countPartitionsWithOnlyStaticData;
++
+ /**
+ * The key of the partition being processed.
+ */
+ protected DecoratedKey currentPartitionKey;
+
+ /**
+ * The number of rows counted so far.
+ */
+ protected int rowCounted;
+
+ /**
+ * The number of rows counted so far in the current partition.
+ */
+ protected int rowCountedInCurrentPartition;
+
+ /**
+ * The number of groups counted so far. A group is counted only once it is complete
+ * (e.g the next one has been reached).
+ */
+ protected int groupCounted;
+
+ /**
+ * The number of groups in the current partition.
+ */
+ protected int groupInCurrentPartition;
+
+ protected boolean hasGroupStarted;
+
+ protected boolean hasLiveStaticRow;
+
+ protected boolean hasReturnedRowsFromCurrentPartition;
+
- private GroupByAwareCounter(int nowInSec, boolean assumeLiveData)
++ private GroupByAwareCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
+ {
+ super(nowInSec, assumeLiveData);
+ this.groupMaker = groupBySpec.newGroupMaker(state);
++ this.countPartitionsWithOnlyStaticData = countPartitionsWithOnlyStaticData;
+
+ // If the end of the partition was reached at the same time than the row limit, the last group might
+ // not have been counted yet. Due to that we need to guess, based on the state, if the previous group
+ // is still open.
+ hasGroupStarted = state.hasClustering();
+ }
+
+ @Override
+ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
+ {
+ if (partitionKey.getKey().equals(state.partitionKey()))
+ {
+ // The only case were we could have state.partitionKey() equals to the partition key
+ // is if some of the partition rows have been returned in the previous page but the
+ // partition was not exhausted (as the state partition key has not been updated yet).
+ // Since we know we have returned rows, we know we have accounted for
+ // the static row if any already, so force hasLiveStaticRow to false so we make sure to not count it
+ // once more.
+ hasLiveStaticRow = false;
+ hasReturnedRowsFromCurrentPartition = true;
+ hasGroupStarted = true;
+ }
+ else
+ {
+ // We need to increment our count of groups if we have reached a new one and unless we had no new
+ // content added since we closed our last group (that is, if hasGroupStarted). Note that we may get
+ // here with hasGroupStarted == false in the following cases:
+ // * the partition limit was reached for the previous partition
+ // * the previous partition was containing only one static row
+ // * the rows of the last group of the previous partition were all marked as deleted
+ if (hasGroupStarted && groupMaker.isNewGroup(partitionKey, Clustering.STATIC_CLUSTERING))
+ {
+ incrementGroupCount();
+ // If we detect, before starting the new partition, that we are done, we need to increase
+ // the per partition group count of the previous partition as the next page will start from
+ // there.
+ if (isDone())
+ incrementGroupInCurrentPartitionCount();
+ hasGroupStarted = false;
+ }
+ hasReturnedRowsFromCurrentPartition = false;
+ hasLiveStaticRow = !staticRow.isEmpty() && isLive(staticRow);
+ }
+ currentPartitionKey = partitionKey;
+ // If we are done we need to preserve the groupInCurrentPartition and rowCountedInCurrentPartition
+ // because the pager need to retrieve the count associated to the last value it has returned.
+ if (!isDone())
+ {
+ groupInCurrentPartition = 0;
+ rowCountedInCurrentPartition = 0;
+ }
+ }
+
+ @Override
+ protected Row applyToStatic(Row row)
+ {
+ // It's possible that we're "done" if the partition we just started bumped the number of groups (in
+ // applyToPartition() above), in which case Transformation will still call this method. In that case, we
+ // want to ignore the static row, it should (and will) be returned with the next page/group if needs be.
+ if (isDone())
+ {
+ hasLiveStaticRow = false; // The row has not been returned
+ return Rows.EMPTY_STATIC_ROW;
+ }
+ return row;
+ }
+
+ @Override
+ public Row applyToRow(Row row)
+ {
+ // We want to check if the row belongs to a new group even if it has been deleted. The goal being
+ // to minimize the chances of having to go through the same data twice if we detect on the next
+ // non deleted row that we have reached the limit.
+ if (groupMaker.isNewGroup(currentPartitionKey, row.clustering()))
+ {
+ if (hasGroupStarted)
+ {
+ incrementGroupCount();
+ incrementGroupInCurrentPartitionCount();
+ }
+ hasGroupStarted = false;
+ }
+
+ // That row may have made us increment the group count, which may mean we're done for this partition, in
+ // which case we shouldn't count this row (it won't be returned).
+ if (isDoneForPartition())
+ {
+ hasGroupStarted = false;
+ return null;
+ }
+
+ if (isLive(row))
+ {
+ hasGroupStarted = true;
+ incrementRowCount();
+ hasReturnedRowsFromCurrentPartition = true;
+ }
+
+ return row;
+ }
+
+ @Override
+ public int counted()
+ {
+ return groupCounted;
+ }
+
+ @Override
+ public int countedInCurrentPartition()
+ {
+ return groupInCurrentPartition;
+ }
+
+ @Override
+ public int rowCounted()
+ {
+ return rowCounted;
+ }
+
+ @Override
+ public int rowCountedInCurrentPartition()
+ {
+ return rowCountedInCurrentPartition;
+ }
+
+ protected void incrementRowCount()
+ {
+ rowCountedInCurrentPartition++;
+ if (++rowCounted >= rowLimit)
+ stop();
+ }
+
+ private void incrementGroupCount()
+ {
+ groupCounted++;
+ if (groupCounted >= groupLimit)
+ stop();
+ }
+
+ private void incrementGroupInCurrentPartitionCount()
+ {
+ groupInCurrentPartition++;
+ if (groupInCurrentPartition >= groupPerPartitionLimit)
+ stopInPartition();
+ }
+
+ @Override
+ public boolean isDoneForPartition()
+ {
+ return isDone() || groupInCurrentPartition >= groupPerPartitionLimit;
+ }
+
+ @Override
+ public boolean isDone()
+ {
+ return groupCounted >= groupLimit;
+ }
+
+ @Override
+ public void onPartitionClose()
+ {
+ // Normally, we don't count static rows as from a CQL point of view, it will be merge with other
+ // rows in the partition. However, if we only have the static row, it will be returned as one group
+ // so count it.
- if (hasLiveStaticRow && !hasReturnedRowsFromCurrentPartition)
++ if (countPartitionsWithOnlyStaticData && hasLiveStaticRow && !hasReturnedRowsFromCurrentPartition)
+ {
+ incrementRowCount();
+ incrementGroupCount();
+ incrementGroupInCurrentPartitionCount();
+ hasGroupStarted = false;
+ }
+ super.onPartitionClose();
+ }
+
+ @Override
+ public void onClose()
+ {
+ // Groups are only counted when the end of the group is reached.
+ // The end of a group is detected by 2 ways:
+ // 1) a new group is reached
+ // 2) the end of the data is reached
+ // We know that the end of the data is reached if the group limit has not been reached
+ // and the number of rows counted is smaller than the internal page size.
+ if (hasGroupStarted && groupCounted < groupLimit && rowCounted < rowLimit)
+ {
+ incrementGroupCount();
+ incrementGroupInCurrentPartitionCount();
+ }
+
+ super.onClose();
+ }
+ }
+ }
+
+ private static class CQLGroupByPagingLimits extends CQLGroupByLimits
+ {
+ private final ByteBuffer lastReturnedKey;
+
+ private final int lastReturnedKeyRemaining;
+
+ public CQLGroupByPagingLimits(int groupLimit,
+ int groupPerPartitionLimit,
+ int rowLimit,
+ AggregationSpecification groupBySpec,
+ GroupingState state,
+ ByteBuffer lastReturnedKey,
+ int lastReturnedKeyRemaining)
+ {
+ super(groupLimit,
+ groupPerPartitionLimit,
+ rowLimit,
+ groupBySpec,
+ state);
+
+ this.lastReturnedKey = lastReturnedKey;
+ this.lastReturnedKeyRemaining = lastReturnedKeyRemaining;
+ }
+
+ @Override
+ public Kind kind()
+ {
+ return Kind.CQL_GROUP_BY_PAGING_LIMIT;
+ }
+
+ @Override
+ public DataLimits forPaging(int pageSize)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataLimits forPaging(int pageSize, ByteBuffer lastReturnedKey, int lastReturnedKeyRemaining)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataLimits forGroupByInternalPaging(GroupingState state)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
- public Counter newCounter(int nowInSec, boolean assumeLiveData)
++ public Counter newCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
+ {
+ assert state == GroupingState.EMPTY_STATE || lastReturnedKey.equals(state.partitionKey());
- return new PagingGroupByAwareCounter(nowInSec, assumeLiveData);
++ return new PagingGroupByAwareCounter(nowInSec, assumeLiveData, countPartitionsWithOnlyStaticData);
+ }
+
+ @Override
+ public DataLimits withoutState()
+ {
+ return new CQLGroupByLimits(groupLimit, groupPerPartitionLimit, rowLimit, groupBySpec);
+ }
+
+ private class PagingGroupByAwareCounter extends GroupByAwareCounter
+ {
- private PagingGroupByAwareCounter(int nowInSec, boolean assumeLiveData)
++ private PagingGroupByAwareCounter(int nowInSec, boolean assumeLiveData, boolean countPartitionsWithOnlyStaticData)
+ {
- super(nowInSec, assumeLiveData);
++ super(nowInSec, assumeLiveData, countPartitionsWithOnlyStaticData);
+ }
+
+ @Override
+ public void applyToPartition(DecoratedKey partitionKey, Row staticRow)
+ {
+ if (partitionKey.getKey().equals(lastReturnedKey))
+ {
+ currentPartitionKey = partitionKey;
+ groupInCurrentPartition = groupPerPartitionLimit - lastReturnedKeyRemaining;
+ hasReturnedRowsFromCurrentPartition = true;
+ hasLiveStaticRow = false;
+ hasGroupStarted = state.hasClustering();
+ }
+ else
+ {
+ super.applyToPartition(partitionKey, staticRow);
+ }
+ }
+ }
+ }
+
+ /**
* Limits used by thrift; this count partition and cells.
*/
private static class ThriftLimits extends DataLimits
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/db/filter/RowFilter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/service/CacheService.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/CacheService.java
index a4e18c0,d23bdb0..e209450
--- a/src/java/org/apache/cassandra/service/CacheService.java
+++ b/src/java/org/apache/cassandra/service/CacheService.java
@@@ -439,10 -441,9 +439,10 @@@ public class CacheService implements Ca
{
DecoratedKey key = cfs.decorateKey(buffer);
int nowInSec = FBUtilities.nowInSeconds();
- try (OpOrder.Group op = cfs.readOrdering.start(); UnfilteredRowIterator iter = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata, nowInSec, key).queryMemtableAndDisk(cfs, op))
+ SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata, nowInSec, key);
+ try (ReadExecutionController controller = cmd.executionController(); UnfilteredRowIterator iter = cmd.queryMemtableAndDisk(cfs, controller))
{
- CachedPartition toCache = CachedBTreePartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec), nowInSec);
+ CachedPartition toCache = CachedBTreePartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec, true), nowInSec);
return Pair.create(new RowCacheKey(cfs.metadata.ksAndCFName, key), (IRowCacheEntry)toCache);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/service/DataResolver.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/DataResolver.java
index f1c4dd1,c96a893..0dd1e7e
--- a/src/java/org/apache/cassandra/service/DataResolver.java
+++ b/src/java/org/apache/cassandra/service/DataResolver.java
@@@ -406,14 -390,12 +406,14 @@@ public class DataResolver extends Respo
private final InetAddress source;
private final DataLimits.Counter counter;
private final DataLimits.Counter postReconciliationCounter;
+ private final long queryStartNanoTime;
- private ShortReadProtection(InetAddress source, DataLimits.Counter postReconciliationCounter)
+ private ShortReadProtection(InetAddress source, DataLimits.Counter postReconciliationCounter, long queryStartNanoTime)
{
this.source = source;
- this.counter = command.limits().newCounter(command.nowInSec(), false).onlyCount();
+ this.counter = command.limits().newCounter(command.nowInSec(), false, command.selectsFullPartition()).onlyCount();
this.postReconciliationCounter = postReconciliationCounter;
+ this.queryStartNanoTime = queryStartNanoTime;
}
@Override
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/service/StorageProxy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/StorageProxy.java
index 0da25a9,6610cf7..b8f87d9
--- a/src/java/org/apache/cassandra/service/StorageProxy.java
+++ b/src/java/org/apache/cassandra/service/StorageProxy.java
@@@ -2281,7 -2164,9 +2281,9 @@@ public class StorageProxy implements St
// Note that in general, a RangeCommandIterator will honor the command limit for each range, but will not enforce it globally.
- return command.limits().filter(command.postReconciliationProcessing(new RangeCommandIterator(ranges, command, concurrencyFactor, keyspace, consistencyLevel, queryStartNanoTime)), command.nowInSec());
- return command.limits().filter(command.postReconciliationProcessing(new RangeCommandIterator(ranges, command, concurrencyFactor, keyspace, consistencyLevel)),
++ return command.limits().filter(command.postReconciliationProcessing(new RangeCommandIterator(ranges, command, concurrencyFactor, keyspace, consistencyLevel, queryStartNanoTime)),
+ command.nowInSec(),
+ command.selectsFullPartition());
}
public Map<String, List<String>> getSchemaVersions()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
index da388d0,11bbc0e..9dae11c
--- a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
+++ b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
@@@ -84,27 -84,6 +85,31 @@@ public class MultiPartitionPager implem
remaining = state == null ? limit.count() : state.remaining;
}
- private MultiPartitionPager(SinglePartitionPager[] pagers, DataLimits limit, int nowInSec, int remaining, int current)
++ private MultiPartitionPager(SinglePartitionPager[] pagers,
++ DataLimits limit,
++ int nowInSec,
++ int remaining,
++ int current)
+ {
+ this.pagers = pagers;
+ this.limit = limit;
+ this.nowInSec = nowInSec;
+ this.remaining = remaining;
+ this.current = current;
+ }
+
+ public QueryPager withUpdatedLimit(DataLimits newLimits)
+ {
+ SinglePartitionPager[] newPagers = Arrays.copyOf(pagers, pagers.length);
+ newPagers[current] = newPagers[current].withUpdatedLimit(newLimits);
+
+ return new MultiPartitionPager(newPagers,
+ newLimits,
+ nowInSec,
+ remaining,
+ current);
+ }
+
public PagingState state()
{
// Sets current to the first non-exhausted pager
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/src/java/org/apache/cassandra/service/pager/QueryPagers.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/pager/QueryPagers.java
index 15311ab,c26bf3f..1b56417
--- a/src/java/org/apache/cassandra/service/pager/QueryPagers.java
+++ b/src/java/org/apache/cassandra/service/pager/QueryPagers.java
@@@ -54,9 -53,9 +54,9 @@@ public class QueryPager
int count = 0;
while (!pager.isExhausted())
{
- try (PartitionIterator iter = pager.fetchPage(pageSize, consistencyLevel, state))
+ try (PartitionIterator iter = pager.fetchPage(pageSize, consistencyLevel, state, queryStartNanoTime))
{
- DataLimits.Counter counter = limits.newCounter(nowInSec, true);
+ DataLimits.Counter counter = limits.newCounter(nowInSec, true, command.selectsFullPartition());
PartitionIterators.consume(counter.applyTo(iter));
count += counter.counted();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/test/unit/org/apache/cassandra/cql3/CQLTester.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/CQLTester.java
index 2dbcfff,ba23c67..2542db0
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@@ -724,17 -719,17 +724,17 @@@ public abstract class CQLTeste
return sessionNet(protocolVersion).execute(formatQuery(query), values);
}
- protected Session sessionNet()
+ protected com.datastax.driver.core.ResultSet executeNetWithPaging(String query, int pageSize) throws Throwable
{
- return sessionNet(getDefaultVersion());
+ return sessionNet().execute(new SimpleStatement(formatQuery(query)).setFetchSize(pageSize));
}
- protected com.datastax.driver.core.ResultSet executeNetWithPaging(String query, int pageSize) throws Throwable
+ protected Session sessionNet()
{
- return sessionNet().execute(new SimpleStatement(formatQuery(query)).setFetchSize(pageSize));
- return sessionNet(PROTOCOL_VERSIONS.get(PROTOCOL_VERSIONS.size() - 1));
++ return sessionNet(getDefaultVersion());
}
- protected Session sessionNet(int protocolVersion)
+ protected Session sessionNet(ProtocolVersion protocolVersion)
{
requireNetwork();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
index 21c48dd,7e90c0a..2442c2b
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/SelectLimitTest.java
@@@ -275,4 -165,257 +277,281 @@@ public class SelectLimitTest extends CQ
row(1, -1, 1, 1),
row(3, -1, 1, 1));
}
+
+ @Test
+ public void testFilteringOnClusteringColumnsWithLimitAndStaticColumns() throws Throwable
+ {
+ // With only one clustering column
+ createTable("CREATE TABLE %s (a int, b int, s int static, c int, primary key (a, b))"
- + " WITH caching = {'keys': 'ALL', 'rows_per_partition' : 'ALL'}");
++ + " WITH caching = {'keys': 'ALL', 'rows_per_partition' : 'ALL'}");
+
+ for (int i = 0; i < 4; i++)
+ {
+ execute("INSERT INTO %s (a, s) VALUES (?, ?)", i, i);
+ for (int j = 0; j < 3; j++)
+ if (!((i == 0 || i == 3) && j == 1))
+ execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", i, j, i + j);
+ }
+
+ beforeAndAfterFlush(() ->
+ {
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, 0, 0, 0),
+ row(0, 2, 0, 2),
+ row(1, 0, 1, 1),
+ row(1, 1, 1, 2),
+ row(1, 2, 1, 3),
+ row(2, 0, 2, 2),
+ row(2, 1, 2, 3),
+ row(2, 2, 2, 4),
+ row(3, 0, 3, 3),
+ row(3, 2, 3, 5));
+
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ // The problem was that the static row of the partition 0 used to be only filtered in SelectStatement and was
+ // by consequence counted as a row. In which case the query was returning one row less.
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
++
++ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 GROUP BY a LIMIT 2 ALLOW FILTERING", pageSize),
++ row(1, 1, 1, 2),
++ row(2, 1, 2, 3));
++
++ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b >= 1 AND b <= 1 GROUP BY a LIMIT 2 ALLOW FILTERING", pageSize),
++ row(1, 1, 1, 2),
++ row(2, 1, 2, 3));
+ }
+ });
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b = 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b = 1 ORDER BY b DESC LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3) AND b >= 1 AND b <= 1 ORDER BY b DESC LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ execute("SELECT * FROM %s WHERE a IN (0, 1, 2, 3)"); // Load all data in the row cache
+
+ // Partition range queries
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ // Multiple partitions queries
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2) AND b = 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (0, 1, 2) AND b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b >= 1 AND b <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 2),
+ row(2, 1, 2, 3));
+ }
+
+ // With multiple clustering columns
+ createTable("CREATE TABLE %s (a int, b int, c int, s int static, d int, primary key (a, b, c))"
+ + " WITH caching = {'keys': 'ALL', 'rows_per_partition' : 'ALL'}");
+
+ for (int i = 0; i < 3; i++)
+ {
+ execute("INSERT INTO %s (a, s) VALUES (?, ?)", i, i);
+ for (int j = 0; j < 3; j++)
+ if (!(i == 0 && j == 1))
+ execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", i, j, j, i + j);
+ }
+
+ beforeAndAfterFlush(() ->
+ {
+ assertRows(execute("SELECT * FROM %s"),
+ row(0, 0, 0, 0, 0),
+ row(0, 2, 2, 0, 2),
+ row(1, 0, 0, 1, 1),
+ row(1, 1, 1, 1, 2),
+ row(1, 2, 2, 1, 3),
+ row(2, 0, 0, 2, 2),
+ row(2, 1, 1, 2, 3),
+ row(2, 2, 2, 2, 4));
+
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
++
++ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 GROUP BY a, b LIMIT 2 ALLOW FILTERING", pageSize),
++ row(1, 1, 1, 1, 2),
++ row(2, 1, 1, 2, 3));
++
++ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 GROUP BY a, b LIMIT 2 ALLOW FILTERING", pageSize),
++ row(1, 1, 1, 1, 2),
++ row(2, 1, 1, 2, 3));
+ }
+ });
+
+ execute("SELECT * FROM %s WHERE a IN (0, 1, 2)"); // Load data in the row cache
+
+ assertRows(execute("SELECT * FROM %s WHERE b = 1 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 3, 4) AND b = 1 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRows(execute("SELECT * FROM %s WHERE a IN (1, 2, 3, 4) AND b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING"),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE b IN (1, 2, 3, 4) AND c >= 1 AND c <= 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 1, 1, 2),
+ row(2, 1, 1, 2, 3));
+ }
+ }
+
+ @Test
+ public void testIndexOnRegularColumnWithPartitionWithoutRows() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, c int, s int static, v int, PRIMARY KEY(pk, c))");
+ createIndex("CREATE INDEX ON %s (v)");
+
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 1, 9, 1);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 2, 9, 2);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 3, 1, 9, 1);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 4, 1, 9, 1);
+ flush();
+
+ assertRows(execute("SELECT * FROM %s WHERE v = ?", 1),
+ row(1, 1, 9, 1),
+ row(3, 1, 9, 1),
+ row(4, 1, 9, 1));
+
+ execute("DELETE FROM %s WHERE pk = ? and c = ?", 3, 1);
+
+ // Test without paging
+ assertRows(execute("SELECT * FROM %s WHERE v = ? LIMIT 2", 1),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+
++ assertRows(execute("SELECT * FROM %s WHERE v = ? GROUP BY pk LIMIT 2", 1),
++ row(1, 1, 9, 1),
++ row(4, 1, 9, 1));
++
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE v = 1 LIMIT 2", pageSize),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
++
++ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE v = 1 GROUP BY pk LIMIT 2", pageSize),
++ row(1, 1, 9, 1),
++ row(4, 1, 9, 1));
+ }
+ }
+
+ @Test
+ public void testFilteringWithPartitionWithoutRows() throws Throwable
+ {
+ createTable("CREATE TABLE %s (pk int, c int, s int static, v int, PRIMARY KEY(pk, c))");
+
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 1, 9, 1);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 1, 2, 9, 2);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 3, 1, 9, 1);
+ execute("INSERT INTO %s (pk, c, s, v) VALUES (?, ?, ?, ?)", 4, 1, 9, 1);
+ flush();
+
+ assertRows(execute("SELECT * FROM %s WHERE v = ? ALLOW FILTERING", 1),
+ row(1, 1, 9, 1),
+ row(3, 1, 9, 1),
+ row(4, 1, 9, 1));
+
+ execute("DELETE FROM %s WHERE pk = ? and c = ?", 3, 1);
+
+ // Test without paging
+ assertRows(execute("SELECT * FROM %s WHERE v = ? LIMIT 2 ALLOW FILTERING", 1),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+
+ assertRows(execute("SELECT * FROM %s WHERE pk IN ? AND v = ? LIMIT 2 ALLOW FILTERING", list(1, 3, 4), 1),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+
+ // Test with paging
+ for (int pageSize = 1; pageSize < 4; pageSize++)
+ {
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE v = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+
+ assertRowsNet(executeNetWithPaging("SELECT * FROM %s WHERE pk IN (1, 3, 4) AND v = 1 LIMIT 2 ALLOW FILTERING", pageSize),
+ row(1, 1, 9, 1),
+ row(4, 1, 9, 1));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7aa89a64/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[3/7] cassandra git commit: Merge branch cassandra-2.2 into
cassandra-3.0
Posted by bl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/service/DataResolver.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/DataResolver.java
index 60cfbba,0000000..c96a893
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/service/DataResolver.java
+++ b/src/java/org/apache/cassandra/service/DataResolver.java
@@@ -1,498 -1,0 +1,498 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service;
+
+import java.net.InetAddress;
+import java.util.*;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.concurrent.Stage;
+import org.apache.cassandra.concurrent.StageManager;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.transform.MoreRows;
+import org.apache.cassandra.db.transform.Transformation;
+import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.net.*;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class DataResolver extends ResponseResolver
+{
+ @VisibleForTesting
+ final List<AsyncOneResponse> repairResults = Collections.synchronizedList(new ArrayList<>());
+
+ public DataResolver(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, int maxResponseCount)
+ {
+ super(keyspace, command, consistency, maxResponseCount);
+ }
+
+ public PartitionIterator getData()
+ {
+ ReadResponse response = responses.iterator().next().payload;
+ return UnfilteredPartitionIterators.filter(response.makeIterator(command), command.nowInSec());
+ }
+
+ public PartitionIterator resolve()
+ {
+ // We could get more responses while this method runs, which is ok (we're happy to ignore any response not here
+ // at the beginning of this method), so grab the response count once and use that through the method.
+ int count = responses.size();
+ List<UnfilteredPartitionIterator> iters = new ArrayList<>(count);
+ InetAddress[] sources = new InetAddress[count];
+ for (int i = 0; i < count; i++)
+ {
+ MessageIn<ReadResponse> msg = responses.get(i);
+ iters.add(msg.payload.makeIterator(command));
+ sources[i] = msg.from;
+ }
+
+ // Even though every responses should honor the limit, we might have more than requested post reconciliation,
+ // so ensure we're respecting the limit.
- DataLimits.Counter counter = command.limits().newCounter(command.nowInSec(), true);
++ DataLimits.Counter counter = command.limits().newCounter(command.nowInSec(), true, command.selectsFullPartition());
+ return counter.applyTo(mergeWithShortReadProtection(iters, sources, counter));
+ }
+
+ public void compareResponses()
+ {
+ // We need to fully consume the results to trigger read repairs if appropriate
+ try (PartitionIterator iterator = resolve())
+ {
+ PartitionIterators.consume(iterator);
+ }
+ }
+
+ private PartitionIterator mergeWithShortReadProtection(List<UnfilteredPartitionIterator> results, InetAddress[] sources, DataLimits.Counter resultCounter)
+ {
+ // If we have only one results, there is no read repair to do and we can't get short reads
+ if (results.size() == 1)
+ return UnfilteredPartitionIterators.filter(results.get(0), command.nowInSec());
+
+ UnfilteredPartitionIterators.MergeListener listener = new RepairMergeListener(sources);
+
+ // So-called "short reads" stems from nodes returning only a subset of the results they have for a partition due to the limit,
+ // but that subset not being enough post-reconciliation. So if we don't have limit, don't bother.
+ if (!command.limits().isUnlimited())
+ {
+ for (int i = 0; i < results.size(); i++)
+ results.set(i, Transformation.apply(results.get(i), new ShortReadProtection(sources[i], resultCounter)));
+ }
+
+ return UnfilteredPartitionIterators.mergeAndFilter(results, command.nowInSec(), listener);
+ }
+
+ private class RepairMergeListener implements UnfilteredPartitionIterators.MergeListener
+ {
+ private final InetAddress[] sources;
+
+ public RepairMergeListener(InetAddress[] sources)
+ {
+ this.sources = sources;
+ }
+
+ public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions)
+ {
+ return new MergeListener(partitionKey, columns(versions), isReversed(versions));
+ }
+
+ private PartitionColumns columns(List<UnfilteredRowIterator> versions)
+ {
+ Columns statics = Columns.NONE;
+ Columns regulars = Columns.NONE;
+ for (UnfilteredRowIterator iter : versions)
+ {
+ if (iter == null)
+ continue;
+
+ PartitionColumns cols = iter.columns();
+ statics = statics.mergeTo(cols.statics);
+ regulars = regulars.mergeTo(cols.regulars);
+ }
+ return new PartitionColumns(statics, regulars);
+ }
+
+ private boolean isReversed(List<UnfilteredRowIterator> versions)
+ {
+ for (UnfilteredRowIterator iter : versions)
+ {
+ if (iter == null)
+ continue;
+
+ // Everything will be in the same order
+ return iter.isReverseOrder();
+ }
+
+ assert false : "Expected at least one iterator";
+ return false;
+ }
+
+ public void close()
+ {
+ try
+ {
+ FBUtilities.waitOnFutures(repairResults, DatabaseDescriptor.getWriteRpcTimeout());
+ }
+ catch (TimeoutException ex)
+ {
+ // We got all responses, but timed out while repairing
+ int blockFor = consistency.blockFor(keyspace);
+ if (Tracing.isTracing())
+ Tracing.trace("Timed out while read-repairing after receiving all {} data and digest responses", blockFor);
+ else
+ logger.debug("Timeout while read-repairing after receiving all {} data and digest responses", blockFor);
+
+ throw new ReadTimeoutException(consistency, blockFor-1, blockFor, true);
+ }
+ }
+
+ private class MergeListener implements UnfilteredRowIterators.MergeListener
+ {
+ private final DecoratedKey partitionKey;
+ private final PartitionColumns columns;
+ private final boolean isReversed;
+ private final PartitionUpdate[] repairs = new PartitionUpdate[sources.length];
+
+ private final Row.Builder[] currentRows = new Row.Builder[sources.length];
+ private final RowDiffListener diffListener;
+
+ // The partition level deletion for the merge row.
+ private DeletionTime partitionLevelDeletion;
+ // When merged has a currently open marker, its time. null otherwise.
+ private DeletionTime mergedDeletionTime;
+ // For each source, the time of the current deletion as known by the source.
+ private final DeletionTime[] sourceDeletionTime = new DeletionTime[sources.length];
+ // For each source, record if there is an open range to send as repair, and from where.
+ private final Slice.Bound[] markerToRepair = new Slice.Bound[sources.length];
+
+ public MergeListener(DecoratedKey partitionKey, PartitionColumns columns, boolean isReversed)
+ {
+ this.partitionKey = partitionKey;
+ this.columns = columns;
+ this.isReversed = isReversed;
+
+ this.diffListener = new RowDiffListener()
+ {
+ public void onPrimaryKeyLivenessInfo(int i, Clustering clustering, LivenessInfo merged, LivenessInfo original)
+ {
+ if (merged != null && !merged.equals(original))
+ currentRow(i, clustering).addPrimaryKeyLivenessInfo(merged);
+ }
+
+ public void onDeletion(int i, Clustering clustering, Row.Deletion merged, Row.Deletion original)
+ {
+ if (merged != null && !merged.equals(original))
+ currentRow(i, clustering).addRowDeletion(merged);
+ }
+
+ public void onComplexDeletion(int i, Clustering clustering, ColumnDefinition column, DeletionTime merged, DeletionTime original)
+ {
+ if (merged != null && !merged.equals(original))
+ currentRow(i, clustering).addComplexDeletion(column, merged);
+ }
+
+ public void onCell(int i, Clustering clustering, Cell merged, Cell original)
+ {
+ if (merged != null && !merged.equals(original))
+ currentRow(i, clustering).addCell(merged);
+ }
+
+ };
+ }
+
+ private PartitionUpdate update(int i)
+ {
+ if (repairs[i] == null)
+ repairs[i] = new PartitionUpdate(command.metadata(), partitionKey, columns, 1);
+ return repairs[i];
+ }
+
+ private Row.Builder currentRow(int i, Clustering clustering)
+ {
+ if (currentRows[i] == null)
+ {
+ currentRows[i] = BTreeRow.sortedBuilder();
+ currentRows[i].newRow(clustering);
+ }
+ return currentRows[i];
+ }
+
+ public void onMergedPartitionLevelDeletion(DeletionTime mergedDeletion, DeletionTime[] versions)
+ {
+ this.partitionLevelDeletion = mergedDeletion;
+ for (int i = 0; i < versions.length; i++)
+ {
+ if (mergedDeletion.supersedes(versions[i]))
+ update(i).addPartitionDeletion(mergedDeletion);
+ }
+ }
+
+ public void onMergedRows(Row merged, Row[] versions)
+ {
+ // If a row was shadowed post merged, it must be by a partition level or range tombstone, and we handle
+ // those case directly in their respective methods (in other words, it would be inefficient to send a row
+ // deletion as repair when we know we've already send a partition level or range tombstone that covers it).
+ if (merged.isEmpty())
+ return;
+
+ Rows.diff(diffListener, merged, versions);
+ for (int i = 0; i < currentRows.length; i++)
+ {
+ if (currentRows[i] != null)
+ update(i).add(currentRows[i].build());
+ }
+ Arrays.fill(currentRows, null);
+ }
+
+ private DeletionTime currentDeletion()
+ {
+ return mergedDeletionTime == null ? partitionLevelDeletion : mergedDeletionTime;
+ }
+
+ public void onMergedRangeTombstoneMarkers(RangeTombstoneMarker merged, RangeTombstoneMarker[] versions)
+ {
+ // The current deletion as of dealing with this marker.
+ DeletionTime currentDeletion = currentDeletion();
+
+ for (int i = 0; i < versions.length; i++)
+ {
+ RangeTombstoneMarker marker = versions[i];
+
+ // Update what the source now thinks is the current deletion
+ if (marker != null)
+ sourceDeletionTime[i] = marker.isOpen(isReversed) ? marker.openDeletionTime(isReversed) : null;
+
+ // If merged == null, some of the source is opening or closing a marker
+ if (merged == null)
+ {
+ // but if it's not this source, move to the next one
+ if (marker == null)
+ continue;
+
+ // We have a close and/or open marker for a source, with nothing corresponding in merged.
+ // Because merged is a superset, this imply that we have a current deletion (being it due to an
+ // early opening in merged or a partition level deletion) and that this deletion will still be
+ // active after that point. Further whatever deletion was open or is open by this marker on the
+ // source, that deletion cannot supersedes the current one.
+ //
+ // But while the marker deletion (before and/or after this point) cannot supersed the current
+ // deletion, we want to know if it's equal to it (both before and after), because in that case
+ // the source is up to date and we don't want to include repair.
+ //
+ // So in practice we have 2 possible case:
+ // 1) the source was up-to-date on deletion up to that point (markerToRepair[i] == null). Then
+ // it won't be from that point on unless it's a boundary and the new opened deletion time
+ // is also equal to the current deletion (note that this implies the boundary has the same
+ // closing and opening deletion time, which should generally not happen, but can due to legacy
+ // reading code not avoiding this for a while, see CASSANDRA-13237).
+ // 2) the source wasn't up-to-date on deletion up to that point (markerToRepair[i] != null), and
+ // it may now be (if it isn't we just have nothing to do for that marker).
+ assert !currentDeletion.isLive() : currentDeletion.toString();
+
+ if (markerToRepair[i] == null)
+ {
+ // Since there is an ongoing merged deletion, the only way we don't have an open repair for
+ // this source is that it had a range open with the same deletion as current and it's
+ // closing it.
+ assert marker.isClose(isReversed) && currentDeletion.equals(marker.closeDeletionTime(isReversed))
+ : String.format("currentDeletion=%s, marker=%s", currentDeletion, marker.toString(command.metadata()));
+
+ // and so unless it's a boundary whose opening deletion time is still equal to the current
+ // deletion (see comment above for why this can actually happen), we have to repair the source
+ // from that point on.
+ if (!(marker.isOpen(isReversed) && currentDeletion.equals(marker.openDeletionTime(isReversed))))
+ markerToRepair[i] = marker.closeBound(isReversed).invert();
+ }
+ // In case 2) above, we only have something to do if the source is up-to-date after that point
+ else if (marker.isOpen(isReversed) && currentDeletion.equals(marker.openDeletionTime(isReversed)))
+ {
+ closeOpenMarker(i, marker.openBound(isReversed).invert());
+ }
+ }
+ else
+ {
+ // We have a change of current deletion in merged (potentially to/from no deletion at all).
+
+ if (merged.isClose(isReversed))
+ {
+ // We're closing the merged range. If we've marked the source as needing to be repaired for
+ // that range, close and add it to the repair to be sent.
+ if (markerToRepair[i] != null)
+ closeOpenMarker(i, merged.closeBound(isReversed));
+
+ }
+
+ if (merged.isOpen(isReversed))
+ {
+ // If we're opening a new merged range (or just switching deletion), then unless the source
+ // is up to date on that deletion (note that we've updated what the source deleteion is
+ // above), we'll have to sent the range to the source.
+ DeletionTime newDeletion = merged.openDeletionTime(isReversed);
+ DeletionTime sourceDeletion = sourceDeletionTime[i];
+ if (!newDeletion.equals(sourceDeletion))
+ markerToRepair[i] = merged.openBound(isReversed);
+ }
+ }
+ }
+
+ if (merged != null)
+ mergedDeletionTime = merged.isOpen(isReversed) ? merged.openDeletionTime(isReversed) : null;
+ }
+
+ private void closeOpenMarker(int i, Slice.Bound close)
+ {
+ Slice.Bound open = markerToRepair[i];
+ update(i).add(new RangeTombstone(Slice.make(isReversed ? close : open, isReversed ? open : close), currentDeletion()));
+ markerToRepair[i] = null;
+ }
+
+ public void close()
+ {
+ for (int i = 0; i < repairs.length; i++)
+ {
+ if (repairs[i] == null)
+ continue;
+
+ // use a separate verb here because we don't want these to be get the white glove hint-
+ // on-timeout behavior that a "real" mutation gets
+ Tracing.trace("Sending read-repair-mutation to {}", sources[i]);
+ MessageOut<Mutation> msg = new Mutation(repairs[i]).createMessage(MessagingService.Verb.READ_REPAIR);
+ repairResults.add(MessagingService.instance().sendRR(msg, sources[i]));
+ }
+ }
+ }
+ }
+
+ private class ShortReadProtection extends Transformation<UnfilteredRowIterator>
+ {
+ private final InetAddress source;
+ private final DataLimits.Counter counter;
+ private final DataLimits.Counter postReconciliationCounter;
+
+ private ShortReadProtection(InetAddress source, DataLimits.Counter postReconciliationCounter)
+ {
+ this.source = source;
- this.counter = command.limits().newCounter(command.nowInSec(), false).onlyCount();
++ this.counter = command.limits().newCounter(command.nowInSec(), false, command.selectsFullPartition()).onlyCount();
+ this.postReconciliationCounter = postReconciliationCounter;
+ }
+
+ @Override
+ public UnfilteredRowIterator applyToPartition(UnfilteredRowIterator partition)
+ {
+ partition = Transformation.apply(partition, counter);
+ // must apply and extend with same protection instance
+ ShortReadRowProtection protection = new ShortReadRowProtection(partition.metadata(), partition.partitionKey());
+ partition = MoreRows.extend(partition, protection);
+ partition = Transformation.apply(partition, protection); // apply after, so it is retained when we extend (in case we need to reextend)
+ return partition;
+ }
+
+ private class ShortReadRowProtection extends Transformation implements MoreRows<UnfilteredRowIterator>
+ {
+ final CFMetaData metadata;
+ final DecoratedKey partitionKey;
+ Clustering lastClustering;
+ int lastCount = 0;
+
+ private ShortReadRowProtection(CFMetaData metadata, DecoratedKey partitionKey)
+ {
+ this.metadata = metadata;
+ this.partitionKey = partitionKey;
+ }
+
+ @Override
+ public Row applyToRow(Row row)
+ {
+ lastClustering = row.clustering();
+ return row;
+ }
+
+ @Override
+ public UnfilteredRowIterator moreContents()
+ {
+ // We have a short read if the node this is the result of has returned the requested number of
+ // rows for that partition (i.e. it has stopped returning results due to the limit), but some of
+ // those results haven't made it in the final result post-reconciliation due to other nodes
+ // tombstones. If that is the case, then the node might have more results that we should fetch
+ // as otherwise we might return less results than required, or results that shouldn't be returned
+ // (because the node has tombstone that hides future results from other nodes but that haven't
+ // been returned due to the limit).
+ // Also note that we only get here once all the results for this node have been returned, and so
+ // if the node had returned the requested number but we still get there, it imply some results were
+ // skipped during reconciliation.
+ if (lastCount == counter.counted() || !counter.isDoneForPartition())
+ return null;
+ lastCount = counter.counted();
+
+ assert !postReconciliationCounter.isDoneForPartition();
+
+ // We need to try to query enough additional results to fulfill our query, but because we could still
+ // get short reads on that additional query, just querying the number of results we miss may not be
+ // enough. But we know that when this node answered n rows (counter.countedInCurrentPartition), only
+ // x rows (postReconciliationCounter.countedInCurrentPartition()) made it in the final result.
+ // So our ratio of live rows to requested rows is x/n, so since we miss n-x rows, we estimate that
+ // we should request m rows so that m * x/n = n-x, that is m = (n^2/x) - n.
+ // Also note that it's ok if we retrieve more results that necessary since our top level iterator is a
+ // counting iterator.
+ int n = postReconciliationCounter.countedInCurrentPartition();
+ int x = counter.countedInCurrentPartition();
+ int toQuery = Math.max(((n * n) / x) - n, 1);
+
+ DataLimits retryLimits = command.limits().forShortReadRetry(toQuery);
+ ClusteringIndexFilter filter = command.clusteringIndexFilter(partitionKey);
+ ClusteringIndexFilter retryFilter = lastClustering == null ? filter : filter.forPaging(metadata.comparator, lastClustering, false);
+ SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(command.metadata(),
+ command.nowInSec(),
+ command.columnFilter(),
+ command.rowFilter(),
+ retryLimits,
+ partitionKey,
+ retryFilter);
+
+ return doShortReadRetry(cmd);
+ }
+
+ private UnfilteredRowIterator doShortReadRetry(SinglePartitionReadCommand retryCommand)
+ {
+ DataResolver resolver = new DataResolver(keyspace, retryCommand, ConsistencyLevel.ONE, 1);
+ ReadCallback handler = new ReadCallback(resolver, ConsistencyLevel.ONE, retryCommand, Collections.singletonList(source));
+ if (StorageProxy.canDoLocalRequest(source))
+ StageManager.getStage(Stage.READ).maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(retryCommand, handler));
+ else
+ MessagingService.instance().sendRRWithFailure(retryCommand.createMessage(MessagingService.current_version), source, handler);
+
+ // We don't call handler.get() because we want to preserve tombstones since we're still in the middle of merging node results.
+ handler.awaitResults();
+ assert resolver.responses.size() == 1;
+ return UnfilteredPartitionIterators.getOnlyElement(resolver.responses.get(0).payload.makeIterator(command), retryCommand);
+ }
+ }
+ }
+
+ public boolean isDataPresent()
+ {
+ return !responses.isEmpty();
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/service/StorageProxy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/StorageProxy.java
index ea082d5,7b7979d..6610cf7
--- a/src/java/org/apache/cassandra/service/StorageProxy.java
+++ b/src/java/org/apache/cassandra/service/StorageProxy.java
@@@ -1598,14 -1358,11 +1598,14 @@@ public class StorageProxy implements St
throws UnavailableException, ReadFailureException, ReadTimeoutException
{
long start = System.nanoTime();
- List<Row> rows = null;
-
try
{
- rows = fetchRows(commands, consistencyLevel);
+ PartitionIterator result = fetchRows(group.commands, consistencyLevel);
+ // If we have more than one command, then despite each read command honoring the limit, the total result
+ // might not honor it and so we should enforce it
+ if (group.commands.size() > 1)
- result = group.limits().filter(result, group.nowInSec());
++ result = group.limits().filter(result, group.nowInSec(), group.selectsFullPartition());
+ return result;
}
catch (UnavailableException e)
{
@@@ -1953,218 -1716,267 +1953,220 @@@
}
}
- public static List<Row> getRangeSlice(AbstractRangeCommand command, ConsistencyLevel consistency_level)
- throws UnavailableException, ReadFailureException, ReadTimeoutException
+ private static class SingleRangeResponse extends AbstractIterator<RowIterator> implements PartitionIterator
{
- Tracing.trace("Computing ranges to query");
- long startTime = System.nanoTime();
+ private final ReadCallback handler;
+ private PartitionIterator result;
- Keyspace keyspace = Keyspace.open(command.keyspace);
- List<Row> rows;
- // now scan until we have enough results
- try
+ private SingleRangeResponse(ReadCallback handler)
{
- int liveRowCount = 0;
- boolean countLiveRows = command.countCQL3Rows() || command.ignoredTombstonedPartitions();
- rows = new ArrayList<>();
+ this.handler = handler;
+ }
- // when dealing with LocalStrategy keyspaces, we can skip the range splitting and merging (which can be
- // expensive in clusters with vnodes)
- List<? extends AbstractBounds<RowPosition>> ranges;
- if (keyspace.getReplicationStrategy() instanceof LocalStrategy)
- ranges = command.keyRange.unwrap();
- else
- ranges = getRestrictedRanges(command.keyRange);
-
- // determine the number of rows to be fetched and the concurrency factor
- int rowsToBeFetched = command.limit();
- int concurrencyFactor;
- if (command.requiresScanningAllRanges())
- {
- // all nodes must be queried
- rowsToBeFetched *= ranges.size();
- concurrencyFactor = ranges.size();
- logger.debug("Requested rows: {}, ranges.size(): {}; concurrent range requests: {}",
- command.limit(),
- ranges.size(),
- concurrencyFactor);
- Tracing.trace("Submitting range requests on {} ranges with a concurrency of {}",
- ranges.size(), concurrencyFactor);
+ private void waitForResponse() throws ReadTimeoutException
+ {
+ if (result != null)
+ return;
+
+ try
+ {
+ result = handler.get();
}
- else
+ catch (DigestMismatchException e)
{
- // our estimate of how many result rows there will be per-range
- float resultRowsPerRange = estimateResultRowsPerRange(command, keyspace);
- // underestimate how many rows we will get per-range in order to increase the likelihood that we'll
- // fetch enough rows in the first round
- resultRowsPerRange -= resultRowsPerRange * CONCURRENT_SUBREQUESTS_MARGIN;
- concurrencyFactor = resultRowsPerRange == 0.0
- ? 1
- : Math.max(1, Math.min(ranges.size(), (int) Math.ceil(command.limit() / resultRowsPerRange)));
-
- logger.trace("Estimated result rows per range: {}; requested rows: {}, ranges.size(): {}; concurrent range requests: {}",
- resultRowsPerRange,
- command.limit(),
- ranges.size(),
- concurrencyFactor);
- Tracing.trace("Submitting range requests on {} ranges with a concurrency of {} ({} rows per range expected)",
- ranges.size(),
- concurrencyFactor,
- resultRowsPerRange);
- }
-
- boolean haveSufficientRows = false;
- int i = 0;
- AbstractBounds<RowPosition> nextRange = null;
- List<InetAddress> nextEndpoints = null;
- List<InetAddress> nextFilteredEndpoints = null;
- while (i < ranges.size())
- {
- List<Pair<AbstractRangeCommand, ReadCallback<RangeSliceReply, Iterable<Row>>>> scanHandlers = new ArrayList<>(concurrencyFactor);
- int concurrentFetchStartingIndex = i;
- int concurrentRequests = 0;
- while ((i - concurrentFetchStartingIndex) < concurrencyFactor)
- {
- AbstractBounds<RowPosition> range = nextRange == null
- ? ranges.get(i)
- : nextRange;
- List<InetAddress> liveEndpoints = nextEndpoints == null
- ? getLiveSortedEndpoints(keyspace, range.right)
- : nextEndpoints;
- List<InetAddress> filteredEndpoints = nextFilteredEndpoints == null
- ? consistency_level.filterForQuery(keyspace, liveEndpoints)
- : nextFilteredEndpoints;
- ++i;
- ++concurrentRequests;
-
- // getRestrictedRange has broken the queried range into per-[vnode] token ranges, but this doesn't take
- // the replication factor into account. If the intersection of live endpoints for 2 consecutive ranges
- // still meets the CL requirements, then we can merge both ranges into the same RangeSliceCommand.
- while (i < ranges.size())
- {
- nextRange = ranges.get(i);
- nextEndpoints = getLiveSortedEndpoints(keyspace, nextRange.right);
- nextFilteredEndpoints = consistency_level.filterForQuery(keyspace, nextEndpoints);
-
- // If the current range right is the min token, we should stop merging because CFS.getRangeSlice
- // don't know how to deal with a wrapping range.
- // Note: it would be slightly more efficient to have CFS.getRangeSlice on the destination nodes unwraps
- // the range if necessary and deal with it. However, we can't start sending wrapped range without breaking
- // wire compatibility, so It's likely easier not to bother;
- if (range.right.isMinimum())
- break;
-
- List<InetAddress> merged = intersection(liveEndpoints, nextEndpoints);
-
- // Check if there is enough endpoint for the merge to be possible.
- if (!consistency_level.isSufficientLiveNodes(keyspace, merged))
- break;
-
- List<InetAddress> filteredMerged = consistency_level.filterForQuery(keyspace, merged);
-
- // Estimate whether merging will be a win or not
- if (!DatabaseDescriptor.getEndpointSnitch().isWorthMergingForRangeQuery(filteredMerged, filteredEndpoints, nextFilteredEndpoints))
- break;
-
- // If we get there, merge this range and the next one
- range = range.withNewRight(nextRange.right);
- liveEndpoints = merged;
- filteredEndpoints = filteredMerged;
- ++i;
- }
-
- AbstractRangeCommand nodeCmd = command.forSubRange(range);
+ throw new AssertionError(e); // no digests in range slices yet
+ }
+ }
- // collect replies and resolve according to consistency level
- RangeSliceResponseResolver resolver = new RangeSliceResponseResolver(nodeCmd.keyspace, command.timestamp);
- List<InetAddress> minimalEndpoints = filteredEndpoints.subList(0, Math.min(filteredEndpoints.size(), consistency_level.blockFor(keyspace)));
- ReadCallback<RangeSliceReply, Iterable<Row>> handler = new ReadCallback<>(resolver, consistency_level, nodeCmd, minimalEndpoints);
- handler.assureSufficientLiveNodes();
- resolver.setSources(filteredEndpoints);
- if (filteredEndpoints.size() == 1
- && filteredEndpoints.get(0).equals(FBUtilities.getBroadcastAddress()))
- {
- StageManager.getStage(Stage.READ).execute(new LocalRangeSliceRunnable(nodeCmd, handler));
- }
- else
- {
- MessageOut<? extends AbstractRangeCommand> message = nodeCmd.createMessage();
- for (InetAddress endpoint : filteredEndpoints)
- {
- Tracing.trace("Enqueuing request to {}", endpoint);
- MessagingService.instance().sendRRWithFailure(message, endpoint, handler);
- }
- }
- scanHandlers.add(Pair.create(nodeCmd, handler));
- }
- Tracing.trace("Submitted {} concurrent range requests covering {} ranges", concurrentRequests, i - concurrentFetchStartingIndex);
+ protected RowIterator computeNext()
+ {
+ waitForResponse();
+ return result.hasNext() ? result.next() : endOfData();
+ }
- List<AsyncOneResponse> repairResponses = new ArrayList<>();
- for (Pair<AbstractRangeCommand, ReadCallback<RangeSliceReply, Iterable<Row>>> cmdPairHandler : scanHandlers)
- {
- ReadCallback<RangeSliceReply, Iterable<Row>> handler = cmdPairHandler.right;
- RangeSliceResponseResolver resolver = (RangeSliceResponseResolver)handler.resolver;
+ public void close()
+ {
+ if (result != null)
+ result.close();
+ }
+ }
- try
- {
- for (Row row : handler.get())
- {
- rows.add(row);
- if (countLiveRows)
- liveRowCount += row.getLiveCount(command.predicate, command.timestamp);
- }
- repairResponses.addAll(resolver.repairResults);
- }
- catch (ReadTimeoutException|ReadFailureException ex)
- {
- // we timed out or failed waiting for responses
- int blockFor = consistency_level.blockFor(keyspace);
- int responseCount = resolver.responses.size();
- String gotData = responseCount > 0
- ? resolver.isDataPresent() ? " (including data)" : " (only digests)"
- : "";
-
- boolean isTimeout = ex instanceof ReadTimeoutException;
- if (Tracing.isTracing())
- {
- Tracing.trace("{}; received {} of {} responses{} for range {} of {}",
- (isTimeout ? "Timed out" : "Failed"), responseCount, blockFor, gotData, i, ranges.size());
- }
- else if (logger.isDebugEnabled())
- {
- logger.debug("Range slice {}; received {} of {} responses{} for range {} of {}",
- (isTimeout ? "timeout" : "failure"), responseCount, blockFor, gotData, i, ranges.size());
- }
- throw ex;
- }
- catch (DigestMismatchException e)
- {
- throw new AssertionError(e); // no digests in range slices yet
- }
+ private static class RangeCommandIterator extends AbstractIterator<RowIterator> implements PartitionIterator
+ {
+ private final Iterator<RangeForQuery> ranges;
+ private final int totalRangeCount;
+ private final PartitionRangeReadCommand command;
+ private final Keyspace keyspace;
+ private final ConsistencyLevel consistency;
- // if we're done, great, otherwise, move to the next range
- int count = countLiveRows ? liveRowCount : rows.size();
- if (count >= rowsToBeFetched)
- {
- haveSufficientRows = true;
- break;
- }
- }
+ private final long startTime;
+ private DataLimits.Counter counter;
+ private PartitionIterator sentQueryIterator;
- try
- {
- FBUtilities.waitOnFutures(repairResponses, DatabaseDescriptor.getWriteRpcTimeout());
- }
- catch (TimeoutException ex)
- {
- // We got all responses, but timed out while repairing
- int blockFor = consistency_level.blockFor(keyspace);
- if (Tracing.isTracing())
- Tracing.trace("Timed out while read-repairing after receiving all {} data and digest responses", blockFor);
- else
- logger.debug("Range slice timeout while read-repairing after receiving all {} data and digest responses", blockFor);
- throw new ReadTimeoutException(consistency_level, blockFor-1, blockFor, true);
- }
+ private int concurrencyFactor;
+ // The two following "metric" are maintained to improve the concurrencyFactor
+ // when it was not good enough initially.
+ private int liveReturned;
+ private int rangesQueried;
- if (haveSufficientRows)
- return command.postReconciliationProcessing(rows);
+ public RangeCommandIterator(RangeIterator ranges, PartitionRangeReadCommand command, int concurrencyFactor, Keyspace keyspace, ConsistencyLevel consistency)
+ {
+ this.command = command;
+ this.concurrencyFactor = concurrencyFactor;
+ this.startTime = System.nanoTime();
+ this.ranges = new RangeMerger(ranges, keyspace, consistency);
+ this.totalRangeCount = ranges.rangeCount();
+ this.consistency = consistency;
+ this.keyspace = keyspace;
+ }
- // we didn't get enough rows in our concurrent fetch; recalculate our concurrency factor
- // based on the results we've seen so far (as long as we still have ranges left to query)
- if (i < ranges.size())
+ public RowIterator computeNext()
+ {
+ try
+ {
+ while (sentQueryIterator == null || !sentQueryIterator.hasNext())
{
- float fetchedRows = countLiveRows ? liveRowCount : rows.size();
- float remainingRows = rowsToBeFetched - fetchedRows;
- float actualRowsPerRange;
- if (fetchedRows == 0.0)
- {
- // we haven't actually gotten any results, so query all remaining ranges at once
- actualRowsPerRange = 0.0f;
- concurrencyFactor = ranges.size() - i;
- }
- else
+ // If we don't have more range to handle, we're done
+ if (!ranges.hasNext())
+ return endOfData();
+
+ // else, sends the next batch of concurrent queries (after having close the previous iterator)
+ if (sentQueryIterator != null)
{
- actualRowsPerRange = fetchedRows / i;
- concurrencyFactor = Math.max(1, Math.min(ranges.size() - i, Math.round(remainingRows / actualRowsPerRange)));
+ liveReturned += counter.counted();
+ sentQueryIterator.close();
+
+ // It's not the first batch of queries and we're not done, so we we can use what has been
+ // returned so far to improve our rows-per-range estimate and update the concurrency accordingly
+ updateConcurrencyFactor();
}
- logger.trace("Didn't get enough response rows; actual rows per range: {}; remaining rows: {}, new concurrent requests: {}",
- actualRowsPerRange, (int) remainingRows, concurrencyFactor);
+ sentQueryIterator = sendNextRequests();
}
+
+ return sentQueryIterator.next();
+ }
+ catch (UnavailableException e)
+ {
+ rangeMetrics.unavailables.mark();
+ throw e;
+ }
+ catch (ReadTimeoutException e)
+ {
+ rangeMetrics.timeouts.mark();
+ throw e;
+ }
+ catch (ReadFailureException e)
+ {
+ rangeMetrics.failures.mark();
+ throw e;
}
}
- catch (ReadTimeoutException e)
+
+ private void updateConcurrencyFactor()
{
- rangeMetrics.timeouts.mark();
- throw e;
+ if (liveReturned == 0)
+ {
+ // we haven't actually gotten any results, so query all remaining ranges at once
+ concurrencyFactor = totalRangeCount - rangesQueried;
+ return;
+ }
+
+ // Otherwise, compute how many rows per range we got on average and pick a concurrency factor
+ // that should allow us to fetch all remaining rows with the next batch of (concurrent) queries.
+ int remainingRows = command.limits().count() - liveReturned;
+ float rowsPerRange = (float)liveReturned / (float)rangesQueried;
+ concurrencyFactor = Math.max(1, Math.min(totalRangeCount - rangesQueried, Math.round(remainingRows / rowsPerRange)));
+ logger.trace("Didn't get enough response rows; actual rows per range: {}; remaining rows: {}, new concurrent requests: {}",
+ rowsPerRange, (int) remainingRows, concurrencyFactor);
}
- catch (UnavailableException e)
+
+ private SingleRangeResponse query(RangeForQuery toQuery)
{
- rangeMetrics.unavailables.mark();
- throw e;
+ PartitionRangeReadCommand rangeCommand = command.forSubRange(toQuery.range);
+
+ DataResolver resolver = new DataResolver(keyspace, rangeCommand, consistency, toQuery.filteredEndpoints.size());
+
+ int blockFor = consistency.blockFor(keyspace);
+ int minResponses = Math.min(toQuery.filteredEndpoints.size(), blockFor);
+ List<InetAddress> minimalEndpoints = toQuery.filteredEndpoints.subList(0, minResponses);
+ ReadCallback handler = new ReadCallback(resolver, consistency, rangeCommand, minimalEndpoints);
+
+ handler.assureSufficientLiveNodes();
+
+ if (toQuery.filteredEndpoints.size() == 1 && canDoLocalRequest(toQuery.filteredEndpoints.get(0)))
+ {
+ StageManager.getStage(Stage.READ).execute(new LocalReadRunnable(rangeCommand, handler));
+ }
+ else
+ {
+ for (InetAddress endpoint : toQuery.filteredEndpoints)
+ {
+ MessageOut<ReadCommand> message = rangeCommand.createMessage(MessagingService.instance().getVersion(endpoint));
+ Tracing.trace("Enqueuing request to {}", endpoint);
+ MessagingService.instance().sendRRWithFailure(message, endpoint, handler);
+ }
+ }
+
+ return new SingleRangeResponse(handler);
}
- catch (ReadFailureException e)
+
+ private PartitionIterator sendNextRequests()
{
- rangeMetrics.failures.mark();
- throw e;
+ List<PartitionIterator> concurrentQueries = new ArrayList<>(concurrencyFactor);
+ for (int i = 0; i < concurrencyFactor && ranges.hasNext(); i++)
+ {
+ concurrentQueries.add(query(ranges.next()));
+ ++rangesQueried;
+ }
+
+ Tracing.trace("Submitted {} concurrent range requests", concurrentQueries.size());
+ // We want to count the results for the sake of updating the concurrency factor (see updateConcurrencyFactor) but we don't want to
+ // enforce any particular limit at this point (this could break code than rely on postReconciliationProcessing), hence the DataLimits.NONE.
- counter = DataLimits.NONE.newCounter(command.nowInSec(), true);
++ counter = DataLimits.NONE.newCounter(command.nowInSec(), true, command.selectsFullPartition());
+ return counter.applyTo(PartitionIterators.concat(concurrentQueries));
}
- finally
+
+ public void close()
{
- long latency = System.nanoTime() - startTime;
- rangeMetrics.addNano(latency);
- Keyspace.open(command.keyspace).getColumnFamilyStore(command.columnFamily).metric.coordinatorScanLatency.update(latency, TimeUnit.NANOSECONDS);
+ try
+ {
+ if (sentQueryIterator != null)
+ sentQueryIterator.close();
+ }
+ finally
+ {
+ long latency = System.nanoTime() - startTime;
+ rangeMetrics.addNano(latency);
+ Keyspace.openAndGetStore(command.metadata()).metric.coordinatorScanLatency.update(latency, TimeUnit.NANOSECONDS);
+ }
}
- return command.postReconciliationProcessing(rows);
+ }
+
+ @SuppressWarnings("resource")
+ public static PartitionIterator getRangeSlice(PartitionRangeReadCommand command, ConsistencyLevel consistencyLevel)
+ {
+ Tracing.trace("Computing ranges to query");
+
+ Keyspace keyspace = Keyspace.open(command.metadata().ksName);
+ RangeIterator ranges = new RangeIterator(command, keyspace, consistencyLevel);
+
+ // our estimate of how many result rows there will be per-range
+ float resultsPerRange = estimateResultsPerRange(command, keyspace);
+ // underestimate how many rows we will get per-range in order to increase the likelihood that we'll
+ // fetch enough rows in the first round
+ resultsPerRange -= resultsPerRange * CONCURRENT_SUBREQUESTS_MARGIN;
+ int concurrencyFactor = resultsPerRange == 0.0
+ ? 1
+ : Math.max(1, Math.min(ranges.rangeCount(), (int) Math.ceil(command.limits().count() / resultsPerRange)));
+ logger.trace("Estimated result rows per range: {}; requested rows: {}, ranges.size(): {}; concurrent range requests: {}",
+ resultsPerRange, command.limits().count(), ranges.rangeCount(), concurrencyFactor);
+ Tracing.trace("Submitting range requests on {} ranges with a concurrency of {} ({} rows per range expected)", ranges.rangeCount(), concurrencyFactor, resultsPerRange);
+
+ // Note that in general, a RangeCommandIterator will honor the command limit for each range, but will not enforce it globally.
+
- return command.limits().filter(command.postReconciliationProcessing(new RangeCommandIterator(ranges, command, concurrencyFactor, keyspace, consistencyLevel)), command.nowInSec());
++ return command.limits().filter(command.postReconciliationProcessing(new RangeCommandIterator(ranges, command, concurrencyFactor, keyspace, consistencyLevel)),
++ command.nowInSec(),
++ command.selectsFullPartition());
}
public Map<String, List<String>> getSchemaVersions()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
index 74ec47d,02623eb..ffd1b82
--- a/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
+++ b/src/java/org/apache/cassandra/service/pager/AbstractQueryPager.java
@@@ -29,196 -34,367 +29,196 @@@ import org.apache.cassandra.service.Cli
abstract class AbstractQueryPager implements QueryPager
{
- private static final Logger logger = LoggerFactory.getLogger(AbstractQueryPager.class);
-
- private final ConsistencyLevel consistencyLevel;
- private final boolean localQuery;
-
- protected final CFMetaData cfm;
- protected final IDiskAtomFilter columnFilter;
- private final long timestamp;
+ protected final ReadCommand command;
+ protected final DataLimits limits;
+ protected final int protocolVersion;
private int remaining;
- private boolean exhausted;
- private boolean shouldFetchExtraRow;
-
- protected AbstractQueryPager(ConsistencyLevel consistencyLevel,
- int toFetch,
- boolean localQuery,
- String keyspace,
- String columnFamily,
- IDiskAtomFilter columnFilter,
- long timestamp)
- {
- this(consistencyLevel, toFetch, localQuery, Schema.instance.getCFMetaData(keyspace, columnFamily), columnFilter, timestamp);
- }
- protected AbstractQueryPager(ConsistencyLevel consistencyLevel,
- int toFetch,
- boolean localQuery,
- CFMetaData cfm,
- IDiskAtomFilter columnFilter,
- long timestamp)
- {
- this.consistencyLevel = consistencyLevel;
- this.localQuery = localQuery;
-
- this.cfm = cfm;
- this.columnFilter = columnFilter;
- this.timestamp = timestamp;
-
- this.remaining = toFetch;
- }
-
-
- public List<Row> fetchPage(int pageSize) throws RequestValidationException, RequestExecutionException
- {
- if (isExhausted())
- return Collections.emptyList();
-
- int currentPageSize = nextPageSize(pageSize);
- List<Row> rows = filterEmpty(queryNextPage(currentPageSize, consistencyLevel, localQuery));
-
- if (rows.isEmpty())
- {
- logger.debug("Got empty set of rows, considering pager exhausted");
- exhausted = true;
- return Collections.emptyList();
- }
-
- int liveCount = getPageLiveCount(rows);
- logger.debug("Fetched {} live rows", liveCount);
-
- // Because SP.getRangeSlice doesn't trim the result (see SP.trim()), liveCount may be greater than what asked
- // (currentPageSize). This would throw off the paging logic so we trim the excess. It's not extremely efficient
- // but most of the time there should be nothing or very little to trim.
- if (liveCount > currentPageSize)
- {
- rows = discardLast(rows, liveCount - currentPageSize);
- liveCount = currentPageSize;
- }
-
- remaining -= liveCount;
-
- // If we've got less than requested, there is no more query to do (but
- // we still need to return the current page)
- if (liveCount < currentPageSize)
- {
- logger.debug("Got result ({}) smaller than page size ({}), considering pager exhausted", liveCount, currentPageSize);
- exhausted = true;
- }
+ // This is the last key we've been reading from (or can still be reading within). This the key for
+ // which remainingInPartition makes sense: if we're starting another key, we should reset remainingInPartition
+ // (and this is done in PagerIterator). This can be null (when we start).
+ private DecoratedKey lastKey;
+ private int remainingInPartition;
- // If it's not the first query and the first column is the last one returned (likely
- // but not certain since paging can race with deletes/expiration), then remove the
- // first column.
- if (containsPreviousLast(rows.get(0)))
- {
- rows = discardFirst(rows);
- remaining++;
- }
- // Otherwise, if 'shouldFetchExtraRow' was set, we queried for one more than the page size,
- // so if the page is full, trim the last entry
- else if (shouldFetchExtraRow && !exhausted)
- {
- // We've asked for one more than necessary
- rows = discardLast(rows);
- remaining++;
- }
-
- logger.debug("Remaining rows to page: {}", remaining);
-
- if (!isExhausted())
- shouldFetchExtraRow = recordLast(rows.get(rows.size() - 1));
-
- return rows;
- }
+ private boolean exhausted;
- private List<Row> filterEmpty(List<Row> result)
+ protected AbstractQueryPager(ReadCommand command, int protocolVersion)
{
- for (Row row : result)
- {
- if (row.cf == null || !row.cf.hasColumns())
- {
- List<Row> newResult = new ArrayList<Row>(result.size() - 1);
- for (Row row2 : result)
- {
- if (row2.cf == null || !row2.cf.hasColumns())
- continue;
+ this.command = command;
+ this.protocolVersion = protocolVersion;
+ this.limits = command.limits();
- newResult.add(row2);
- }
- return newResult;
- }
- }
- return result;
+ this.remaining = limits.count();
+ this.remainingInPartition = limits.perPartitionCount();
}
- protected void restoreState(int remaining, boolean shouldFetchExtraRow)
+ public ReadOrderGroup startOrderGroup()
{
- this.remaining = remaining;
- this.shouldFetchExtraRow = shouldFetchExtraRow;
+ return command.startOrderGroup();
}
- public boolean isExhausted()
+ public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState) throws RequestValidationException, RequestExecutionException
{
- return exhausted || remaining == 0;
- }
+ if (isExhausted())
+ return EmptyIterators.partition();
- public int maxRemaining()
- {
- return remaining;
+ pageSize = Math.min(pageSize, remaining);
+ Pager pager = new RowPager(limits.forPaging(pageSize), command.nowInSec());
+ return Transformation.apply(nextPageReadCommand(pageSize).execute(consistency, clientState), pager);
}
- public long timestamp()
+ public PartitionIterator fetchPageInternal(int pageSize, ReadOrderGroup orderGroup) throws RequestValidationException, RequestExecutionException
{
- return timestamp;
- }
+ if (isExhausted())
+ return EmptyIterators.partition();
- private int nextPageSize(int pageSize)
- {
- return Math.min(remaining, pageSize) + (shouldFetchExtraRow ? 1 : 0);
+ pageSize = Math.min(pageSize, remaining);
+ RowPager pager = new RowPager(limits.forPaging(pageSize), command.nowInSec());
+ return Transformation.apply(nextPageReadCommand(pageSize).executeInternal(orderGroup), pager);
}
- public ColumnCounter columnCounter()
+ public UnfilteredPartitionIterator fetchPageUnfiltered(CFMetaData cfm, int pageSize, ReadOrderGroup orderGroup)
{
- return columnFilter.columnCounter(cfm.comparator, timestamp);
- }
-
- protected abstract List<Row> queryNextPage(int pageSize, ConsistencyLevel consistency, boolean localQuery) throws RequestValidationException, RequestExecutionException;
-
- /**
- * Checks to see if the first row of a new page contains the last row from the previous page.
- * @param first the first row of the new page
- * @return true if <code>first</code> contains the last from from the previous page and it is live, false otherwise
- */
- protected abstract boolean containsPreviousLast(Row first);
-
- /**
- * Saves the paging state by recording the last seen partition key and cell name (where applicable).
- * @param last the last row in the current page
- * @return true if an extra row should be fetched in the next page,false otherwise
- */
- protected abstract boolean recordLast(Row last);
+ if (isExhausted())
+ return EmptyIterators.unfilteredPartition(cfm, false);
- protected abstract boolean isReversed();
+ pageSize = Math.min(pageSize, remaining);
+ UnfilteredPager pager = new UnfilteredPager(limits.forPaging(pageSize), command.nowInSec());
- private List<Row> discardFirst(List<Row> rows)
- {
- return discardFirst(rows, 1);
+ return Transformation.apply(nextPageReadCommand(pageSize).executeLocally(orderGroup), pager);
}
- @VisibleForTesting
- List<Row> discardFirst(List<Row> rows, int toDiscard)
+ private class UnfilteredPager extends Pager<Unfiltered>
{
- if (toDiscard == 0 || rows.isEmpty())
- return rows;
- int i = 0;
- DecoratedKey firstKey = null;
- ColumnFamily firstCf = null;
- while (toDiscard > 0 && i < rows.size())
+ private UnfilteredPager(DataLimits pageLimits, int nowInSec)
{
- Row first = rows.get(i++);
- firstKey = first.key;
- firstCf = first.cf.cloneMeShallow(isReversed());
- toDiscard -= isReversed()
- ? discardLast(first.cf, toDiscard, firstCf)
- : discardFirst(first.cf, toDiscard, firstCf);
+ super(pageLimits, nowInSec);
}
- // If there is less live data than to discard, all is discarded
- if (toDiscard > 0)
- return Collections.<Row>emptyList();
-
- // i is the index of the first row that we are sure to keep. On top of that,
- // we also keep firstCf is it hasn't been fully emptied by the last iteration above.
- int count = firstCf.getColumnCount();
- int newSize = rows.size() - (count == 0 ? i : i - 1);
- List<Row> newRows = new ArrayList<Row>(newSize);
- if (count != 0)
- newRows.add(new Row(firstKey, firstCf));
- newRows.addAll(rows.subList(i, rows.size()));
-
- return newRows;
- }
-
- private List<Row> discardLast(List<Row> rows)
- {
- return discardLast(rows, 1);
+ protected BaseRowIterator<Unfiltered> apply(BaseRowIterator<Unfiltered> partition)
+ {
+ return Transformation.apply(counter.applyTo((UnfilteredRowIterator) partition), this);
+ }
}
- @VisibleForTesting
- List<Row> discardLast(List<Row> rows, int toDiscard)
+ private class RowPager extends Pager<Row>
{
- if (toDiscard == 0 || rows.isEmpty())
- return rows;
- int i = rows.size()-1;
- DecoratedKey lastKey = null;
- ColumnFamily lastCf = null;
- while (toDiscard > 0 && i >= 0)
+ private RowPager(DataLimits pageLimits, int nowInSec)
{
- Row last = rows.get(i--);
- lastKey = last.key;
- lastCf = last.cf.cloneMeShallow(isReversed());
- toDiscard -= isReversed()
- ? discardFirst(last.cf, toDiscard, lastCf)
- : discardLast(last.cf, toDiscard, lastCf);
+ super(pageLimits, nowInSec);
}
- // If there is less live data than to discard, all is discarded
- if (toDiscard > 0)
- return Collections.<Row>emptyList();
-
- // i is the index of the last row that we are sure to keep. On top of that,
- // we also keep lastCf is it hasn't been fully emptied by the last iteration above.
- int count = lastCf.getColumnCount();
- int newSize = count == 0 ? i+1 : i+2;
- List<Row> newRows = new ArrayList<Row>(newSize);
- newRows.addAll(rows.subList(0, i+1));
- if (count != 0)
- newRows.add(new Row(lastKey, lastCf));
-
- return newRows;
+ protected BaseRowIterator<Row> apply(BaseRowIterator<Row> partition)
+ {
+ return Transformation.apply(counter.applyTo((RowIterator) partition), this);
+ }
}
- private int getPageLiveCount(List<Row> page)
+ private abstract class Pager<T extends Unfiltered> extends Transformation<BaseRowIterator<T>>
{
- int count = 0;
- for (Row row : page)
- count += columnCounter().countAll(row.cf).live();
- return count;
- }
+ private final DataLimits pageLimits;
+ protected final DataLimits.Counter counter;
+ private Row lastRow;
+ private boolean isFirstPartition = true;
- private int discardFirst(ColumnFamily cf, int toDiscard, ColumnFamily newCf)
- {
- boolean isReversed = isReversed();
- DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(isReversed);
- return isReversed
- ? discardTail(cf, toDiscard, newCf, cf.reverseIterator(), tester)
- : discardHead(toDiscard, newCf, cf.iterator(), tester);
- }
+ private Pager(DataLimits pageLimits, int nowInSec)
+ {
- this.counter = pageLimits.newCounter(nowInSec, true);
++ this.counter = pageLimits.newCounter(nowInSec, true, command.selectsFullPartition());
+ this.pageLimits = pageLimits;
+ }
- private int discardLast(ColumnFamily cf, int toDiscard, ColumnFamily newCf)
- {
- boolean isReversed = isReversed();
- DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(isReversed);
- return isReversed
- ? discardHead(toDiscard, newCf, cf.reverseIterator(), tester)
- : discardTail(cf, toDiscard, newCf, cf.iterator(), tester);
- }
+ @Override
+ public BaseRowIterator<T> applyToPartition(BaseRowIterator<T> partition)
+ {
+ DecoratedKey key = partition.partitionKey();
+ if (lastKey == null || !lastKey.equals(key))
+ remainingInPartition = limits.perPartitionCount();
+ lastKey = key;
+
+ // If this is the first partition of this page, this could be the continuation of a partition we've started
+ // on the previous page. In which case, we could have the problem that the partition has no more "regular"
+ // rows (but the page size is such we didn't knew before) but it does has a static row. We should then skip
+ // the partition as returning it would means to the upper layer that the partition has "only" static columns,
+ // which is not the case (and we know the static results have been sent on the previous page).
+ if (isFirstPartition)
+ {
+ isFirstPartition = false;
+ if (isPreviouslyReturnedPartition(key) && !partition.hasNext())
+ {
+ partition.close();
+ return null;
+ }
+ }
- private int discardHead(int toDiscard, ColumnFamily copy, Iterator<Cell> iter, DeletionInfo.InOrderTester tester)
- {
- ColumnCounter counter = columnCounter();
+ return apply(partition);
+ }
- List<Cell> staticCells = new ArrayList<>(cfm.staticColumns().size());
+ protected abstract BaseRowIterator<T> apply(BaseRowIterator<T> partition);
- // Discard the first 'toDiscard' live, non-static cells
- while (iter.hasNext())
+ @Override
+ public void onClose()
{
- Cell c = iter.next();
-
- // if it's a static column, don't count it and save it to add to the trimmed results
- ColumnDefinition columnDef = cfm.getColumnDefinition(c.name());
- if (columnDef != null && columnDef.kind == ColumnDefinition.Kind.STATIC)
+ recordLast(lastKey, lastRow);
+
+ int counted = counter.counted();
+ remaining -= counted;
+ // If the clustering of the last row returned is a static one, it means that the partition was only
+ // containing data within the static columns. If the clustering of the last row returned is empty
+ // it means that there is only one row per partition. Therefore, in both cases there are no data remaining
+ // within the partition.
+ if (lastRow != null && (lastRow.clustering() == Clustering.STATIC_CLUSTERING
+ || lastRow.clustering() == Clustering.EMPTY))
{
- staticCells.add(c);
- continue;
+ remainingInPartition = 0;
}
-
- counter.count(c, tester);
-
- // once we've discarded the required amount, add the rest
- if (counter.live() > toDiscard)
+ else
{
- for (Cell staticCell : staticCells)
- copy.addColumn(staticCell);
-
- copy.addColumn(c);
- while (iter.hasNext())
- copy.addColumn(iter.next());
+ remainingInPartition -= counter.countedInCurrentPartition();
}
+ exhausted = counted < pageLimits.count();
}
- int live = counter.live();
- // We want to take into account the row even if it was containing only static columns
- if (live == 0 && !staticCells.isEmpty())
- live = 1;
- return Math.min(live, toDiscard);
- }
-
- private int discardTail(ColumnFamily cf, int toDiscard, ColumnFamily copy, Iterator<Cell> iter, DeletionInfo.InOrderTester tester)
- {
- // Redoing the counting like that is not extremely efficient.
- // This is called only for reversed slices or in the case of a race between
- // paging and a deletion (pretty unlikely), so this is probably acceptable.
- int liveCount = columnCounter().countAll(cf).live();
- if (liveCount == toDiscard)
- return toDiscard;
-
- ColumnCounter counter = columnCounter();
- // Discard the last 'toDiscard' live (so stop adding as sound as we're past 'liveCount - toDiscard')
- while (iter.hasNext())
+ public Row applyToStatic(Row row)
{
- Cell c = iter.next();
- counter.count(c, tester);
- if (counter.live() > liveCount - toDiscard)
- break;
+ if (!row.isEmpty())
+ lastRow = row;
+ return row;
+ }
- copy.addColumn(c);
+ @Override
+ public Row applyToRow(Row row)
+ {
+ lastRow = row;
+ return row;
}
- return Math.min(liveCount, toDiscard);
}
- /**
- * Returns the first non-static cell in the ColumnFamily. This is necessary to avoid recording a static column
- * as the "last" cell seen in a reversed query. Because we will always query static columns alongside the normal
- * data for a page, they are not a good indicator of where paging should resume. When we begin the next page, we
- * need to start from the last non-static cell.
- */
- protected Cell firstNonStaticCell(ColumnFamily cf)
+ protected void restoreState(DecoratedKey lastKey, int remaining, int remainingInPartition)
{
- for (Cell cell : cf)
- {
- ColumnDefinition def = cfm.getColumnDefinition(cell.name());
- if (def == null || def.kind != ColumnDefinition.Kind.STATIC)
- return cell;
- }
- return null;
+ this.lastKey = lastKey;
+ this.remaining = remaining;
+ this.remainingInPartition = remainingInPartition;
}
- protected static Cell lastCell(ColumnFamily cf)
+ public boolean isExhausted()
{
- return cf.getReverseSortedColumns().iterator().next();
+ return exhausted || remaining == 0 || ((this instanceof SinglePartitionPager) && remainingInPartition == 0);
}
+
+ public int maxRemaining()
+ {
+ return remaining;
+ }
+
+ protected int remainingInPartition()
+ {
+ return remainingInPartition;
+ }
+
+ protected abstract ReadCommand nextPageReadCommand(int pageSize);
+ protected abstract void recordLast(DecoratedKey key, Row row);
+ protected abstract boolean isPreviouslyReturnedPartition(DecoratedKey key);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
index 8caa14d,35d0971..11bbc0e
--- a/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
+++ b/src/java/org/apache/cassandra/service/pager/MultiPartitionPager.java
@@@ -38,24 -36,19 +38,26 @@@ import org.apache.cassandra.service.Cli
*
* For now, we keep it simple (somewhat) and just do one command at a time. Provided that we make sure to not
* create a pager unless we need to, this is probably fine. Though if we later want to get fancy, we could use the
- * cfs meanRowSize to decide if parallelizing some of the command might be worth it while being confident we don't
+ * cfs meanPartitionSize to decide if parallelizing some of the command might be worth it while being confident we don't
* blow out memory.
*/
-class MultiPartitionPager implements QueryPager
+public class MultiPartitionPager implements QueryPager
{
private final SinglePartitionPager[] pagers;
- private final long timestamp;
+ private final DataLimits limit;
++ private final boolean selectsFullPartitions;
+
+ private final int nowInSec;
private int remaining;
private int current;
- MultiPartitionPager(List<ReadCommand> commands, ConsistencyLevel consistencyLevel, ClientState cState, boolean localQuery, PagingState state, int limitForQuery)
+ public MultiPartitionPager(SinglePartitionReadCommand.Group group, PagingState state, int protocolVersion)
{
+ this.limit = group.limits();
+ this.nowInSec = group.nowInSec();
++ this.selectsFullPartitions = group.selectsFullPartition();
+
int i = 0;
// If it's not the beginning (state != null), we need to find where we were and skip previous commands
// since they are done.
@@@ -70,15 -63,29 +72,16 @@@
return;
}
- pagers = new SinglePartitionPager[commands.size() - i];
+ pagers = new SinglePartitionPager[group.commands.size() - i];
// 'i' is on the first non exhausted pager for the previous page (or the first one)
- pagers[0] = group.commands.get(i).getPager(state, protocolVersion);
- pagers[0] = makePager(commands.get(i), consistencyLevel, cState, localQuery, state);
- timestamp = commands.get(i).timestamp;
++ SinglePartitionReadCommand command = group.commands.get(i);
++ pagers[0] = command.getPager(state, protocolVersion);
// Following ones haven't been started yet
- for (int j = i + 1; j < commands.size(); j++)
- {
- ReadCommand command = commands.get(j);
- if (command.timestamp != timestamp)
- throw new IllegalArgumentException("All commands must have the same timestamp or weird results may happen.");
- pagers[j - i] = makePager(command, consistencyLevel, cState, localQuery, null);
- }
+ for (int j = i + 1; j < group.commands.size(); j++)
+ pagers[j - i] = group.commands.get(j).getPager(null, protocolVersion);
- remaining = state == null ? limitForQuery : state.remaining;
- }
-
- private static SinglePartitionPager makePager(ReadCommand command, ConsistencyLevel consistencyLevel, ClientState cState, boolean localQuery, PagingState state)
- {
- return command instanceof SliceFromReadCommand
- ? new SliceQueryPager((SliceFromReadCommand)command, consistencyLevel, cState, localQuery, state)
- : new NamesQueryPager((SliceByNamesReadCommand)command, consistencyLevel, cState, localQuery);
+ remaining = state == null ? limit.count() : state.remaining;
}
public PagingState state()
@@@ -106,93 -113,35 +109,93 @@@
return true;
}
- public List<Row> fetchPage(int pageSize) throws RequestValidationException, RequestExecutionException
+ public ReadOrderGroup startOrderGroup()
{
- List<Row> result = new ArrayList<Row>();
-
- int remainingThisQuery = Math.min(remaining, pageSize);
- while (remainingThisQuery > 0 && !isExhausted())
+ // Note that for all pagers, the only difference is the partition key to which it applies, so in practice we
+ // can use any of the sub-pager ReadOrderGroup group to protect the whole pager
+ for (int i = current; i < pagers.length; i++)
{
- // isExhausted has set us on the first non-exhausted pager
- List<Row> page = pagers[current].fetchPage(remainingThisQuery);
- if (page.isEmpty())
- continue;
-
- Row row = page.get(0);
- int fetched = pagers[current].columnCounter().countAll(row.cf).live();
- remaining -= fetched;
- remainingThisQuery -= fetched;
- result.add(row);
+ if (pagers[i] != null)
+ return pagers[i].startOrderGroup();
}
+ throw new AssertionError("Shouldn't be called on an exhausted pager");
+ }
- return result;
+ @SuppressWarnings("resource") // iter closed via countingIter
+ public PartitionIterator fetchPage(int pageSize, ConsistencyLevel consistency, ClientState clientState) throws RequestValidationException, RequestExecutionException
+ {
+ int toQuery = Math.min(remaining, pageSize);
+ PagersIterator iter = new PagersIterator(toQuery, consistency, clientState, null);
- DataLimits.Counter counter = limit.forPaging(toQuery).newCounter(nowInSec, true);
++ DataLimits.Counter counter = limit.forPaging(toQuery).newCounter(nowInSec, true, selectsFullPartitions);
+ iter.setCounter(counter);
+ return counter.applyTo(iter);
}
- public int maxRemaining()
+ @SuppressWarnings("resource") // iter closed via countingIter
+ public PartitionIterator fetchPageInternal(int pageSize, ReadOrderGroup orderGroup) throws RequestValidationException, RequestExecutionException
{
- return remaining;
+ int toQuery = Math.min(remaining, pageSize);
+ PagersIterator iter = new PagersIterator(toQuery, null, null, orderGroup);
- DataLimits.Counter counter = limit.forPaging(toQuery).newCounter(nowInSec, true);
++ DataLimits.Counter counter = limit.forPaging(toQuery).newCounter(nowInSec, true, selectsFullPartitions);
+ iter.setCounter(counter);
+ return counter.applyTo(iter);
}
- public long timestamp()
+ private class PagersIterator extends AbstractIterator<RowIterator> implements PartitionIterator
{
- return timestamp;
+ private final int pageSize;
+ private PartitionIterator result;
+ private DataLimits.Counter counter;
+
+ // For "normal" queries
+ private final ConsistencyLevel consistency;
+ private final ClientState clientState;
+
+ // For internal queries
+ private final ReadOrderGroup orderGroup;
+
+ public PagersIterator(int pageSize, ConsistencyLevel consistency, ClientState clientState, ReadOrderGroup orderGroup)
+ {
+ this.pageSize = pageSize;
+ this.consistency = consistency;
+ this.clientState = clientState;
+ this.orderGroup = orderGroup;
+ }
+
+ public void setCounter(DataLimits.Counter counter)
+ {
+ this.counter = counter;
+ }
+
+ protected RowIterator computeNext()
+ {
+ while (result == null || !result.hasNext())
+ {
+ if (result != null)
+ result.close();
+
+ // This sets us on the first non-exhausted pager
+ if (isExhausted())
+ return endOfData();
+
+ int toQuery = pageSize - counter.counted();
+ result = consistency == null
+ ? pagers[current].fetchPageInternal(toQuery, orderGroup)
+ : pagers[current].fetchPage(toQuery, consistency, clientState);
+ }
+ return result.next();
+ }
+
+ public void close()
+ {
+ remaining -= counter.counted();
+ if (result != null)
+ result.close();
+ }
+ }
+
+ public int maxRemaining()
+ {
+ return remaining;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/src/java/org/apache/cassandra/service/pager/QueryPagers.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/pager/QueryPagers.java
index 02b5de2,f933ccb..c26bf3f
--- a/src/java/org/apache/cassandra/service/pager/QueryPagers.java
+++ b/src/java/org/apache/cassandra/service/pager/QueryPagers.java
@@@ -36,30 -41,156 +36,30 @@@ public class QueryPager
/**
* Convenience method that count (live) cells/rows for a given slice of a row, but page underneath.
*/
- public static int countPaged(String keyspace,
- String columnFamily,
- ByteBuffer key,
- SliceQueryFilter filter,
+ public static int countPaged(CFMetaData metadata,
+ DecoratedKey key,
+ ColumnFilter columnFilter,
+ ClusteringIndexFilter filter,
+ DataLimits limits,
ConsistencyLevel consistencyLevel,
- ClientState cState,
+ ClientState state,
final int pageSize,
- long now) throws RequestValidationException, RequestExecutionException
+ int nowInSec,
+ boolean isForThrift) throws RequestValidationException, RequestExecutionException
{
- SliceFromReadCommand command = new SliceFromReadCommand(keyspace, key, columnFamily, now, filter);
- final SliceQueryPager pager = new SliceQueryPager(command, consistencyLevel, cState, false);
+ SinglePartitionReadCommand command = SinglePartitionReadCommand.create(isForThrift, metadata, nowInSec, columnFilter, RowFilter.NONE, limits, key, filter);
+ final SinglePartitionPager pager = new SinglePartitionPager(command, null, Server.CURRENT_VERSION);
- ColumnCounter counter = filter.columnCounter(Schema.instance.getCFMetaData(keyspace, columnFamily).comparator, now);
+ int count = 0;
while (!pager.isExhausted())
{
- List<Row> next = pager.fetchPage(pageSize);
- if (!next.isEmpty())
- counter.countAll(next.get(0).cf);
+ try (PartitionIterator iter = pager.fetchPage(pageSize, consistencyLevel, state))
+ {
- DataLimits.Counter counter = limits.newCounter(nowInSec, true);
++ DataLimits.Counter counter = limits.newCounter(nowInSec, true, command.selectsFullPartition());
+ PartitionIterators.consume(counter.applyTo(iter));
+ count += counter.counted();
+ }
}
- return counter.live();
+ return count;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e2445cfb/test/unit/org/apache/cassandra/cql3/CQLTester.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/CQLTester.java
index 11d9e19,416a4b2..ba23c67
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@@ -716,17 -549,14 +716,17 @@@ public abstract class CQLTeste
protected com.datastax.driver.core.ResultSet executeNet(int protocolVersion, String query, Object... values) throws Throwable
{
- requireNetwork();
-
- return session[protocolVersion-1].execute(formatQuery(query), values);
+ return sessionNet(protocolVersion).execute(formatQuery(query), values);
}
- protected Session sessionNet()
+ protected com.datastax.driver.core.ResultSet executeNetWithPaging(String query, int pageSize) throws Throwable
{
- return sessionNet(PROTOCOL_VERSIONS.get(PROTOCOL_VERSIONS.size() - 1));
- return sessionNet(maxProtocolVersion).execute(new SimpleStatement(formatQuery(query)).setFetchSize(pageSize));
++ return sessionNet().execute(new SimpleStatement(formatQuery(query)).setFetchSize(pageSize));
+ }
+
- protected com.datastax.driver.core.ResultSet executeNetWithPaging(String query, int pageSize) throws Throwable
++ protected Session sessionNet()
+ {
- return sessionNet().execute(new SimpleStatement(formatQuery(query)).setFetchSize(pageSize));
++ return sessionNet(PROTOCOL_VERSIONS.get(PROTOCOL_VERSIONS.size() - 1));
}
protected Session sessionNet(int protocolVersion)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org