You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2010/08/20 23:26:25 UTC
svn commit: r987638 - in /cassandra/trunk: CHANGES.txt
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
src/java/org/apache/cassandra/db/Memtable.java
test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
Author: jbellis
Date: Fri Aug 20 21:26:25 2010
New Revision: 987638
URL: http://svn.apache.org/viewvc?rev=987638&view=rev
Log:
allow index expressions against columns that are not part of the SlicePredicate.
patch by jbellis; reviewed by Nate McCall for CASSANDRA-1410
Modified:
cassandra/trunk/CHANGES.txt
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=987638&r1=987637&r2=987638&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Fri Aug 20 21:26:25 2010
@@ -24,6 +24,8 @@ dev
* use JNA, if present, to take snapshots (CASSANDRA-1371)
* truncate hints if starting 0.7 for the first time (CASSANDRA-1414)
* fix FD leak in single-row slicepredicate queries (CASSANDRA-1416)
+ * allow index expressions against columns that are not part of the
+ SlicePredicate (CASSANDRA-1410)
0.7-beta1
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=987638&r1=987637&r2=987638&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Fri Aug 20 21:26:25 2010
@@ -1136,15 +1136,67 @@ public class ColumnFamilyStore implement
public List<Row> scan(IndexClause clause, AbstractBounds range, IFilter dataFilter)
{
+ // Start with the most-restrictive indexed clause, then apply remaining clauses
+ // to each row matching that clause.
// TODO: allow merge join instead of just one index + loop
- IndexExpression first = highestSelectivityPredicate(clause);
- ColumnFamilyStore indexCFS = getIndexedColumnFamilyStore(first.column_name);
+ IndexExpression primary = highestSelectivityPredicate(clause);
+ ColumnFamilyStore indexCFS = getIndexedColumnFamilyStore(primary.column_name);
assert indexCFS != null;
- DecoratedKey indexKey = indexCFS.partitioner.decorateKey(first.value);
+ DecoratedKey indexKey = indexCFS.partitioner.decorateKey(primary.value);
+
+ // if the slicepredicate doesn't contain all the columns for which we have expressions to evaluate,
+ // it needs to be expanded to include those too
+ IFilter firstFilter = dataFilter;
+ NamesQueryFilter extraFilter = null;
+ if (clause.expressions.size() > 1)
+ {
+ if (dataFilter instanceof SliceQueryFilter)
+ {
+ // if we have a high chance of getting all the columns in a single index slice, do that.
+ // otherwise, create an extraFilter to fetch by name the columns referenced by the additional expressions.
+ if (getMaxRowSize() < DatabaseDescriptor.getColumnIndexSize())
+ {
+ firstFilter = new SliceQueryFilter(ArrayUtils.EMPTY_BYTE_ARRAY,
+ ArrayUtils.EMPTY_BYTE_ARRAY,
+ ((SliceQueryFilter) dataFilter).reversed,
+ Integer.MAX_VALUE);
+ }
+ else
+ {
+ SortedSet<byte[]> columns = new TreeSet<byte[]>(getComparator());
+ for (IndexExpression expr : clause.expressions)
+ {
+ if (expr == primary)
+ continue;
+ columns.add(expr.column_name);
+ }
+ extraFilter = new NamesQueryFilter(columns);
+ }
+ }
+ else
+ {
+ // just add in columns that are not part of the resultset
+ assert dataFilter instanceof NamesQueryFilter;
+ SortedSet<byte[]> columns = new TreeSet<byte[]>(getComparator());
+ for (IndexExpression expr : clause.expressions)
+ {
+ if (expr == primary || ((NamesQueryFilter) dataFilter).columns.contains(expr.column_name))
+ continue;
+ columns.add(expr.column_name);
+ }
+ if (columns.size() > 0)
+ {
+ columns.addAll(((NamesQueryFilter) dataFilter).columns);
+ firstFilter = new NamesQueryFilter(columns);
+ }
+ }
+ }
List<Row> rows = new ArrayList<Row>();
byte[] startKey = clause.start_key;
-
+
+ // fetch row keys matching the primary expression, fetch the slice predicate for each
+ // and filter by remaining expressions. repeat until finished w/ assigned range or index row is exhausted.
outer:
while (true)
{
@@ -1176,9 +1228,42 @@ public class ColumnFamilyStore implement
break outer;
if (!range.contains(dk.token))
continue;
- ColumnFamily data = getColumnFamily(new QueryFilter(dk, new QueryPath(columnFamily), dataFilter));
- if (satisfies(data, clause, first))
+
+ // get the row columns requested, and additional columns for the expressions if necessary
+ ColumnFamily data = getColumnFamily(new QueryFilter(dk, new QueryPath(columnFamily), firstFilter));
+ if (extraFilter != null)
+ {
+ // we might have gotten the expression columns in with the main data slice, but
+ // we can't know for sure until that slice is done. So, we'll do the extra query
+ // if we go through and any expression columns are not present.
+ for (IndexExpression expr : clause.expressions)
+ {
+ if (expr != primary && data.getColumn(expr.column_name) == null)
+ {
+ data.addAll(getColumnFamily(new QueryFilter(dk, new QueryPath(columnFamily), extraFilter)));
+ break;
+ }
+ }
+ }
+
+ if (satisfies(data, clause, primary))
+ {
+ // cut the resultset back to what was requested, if necessary
+ if (firstFilter != dataFilter)
+ {
+ ColumnFamily expandedData = data;
+ data = expandedData.cloneMeShallow();
+ IColumnIterator iter = dataFilter.getMemtableColumnIterator(expandedData, dk, getComparator());
+ while (iter.hasNext())
+ {
+ IColumn c = iter.next();
+ data.addColumn(c);
+ }
+ }
+
rows.add(new Row(dk, data));
+ }
+
if (rows.size() == clause.count)
break outer;
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java?rev=987638&r1=987637&r2=987638&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java Fri Aug 20 21:26:25 2010
@@ -200,7 +200,7 @@ public class Memtable implements Compara
/**
* obtain an iterator of columns in this memtable in the specified order starting from a given column.
*/
- public static IColumnIterator getSliceIterator(final DecoratedKey key, final ColumnFamily cf, SliceQueryFilter filter, AbstractType typeComparator)
+ public static IColumnIterator getSliceIterator(final DecoratedKey key, final ColumnFamily cf, final SliceQueryFilter filter, AbstractType typeComparator)
{
assert cf != null;
final boolean isSuper = cf.isSuper();
@@ -221,6 +221,8 @@ public class Memtable implements Compara
return new AbstractColumnIterator()
{
+ private int n = 0;
+
public ColumnFamily getColumnFamily()
{
return cf;
@@ -233,7 +235,7 @@ public class Memtable implements Compara
public boolean hasNext()
{
- return filteredIter.hasNext();
+ return (n++ < filter.count) && filteredIter.hasNext();
}
public IColumn next()
Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java?rev=987638&r1=987637&r2=987638&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java Fri Aug 20 21:26:25 2010
@@ -179,6 +179,7 @@ public class ColumnFamilyStoreTest exten
rm.add(new QueryPath("Indexed1", null, "birthdate".getBytes("UTF8")), FBUtilities.toByteArray(3L), new TimestampClock(0));
rm.apply();
+ // basic single-expression query
IndexExpression expr = new IndexExpression("birthdate".getBytes("UTF8"), IndexOperator.EQ, FBUtilities.toByteArray(1L));
IndexClause clause = new IndexClause(Arrays.asList(expr), ArrayUtils.EMPTY_BYTE_ARRAY, 100);
IFilter filter = new IdentityQueryFilter();
@@ -193,12 +194,28 @@ public class ColumnFamilyStoreTest exten
assert Arrays.equals(FBUtilities.toByteArray(1L), rows.get(0).cf.getColumn("birthdate".getBytes("UTF8")).value());
assert Arrays.equals(FBUtilities.toByteArray(1L), rows.get(1).cf.getColumn("birthdate".getBytes("UTF8")).value());
+ // add a second expression
IndexExpression expr2 = new IndexExpression("notbirthdate".getBytes("UTF8"), IndexOperator.GTE, FBUtilities.toByteArray(2L));
clause = new IndexClause(Arrays.asList(expr, expr2), ArrayUtils.EMPTY_BYTE_ARRAY, 100);
rows = Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range, filter);
assert rows.size() == 1 : StringUtils.join(rows, ",");
assert Arrays.equals("k3".getBytes(), rows.get(0).key.key);
+
+ // same query again, but with resultset not including the subordinate expression
+ rows = Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range, new NamesQueryFilter("birthdate".getBytes("UTF8")));
+
+ assert rows.size() == 1 : StringUtils.join(rows, ",");
+ assert Arrays.equals("k3".getBytes(), rows.get(0).key.key);
+ assert rows.get(0).cf.getColumnCount() == 1;
+
+ // once more, this time with a slice rowset that needs to be expanded
+ SliceQueryFilter sqf = new SliceQueryFilter(ArrayUtils.EMPTY_BYTE_ARRAY, ArrayUtils.EMPTY_BYTE_ARRAY, false, 0);
+ rows = Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range, sqf);
+
+ assert rows.size() == 1 : StringUtils.join(rows, ",");
+ assert Arrays.equals("k3".getBytes(), rows.get(0).key.key);
+ assert rows.get(0).cf.getColumnCount() == 0;
}
@Test