You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ma...@apache.org on 2016/02/24 22:13:59 UTC
[41/50] [abbrv] phoenix git commit: PHOENIX-2666 Performance
regression: Aggregate query with filter on table with multiple column
families
PHOENIX-2666 Performance regression: Aggregate query with filter on table with multiple column families
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/28a8b802
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/28a8b802
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/28a8b802
Branch: refs/heads/calcite
Commit: 28a8b802c2a32f9735bf187f08ef0a9e33baf2dd
Parents: cac0305
Author: James Taylor <jt...@salesforce.com>
Authored: Thu Feb 18 22:16:51 2016 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Thu Feb 18 22:16:51 2016 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/DeleteIT.java | 3 +-
.../StatsCollectorWithSplitsAndMultiCFIT.java | 3 +-
.../apache/phoenix/compile/DeleteCompiler.java | 6 +-
.../apache/phoenix/compile/UpsertCompiler.java | 7 +-
.../phoenix/filter/ColumnProjectionFilter.java | 2 +
.../phoenix/iterate/BaseResultIterators.java | 232 +++++++++++--------
.../apache/phoenix/iterate/ExplainTable.java | 28 +--
.../phoenix/compile/QueryCompilerTest.java | 29 +++
8 files changed, 189 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28a8b802/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java
index 745c730..6b4eead 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java
@@ -205,7 +205,8 @@ public class DeleteIT extends BaseHBaseManagedTimeIT {
deleteStmt = "DELETE FROM IntIntKeyTest WHERE j IS NULL";
stmt = conn.prepareStatement(deleteStmt);
assertIndexUsed(conn, deleteStmt, indexName, createIndex);
- stmt.execute();
+ int deleteCount = stmt.executeUpdate();
+ assertEquals(3, deleteCount);
if (!autoCommit) {
conn.commit();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28a8b802/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
index 13cd54c..d922ad9 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
@@ -111,10 +111,9 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
rs = conn.createStatement().executeQuery(
"SELECT COLUMN_FAMILY,SUM(GUIDE_POSTS_ROW_COUNT),SUM(GUIDE_POSTS_WIDTH),COUNT(*) from SYSTEM.STATS where PHYSICAL_NAME = '"
- + STATS_TEST_TABLE_NAME_NEW + "' GROUP BY COLUMN_FAMILY");
+ + STATS_TEST_TABLE_NAME_NEW + "' GROUP BY COLUMN_FAMILY ORDER BY COLUMN_FAMILY");
assertTrue(rs.next());
- assertTrue(rs.next());
assertEquals("A", rs.getString(1));
assertEquals(25, rs.getInt(2));
assertEquals(12420, rs.getInt(3));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28a8b802/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index 4c41f82..8e9e1de 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -508,7 +508,11 @@ public class DeleteCompiler {
// The coprocessor will delete each row returned from the scan
// Ignoring ORDER BY, since with auto commit on and no limit makes no difference
SelectStatement aggSelect = SelectStatement.create(SelectStatement.COUNT_ONE, delete.getHint());
- final RowProjector projector = ProjectionCompiler.compile(context, aggSelect, GroupBy.EMPTY_GROUP_BY);
+ RowProjector projectorToBe = ProjectionCompiler.compile(context, aggSelect, GroupBy.EMPTY_GROUP_BY);
+ if (plan.getProjector().projectEveryRow()) {
+ projectorToBe = new RowProjector(projectorToBe,true);
+ }
+ final RowProjector projector = projectorToBe;
final QueryPlan aggPlan = new AggregatePlan(context, select, tableRef, projector, null, OrderBy.EMPTY_ORDER_BY, null, GroupBy.EMPTY_GROUP_BY, null);
mutationPlans.add(new MutationPlan() {
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28a8b802/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
index 0f7f6f9..6ec7f70 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
@@ -636,7 +636,12 @@ public class UpsertCompiler {
PTable projectedTable = PTableImpl.makePTable(table, projectedColumns);
SelectStatement select = SelectStatement.create(SelectStatement.COUNT_ONE, upsert.getHint());
- final RowProjector aggProjector = ProjectionCompiler.compile(queryPlan.getContext(), select, GroupBy.EMPTY_GROUP_BY);
+ RowProjector aggProjectorToBe = ProjectionCompiler.compile(queryPlan.getContext(), select, GroupBy.EMPTY_GROUP_BY);
+ if (queryPlan.getProjector().projectEveryRow()) {
+ aggProjectorToBe = new RowProjector(aggProjectorToBe,true);
+ }
+ final RowProjector aggProjector = aggProjectorToBe;
+
/*
* Transfer over PTable representing subset of columns selected, but all PK columns.
* Move columns setting PK first in pkSlot order, adding LiteralExpression of null for any missing ones.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28a8b802/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
index cf9f7ab..b8b0350 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
@@ -66,6 +66,7 @@ public class ColumnProjectionFilter extends FilterBase implements Writable {
this.conditionOnlyCfs = conditionOnlyCfs;
}
+ @Override
public void readFields(DataInput input) throws IOException {
this.emptyCFName = WritableUtils.readCompressedByteArray(input);
int familyMapSize = WritableUtils.readVInt(input);
@@ -93,6 +94,7 @@ public class ColumnProjectionFilter extends FilterBase implements Writable {
}
}
+ @Override
public void write(DataOutput output) throws IOException {
WritableUtils.writeCompressedByteArray(output, this.emptyCFName);
WritableUtils.writeVInt(output, this.columnsTracker.size());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28a8b802/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index d8256d7..fa09704 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -64,6 +64,7 @@ import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.filter.ColumnProjectionFilter;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.parse.FilterableStatement;
import org.apache.phoenix.parse.HintNode.Hint;
import org.apache.phoenix.query.ConnectionQueryServices;
@@ -107,6 +108,7 @@ import com.google.common.collect.Lists;
public abstract class BaseResultIterators extends ExplainTable implements ResultIterators {
private static final Logger logger = LoggerFactory.getLogger(BaseResultIterators.class);
private static final int ESTIMATED_GUIDEPOSTS_PER_REGION = 20;
+ private static final int MIN_SEEK_TO_COLUMN_VERSION = VersionUtil.encodeVersion("0", "94", "12");
private final List<List<Scan>> scans;
private final List<KeyRange> splits;
@@ -167,37 +169,40 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
} else {
FilterableStatement statement = plan.getStatement();
RowProjector projector = plan.getProjector();
+ boolean optimizeProjection = false;
boolean keyOnlyFilter = familyMap.isEmpty() && context.getWhereConditionColumns().isEmpty();
if (!projector.projectEverything()) {
// If nothing projected into scan and we only have one column family, just allow everything
// to be projected and use a FirstKeyOnlyFilter to skip from row to row. This turns out to
// be quite a bit faster.
// Where condition columns also will get added into familyMap
- // When where conditions are present, we can not add FirstKeyOnlyFilter at beginning.
- if (familyMap.isEmpty() && context.getWhereConditionColumns().isEmpty()
- && table.getColumnFamilies().size() == 1) {
+ // When where conditions are present, we cannot add FirstKeyOnlyFilter at beginning.
+ // FIXME: we only enter this if the number of column families is 1 because otherwise
+ // local indexes break because it appears that the column families in the PTable do
+ // not match the actual column families of the table (which is bad).
+ if (keyOnlyFilter && table.getColumnFamilies().size() == 1) {
// Project the one column family. We must project a column family since it's possible
// that there are other non declared column families that we need to ignore.
scan.addFamily(table.getColumnFamilies().get(0).getName().getBytes());
- } else if (projector.projectEveryRow()) {
- if (table.getViewType() == ViewType.MAPPED) {
- // Since we don't have the empty key value in MAPPED tables,
- // we must select all CFs in HRS. However, only the
- // selected column values are returned back to client.
- for (PColumnFamily family : table.getColumnFamilies()) {
- scan.addFamily(family.getName().getBytes());
- }
- } else {
- byte[] ecf = SchemaUtil.getEmptyColumnFamily(table);
- // Project empty key value unless the column family containing it has
- // been projected in its entirety.
- if (!familyMap.containsKey(ecf) || familyMap.get(ecf) != null) {
- scan.addColumn(ecf, QueryConstants.EMPTY_COLUMN_BYTES);
- }
- }
} else {
- for (Pair<byte[], byte[]> whereColumn : context.getWhereConditionColumns()) {
- scan.addColumn(whereColumn.getFirst(), whereColumn.getSecond());
+ optimizeProjection = true;
+ if (projector.projectEveryRow()) {
+ if (table.getViewType() == ViewType.MAPPED) {
+ // Since we don't have the empty key value in MAPPED tables,
+ // we must project all CFs in HRS. However, only the
+ // selected column values are returned back to client.
+ context.getWhereConditionColumns().clear();
+ for (PColumnFamily family : table.getColumnFamilies()) {
+ context.addWhereCoditionColumn(family.getName().getBytes(), null);
+ }
+ } else {
+ byte[] ecf = SchemaUtil.getEmptyColumnFamily(table);
+ // Project empty key value unless the column family containing it has
+ // been projected in its entirety.
+ if (!familyMap.containsKey(ecf) || familyMap.get(ecf) != null) {
+ scan.addColumn(ecf, QueryConstants.EMPTY_COLUMN_BYTES);
+ }
+ }
}
}
}
@@ -210,10 +215,113 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
ScanUtil.andFilterAtEnd(scan, new PageFilter(perScanLimit));
}
- doColumnProjectionOptimization(context, scan, table, statement);
+ if (optimizeProjection) {
+ optimizeProjection(context, scan, table, statement);
+ }
}
}
+ private static void optimizeProjection(StatementContext context, Scan scan, PTable table, FilterableStatement statement) {
+ Map<byte[], NavigableSet<byte[]>> familyMap = scan.getFamilyMap();
+ // columnsTracker contain cf -> qualifiers which should get returned.
+ Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker =
+ new TreeMap<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>>();
+ Set<byte[]> conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ int referencedCfCount = familyMap.size();
+ boolean filteredColumnNotInProjection = false;
+ for (Pair<byte[], byte[]> whereCol : context.getWhereConditionColumns()) {
+ byte[] filteredFamily = whereCol.getFirst();
+ if (!(familyMap.containsKey(filteredFamily))) {
+ referencedCfCount++;
+ filteredColumnNotInProjection = true;
+ } else if (!filteredColumnNotInProjection) {
+ NavigableSet<byte[]> projectedColumns = familyMap.get(filteredFamily);
+ if (projectedColumns != null) {
+ byte[] filteredColumn = whereCol.getSecond();
+ if (filteredColumn == null) {
+ filteredColumnNotInProjection = true;
+ } else {
+ filteredColumnNotInProjection = !projectedColumns.contains(filteredColumn);
+ }
+ }
+ }
+ }
+ boolean preventSeekToColumn;
+ if (statement.getHint().hasHint(Hint.SEEK_TO_COLUMN)) {
+ // Allow seeking to column during filtering
+ preventSeekToColumn = false;
+ } else if (statement.getHint().hasHint(Hint.NO_SEEK_TO_COLUMN)) {
+ // Prevent seeking to column during filtering
+ preventSeekToColumn = true;
+ } else {
+ int hbaseServerVersion = context.getConnection().getQueryServices().getLowestClusterHBaseVersion();
+ // When only a single column family is referenced, there are no hints, and HBase server version
+ // is less than when the fix for HBASE-13109 went in (0.98.12), then we prevent seeking to a
+ // column.
+ preventSeekToColumn = referencedCfCount == 1 && hbaseServerVersion < MIN_SEEK_TO_COLUMN_VERSION;
+ }
+ for (Entry<byte[], NavigableSet<byte[]>> entry : familyMap.entrySet()) {
+ ImmutableBytesPtr cf = new ImmutableBytesPtr(entry.getKey());
+ NavigableSet<byte[]> qs = entry.getValue();
+ NavigableSet<ImmutableBytesPtr> cols = null;
+ if (qs != null) {
+ cols = new TreeSet<ImmutableBytesPtr>();
+ for (byte[] q : qs) {
+ cols.add(new ImmutableBytesPtr(q));
+ }
+ }
+ columnsTracker.put(cf, cols);
+ }
+ // Making sure that where condition CFs are getting scanned at HRS.
+ for (Pair<byte[], byte[]> whereCol : context.getWhereConditionColumns()) {
+ byte[] family = whereCol.getFirst();
+ if (preventSeekToColumn) {
+ if (!(familyMap.containsKey(family))) {
+ conditionOnlyCfs.add(family);
+ }
+ scan.addFamily(family);
+ } else {
+ if (familyMap.containsKey(family)) {
+ // where column's CF is present. If there are some specific columns added against this CF, we
+ // need to ensure this where column also getting added in it.
+ // If the select was like select cf1.*, then that itself will select the whole CF. So no need to
+ // specifically add the where column. Adding that will remove the cf1.* stuff and only this
+ // where condition column will get returned!
+ NavigableSet<byte[]> cols = familyMap.get(family);
+ // cols is null means the whole CF will get scanned.
+ if (cols != null) {
+ if (whereCol.getSecond() == null) {
+ scan.addFamily(family);
+ } else {
+ scan.addColumn(family, whereCol.getSecond());
+ }
+ }
+ } else if (whereCol.getSecond() == null) {
+ scan.addFamily(family);
+ } else {
+ // where column's CF itself is not present in family map. We need to add the column
+ scan.addColumn(family, whereCol.getSecond());
+ }
+ }
+ }
+ if (!columnsTracker.isEmpty()) {
+ if (preventSeekToColumn) {
+ for (ImmutableBytesPtr f : columnsTracker.keySet()) {
+ // This addFamily will remove explicit cols in scan familyMap and make it as entire row.
+ // We don't want the ExplicitColumnTracker to be used. Instead we have the ColumnProjectionFilter
+ scan.addFamily(f.get());
+ }
+ }
+ // We don't need this filter for aggregates, as we're not returning back what's
+ // in the scan in this case. We still want the other optimization that causes
+ // the ExplicitColumnTracker not to be used, though.
+ if (!statement.isAggregate() && filteredColumnNotInProjection) {
+ ScanUtil.andFilterAtEnd(scan, new ColumnProjectionFilter(SchemaUtil.getEmptyColumnFamily(table),
+ columnsTracker, conditionOnlyCfs));
+ }
+ }
+ }
+
public BaseResultIterators(QueryPlan plan, Integer perScanLimit, ParallelScanGrouper scanGrouper) throws SQLException {
super(plan.getContext(), plan.getTableRef(), plan.getGroupBy(), plan.getOrderBy(), plan.getStatement().getHint(), plan.getLimit());
this.plan = plan;
@@ -243,86 +351,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
this.allFutures = Lists.newArrayListWithExpectedSize(1);
}
- private static void doColumnProjectionOptimization(StatementContext context, Scan scan, PTable table, FilterableStatement statement) {
- Map<byte[], NavigableSet<byte[]>> familyMap = scan.getFamilyMap();
- if (familyMap != null && !familyMap.isEmpty()) {
- // columnsTracker contain cf -> qualifiers which should get returned.
- Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker =
- new TreeMap<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>>();
- Set<byte[]> conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- int referencedCfCount = familyMap.size();
- for (Pair<byte[], byte[]> whereCol : context.getWhereConditionColumns()) {
- if (!(familyMap.containsKey(whereCol.getFirst()))) {
- referencedCfCount++;
- }
- }
- boolean useOptimization;
- if (statement.getHint().hasHint(Hint.SEEK_TO_COLUMN)) {
- // Do not use the optimization
- useOptimization = false;
- } else if (statement.getHint().hasHint(Hint.NO_SEEK_TO_COLUMN)) {
- // Strictly use the optimization
- useOptimization = true;
- } else {
- // when referencedCfCount is >1 and no Hints, we are not using the optimization
- useOptimization = referencedCfCount == 1;
- }
- if (useOptimization) {
- for (Entry<byte[], NavigableSet<byte[]>> entry : familyMap.entrySet()) {
- ImmutableBytesPtr cf = new ImmutableBytesPtr(entry.getKey());
- NavigableSet<byte[]> qs = entry.getValue();
- NavigableSet<ImmutableBytesPtr> cols = null;
- if (qs != null) {
- cols = new TreeSet<ImmutableBytesPtr>();
- for (byte[] q : qs) {
- cols.add(new ImmutableBytesPtr(q));
- }
- }
- columnsTracker.put(cf, cols);
- }
- }
- // Making sure that where condition CFs are getting scanned at HRS.
- for (Pair<byte[], byte[]> whereCol : context.getWhereConditionColumns()) {
- if (useOptimization) {
- if (!(familyMap.containsKey(whereCol.getFirst()))) {
- scan.addFamily(whereCol.getFirst());
- conditionOnlyCfs.add(whereCol.getFirst());
- }
- } else {
- if (familyMap.containsKey(whereCol.getFirst())) {
- // where column's CF is present. If there are some specific columns added against this CF, we
- // need to ensure this where column also getting added in it.
- // If the select was like select cf1.*, then that itself will select the whole CF. So no need to
- // specifically add the where column. Adding that will remove the cf1.* stuff and only this
- // where condition column will get returned!
- NavigableSet<byte[]> cols = familyMap.get(whereCol.getFirst());
- // cols is null means the whole CF will get scanned.
- if (cols != null) {
- scan.addColumn(whereCol.getFirst(), whereCol.getSecond());
- }
- } else {
- // where column's CF itself is not present in family map. We need to add the column
- scan.addColumn(whereCol.getFirst(), whereCol.getSecond());
- }
- }
- }
- if (useOptimization && !columnsTracker.isEmpty()) {
- for (ImmutableBytesPtr f : columnsTracker.keySet()) {
- // This addFamily will remove explicit cols in scan familyMap and make it as entire row.
- // We don't want the ExplicitColumnTracker to be used. Instead we have the ColumnProjectionFilter
- scan.addFamily(f.get());
- }
- // We don't need this filter for aggregates, as we're not returning back what's
- // in the scan in this case. We still want the other optimization that causes
- // the ExplicitColumnTracker not to be used, though.
- if (!(statement.isAggregate())) {
- ScanUtil.andFilterAtEnd(scan, new ColumnProjectionFilter(SchemaUtil.getEmptyColumnFamily(table),
- columnsTracker, conditionOnlyCfs));
- }
- }
- }
- }
-
@Override
public List<KeyRange> getSplits() {
if (splits == null)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28a8b802/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
index 1fa4526..4a71483 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
@@ -130,11 +130,11 @@ public abstract class ExplainTable {
planSteps.add(" ROW TIMESTAMP FILTER [" + range.getMin() + ", " + range.getMax() + ")");
}
+ PageFilter pageFilter = null;
+ FirstKeyOnlyFilter firstKeyOnlyFilter = null;
+ BooleanExpressionFilter whereFilter = null;
Iterator<Filter> filterIterator = ScanUtil.getFilterIterator(scan);
if (filterIterator.hasNext()) {
- PageFilter pageFilter = null;
- FirstKeyOnlyFilter firstKeyOnlyFilter = null;
- BooleanExpressionFilter whereFilter = null;
do {
Filter filter = filterIterator.next();
if (filter instanceof FirstKeyOnlyFilter) {
@@ -145,17 +145,17 @@ public abstract class ExplainTable {
whereFilter = (BooleanExpressionFilter)filter;
}
} while (filterIterator.hasNext());
- if (whereFilter != null) {
- planSteps.add(" SERVER FILTER BY " + (firstKeyOnlyFilter == null ? "" : "FIRST KEY ONLY AND ") + whereFilter.toString());
- } else if (firstKeyOnlyFilter != null) {
- planSteps.add(" SERVER FILTER BY FIRST KEY ONLY");
- }
- if (!orderBy.getOrderByExpressions().isEmpty() && groupBy.isEmpty()) { // with GROUP BY, sort happens client-side
- planSteps.add(" SERVER" + (limit == null ? "" : " TOP " + limit + " ROW" + (limit == 1 ? "" : "S"))
- + " SORTED BY " + orderBy.getOrderByExpressions().toString());
- } else if (pageFilter != null) {
- planSteps.add(" SERVER " + pageFilter.getPageSize() + " ROW LIMIT");
- }
+ }
+ if (whereFilter != null) {
+ planSteps.add(" SERVER FILTER BY " + (firstKeyOnlyFilter == null ? "" : "FIRST KEY ONLY AND ") + whereFilter.toString());
+ } else if (firstKeyOnlyFilter != null) {
+ planSteps.add(" SERVER FILTER BY FIRST KEY ONLY");
+ }
+ if (!orderBy.getOrderByExpressions().isEmpty() && groupBy.isEmpty()) { // with GROUP BY, sort happens client-side
+ planSteps.add(" SERVER" + (limit == null ? "" : " TOP " + limit + " ROW" + (limit == 1 ? "" : "S"))
+ + " SORTED BY " + orderBy.getOrderByExpressions().toString());
+ } else if (pageFilter != null) {
+ planSteps.add(" SERVER " + pageFilter.getPageSize() + " ROW LIMIT");
}
Integer groupByLimit = null;
byte[] groupByLimitBytes = scan.getAttribute(BaseScannerRegionObserver.GROUP_BY_LIMIT);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28a8b802/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index cfec967..ce38cfd 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -37,10 +37,12 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.OrderByCompiler.OrderBy;
@@ -52,6 +54,7 @@ import org.apache.phoenix.expression.aggregator.Aggregator;
import org.apache.phoenix.expression.aggregator.CountAggregator;
import org.apache.phoenix.expression.aggregator.ServerAggregators;
import org.apache.phoenix.expression.function.TimeUnit;
+import org.apache.phoenix.filter.ColumnProjectionFilter;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
@@ -68,6 +71,7 @@ import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.junit.Test;
@@ -2213,5 +2217,30 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
}
}
+ private static boolean hasColumnProjectionFilter(Scan scan) {
+ Iterator<Filter> iterator = ScanUtil.getFilterIterator(scan);
+ while (iterator.hasNext()) {
+ Filter filter = iterator.next();
+ if (filter instanceof ColumnProjectionFilter) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Test
+ public void testColumnProjectionOptimized() throws SQLException {
+ Connection conn = DriverManager.getConnection(getUrl());
+ try {
+ conn.createStatement().execute("CREATE TABLE t(k INTEGER PRIMARY KEY, a.v1 VARCHAR, a.v1b VARCHAR, b.v2 VARCHAR, c.v3 VARCHAR)");
+ assertTrue(hasColumnProjectionFilter(projectQuery("SELECT k, v1 FROM t WHERE v2 = 'foo'")));
+ assertFalse(hasColumnProjectionFilter(projectQuery("SELECT k, v1 FROM t WHERE v1 = 'foo'")));
+ assertFalse(hasColumnProjectionFilter(projectQuery("SELECT v1,v2 FROM t WHERE v1 = 'foo'")));
+ assertTrue(hasColumnProjectionFilter(projectQuery("SELECT v1,v2 FROM t WHERE v1 = 'foo' and v2 = 'bar' and v3 = 'bas'")));
+ assertFalse(hasColumnProjectionFilter(projectQuery("SELECT a.* FROM t WHERE v1 = 'foo' and v1b = 'bar'")));
+ } finally {
+ conn.close();
+ }
+ }
}