You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by an...@apache.org on 2014/03/01 08:54:11 UTC
[1/3] git commit: PHOENIX-76
Repository: incubator-phoenix
Updated Branches:
refs/heads/master c692e71ef -> c2010d174
PHOENIX-76
Project: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/commit/348b7517
Tree: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/tree/348b7517
Diff: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/diff/348b7517
Branch: refs/heads/master
Commit: 348b7517ade37b3e8b879899b3b3a3729efcb63a
Parents: e781eba
Author: anoopsjohn <an...@gmail.com>
Authored: Fri Feb 28 12:05:00 2014 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Fri Feb 28 12:05:00 2014 +0530
----------------------------------------------------------------------
.../phoenix/filter/ColumnProjectionFilter.java | 24 +++++++++++++++--
.../phoenix/iterate/ParallelIterators.java | 27 +++++++++++++++++---
2 files changed, 45 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/348b7517/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
index bf9bc7e..8c5225f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
@@ -21,9 +21,11 @@ import java.io.*;
import java.util.*;
import java.util.Map.Entry;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.query.QueryConstants;
@@ -37,15 +39,17 @@ public class ColumnProjectionFilter extends FilterBase {
private byte[] emptyCFName;
private Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker;
+ private Set<byte[]> conditionOnlyCfs;
public ColumnProjectionFilter() {
}
public ColumnProjectionFilter(byte[] emptyCFName,
- Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker) {
+ Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker, Set<byte[]> conditionOnlyCfs) {
this.emptyCFName = emptyCFName;
this.columnsTracker = columnsTracker;
+ this.conditionOnlyCfs = conditionOnlyCfs;
}
@Override
@@ -68,6 +72,12 @@ public class ColumnProjectionFilter extends FilterBase {
columnsTracker.put(new ImmutableBytesPtr(cf), qualifiers);
familyMapSize--;
}
+ int conditionOnlyCfsSize = WritableUtils.readVInt(input);
+ this.conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ while (conditionOnlyCfsSize > 0) {
+ this.conditionOnlyCfs.add(WritableUtils.readCompressedByteArray(input));
+ conditionOnlyCfsSize--;
+ }
}
@Override
@@ -86,6 +96,11 @@ public class ColumnProjectionFilter extends FilterBase {
}
}
}
+ // Write conditionOnlyCfs
+ WritableUtils.writeVInt(output, this.conditionOnlyCfs.size());
+ for (byte[] f : this.conditionOnlyCfs) {
+ WritableUtils.writeCompressedByteArray(output, f);
+ }
}
@Override
@@ -120,6 +135,11 @@ public class ColumnProjectionFilter extends FilterBase {
}
@Override
+ public boolean isFamilyEssential(byte[] name) {
+ return conditionOnlyCfs.isEmpty() || this.conditionOnlyCfs.contains(name);
+ }
+
+ @Override
public String toString() {
return "";
}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/348b7517/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index 563d97f..0be2a11 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -105,11 +105,19 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
ScanUtil.andFilterAtEnd(scan, new PageFilter(limit));
}
+ if (!(statement.isAggregate())) {
+ doColumnProjectionOptimization(context, scan, table);
+ }
+ }
+
+ private void doColumnProjectionOptimization(StatementContext context, Scan scan, PTable table) {
Map<byte[], NavigableSet<byte[]>> familyMap = scan.getFamilyMap();
if (familyMap != null && !familyMap.isEmpty()) {
// columnsTracker contain cf -> qualifiers which should get returned.
Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker =
new TreeMap<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>>();
+ Set<byte[]> conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ boolean useOptimization = true;
for (Entry<byte[], NavigableSet<byte[]>> entry : familyMap.entrySet()) {
ImmutableBytesPtr cf = new ImmutableBytesPtr(entry.getKey());
NavigableSet<byte[]> qs = entry.getValue();
@@ -122,20 +130,31 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
}
columnsTracker.put(cf, cols);
}
+ if (familyMap.size() > 1) {
+ useOptimization = false;
+ }
// Making sure that where condition CFs are getting scanned at HRS.
for (Pair<byte[], byte[]> whereCol : context.getWhereCoditionColumns()) {
- if (!(familyMap.containsKey(whereCol.getFirst()))) {
- scan.addFamily(whereCol.getFirst());
+ if (useOptimization) {
+ if (!(familyMap.containsKey(whereCol.getFirst()))) {
+ scan.addFamily(whereCol.getFirst());
+ conditionOnlyCfs.add(whereCol.getFirst());
+ }
+ } else {
+ scan.addColumn(whereCol.getFirst(), whereCol.getSecond());
}
}
- if (!columnsTracker.isEmpty()) {
+ if (familyMap.size() > 1) {
+ useOptimization = false;
+ }
+ if (useOptimization && !columnsTracker.isEmpty()) {
for (ImmutableBytesPtr f : columnsTracker.keySet()) {
// This addFamily will remove explicit cols in scan familyMap and make it as entire row.
// We don't want the ExplicitColumnTracker to be used. Instead we have the ColumnProjectionFilter
scan.addFamily(f.get());
}
ScanUtil.andFilterAtEnd(scan, new ColumnProjectionFilter(SchemaUtil.getEmptyColumnFamily(table),
- columnsTracker));
+ columnsTracker, conditionOnlyCfs));
}
if (table.getViewType() == ViewType.MAPPED) {
// Since we don't have the empty key value in MAPPED tables, we must select all CFs in HRS. But only the
[2/3] git commit: PHOENIX-76
Posted by an...@apache.org.
PHOENIX-76
Project: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/commit/2ca85ab6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/tree/2ca85ab6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/diff/2ca85ab6
Branch: refs/heads/master
Commit: 2ca85ab681f3080e52db632b1a78d45445fd9cea
Parents: 348b751
Author: anoopsjohn <an...@gmail.com>
Authored: Fri Feb 28 12:14:54 2014 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Fri Feb 28 12:14:54 2014 +0530
----------------------------------------------------------------------
.../phoenix/iterate/ParallelIterators.java | 36 +++++++++++---------
1 file changed, 20 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/2ca85ab6/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index 0be2a11..b82a7ea 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -117,21 +117,28 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker =
new TreeMap<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>>();
Set<byte[]> conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- boolean useOptimization = true;
- for (Entry<byte[], NavigableSet<byte[]>> entry : familyMap.entrySet()) {
- ImmutableBytesPtr cf = new ImmutableBytesPtr(entry.getKey());
- NavigableSet<byte[]> qs = entry.getValue();
- NavigableSet<ImmutableBytesPtr> cols = null;
- if (qs != null) {
- cols = new TreeSet<ImmutableBytesPtr>();
- for (byte[] q : qs) {
- cols.add(new ImmutableBytesPtr(q));
- }
+ int referencedCfCount = familyMap.size();
+ for (Pair<byte[], byte[]> whereCol : context.getWhereCoditionColumns()) {
+ if (!(familyMap.containsKey(whereCol.getFirst()))) {
+ referencedCfCount++;
}
- columnsTracker.put(cf, cols);
}
- if (familyMap.size() > 1) {
- useOptimization = false;
+ boolean useOptimization = referencedCfCount == 1;
+ // when referencedCfCount is >1 we are not using the optimization as of now
+ // TODO support adding a HINT in query so that in case of cf count>1 also, this optimization can be applied.
+ if (useOptimization) {
+ for (Entry<byte[], NavigableSet<byte[]>> entry : familyMap.entrySet()) {
+ ImmutableBytesPtr cf = new ImmutableBytesPtr(entry.getKey());
+ NavigableSet<byte[]> qs = entry.getValue();
+ NavigableSet<ImmutableBytesPtr> cols = null;
+ if (qs != null) {
+ cols = new TreeSet<ImmutableBytesPtr>();
+ for (byte[] q : qs) {
+ cols.add(new ImmutableBytesPtr(q));
+ }
+ }
+ columnsTracker.put(cf, cols);
+ }
}
// Making sure that where condition CFs are getting scanned at HRS.
for (Pair<byte[], byte[]> whereCol : context.getWhereCoditionColumns()) {
@@ -144,9 +151,6 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
scan.addColumn(whereCol.getFirst(), whereCol.getSecond());
}
}
- if (familyMap.size() > 1) {
- useOptimization = false;
- }
if (useOptimization && !columnsTracker.isEmpty()) {
for (ImmutableBytesPtr f : columnsTracker.keySet()) {
// This addFamily will remove explicit cols in scan familyMap and make it as entire row.
[3/3] git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-phoenix
Posted by an...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-phoenix
Project: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/commit/c2010d17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/tree/c2010d17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/diff/c2010d17
Branch: refs/heads/master
Commit: c2010d1749201d8220c0830b4d56e4e1fb1d4fe6
Parents: 2ca85ab c692e71
Author: anoopsjohn <an...@gmail.com>
Authored: Sat Mar 1 13:23:15 2014 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Sat Mar 1 13:23:15 2014 +0530
----------------------------------------------------------------------
KEYS | 73 ++
.../org/apache/phoenix/cache/GlobalCache.java | 60 +-
.../apache/phoenix/cache/ServerCacheClient.java | 2 +
.../apache/phoenix/cache/TenantCacheImpl.java | 14 +-
.../apache/phoenix/client/ClientKeyValue.java | 16 +
.../phoenix/client/ClientKeyValueBuilder.java | 15 +-
.../phoenix/client/GenericKeyValueBuilder.java | 13 +
.../apache/phoenix/client/KeyValueBuilder.java | 25 +-
.../phoenix/compile/CreateTableCompiler.java | 4 +-
.../apache/phoenix/compile/UpsertCompiler.java | 13 +-
.../apache/phoenix/compile/WhereOptimizer.java | 9 +-
.../coprocessor/HashJoinRegionScanner.java | 8 +-
.../coprocessor/MetaDataEndpointImpl.java | 66 +-
.../phoenix/coprocessor/MetaDataProtocol.java | 14 +-
.../coprocessor/MetaDataRegionObserver.java | 3 +-
.../apache/phoenix/execute/HashJoinPlan.java | 31 +-
.../expression/ArrayConstructorExpression.java | 120 +++-
.../phoenix/expression/BaseExpression.java | 149 ++++
.../phoenix/expression/CoerceExpression.java | 7 +
.../phoenix/expression/InListExpression.java | 69 +-
.../RowValueConstructorExpression.java | 139 ----
.../expression/function/ArrayIndexFunction.java | 14 +-
.../expression/function/CoalesceFunction.java | 9 +-
.../filter/EvaluateOnCompletionVisitor.java | 10 +-
.../apache/phoenix/filter/SkipScanFilter.java | 11 +-
.../apache/phoenix/iterate/ExplainTable.java | 18 +-
.../apache/phoenix/jdbc/PhoenixConnection.java | 14 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 8 +-
.../org/apache/phoenix/jdbc/PhoenixDriver.java | 27 +-
.../phoenix/jdbc/PhoenixEmbeddedDriver.java | 27 +-
.../phoenix/memory/GlobalMemoryManager.java | 2 +-
.../query/ConnectionQueryServicesImpl.java | 30 +-
.../org/apache/phoenix/query/QueryServices.java | 4 +-
.../phoenix/query/QueryServicesOptions.java | 9 +-
.../apache/phoenix/schema/DelegateColumn.java | 7 +
.../apache/phoenix/schema/MetaDataClient.java | 78 ++-
.../apache/phoenix/schema/PArrayDataType.java | 671 ++++++++++++-------
.../org/apache/phoenix/schema/PBaseColumn.java | 9 +-
.../java/org/apache/phoenix/schema/PColumn.java | 2 +
.../apache/phoenix/schema/PColumnFamily.java | 2 +
.../phoenix/schema/PColumnFamilyImpl.java | 17 +-
.../org/apache/phoenix/schema/PColumnImpl.java | 8 +
.../org/apache/phoenix/schema/PDataType.java | 65 +-
.../java/org/apache/phoenix/schema/PName.java | 12 +
.../org/apache/phoenix/schema/PNameFactory.java | 4 +-
.../org/apache/phoenix/schema/PNameImpl.java | 9 +-
.../java/org/apache/phoenix/schema/PTable.java | 2 +
.../org/apache/phoenix/schema/PTableImpl.java | 40 +-
.../org/apache/phoenix/schema/PhoenixArray.java | 16 +
.../org/apache/phoenix/schema/ValueSchema.java | 13 +-
.../org/apache/phoenix/util/MetaDataUtil.java | 62 +-
.../java/org/apache/phoenix/util/SizedUtil.java | 10 +-
.../org/apache/phoenix/end2end/ArrayTest.java | 176 ++++-
.../BaseTenantSpecificViewIndexTest.java | 85 +++
.../apache/phoenix/end2end/BaseViewTest.java | 133 ++++
.../end2end/QueryDatabaseMetaDataTest.java | 2 +
.../org/apache/phoenix/end2end/QueryTest.java | 59 +-
.../apache/phoenix/end2end/SaltedViewTest.java | 35 +
.../phoenix/end2end/SkipScanQueryTest.java | 70 +-
.../TenantSpecificViewIndexSaltedTest.java | 13 +
.../end2end/TenantSpecificViewIndexTest.java | 56 +-
.../org/apache/phoenix/end2end/ViewTest.java | 59 +-
.../apache/phoenix/jdbc/PhoenixTestDriver.java | 20 +-
.../java/org/apache/phoenix/query/BaseTest.java | 12 +-
.../phoenix/schema/PDataTypeForArraysTest.java | 613 ++++++++++++++++-
.../apache/phoenix/util/MetaDataUtilTest.java | 56 +-
pom.xml | 2 +-
67 files changed, 2627 insertions(+), 824 deletions(-)
----------------------------------------------------------------------