You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by an...@apache.org on 2014/03/01 08:54:11 UTC

[1/3] git commit: PHOENIX-76

Repository: incubator-phoenix
Updated Branches:
  refs/heads/master c692e71ef -> c2010d174


PHOENIX-76


Project: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/commit/348b7517
Tree: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/tree/348b7517
Diff: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/diff/348b7517

Branch: refs/heads/master
Commit: 348b7517ade37b3e8b879899b3b3a3729efcb63a
Parents: e781eba
Author: anoopsjohn <an...@gmail.com>
Authored: Fri Feb 28 12:05:00 2014 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Fri Feb 28 12:05:00 2014 +0530

----------------------------------------------------------------------
 .../phoenix/filter/ColumnProjectionFilter.java  | 24 +++++++++++++++--
 .../phoenix/iterate/ParallelIterators.java      | 27 +++++++++++++++++---
 2 files changed, 45 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/348b7517/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
index bf9bc7e..8c5225f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java
@@ -21,9 +21,11 @@ import java.io.*;
 import java.util.*;
 import java.util.Map.Entry;
 
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.query.QueryConstants;
@@ -37,15 +39,17 @@ public class ColumnProjectionFilter extends FilterBase {
 
     private byte[] emptyCFName;
     private Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker;
+    private Set<byte[]> conditionOnlyCfs;
 
     public ColumnProjectionFilter() {
 
     }
 
     public ColumnProjectionFilter(byte[] emptyCFName,
-            Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker) {
+            Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker, Set<byte[]> conditionOnlyCfs) {
         this.emptyCFName = emptyCFName;
         this.columnsTracker = columnsTracker;
+        this.conditionOnlyCfs = conditionOnlyCfs;
     }
 
     @Override
@@ -68,6 +72,12 @@ public class ColumnProjectionFilter extends FilterBase {
             columnsTracker.put(new ImmutableBytesPtr(cf), qualifiers);
             familyMapSize--;
         }
+        int conditionOnlyCfsSize = WritableUtils.readVInt(input);
+        this.conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+        while (conditionOnlyCfsSize > 0) {
+            this.conditionOnlyCfs.add(WritableUtils.readCompressedByteArray(input));
+            conditionOnlyCfsSize--;
+        }
     }
 
     @Override
@@ -86,6 +96,11 @@ public class ColumnProjectionFilter extends FilterBase {
                 }
             }
         }
+        // Write conditionOnlyCfs
+        WritableUtils.writeVInt(output, this.conditionOnlyCfs.size());
+        for (byte[] f : this.conditionOnlyCfs) {
+            WritableUtils.writeCompressedByteArray(output, f);
+        }
     }
 
     @Override
@@ -120,6 +135,11 @@ public class ColumnProjectionFilter extends FilterBase {
     }
 
     @Override
+    public boolean isFamilyEssential(byte[] name) {
+        return conditionOnlyCfs.isEmpty() || this.conditionOnlyCfs.contains(name);
+    }
+
+    @Override
     public String toString() {
         return "";
     }

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/348b7517/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index 563d97f..0be2a11 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -105,11 +105,19 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
             ScanUtil.andFilterAtEnd(scan, new PageFilter(limit));
         }
 
+        if (!(statement.isAggregate())) {
+            doColumnProjectionOptimization(context, scan, table);
+        }
+    }
+
+    private void doColumnProjectionOptimization(StatementContext context, Scan scan, PTable table) {
         Map<byte[], NavigableSet<byte[]>> familyMap = scan.getFamilyMap();
         if (familyMap != null && !familyMap.isEmpty()) {
             // columnsTracker contain cf -> qualifiers which should get returned.
             Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker = 
                     new TreeMap<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>>();
+            Set<byte[]> conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+            boolean useOptimization = true;
             for (Entry<byte[], NavigableSet<byte[]>> entry : familyMap.entrySet()) {
                 ImmutableBytesPtr cf = new ImmutableBytesPtr(entry.getKey());
                 NavigableSet<byte[]> qs = entry.getValue();
@@ -122,20 +130,31 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
                 }
                 columnsTracker.put(cf, cols);
             }
+            if (familyMap.size() > 1) {
+                useOptimization = false;
+            }
             // Making sure that where condition CFs are getting scanned at HRS.
             for (Pair<byte[], byte[]> whereCol : context.getWhereCoditionColumns()) {
-                if (!(familyMap.containsKey(whereCol.getFirst()))) {
-                    scan.addFamily(whereCol.getFirst());
+                if (useOptimization) {
+                    if (!(familyMap.containsKey(whereCol.getFirst()))) {
+                        scan.addFamily(whereCol.getFirst());
+                        conditionOnlyCfs.add(whereCol.getFirst());
+                    }
+                } else {
+                    scan.addColumn(whereCol.getFirst(), whereCol.getSecond());
                 }
             }
-            if (!columnsTracker.isEmpty()) {
+            if (familyMap.size() > 1) {
+                useOptimization = false;
+            }
+            if (useOptimization && !columnsTracker.isEmpty()) {
                 for (ImmutableBytesPtr f : columnsTracker.keySet()) {
                     // This addFamily will remove explicit cols in scan familyMap and make it as entire row.
                     // We don't want the ExplicitColumnTracker to be used. Instead we have the ColumnProjectionFilter
                     scan.addFamily(f.get());
                 }
                 ScanUtil.andFilterAtEnd(scan, new ColumnProjectionFilter(SchemaUtil.getEmptyColumnFamily(table),
-                        columnsTracker));
+                        columnsTracker, conditionOnlyCfs));
             }
             if (table.getViewType() == ViewType.MAPPED) {
                 // Since we don't have the empty key value in MAPPED tables, we must select all CFs in HRS. But only the


[2/3] git commit: PHOENIX-76

Posted by an...@apache.org.
PHOENIX-76


Project: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/commit/2ca85ab6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/tree/2ca85ab6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/diff/2ca85ab6

Branch: refs/heads/master
Commit: 2ca85ab681f3080e52db632b1a78d45445fd9cea
Parents: 348b751
Author: anoopsjohn <an...@gmail.com>
Authored: Fri Feb 28 12:14:54 2014 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Fri Feb 28 12:14:54 2014 +0530

----------------------------------------------------------------------
 .../phoenix/iterate/ParallelIterators.java      | 36 +++++++++++---------
 1 file changed, 20 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/2ca85ab6/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
index 0be2a11..b82a7ea 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ParallelIterators.java
@@ -117,21 +117,28 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
             Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker = 
                     new TreeMap<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>>();
             Set<byte[]> conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
-            boolean useOptimization = true;
-            for (Entry<byte[], NavigableSet<byte[]>> entry : familyMap.entrySet()) {
-                ImmutableBytesPtr cf = new ImmutableBytesPtr(entry.getKey());
-                NavigableSet<byte[]> qs = entry.getValue();
-                NavigableSet<ImmutableBytesPtr> cols = null;
-                if (qs != null) {
-                    cols = new TreeSet<ImmutableBytesPtr>();
-                    for (byte[] q : qs) {
-                        cols.add(new ImmutableBytesPtr(q));
-                    }
+            int referencedCfCount = familyMap.size();
+            for (Pair<byte[], byte[]> whereCol : context.getWhereCoditionColumns()) {
+                if (!(familyMap.containsKey(whereCol.getFirst()))) {
+                    referencedCfCount++;
                 }
-                columnsTracker.put(cf, cols);
             }
-            if (familyMap.size() > 1) {
-                useOptimization = false;
+            boolean useOptimization = referencedCfCount == 1;
+            // when referencedCfCount is >1 we are not using the optimization as of now
+            // TODO support adding a HINT in query so that in case of cf count>1 also, this optimization can be applied.
+            if (useOptimization) {
+                for (Entry<byte[], NavigableSet<byte[]>> entry : familyMap.entrySet()) {
+                    ImmutableBytesPtr cf = new ImmutableBytesPtr(entry.getKey());
+                    NavigableSet<byte[]> qs = entry.getValue();
+                    NavigableSet<ImmutableBytesPtr> cols = null;
+                    if (qs != null) {
+                        cols = new TreeSet<ImmutableBytesPtr>();
+                        for (byte[] q : qs) {
+                            cols.add(new ImmutableBytesPtr(q));
+                        }
+                    }
+                    columnsTracker.put(cf, cols);
+                }
             }
             // Making sure that where condition CFs are getting scanned at HRS.
             for (Pair<byte[], byte[]> whereCol : context.getWhereCoditionColumns()) {
@@ -144,9 +151,6 @@ public class ParallelIterators extends ExplainTable implements ResultIterators {
                     scan.addColumn(whereCol.getFirst(), whereCol.getSecond());
                 }
             }
-            if (familyMap.size() > 1) {
-                useOptimization = false;
-            }
             if (useOptimization && !columnsTracker.isEmpty()) {
                 for (ImmutableBytesPtr f : columnsTracker.keySet()) {
                     // This addFamily will remove explicit cols in scan familyMap and make it as entire row.


[3/3] git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-phoenix

Posted by an...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-phoenix


Project: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/commit/c2010d17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/tree/c2010d17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/diff/c2010d17

Branch: refs/heads/master
Commit: c2010d1749201d8220c0830b4d56e4e1fb1d4fe6
Parents: 2ca85ab c692e71
Author: anoopsjohn <an...@gmail.com>
Authored: Sat Mar 1 13:23:15 2014 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Sat Mar 1 13:23:15 2014 +0530

----------------------------------------------------------------------
 KEYS                                            |  73 ++
 .../org/apache/phoenix/cache/GlobalCache.java   |  60 +-
 .../apache/phoenix/cache/ServerCacheClient.java |   2 +
 .../apache/phoenix/cache/TenantCacheImpl.java   |  14 +-
 .../apache/phoenix/client/ClientKeyValue.java   |  16 +
 .../phoenix/client/ClientKeyValueBuilder.java   |  15 +-
 .../phoenix/client/GenericKeyValueBuilder.java  |  13 +
 .../apache/phoenix/client/KeyValueBuilder.java  |  25 +-
 .../phoenix/compile/CreateTableCompiler.java    |   4 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |  13 +-
 .../apache/phoenix/compile/WhereOptimizer.java  |   9 +-
 .../coprocessor/HashJoinRegionScanner.java      |   8 +-
 .../coprocessor/MetaDataEndpointImpl.java       |  66 +-
 .../phoenix/coprocessor/MetaDataProtocol.java   |  14 +-
 .../coprocessor/MetaDataRegionObserver.java     |   3 +-
 .../apache/phoenix/execute/HashJoinPlan.java    |  31 +-
 .../expression/ArrayConstructorExpression.java  | 120 +++-
 .../phoenix/expression/BaseExpression.java      | 149 ++++
 .../phoenix/expression/CoerceExpression.java    |   7 +
 .../phoenix/expression/InListExpression.java    |  69 +-
 .../RowValueConstructorExpression.java          | 139 ----
 .../expression/function/ArrayIndexFunction.java |  14 +-
 .../expression/function/CoalesceFunction.java   |   9 +-
 .../filter/EvaluateOnCompletionVisitor.java     |  10 +-
 .../apache/phoenix/filter/SkipScanFilter.java   |  11 +-
 .../apache/phoenix/iterate/ExplainTable.java    |  18 +-
 .../apache/phoenix/jdbc/PhoenixConnection.java  |  14 +-
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |   8 +-
 .../org/apache/phoenix/jdbc/PhoenixDriver.java  |  27 +-
 .../phoenix/jdbc/PhoenixEmbeddedDriver.java     |  27 +-
 .../phoenix/memory/GlobalMemoryManager.java     |   2 +-
 .../query/ConnectionQueryServicesImpl.java      |  30 +-
 .../org/apache/phoenix/query/QueryServices.java |   4 +-
 .../phoenix/query/QueryServicesOptions.java     |   9 +-
 .../apache/phoenix/schema/DelegateColumn.java   |   7 +
 .../apache/phoenix/schema/MetaDataClient.java   |  78 ++-
 .../apache/phoenix/schema/PArrayDataType.java   | 671 ++++++++++++-------
 .../org/apache/phoenix/schema/PBaseColumn.java  |   9 +-
 .../java/org/apache/phoenix/schema/PColumn.java |   2 +
 .../apache/phoenix/schema/PColumnFamily.java    |   2 +
 .../phoenix/schema/PColumnFamilyImpl.java       |  17 +-
 .../org/apache/phoenix/schema/PColumnImpl.java  |   8 +
 .../org/apache/phoenix/schema/PDataType.java    |  65 +-
 .../java/org/apache/phoenix/schema/PName.java   |  12 +
 .../org/apache/phoenix/schema/PNameFactory.java |   4 +-
 .../org/apache/phoenix/schema/PNameImpl.java    |   9 +-
 .../java/org/apache/phoenix/schema/PTable.java  |   2 +
 .../org/apache/phoenix/schema/PTableImpl.java   |  40 +-
 .../org/apache/phoenix/schema/PhoenixArray.java |  16 +
 .../org/apache/phoenix/schema/ValueSchema.java  |  13 +-
 .../org/apache/phoenix/util/MetaDataUtil.java   |  62 +-
 .../java/org/apache/phoenix/util/SizedUtil.java |  10 +-
 .../org/apache/phoenix/end2end/ArrayTest.java   | 176 ++++-
 .../BaseTenantSpecificViewIndexTest.java        |  85 +++
 .../apache/phoenix/end2end/BaseViewTest.java    | 133 ++++
 .../end2end/QueryDatabaseMetaDataTest.java      |   2 +
 .../org/apache/phoenix/end2end/QueryTest.java   |  59 +-
 .../apache/phoenix/end2end/SaltedViewTest.java  |  35 +
 .../phoenix/end2end/SkipScanQueryTest.java      |  70 +-
 .../TenantSpecificViewIndexSaltedTest.java      |  13 +
 .../end2end/TenantSpecificViewIndexTest.java    |  56 +-
 .../org/apache/phoenix/end2end/ViewTest.java    |  59 +-
 .../apache/phoenix/jdbc/PhoenixTestDriver.java  |  20 +-
 .../java/org/apache/phoenix/query/BaseTest.java |  12 +-
 .../phoenix/schema/PDataTypeForArraysTest.java  | 613 ++++++++++++++++-
 .../apache/phoenix/util/MetaDataUtilTest.java   |  56 +-
 pom.xml                                         |   2 +-
 67 files changed, 2627 insertions(+), 824 deletions(-)
----------------------------------------------------------------------