You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by od...@apache.org on 2017/04/14 00:01:48 UTC
incubator-hawq git commit: HAWQ-1431. Do not use StatsAccessor when
column specified in SELECT clause.
Repository: incubator-hawq
Updated Branches:
refs/heads/master c7a76523d -> d6dd24fd3
HAWQ-1431. Do not use StatsAccessor when column specified in SELECT clause.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d6dd24fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d6dd24fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d6dd24fd
Branch: refs/heads/master
Commit: d6dd24fd31dd3cb6a7c9235f68d1c5dc32f11206
Parents: c7a7652
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Thu Apr 13 17:01:39 2017 -0700
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Thu Apr 13 17:01:39 2017 -0700
----------------------------------------------------------------------
.../hawq/pxf/api/utilities/InputData.java | 26 ++++++++++++++++++++
.../hawq/pxf/api/utilities/Utilities.java | 14 +++++------
.../hawq/pxf/api/utilities/UtilitiesTest.java | 7 ++++++
.../pxf/plugins/hive/HiveORCAccessorTest.java | 1 +
.../pxf/service/utilities/ProtocolData.java | 1 +
5 files changed, 42 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index 959cda6..7bf57ed 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -75,6 +75,16 @@ public class InputData {
protected ColumnDescriptor recordkeyColumn;
/**
+ * Number of attributes projected in query.
+ *
+ * Example:
+ * SELECT col1, col2, col3... : number of attributes projected - 3
+ * SELECT col1, col2, col3... WHERE col4=a : number of attributes projected - 4
+ * SELECT *... : number of attributes projected - 0
+ */
+ protected int numAttrsProjected;
+
+ /**
* Constructs an empty InputData
*/
public InputData() {
@@ -369,4 +379,20 @@ public class InputData {
this.fragmentIndex = fragmentIndex;
}
+ /**
+ * Returns number of attributes projected in a query
+ * @return number of attributes projected
+ */
+ public int getNumAttrsProjected() {
+ return numAttrsProjected;
+ }
+
+ /**
+ * Sets number of attributes projected
+ * @param numAttrsProjected number of attrivutes projected
+ */
+ public void setNumAttrsProjected(int numAttrsProjected) {
+ this.numAttrsProjected = numAttrsProjected;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
index 6dd0044..916c2b5 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
@@ -204,6 +204,9 @@ public class Utilities {
public static boolean useAggBridge(InputData inputData) {
boolean isStatsAccessor = false;
try {
+ if (inputData == null || inputData.getAccessor() == null) {
+ throw new IllegalArgumentException("Missing accessor information");
+ }
isStatsAccessor = ArrayUtils.contains(Class.forName(inputData.getAccessor()).getInterfaces(), StatsAccessor.class);
} catch (ClassNotFoundException e) {
LOG.error("Unable to load accessor class: " + e.getMessage());
@@ -213,6 +216,7 @@ public class Utilities {
return (inputData != null) && !inputData.hasFilter()
&& (inputData.getAggType() != null)
&& inputData.getAggType().isOptimizationSupported()
+ && inputData.getNumAttrsProjected() == 0
&& isStatsAccessor;
}
@@ -224,14 +228,10 @@ public class Utilities {
* @return true if this accessor should use statistic information
*/
public static boolean useStats(ReadAccessor accessor, InputData inputData) {
- if (accessor instanceof StatsAccessor) {
- /* Make sure filter is not present and aggregate operation supports optimization */
- if (inputData != null && !inputData.hasFilter()
- && inputData.getAggType() != null
- && inputData.getAggType().isOptimizationSupported()) {
+ if (accessor instanceof StatsAccessor && useAggBridge(inputData)) {
return true;
- }
+ } else {
+ return false;
}
- return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
index 12ba3d1..01c09bf 100644
--- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/UtilitiesTest.java
@@ -189,6 +189,7 @@ public class UtilitiesTest {
InputData metaData = mock(InputData.class);
when(metaData.getAccessor()).thenReturn(StatsAccessorImpl.class.getName());
when(metaData.getAggType()).thenReturn(EnumAggregationType.COUNT);
+ when(metaData.getAccessor()).thenReturn("org.apache.hawq.pxf.api.utilities.UtilitiesTest$StatsAccessorImpl");
assertTrue(Utilities.useAggBridge(metaData));
when(metaData.getAccessor()).thenReturn(UtilitiesTest.class.getName());
@@ -207,6 +208,7 @@ public class UtilitiesTest {
InputData metaData = mock(InputData.class);
ReadAccessor accessor = new StatsAccessorImpl();
when(metaData.getAggType()).thenReturn(EnumAggregationType.COUNT);
+ when(metaData.getAccessor()).thenReturn("org.apache.hawq.pxf.api.utilities.UtilitiesTest$StatsAccessorImpl");
assertTrue(Utilities.useStats(accessor, metaData));
ReadAccessor nonStatusAccessor = new NonStatsAccessorImpl();
assertFalse(Utilities.useStats(nonStatusAccessor, metaData));
@@ -214,5 +216,10 @@ public class UtilitiesTest {
//Do not use stats when input data has filter
when(metaData.hasFilter()).thenReturn(true);
assertFalse(Utilities.useStats(accessor, metaData));
+
+ //Do not use stats when more than one column is projected
+ when(metaData.hasFilter()).thenReturn(false);
+ when(metaData.getNumAttrsProjected()).thenReturn(1);
+ assertFalse(Utilities.useStats(accessor, metaData));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessorTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessorTest.java b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessorTest.java
index daee331..b1ee3f3 100644
--- a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessorTest.java
+++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveORCAccessorTest.java
@@ -82,6 +82,7 @@ public class HiveORCAccessorTest {
PowerMockito.whenNew(OrcInputFormat.class).withNoArguments().thenReturn(orcInputFormat);
RecordReader recordReader = mock(RecordReader.class);
PowerMockito.when(orcInputFormat.getRecordReader(any(InputSplit.class), any(JobConf.class), any(Reporter.class))).thenReturn(recordReader);
+ PowerMockito.when(inputData.getAccessor()).thenReturn(HiveORCAccessor.class.getName());
accessor = new HiveORCAccessor(inputData);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d6dd24fd/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
index 0cb6d47..8456165 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
@@ -383,6 +383,7 @@ public class ProtocolData extends InputData {
List<Integer> columnProjList = new ArrayList<Integer>();
if(columnProjStr != null) {
int columnProj = Integer.parseInt(columnProjStr);
+ numAttrsProjected = columnProj;
if(columnProj > 0) {
String columnProjIndexStr = getProperty("ATTRS-PROJ-IDX");
String columnProjIdx[] = columnProjIndexStr.split(",");