You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by go...@apache.org on 2016/09/14 19:31:25 UTC
incubator-hawq git commit: HAWQ-779 support pxf filter pushdwon at
the 'CREATE PLAN' stage , and more filter type & op
Repository: incubator-hawq
Updated Branches:
refs/heads/master cd81289bc -> 6416b536d
HAWQ-779 support pxf filter pushdwon at the 'CREATE PLAN' stage ,and more filter type & op
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6416b536
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6416b536
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6416b536
Branch: refs/heads/master
Commit: 6416b536d273b5add89482d50ea403c70572da80
Parents: cd81289
Author: Devin Jia <ji...@inspur.com>
Authored: Fri Aug 5 15:05:50 2016 +0800
Committer: Goden Yao <go...@apache.org>
Committed: Wed Sep 14 12:30:25 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hawq/pxf/api/FilterParser.java | 4 +++-
.../org/apache/hawq/pxf/api/FilterParserTest.java | 8 ++++++++
.../hawq/pxf/plugins/hbase/HBaseFilterBuilder.java | 8 ++++++++
.../hawq/pxf/plugins/hive/HiveDataFragmenter.java | 12 ++++++++++++
src/backend/access/external/pxffilters.c | 15 +++++++++++++--
src/backend/access/external/test/pxffilters_test.c | 14 +++++++++++++-
src/backend/optimizer/plan/createplan.c | 8 +++++++-
src/include/access/pxffilters.h | 3 ++-
8 files changed, 66 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
index ae96d22..00fbf2b 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
@@ -61,7 +61,8 @@ public class FilterParser {
HDOP_GE,
HDOP_EQ,
HDOP_NE,
- HDOP_AND
+ HDOP_AND,
+ HDOP_LIKE
}
/**
@@ -392,6 +393,7 @@ public class FilterParser {
operatorTranslationMap.put(5, Operation.HDOP_EQ);
operatorTranslationMap.put(6, Operation.HDOP_NE);
operatorTranslationMap.put(7, Operation.HDOP_AND);
+ operatorTranslationMap.put(8, Operation.HDOP_LIKE);
return operatorTranslationMap;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
index 0308f7f..1ded4a3 100644
--- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
@@ -215,6 +215,10 @@ public class FilterParserTest {
filter = "a1c2o7";
op = Operation.HDOP_AND;
runParseOneOperation("this filter was build from HDOP_AND", filter, op);
+
+ filter = "a1c2o8";
+ op = Operation.HDOP_LIKE;
+ runParseOneOperation("this filter was built from HDOP_LIKE", filter, op);
}
@Test
@@ -247,6 +251,10 @@ public class FilterParserTest {
filter = "c2a1o7";
op = Operation.HDOP_AND;
runParseOneOperation("this filter was build from HDOP_AND using reverse!", filter, op);
+
+ filter = "c2a1o8";
+ op = Operation.HDOP_LIKE;
+ runParseOneOperation("this filter was build from HDOP_LIKE using reverse!", filter, op);
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
index 6dc271e..8eadc88 100644
--- a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
+++ b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
@@ -165,6 +165,14 @@ public class HBaseFilterBuilder implements FilterParser.FilterBuilder {
ByteArrayComparable comparator = getComparator(hbaseColumn.columnTypeCode(),
constant.constant());
+ if(operatorsMap.get(opId) == null){
+ //HBase does not support HDOP_LIKE, use 'NOT NULL' comparator
+ return new SingleColumnValueFilter(hbaseColumn.columnFamilyBytes(),
+ hbaseColumn.qualifierBytes(),
+ CompareFilter.CompareOp.NOT_EQUAL,
+ new NullComparator());
+ }
+
/**
* If row key is of type TEXT, allow filter in start/stop row key API in
* HBaseAccessor/Scan object.
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
index ded9627..2fe31cd 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
@@ -25,6 +25,8 @@ import java.util.ListIterator;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
+import java.util.Map;
+import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -37,6 +39,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputFormat;
@@ -48,6 +51,7 @@ import org.apache.hawq.pxf.api.Fragment;
import org.apache.hawq.pxf.api.Fragmenter;
import org.apache.hawq.pxf.api.FragmentsStats;
import org.apache.hawq.pxf.api.Metadata;
+import org.apache.hawq.pxf.api.io.DataType;
import org.apache.hawq.pxf.api.utilities.ColumnDescriptor;
import org.apache.hawq.pxf.api.utilities.InputData;
import org.apache.hawq.pxf.plugins.hdfs.utilities.HdfsUtilities;
@@ -87,6 +91,7 @@ public class HiveDataFragmenter extends Fragmenter {
// partition filtering
private Set<String> setPartitions = new TreeSet<String>(
String.CASE_INSENSITIVE_ORDER);
+ private Map<String, String> partitionkeyTypes = new HashMap<>();
/**
* A Hive table unit - means a subset of the HIVE table, where we can say
@@ -191,6 +196,7 @@ public class HiveDataFragmenter extends Fragmenter {
// Save all hive partition names in a set for later filter match
for (FieldSchema fs : tbl.getPartitionKeys()) {
setPartitions.add(fs.getName());
+ partitionkeyTypes.put(fs.getName(), fs.getType());
}
LOG.debug("setPartitions :" + setPartitions);
@@ -451,6 +457,12 @@ public class HiveDataFragmenter extends Fragmenter {
return false;
}
+ if (!partitionkeyTypes.get(filterColumnName).equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ LOG.debug("Filter type is not string type , ignore this filter for hive: "
+ + filter);
+ return false;
+ }
+
filtersString.append(prefix);
filtersString.append(filterColumnName);
filtersString.append(HIVE_API_EQ);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/backend/access/external/pxffilters.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/pxffilters.c b/src/backend/access/external/pxffilters.c
index 405e6d7..cdb2cb6 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -79,6 +79,7 @@ dbop_pxfop_map pxf_supported_opr[] =
{665 /* text_le */, PXFOP_LE},
{667 /* text_ge */, PXFOP_GE},
{531 /* textlt */, PXFOP_NE},
+ {1209 /* textlike */, PXFOP_LIKE},
/* int2 to int4 */
{Int24EqualOperator /* int24eq */, PXFOP_EQ},
@@ -126,7 +127,15 @@ dbop_pxfop_map pxf_supported_opr[] =
{1871 /* int82gt */, PXFOP_GT},
{1872 /* int82le */, PXFOP_LE},
{1873 /* int82ge */, PXFOP_GE},
- {1869 /* int82ne */, PXFOP_NE}
+ {1869 /* int82ne */, PXFOP_NE},
+
+ /* date */
+ {DateEqualOperator /* eq */, PXFOP_EQ},
+ {1095 /* date_lt */, PXFOP_LT},
+ {1097 /* date_gt */, PXFOP_GT},
+ {1096 /* date_le */, PXFOP_LE},
+ {1098 /* date_ge */, PXFOP_GE},
+ {1094 /* date_ne */, PXFOP_NE}
};
@@ -143,7 +152,8 @@ Oid pxf_supported_types[] =
BPCHAROID,
CHAROID,
BYTEAOID,
- BOOLOID
+ BOOLOID,
+ DATEOID
};
/*
@@ -559,6 +569,7 @@ const_to_str(Const *constval, StringInfo buf)
case BPCHAROID:
case CHAROID:
case BYTEAOID:
+ case DATEOID:
appendStringInfo(buf, "\\\"%s\\\"", extval);
break;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/backend/access/external/test/pxffilters_test.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/test/pxffilters_test.c b/src/backend/access/external/test/pxffilters_test.c
index a5b3fb0..65b1aef 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -42,6 +42,7 @@ test__supported_filter_type(void **state)
CHAROID,
BYTEAOID,
BOOLOID,
+ DATEOID,
CIRCLEOID /* unsupported type */
};
@@ -61,7 +62,7 @@ test__supported_filter_type(void **state)
/* go over pxf_supported_types array */
int nargs = sizeof(pxf_supported_types) / sizeof(Oid);
- assert_int_equal(nargs, 12);
+ assert_int_equal(nargs, 13);
for (i = 0; i < nargs; ++i)
{
assert_true(supported_filter_type(pxf_supported_types[i]));
@@ -184,6 +185,7 @@ test__const_to_str__text(void **state)
verify__const_to_str(false, "isn't", BPCHAROID, "\\\"isn't\\\"");
verify__const_to_str(false, "funny", CHAROID, "\\\"funny\\\"");
verify__const_to_str(false, "anymore", BYTEAOID, "\\\"anymore\\\"");
+ verify__const_to_str(false, "iamdate", DATEOID, "\\\"iamdate\\\"");
}
void
@@ -523,6 +525,10 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
PXF_ATTR_CODE, 5, NULL,
PXF_CONST_CODE, 0, "\"Winston\"",
PXFOP_GE);
+ PxfFilterDesc* filter5 = build_filter(
+ PXF_ATTR_CODE, 6, NULL,
+ PXF_CONST_CODE, 0, "\"Eric-%\"",
+ PXFOP_LIKE);
filter_list = lappend(filter_list, filter1);
filter_list = lappend(filter_list, filter2);
@@ -543,6 +549,12 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
assert_string_equal(result, "a1c1983o2a2c1985o1o7a3c\"George Orwell\"o5o7a4c\"Winston\"o4o7");
pfree(result);
+ filter_list = lappend(filter_list, filter5);
+
+ result = pxf_serialize_filter_list(filter_list);
+ assert_string_equal(result, "a1c1983o2a2c1985o1o7a3c\"George Orwell\"o5o7a4c\"Winston\"o4o7a5c\"Eric-%\"o8o7");
+ pfree(result);
+
pxf_free_filter_list(filter_list);
filter_list = NIL;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/backend/optimizer/plan/createplan.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c
index b5a5cea..d7efa62 100644
--- a/src/backend/optimizer/plan/createplan.c
+++ b/src/backend/optimizer/plan/createplan.c
@@ -1144,9 +1144,15 @@ static char** create_pxf_plan(char **segdb_file_map, RelOptInfo *rel, int total_
Relation relation = RelationIdGetRelation(planner_rt_fetch(scan_relid, ctx->root)->relid);
- segdb_work_map = map_hddata_2gp_segments(uri_str,
+ if (pxf_enable_filter_pushdown){
+ segdb_work_map = map_hddata_2gp_segments(uri_str,
+ total_segs, segs_participating,
+ relation, ctx->root->parse->jointree->quals);
+ }else{
+ segdb_work_map = map_hddata_2gp_segments(uri_str,
total_segs, segs_participating,
relation, NULL);
+ }
Assert(segdb_work_map != NULL);
RelationClose(relation);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/include/access/pxffilters.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxffilters.h b/src/include/access/pxffilters.h
index 1409e39..f54c47c 100644
--- a/src/include/access/pxffilters.h
+++ b/src/include/access/pxffilters.h
@@ -44,7 +44,8 @@ typedef enum PxfOperatorCode
PXFOP_GE,
PXFOP_EQ,
PXFOP_NE,
- PXFOP_AND
+ PXFOP_AND,
+ PXFOP_LIKE
} PxfOperatorCode;