You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by sh...@apache.org on 2016/10/26 17:13:31 UTC
incubator-hawq git commit: HAWQ-1116. HAWQ/PXF bridge support for PPD
for operators isNull and isNotNull
Repository: incubator-hawq
Updated Branches:
refs/heads/master 838cf50d3 -> c9072d2ad
HAWQ-1116. HAWQ/PXF bridge support for PPD for operators isNull and isNotNull
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c9072d2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c9072d2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c9072d2a
Branch: refs/heads/master
Commit: c9072d2ad78421b9b89d9df643f046b4a3dd3c78
Parents: 838cf50
Author: Shivram Mani <sh...@gmail.com>
Authored: Wed Oct 26 10:13:18 2016 -0700
Committer: Shivram Mani <sh...@gmail.com>
Committed: Wed Oct 26 10:13:18 2016 -0700
----------------------------------------------------------------------
.../hawq/pxf/plugins/hive/HiveAccessor.java | 2 +-
src/backend/access/external/pxffilters.c | 28 ++++++-
.../access/external/test/pxffilters_test.c | 86 ++++++++++++++++++--
src/include/access/pxffilters.h | 4 +-
4 files changed, 109 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c9072d2a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
index b757371..ef9f76e 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveAccessor.java
@@ -361,7 +361,7 @@ public class HiveAccessor extends HdfsSplittableDataAccessor {
BasicFilter bFilter = (BasicFilter) filter;
boolean isOperationEqual = (bFilter.getOperation() == FilterParser.Operation.HDOP_EQ);
int columnIndex = bFilter.getColumn().index();
- String value = bFilter.getConstant().constant().toString();
+ String value = bFilter.getConstant() == null ? null : bFilter.getConstant().constant().toString();
LOG.debug("isOperationEqual: " + isOperationEqual + " columnIndex: "
+ columnIndex + " value: " + value);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c9072d2a/src/backend/access/external/pxffilters.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/pxffilters.c b/src/backend/access/external/pxffilters.c
index f06b07f..3961b48 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -41,7 +41,7 @@ static List* append_attr_from_var(Var* var, List* attrs);
static void enrich_trivial_expression(List *expressionItems);
/*
- * All supported HAWQ operators, and their respective HFDS operator code.
+ * All supported HAWQ operators, and their respective HDFS operator code.
* Note that it is OK to use hardcoded OIDs, since these are all pinned
* down system catalog operators.
* see pg_operator.h
@@ -249,6 +249,7 @@ pxf_make_expression_items_list(List *quals, Node *parent, int *logicalOpsNum)
switch (tag)
{
case T_OpExpr:
+ case T_NullTest:
{
result = lappend(result, expressionItem);
break;
@@ -321,7 +322,7 @@ pxf_free_filter(PxfFilterDesc* filter)
* as flattened tree. Operands and operators are represented with their
* respective codes. Each filter is serialized as follows:
*
- * <attcode><attnum><constcode><constval><opercode><opernum>
+ * <attcode><attnum><constcode><constval><constsizecode><constsize><constdata><constvalue><opercode><opernum>
*
* Example filter list:
*
@@ -429,6 +430,29 @@ pxf_serialize_filter_list(List *expressionItems)
appendStringInfo(resbuf, "%c%d", PXF_LOGICAL_OPERATOR_CODE, boolType);
break;
}
+ case T_NullTest:
+ {
+ elog(DEBUG1, "pxf_serialize_filter_list: node tag %d (T_NullTest)", tag);
+ NullTest *expr = (NullTest *) node;
+
+ /* filter expression for T_NullTest will not have any constant value */
+ if (expr->nulltesttype == IS_NULL)
+ {
+ appendStringInfo(resbuf, "%c%d%c%d", PXF_ATTR_CODE, ((Var *) expr->arg)->varattno - 1, PXF_OPERATOR_CODE, PXFOP_IS_NULL);
+ }
+ else if (expr->nulltesttype == IS_NOT_NULL)
+ {
+ appendStringInfo(resbuf, "%c%d%c%d", PXF_ATTR_CODE, ((Var *) expr->arg)->varattno - 1, PXF_OPERATOR_CODE, PXFOP_IS_NOTNULL);
+ }
+ else
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("internal error in pxffilters.c:pxf_serialize_"
+ "filter_list. Found a NullTest filter with incorrect NullTestType")));
+ }
+ break;
+ }
default:
{
elog(DEBUG5, "Skipping tag: %d", tag);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c9072d2a/src/backend/access/external/test/pxffilters_test.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/test/pxffilters_test.c b/src/backend/access/external/test/pxffilters_test.c
index fa1db94..235a1d7 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -323,10 +323,32 @@ OpExpr* build_op_expr(void* left, void* right, int op)
return expr;
}
-ExpressionItem* build_expression_item(int lattnum, Oid lattrtype, char* rconststr, Oid rattrtype, int op) {
+NullTest* build_null_expr(Expr* arg, NullTestType nullType)
+{
+ NullTest *expr = (NullTest*) palloc0(sizeof(NullTest));
+ expr->nulltesttype = nullType;
+ expr->arg = arg;
+
+ expr->xpr.type = T_NullTest;
+ return expr;
+}
+ExpressionItem* build_null_expression_item(int attnum, Oid attrtype, NullTestType nullType)
+{
ExpressionItem *expressionItem = (ExpressionItem*) palloc0(sizeof(ExpressionItem));
+ Var *vararg = build_var(attrtype, attnum);
+ OpExpr *operationExpression = build_null_expr(vararg, nullType);
+ expressionItem->node = operationExpression;
+ expressionItem->processed = false;
+ expressionItem->parent = NULL;
+
+ return expressionItem;
+}
+
+ExpressionItem* build_expression_item(int lattnum, Oid lattrtype, char* rconststr, Oid rattrtype, int op)
+{
+ ExpressionItem *expressionItem = (ExpressionItem*) palloc0(sizeof(ExpressionItem));
Var *leftop = build_var(lattrtype, lattnum);
Const *rightop = build_const(rattrtype, strdup(rconststr));
OpExpr *operationExpression = build_op_expr(leftop, rightop, op);
@@ -387,7 +409,7 @@ test__opexpr_to_pxffilter__allSupportedTypes(void **state)
/* NOTE: this test is not a use case - when the query includes
* 'is null' or 'is not null' the qualifier code is T_NullTest and not T_OpExpr */
void
-test__opexpr_to_pxffilter__attributeIsNull(void **state)
+test__opexpr_to_pxffilter__attributeEqualsNull(void **state)
{
PxfFilterDesc *filter = (PxfFilterDesc*) palloc0(sizeof(PxfFilterDesc));
Var *arg_var = build_var(INT2OID, 1);
@@ -410,6 +432,17 @@ test__opexpr_to_pxffilter__attributeIsNull(void **state)
pfree(expr);
}
+void
+test__opexpr_to_pxffilter__attributeIsNull(void **state)
+{
+ PxfFilterDesc *filter = (PxfFilterDesc*) palloc0(sizeof(PxfFilterDesc));
+ Var *arg_var = build_var(INT2OID, 1);
+ NullTest *expr = build_null_expr(arg_var, IS_NULL);
+
+ free(expr->arg);
+ pfree(expr);
+}
+
/*
* Test for a query with different types.
* Types pairing are not checked, it is covered by the
@@ -490,8 +523,8 @@ test__opexpr_to_pxffilter__unsupportedOpNot(void **state)
pfree(expr);
}
-void test__pxf_serialize_filter_list__oneFilter(void **state) {
-
+void test__pxf_serialize_filter_list__oneFilter(void **state)
+{
List* expressionItems = NIL;
ExpressionItem* filterExpressionItem = build_expression_item(1, TEXTOID, "1984", TEXTOID, TextEqualOperator);
@@ -507,6 +540,42 @@ void test__pxf_serialize_filter_list__oneFilter(void **state) {
}
+void test__pxf_serialize_fillter_list__nullFilter(void **state)
+{
+
+ List* expressionItems = NIL;
+
+ ExpressionItem* filterExpressionItem = build_null_expression_item(1, TEXTOID, IS_NULL);
+
+ expressionItems = lappend(expressionItems, filterExpressionItem);
+
+ char* result = pxf_serialize_filter_list(expressionItems);
+ assert_string_equal(result, "a0o8");
+
+ pxf_free_expression_items_list(expressionItems, true);
+ expressionItems = NIL;
+ pfree(result);
+
+}
+
+void test__pxf_serialize_fillter_list__nullFilter(void **state)
+{
+
+ List* expressionItems = NIL;
+
+ ExpressionItem* filterExpressionItem = build_null_expression_item(1, TEXTOID, IS_NULL);
+
+ expressionItems = lappend(expressionItems, filterExpressionItem);
+
+ char* result = pxf_serialize_filter_list(expressionItems);
+ assert_string_equal(result, "a0o8");
+
+ pxf_free_expression_items_list(expressionItems, true);
+ expressionItems = NIL;
+ pfree(result);
+
+}
+
void
test__pxf_serialize_filter_list__manyFilters(void **state)
{
@@ -519,7 +588,7 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID, "Eric-%", TEXTOID, 1209);
ExpressionItem* expressionItem5 = build_expression_item(5, TEXTOID, "\"Ugly\" string with quotes", TEXTOID, TextEqualOperator);
ExpressionItem* expressionItem6 = build_expression_item(6, TEXTOID, "", TEXTOID, TextEqualOperator);
-
+ ExpressionItem* expressionItem7 = build_null_expression_item(7, TEXTOID, IS_NOT_NULL);
expressionItems = lappend(expressionItems, expressionItem1);
expressionItems = lappend(expressionItems, expressionItem2);
@@ -527,9 +596,10 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
expressionItems = lappend(expressionItems, expressionItem4);
expressionItems = lappend(expressionItems, expressionItem5);
expressionItems = lappend(expressionItems, expressionItem6);
+ expressionItems = lappend(expressionItems, expressionItem7);
result = pxf_serialize_filter_list(expressionItems);
- assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7a4c25s25d\"Ugly\" string with quoteso5a5c25s0do5");
+ assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7a4c25s25d\"Ugly\" string with quoteso5a5c25s0do5a4o9");
pfree(result);
int trivialExpressionItems = expressionItems->length;
@@ -557,12 +627,14 @@ main(int argc, char* argv[])
unit_test(test__opexpr_to_pxffilter__unary_expr),
unit_test(test__opexpr_to_pxffilter__intGT),
unit_test(test__opexpr_to_pxffilter__allSupportedTypes),
- unit_test(test__opexpr_to_pxffilter__attributeIsNull),
+ unit_test(test__opexpr_to_pxffilter__attributeEqualsNull),
unit_test(test__opexpr_to_pxffilter__differentTypes),
unit_test(test__opexpr_to_pxffilter__unsupportedTypeCircle),
unit_test(test__opexpr_to_pxffilter__twoVars),
unit_test(test__opexpr_to_pxffilter__unsupportedOpNot),
+ unit_test(test__opexpr_to_pxffilter__attributeIsNull),
unit_test(test__pxf_serialize_filter_list__oneFilter),
+ unit_test(test__pxf_serialize_fillter_list__nullFilter),
unit_test(test__pxf_serialize_filter_list__manyFilters)
};
return run_tests(tests);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c9072d2a/src/include/access/pxffilters.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxffilters.h b/src/include/access/pxffilters.h
index 00c77d3..f194966 100644
--- a/src/include/access/pxffilters.h
+++ b/src/include/access/pxffilters.h
@@ -44,7 +44,9 @@ typedef enum PxfOperatorCode
PXFOP_GE,
PXFOP_EQ,
PXFOP_NE,
- PXFOP_LIKE
+ PXFOP_LIKE,
+ PXFOP_IS_NULL,
+ PXFOP_IS_NOTNULL
} PxfOperatorCode;