You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by od...@apache.org on 2016/10/24 19:36:55 UTC
incubator-hawq git commit: HAWQ-1103. Send constant datatype and
length in filter to PXF.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 402e08da6 -> 55f0ff7e4
HAWQ-1103. Send constant datatype and length in filter to PXF.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/55f0ff7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/55f0ff7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/55f0ff7e
Branch: refs/heads/master
Commit: 55f0ff7e4e03f5d90dc963f287a78a426f66495d
Parents: 402e08d
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Mon Oct 24 12:36:29 2016 -0700
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Mon Oct 24 12:36:48 2016 -0700
----------------------------------------------------------------------
src/backend/access/external/pxffilters.c | 58 +++++++++++++++-----
.../access/external/test/pxffilters_test.c | 33 ++++++-----
src/include/access/pxffilters.h | 9 ++-
3 files changed, 68 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/55f0ff7e/src/backend/access/external/pxffilters.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/pxffilters.c b/src/backend/access/external/pxffilters.c
index 56aea55..f06b07f 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -31,7 +31,7 @@
#include "utils/guc.h"
#include "utils/lsyscache.h"
-static List* pxf_make_expression_items_list(List *quals, Node *parent, bool *logicalOpsNum);
+static List* pxf_make_expression_items_list(List *quals, Node *parent, int *logicalOpsNum);
static void pxf_free_filter(PxfFilterDesc* filter);
static char* pxf_serialize_filter_list(List *filters);
static bool opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter);
@@ -193,7 +193,6 @@ Oid pxf_supported_types[] =
static void
pxf_free_expression_items_list(List *expressionItems, bool freeBoolExprNodes)
{
- ListCell *lc = NULL;
ExpressionItem *expressionItem = NULL;
int previousLength;
@@ -228,7 +227,7 @@ pxf_free_expression_items_list(List *expressionItems, bool freeBoolExprNodes)
*
*/
static List *
-pxf_make_expression_items_list(List *quals, Node *parent, bool *logicalOpsNum)
+pxf_make_expression_items_list(List *quals, Node *parent, int *logicalOpsNum)
{
ExpressionItem *expressionItem = NULL;
List *result = NIL;
@@ -330,7 +329,27 @@ pxf_free_filter(PxfFilterDesc* filter)
*
* Yields the following serialized string:
*
- * a0c1o2a0c5o1o7a2c"third"o5o7
+ * a0c23s1d1o2a1c23s1d5o1a2c25s5dthirdo5l0l0
+ *
+ * Where:
+ *
+ * a0 - first column of table
+ * c23 - scalar constant with type oid 23(INT4)
+ * s1 - size of constant in bytes
+ * d1 - serialized constant value
+ * o2 - greater than operation
+ * a1 - second column of table
+ * c23 - scalar constant with type oid 23(INT4)
+ * s1 - size of constant in bytes
+ * d5 - serialized constant value
+ * o1 - less than operation
+ * a2 - third column of table
+ * c25 - scalar constant with type oid 25(TEXT)
+ * s5 - size of constant in bytes
+ * dthird - serialized constant value
+ * o5 - equals operation
+ * l0 - AND operator
+ * l0 - AND operator
*
*/
static char *
@@ -369,14 +388,18 @@ pxf_serialize_filter_list(List *expressionItems)
PxfOperatorCode o = filter->op;
if (pxfoperand_is_attr(l) && pxfoperand_is_const(r))
{
- appendStringInfo(resbuf, "%c%d%c%s",
+ appendStringInfo(resbuf, "%c%d%c%d%c%lu%c%s",
PXF_ATTR_CODE, l.attnum - 1, /* Java attrs are 0-based */
- PXF_CONST_CODE, (r.conststr)->data);
+ PXF_CONST_CODE, r.consttype,
+ PXF_SIZE_BYTES, strlen(r.conststr->data),
+ PXF_CONST_DATA, (r.conststr)->data);
}
else if (pxfoperand_is_const(l) && pxfoperand_is_attr(r))
{
- appendStringInfo(resbuf, "%c%s%c%d",
- PXF_CONST_CODE, (l.conststr)->data,
+ appendStringInfo(resbuf, "%c%d%c%lu%c%s%c%d",
+ PXF_CONST_CODE, l.consttype,
+ PXF_SIZE_BYTES, strlen(l.conststr->data),
+ PXF_CONST_DATA, (l.conststr)->data,
PXF_ATTR_CODE, r.attnum - 1); /* Java attrs are 0-based */
}
else
@@ -406,6 +429,10 @@ pxf_serialize_filter_list(List *expressionItems)
appendStringInfo(resbuf, "%c%d", PXF_LOGICAL_OPERATOR_CODE, boolType);
break;
}
+ default:
+ {
+ elog(DEBUG5, "Skipping tag: %d", tag);
+ }
}
}
@@ -468,6 +495,7 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
{
filter->l.opcode = PXF_ATTR_CODE;
filter->l.attnum = ((Var *) leftop)->varattno;
+ filter->l.consttype = InvalidOid;
if (filter->l.attnum <= InvalidAttrNumber)
return false; /* system attr not supported */
@@ -476,6 +504,7 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
filter->r.conststr = makeStringInfo();
initStringInfo(filter->r.conststr);
const_to_str((Const *)rightop, filter->r.conststr);
+ filter->r.consttype = ((Const *)rightop)->consttype;
}
else if (IsA(leftop, Const) && IsA(rightop, Var))
{
@@ -484,9 +513,11 @@ opexpr_to_pxffilter(OpExpr *expr, PxfFilterDesc *filter)
filter->l.conststr = makeStringInfo();
initStringInfo(filter->l.conststr);
const_to_str((Const *)leftop, filter->l.conststr);
+ filter->l.consttype = ((Const *)leftop)->consttype;
filter->r.opcode = PXF_ATTR_CODE;
filter->r.attnum = ((Var *) rightop)->varattno;
+ filter->r.consttype = InvalidOid;
if (filter->r.attnum <= InvalidAttrNumber)
return false; /* system attr not supported */
}
@@ -632,9 +663,6 @@ const_to_str(Const *constval, StringInfo buf)
case FLOAT4OID:
case FLOAT8OID:
case NUMERICOID:
- appendStringInfo(buf, "%s", extval);
- break;
-
case TEXTOID:
case VARCHAROID:
case BPCHAROID:
@@ -642,14 +670,14 @@ const_to_str(Const *constval, StringInfo buf)
case BYTEAOID:
case DATEOID:
case TIMESTAMPOID:
- appendStringInfo(buf, "\\\"%s\\\"", extval);
+ appendStringInfo(buf, "%s", extval);
break;
case BOOLOID:
if (strcmp(extval, "t") == 0)
- appendStringInfo(buf, "\"true\"");
+ appendStringInfo(buf, "true");
else
- appendStringInfo(buf, "\"false\"");
+ appendStringInfo(buf, "false");
break;
default:
@@ -721,7 +749,7 @@ void enrich_trivial_expression(List *expressionItems) {
andExpr->boolop = AND_EXPR;
- andExpressionItem->node = andExpr;
+ andExpressionItem->node = (Node *) andExpr;
andExpressionItem->parent = NULL;
andExpressionItem->processed = false;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/55f0ff7e/src/backend/access/external/test/pxffilters_test.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/test/pxffilters_test.c b/src/backend/access/external/test/pxffilters_test.c
index 76be330..fa1db94 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -183,19 +183,19 @@ test__const_to_str__int(void **state)
void
test__const_to_str__text(void **state)
{
- verify__const_to_str(false, "that", TEXTOID, "\\\"that\\\"");
- verify__const_to_str(false, "joke", VARCHAROID, "\\\"joke\\\"");
- verify__const_to_str(false, "isn't", BPCHAROID, "\\\"isn't\\\"");
- verify__const_to_str(false, "funny", CHAROID, "\\\"funny\\\"");
- verify__const_to_str(false, "anymore", BYTEAOID, "\\\"anymore\\\"");
- verify__const_to_str(false, "iamdate", DATEOID, "\\\"iamdate\\\"");
+ verify__const_to_str(false, "that", TEXTOID, "that");
+ verify__const_to_str(false, "joke", VARCHAROID, "joke");
+ verify__const_to_str(false, "isn't", BPCHAROID, "isn't");
+ verify__const_to_str(false, "funny", CHAROID, "funny");
+ verify__const_to_str(false, "anymore", BYTEAOID, "anymore");
+ verify__const_to_str(false, "iamdate", DATEOID, "iamdate");
}
void
test__const_to_str__boolean(void **state)
{
- verify__const_to_str(false, "t", BOOLOID, "\"true\"");
- verify__const_to_str(false, "f", BOOLOID, "\"false\"");
+ verify__const_to_str(false, "t", BOOLOID, "true");
+ verify__const_to_str(false, "f", BOOLOID, "false");
}
void
@@ -499,7 +499,7 @@ void test__pxf_serialize_filter_list__oneFilter(void **state) {
expressionItems = lappend(expressionItems, filterExpressionItem);
char* result = pxf_serialize_filter_list(expressionItems);
- assert_string_equal(result, "a0c\\\"1984\\\"o5");
+ assert_string_equal(result, "a0c25s4d1984o5");
pxf_free_expression_items_list(expressionItems, true);
expressionItems = NIL;
@@ -514,23 +514,28 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
List* expressionItems = NIL;
ExpressionItem* expressionItem1 = build_expression_item(1, TEXTOID, "1984", TEXTOID, TextEqualOperator);
- ExpressionItem* expressionItem2 = build_expression_item(2, TEXTOID, "\"George Orwell\"", TEXTOID, TextEqualOperator);
- ExpressionItem* expressionItem3 = build_expression_item(3, TEXTOID, "\"Winston\"", TEXTOID, TextEqualOperator);
- ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID, "\"Eric-%\"", TEXTOID, 1209);
+ ExpressionItem* expressionItem2 = build_expression_item(2, TEXTOID, "George Orwell", TEXTOID, TextEqualOperator);
+ ExpressionItem* expressionItem3 = build_expression_item(3, TEXTOID, "Winston", TEXTOID, TextEqualOperator);
+ ExpressionItem* expressionItem4 = build_expression_item(4, TEXTOID, "Eric-%", TEXTOID, 1209);
+ ExpressionItem* expressionItem5 = build_expression_item(5, TEXTOID, "\"Ugly\" string with quotes", TEXTOID, TextEqualOperator);
+ ExpressionItem* expressionItem6 = build_expression_item(6, TEXTOID, "", TEXTOID, TextEqualOperator);
expressionItems = lappend(expressionItems, expressionItem1);
expressionItems = lappend(expressionItems, expressionItem2);
expressionItems = lappend(expressionItems, expressionItem3);
expressionItems = lappend(expressionItems, expressionItem4);
+ expressionItems = lappend(expressionItems, expressionItem5);
+ expressionItems = lappend(expressionItems, expressionItem6);
result = pxf_serialize_filter_list(expressionItems);
- assert_string_equal(result, "a0c\\\"1984\\\"o5a1c\\\"\"George Orwell\"\\\"o5a2c\\\"\"Winston\"\\\"o5a3c\\\"\"Eric-%\"\\\"o7");
+ assert_string_equal(result, "a0c25s4d1984o5a1c25s13dGeorge Orwello5a2c25s7dWinstono5a3c25s6dEric-%o7a4c25s25d\"Ugly\" string with quoteso5a5c25s0do5");
pfree(result);
+ int trivialExpressionItems = expressionItems->length;
enrich_trivial_expression(expressionItems);
- assert_int_equal(expressionItems->length, 7);
+ assert_int_equal(expressionItems->length, 2*trivialExpressionItems - 1);
pxf_free_expression_items_list(expressionItems, true);
expressionItems = NIL;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/55f0ff7e/src/include/access/pxffilters.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxffilters.h b/src/include/access/pxffilters.h
index 894337b..00c77d3 100644
--- a/src/include/access/pxffilters.h
+++ b/src/include/access/pxffilters.h
@@ -53,9 +53,11 @@ typedef enum PxfOperatorCode
* by a code that will describe the operator type in the final serialized
* string that gets pushed down.
*/
-#define PXF_ATTR_CODE 'a'
-#define PXF_CONST_CODE 'c'
-#define PXF_OPERATOR_CODE 'o'
+#define PXF_ATTR_CODE 'a'
+#define PXF_CONST_CODE 'c'
+#define PXF_SIZE_BYTES 's'
+#define PXF_CONST_DATA 'd'
+#define PXF_OPERATOR_CODE 'o'
#define PXF_LOGICAL_OPERATOR_CODE 'l'
/*
@@ -68,6 +70,7 @@ typedef struct PxfOperand
char opcode; /* PXF_ATTR_CODE or PXF_CONST_CODE*/
AttrNumber attnum; /* used when opcode is PXF_ATTR_CODE */
StringInfo conststr; /* used when opcode is PXF_CONST_CODE */
+ Oid consttype; /* used when opcode is PXF_CONST_CODE */
} PxfOperand;