You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hawq.apache.org by zhangjackey <gi...@git.apache.org> on 2018/03/19 08:21:47 UTC
[GitHub] incubator-hawq pull request #1348: HAWQ-1593. Vectorized execution condition...
GitHub user zhangjackey opened a pull request:
https://github.com/apache/incubator-hawq/pull/1348
HAWQ-1593. Vectorized execution condition check in plan tree
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/zhangjackey/incubator-hawq vec_chk
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/incubator-hawq/pull/1348.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #1348
----
commit fe7e45dbe79687f9fa057c64515e04c046a06d91
Author: Shujie Zhang <sh...@...>
Date: 2018-03-19T08:14:59Z
add the check phase for vectorized executor
----
---
[GitHub] incubator-hawq pull request #1348: HAWQ-1593. Vectorized execution condition...
Posted by zhangjackey <gi...@git.apache.org>.
Github user zhangjackey commented on a diff in the pull request:
https://github.com/apache/incubator-hawq/pull/1348#discussion_r175992596
--- Diff: contrib/vexecutor/vcheck.c ---
@@ -54,7 +61,274 @@ typedef struct VecFuncHashEntry
vFuncMap *vFunc;
} VecFuncHashEntry;
+typedef struct VecTypeHashEntry
+{
+ Oid src;
+ Oid dest;
+}VecTypeHashEntry;
+
+/* Map between the vectorized types and non-vectorized types */
+static HTAB *hashMapN2V = NULL;
+
+/*
+ * We check the expressions tree recursively becuase the args can be a sub expression,
+ * we must check the return type of sub expression to fit the parent expressions.
+ * so the retType in Vectorized is a temporary values, after we check on expression,
+ * we set the retType of this expression, and transfer this value to his parent.
+ */
+typedef struct VectorizedContext
+{
+ plan_tree_base_prefix base; /* Required prefix for plan_tree_walker/mutator */
+ Oid retType;
+ bool replace;
+}VectorizedContext;
+
+/*
+ * Check all the expressions if they can be vectorized
+ * NOTE: if an expressions is vectorized, we return false...,because we should check
+ * all the expressions in the Plan node, if we return true, then the walker will be
+ * over...
+ */
+static bool
+CheckVectorizedExpression(Node *node, VectorizedContext *ctx)
+{
+ if(NULL == node)
+ return false;
+
+ if(is_plan_node(node))
+ return false;
+
+ //check the type of Var if it can be vectorized
+ if(IsA(node, Var))
+ {
+ Var *var = (Var*)node;
+ Oid vtype = GetVtype(var->vartype);
+ if(InvalidOid == vtype)
+ return true;
+ ctx->retType = vtype;
+ if(ctx->replace)
+ var->vartype = vtype;
+ return false;
+ }
+
+ //Const treat as can be vectorzied, its return type is non-vectorized type
+ //because we support the function like this: vtype op(vtype, const);
+ if(IsA(node, Const))
+ {
+ Const *c = (Const*)node;
+ ctx->retType = c->consttype;
+ return false;
+ }
+
+ //OpExpr:args, return types should can be vectorized,
+ //and there must exists an vectorized function to implement the operator
+ if(IsA(node, OpExpr))
+ {
+ OpExpr *op = (OpExpr*)node;
+ Node *argnode = NULL;
+ Oid ltype, rtype, rettype;
+ Form_pg_operator voper;
+ HeapTuple tuple;
+
+ //OpExpr mostly have two args, check the first one
+ argnode = linitial(op->args);
+ if(CheckVectorizedExpression(argnode, ctx))
+ return true;
+
+ ltype = ctx->retType;
+
+ //check the second one
+ argnode = lsecond(op->args);
+ if(CheckVectorizedExpression(argnode, ctx))
+ return true;
+
+ rtype = ctx->retType;
+
+ //check the return type
+ rettype = GetVtype(op->opresulttype);
+ if(InvalidOid == rettype)
+ return true;
+
+
+ //get the vectorized operator functions
+ //NOTE:we have no ParseState now, Give the NULL value is OK but not good...
+ tuple = oper(NULL, list_make1(makeString(get_opname(op->opno))),
+ ltype, rtype, false, -1);
+ if(NULL == tuple)
+ return true;
+
+ voper = (Form_pg_operator)GETSTRUCT(tuple);
+ if(voper->oprresult != rettype)
+ return true;
+
+ if(ctx->replace)
+ {
+ op->opresulttype = rettype;
+ op->opfuncid = voper->oprcode;
+ }
+
+ ctx->retType = rettype;
+ return false;
+ }
+
+ //now, other nodes treat as can not be vectorized
+ return plan_tree_walker(node, CheckVectorizedExpression, ctx);;
+}
+
+/*
+ * check an plan node, all the expressions in it should be checked
+ * set the flag if an plan node can be vectorized
+ */
+static bool
+CheckPlanNodeWalker(PlannerInfo *root, Plan *plan)
+{
+ VectorizedContext ctx;
+
+ if(plan->vectorized)
+ return true;
+
+ ctx.replace =false;
+
+ ctx.retType = InvalidOid;
+ plan->vectorized = !plan_tree_walker((Node*)plan,
+ CheckVectorizedExpression,
+ &ctx);
+
+
+ return false;
+}
+
+/*
+ * check the plan tree
+ */
+static Plan*
+CheckPlanVectorzied(PlannerInfo *root, Plan *plan)
+{
+ if(NULL == plan)
+ return plan;
+
+ CheckPlanVectorzied(root, plan->lefttree);
+ CheckPlanVectorzied(root, plan->righttree);
+ CheckPlanNodeWalker(root, plan);
+
+ return plan;
+}
+
+/*
+ * Replace the non-vectorirzed type to vectorized type
+ */
+static bool
+ReplacePlanNodeWalker(PlannerInfo *root, Plan *plan)
+{
+ VectorizedContext ctx;
+
+ if(!plan->vectorized)
+ return false;
+
+ if(!HasVecExecOprator(nodeTag(plan)))
+ {
+ plan->vectorized = false;
+ return false;
+ }
+
+ ctx.replace =true;
+
+ ctx.retType = InvalidOid;
+ plan_tree_walker((Node*)plan,
+ CheckVectorizedExpression,
+ &ctx);
+
+ return false;
+}
+
+/*
+ * check the plan tree
+ */
+static Plan*
+ReplacePlanVectorzied(PlannerInfo *root, Plan *plan)
+{
+ if(NULL == plan)
+ return plan;
+
+ ReplacePlanVectorzied(root, plan->lefttree);
+ ReplacePlanVectorzied(root, plan->righttree);
+ ReplacePlanNodeWalker(root, plan);
+
+ return plan;
+}
+
+Plan*
+CheckAndReplacePlanVectorized(PlannerInfo *root, Plan *plan)
--- End diff --
So far it is not supported, but it does not seem to affect the normal running.
---
[GitHub] incubator-hawq issue #1348: HAWQ-1593. Vectorized execution condition check ...
Posted by wengyanqing <gi...@git.apache.org>.
Github user wengyanqing commented on the issue:
https://github.com/apache/incubator-hawq/pull/1348
LGTM.
---
[GitHub] incubator-hawq pull request #1348: HAWQ-1593. Vectorized execution condition...
Posted by zhangjackey <gi...@git.apache.org>.
Github user zhangjackey commented on a diff in the pull request:
https://github.com/apache/incubator-hawq/pull/1348#discussion_r175992712
--- Diff: contrib/vexecutor/vcheck.c ---
@@ -54,7 +61,274 @@ typedef struct VecFuncHashEntry
vFuncMap *vFunc;
} VecFuncHashEntry;
+typedef struct VecTypeHashEntry
+{
+ Oid src;
+ Oid dest;
+}VecTypeHashEntry;
+
+/* Map between the vectorized types and non-vectorized types */
+static HTAB *hashMapN2V = NULL;
+
+/*
+ * We check the expressions tree recursively becuase the args can be a sub expression,
+ * we must check the return type of sub expression to fit the parent expressions.
+ * so the retType in Vectorized is a temporary values, after we check on expression,
+ * we set the retType of this expression, and transfer this value to his parent.
+ */
+typedef struct VectorizedContext
+{
+ plan_tree_base_prefix base; /* Required prefix for plan_tree_walker/mutator */
+ Oid retType;
+ bool replace;
+}VectorizedContext;
+
+/*
+ * Check all the expressions if they can be vectorized
+ * NOTE: if an expressions is vectorized, we return false...,because we should check
+ * all the expressions in the Plan node, if we return true, then the walker will be
+ * over...
+ */
+static bool
+CheckVectorizedExpression(Node *node, VectorizedContext *ctx)
+{
+ if(NULL == node)
+ return false;
+
+ if(is_plan_node(node))
+ return false;
+
+ //check the type of Var if it can be vectorized
+ if(IsA(node, Var))
+ {
+ Var *var = (Var*)node;
+ Oid vtype = GetVtype(var->vartype);
+ if(InvalidOid == vtype)
+ return true;
+ ctx->retType = vtype;
+ if(ctx->replace)
+ var->vartype = vtype;
+ return false;
+ }
+
+ //Const treat as can be vectorzied, its return type is non-vectorized type
+ //because we support the function like this: vtype op(vtype, const);
+ if(IsA(node, Const))
+ {
+ Const *c = (Const*)node;
+ ctx->retType = c->consttype;
+ return false;
+ }
+
+ //OpExpr:args, return types should can be vectorized,
+ //and there must exists an vectorized function to implement the operator
+ if(IsA(node, OpExpr))
+ {
+ OpExpr *op = (OpExpr*)node;
+ Node *argnode = NULL;
+ Oid ltype, rtype, rettype;
+ Form_pg_operator voper;
+ HeapTuple tuple;
+
+ //OpExpr mostly have two args, check the first one
+ argnode = linitial(op->args);
+ if(CheckVectorizedExpression(argnode, ctx))
+ return true;
+
+ ltype = ctx->retType;
+
+ //check the second one
+ argnode = lsecond(op->args);
+ if(CheckVectorizedExpression(argnode, ctx))
+ return true;
+
+ rtype = ctx->retType;
+
+ //check the return type
+ rettype = GetVtype(op->opresulttype);
+ if(InvalidOid == rettype)
+ return true;
+
+
+ //get the vectorized operator functions
+ //NOTE:we have no ParseState now, Give the NULL value is OK but not good...
+ tuple = oper(NULL, list_make1(makeString(get_opname(op->opno))),
+ ltype, rtype, false, -1);
+ if(NULL == tuple)
+ return true;
+
+ voper = (Form_pg_operator)GETSTRUCT(tuple);
+ if(voper->oprresult != rettype)
+ return true;
+
+ if(ctx->replace)
+ {
+ op->opresulttype = rettype;
+ op->opfuncid = voper->oprcode;
+ }
+
+ ctx->retType = rettype;
+ return false;
+ }
--- End diff --
Yes, we will support more expression nodes step by step.
---
[GitHub] incubator-hawq pull request #1348: HAWQ-1593. Vectorized execution condition...
Posted by wengyanqing <gi...@git.apache.org>.
Github user wengyanqing commented on a diff in the pull request:
https://github.com/apache/incubator-hawq/pull/1348#discussion_r175950023
--- Diff: contrib/vexecutor/vcheck.c ---
@@ -54,7 +61,274 @@ typedef struct VecFuncHashEntry
vFuncMap *vFunc;
} VecFuncHashEntry;
+typedef struct VecTypeHashEntry
+{
+ Oid src;
+ Oid dest;
+}VecTypeHashEntry;
+
+/* Map between the vectorized types and non-vectorized types */
+static HTAB *hashMapN2V = NULL;
+
+/*
+ * We check the expressions tree recursively becuase the args can be a sub expression,
+ * we must check the return type of sub expression to fit the parent expressions.
+ * so the retType in Vectorized is a temporary values, after we check on expression,
+ * we set the retType of this expression, and transfer this value to his parent.
+ */
+typedef struct VectorizedContext
+{
+ plan_tree_base_prefix base; /* Required prefix for plan_tree_walker/mutator */
+ Oid retType;
+ bool replace;
+}VectorizedContext;
+
+/*
+ * Check all the expressions if they can be vectorized
+ * NOTE: if an expressions is vectorized, we return false...,because we should check
+ * all the expressions in the Plan node, if we return true, then the walker will be
+ * over...
+ */
+static bool
+CheckVectorizedExpression(Node *node, VectorizedContext *ctx)
+{
+ if(NULL == node)
+ return false;
+
+ if(is_plan_node(node))
+ return false;
+
+ //check the type of Var if it can be vectorized
+ if(IsA(node, Var))
+ {
+ Var *var = (Var*)node;
+ Oid vtype = GetVtype(var->vartype);
+ if(InvalidOid == vtype)
+ return true;
+ ctx->retType = vtype;
+ if(ctx->replace)
+ var->vartype = vtype;
+ return false;
+ }
+
+ //Const treat as can be vectorzied, its return type is non-vectorized type
+ //because we support the function like this: vtype op(vtype, const);
+ if(IsA(node, Const))
+ {
+ Const *c = (Const*)node;
+ ctx->retType = c->consttype;
+ return false;
+ }
+
+ //OpExpr:args, return types should can be vectorized,
+ //and there must exists an vectorized function to implement the operator
+ if(IsA(node, OpExpr))
+ {
+ OpExpr *op = (OpExpr*)node;
+ Node *argnode = NULL;
+ Oid ltype, rtype, rettype;
+ Form_pg_operator voper;
+ HeapTuple tuple;
+
+ //OpExpr mostly have two args, check the first one
+ argnode = linitial(op->args);
+ if(CheckVectorizedExpression(argnode, ctx))
+ return true;
+
+ ltype = ctx->retType;
+
+ //check the second one
+ argnode = lsecond(op->args);
+ if(CheckVectorizedExpression(argnode, ctx))
+ return true;
+
+ rtype = ctx->retType;
+
+ //check the return type
+ rettype = GetVtype(op->opresulttype);
+ if(InvalidOid == rettype)
+ return true;
+
+
+ //get the vectorized operator functions
+ //NOTE:we have no ParseState now, Give the NULL value is OK but not good...
+ tuple = oper(NULL, list_make1(makeString(get_opname(op->opno))),
+ ltype, rtype, false, -1);
+ if(NULL == tuple)
+ return true;
+
+ voper = (Form_pg_operator)GETSTRUCT(tuple);
+ if(voper->oprresult != rettype)
+ return true;
+
+ if(ctx->replace)
+ {
+ op->opresulttype = rettype;
+ op->opfuncid = voper->oprcode;
+ }
+
+ ctx->retType = rettype;
+ return false;
+ }
+
+ //now, other nodes treat as can not be vectorized
+ return plan_tree_walker(node, CheckVectorizedExpression, ctx);;
+}
+
+/*
+ * check an plan node, all the expressions in it should be checked
+ * set the flag if an plan node can be vectorized
+ */
+static bool
+CheckPlanNodeWalker(PlannerInfo *root, Plan *plan)
+{
+ VectorizedContext ctx;
+
+ if(plan->vectorized)
+ return true;
+
+ ctx.replace =false;
+
+ ctx.retType = InvalidOid;
+ plan->vectorized = !plan_tree_walker((Node*)plan,
+ CheckVectorizedExpression,
+ &ctx);
+
+
+ return false;
+}
+
+/*
+ * check the plan tree
+ */
+static Plan*
+CheckPlanVectorzied(PlannerInfo *root, Plan *plan)
+{
+ if(NULL == plan)
+ return plan;
+
+ CheckPlanVectorzied(root, plan->lefttree);
+ CheckPlanVectorzied(root, plan->righttree);
+ CheckPlanNodeWalker(root, plan);
+
+ return plan;
+}
+
+/*
+ * Replace the non-vectorirzed type to vectorized type
+ */
+static bool
+ReplacePlanNodeWalker(PlannerInfo *root, Plan *plan)
+{
+ VectorizedContext ctx;
+
+ if(!plan->vectorized)
+ return false;
+
+ if(!HasVecExecOprator(nodeTag(plan)))
+ {
+ plan->vectorized = false;
+ return false;
+ }
+
+ ctx.replace =true;
+
+ ctx.retType = InvalidOid;
+ plan_tree_walker((Node*)plan,
+ CheckVectorizedExpression,
+ &ctx);
+
+ return false;
+}
+
+/*
+ * check the plan tree
+ */
+static Plan*
+ReplacePlanVectorzied(PlannerInfo *root, Plan *plan)
+{
+ if(NULL == plan)
+ return plan;
+
+ ReplacePlanVectorzied(root, plan->lefttree);
+ ReplacePlanVectorzied(root, plan->righttree);
+ ReplacePlanNodeWalker(root, plan);
+
+ return plan;
+}
+
+Plan*
+CheckAndReplacePlanVectorized(PlannerInfo *root, Plan *plan)
--- End diff --
How to check initPlans of main Plan node?
---
[GitHub] incubator-hawq pull request #1348: HAWQ-1593. Vectorized execution condition...
Posted by wengyanqing <gi...@git.apache.org>.
Github user wengyanqing commented on a diff in the pull request:
https://github.com/apache/incubator-hawq/pull/1348#discussion_r175950445
--- Diff: contrib/vexecutor/vcheck.c ---
@@ -54,7 +61,274 @@ typedef struct VecFuncHashEntry
vFuncMap *vFunc;
} VecFuncHashEntry;
+typedef struct VecTypeHashEntry
+{
+ Oid src;
+ Oid dest;
+}VecTypeHashEntry;
+
+/* Map between the vectorized types and non-vectorized types */
+static HTAB *hashMapN2V = NULL;
+
+/*
+ * We check the expressions tree recursively becuase the args can be a sub expression,
+ * we must check the return type of sub expression to fit the parent expressions.
+ * so the retType in Vectorized is a temporary values, after we check on expression,
+ * we set the retType of this expression, and transfer this value to his parent.
+ */
+typedef struct VectorizedContext
+{
+ plan_tree_base_prefix base; /* Required prefix for plan_tree_walker/mutator */
+ Oid retType;
+ bool replace;
+}VectorizedContext;
+
+/*
+ * Check all the expressions if they can be vectorized
+ * NOTE: if an expressions is vectorized, we return false...,because we should check
+ * all the expressions in the Plan node, if we return true, then the walker will be
+ * over...
+ */
+static bool
+CheckVectorizedExpression(Node *node, VectorizedContext *ctx)
+{
+ if(NULL == node)
+ return false;
+
+ if(is_plan_node(node))
+ return false;
+
+ //check the type of Var if it can be vectorized
+ if(IsA(node, Var))
+ {
+ Var *var = (Var*)node;
+ Oid vtype = GetVtype(var->vartype);
+ if(InvalidOid == vtype)
+ return true;
+ ctx->retType = vtype;
+ if(ctx->replace)
+ var->vartype = vtype;
+ return false;
+ }
+
+ //Const treat as can be vectorzied, its return type is non-vectorized type
+ //because we support the function like this: vtype op(vtype, const);
+ if(IsA(node, Const))
+ {
+ Const *c = (Const*)node;
+ ctx->retType = c->consttype;
+ return false;
+ }
+
+ //OpExpr:args, return types should can be vectorized,
+ //and there must exists an vectorized function to implement the operator
+ if(IsA(node, OpExpr))
+ {
+ OpExpr *op = (OpExpr*)node;
+ Node *argnode = NULL;
+ Oid ltype, rtype, rettype;
+ Form_pg_operator voper;
+ HeapTuple tuple;
+
+ //OpExpr mostly have two args, check the first one
+ argnode = linitial(op->args);
+ if(CheckVectorizedExpression(argnode, ctx))
+ return true;
+
+ ltype = ctx->retType;
+
+ //check the second one
+ argnode = lsecond(op->args);
+ if(CheckVectorizedExpression(argnode, ctx))
+ return true;
+
+ rtype = ctx->retType;
+
+ //check the return type
+ rettype = GetVtype(op->opresulttype);
+ if(InvalidOid == rettype)
+ return true;
+
+
+ //get the vectorized operator functions
+ //NOTE:we have no ParseState now, Give the NULL value is OK but not good...
+ tuple = oper(NULL, list_make1(makeString(get_opname(op->opno))),
+ ltype, rtype, false, -1);
+ if(NULL == tuple)
+ return true;
+
+ voper = (Form_pg_operator)GETSTRUCT(tuple);
+ if(voper->oprresult != rettype)
+ return true;
+
+ if(ctx->replace)
+ {
+ op->opresulttype = rettype;
+ op->opfuncid = voper->oprcode;
+ }
+
+ ctx->retType = rettype;
+ return false;
+ }
--- End diff --
Maybe need to check more nodes, such as FuncExpr.
---
[GitHub] incubator-hawq pull request #1348: HAWQ-1593. Vectorized execution condition...
Posted by zhangjackey <gi...@git.apache.org>.
Github user zhangjackey closed the pull request at:
https://github.com/apache/incubator-hawq/pull/1348
---