You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/03/23 13:24:29 UTC
[37/50] [abbrv] ambari git commit: AMBARI-20519. Runtime filtering is
not rendering properly (pallavkul)
AMBARI-20519. Runtime filtering is not rendering properly (pallavkul)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0f5d5507
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0f5d5507
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0f5d5507
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 0f5d55077570821c7759a298d05a13104662b390
Parents: 845e921
Author: pallavkul <pa...@gmail.com>
Authored: Wed Mar 22 22:31:47 2017 +0530
Committer: pallavkul <pa...@gmail.com>
Committed: Wed Mar 22 22:31:47 2017 +0530
----------------------------------------------------------------------
.../ui/app/utils/hive-explainer/processor.js | 98 ++++++++++++++++++++
.../ui/app/utils/hive-explainer/transformer.js | 71 +++++++++++---
2 files changed, 154 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f5d5507/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js
index f6d3704..298366f 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js
+++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js
@@ -261,10 +261,108 @@ export function getEdgesWithCorrectedUnion(edges) {
}
+function findAllOutputOperators(vertices, outputOperatorsList, edges, patterns) {
+ vertices.forEach(cEdge => {
+ console.log(cEdge);
+ edges.push(cEdge);
+ let outputOperator = cEdge["OutputOperators:"];
+ if(outputOperator) {
+ patterns.push({outputOperator:outputOperator.substring(1, outputOperator.length-1), cEdge:[edges[edges.length-4], edges[edges.length-3], edges[edges.length-2], edges[edges.length-1]]});
+ outputOperatorsList.push({outputOperator:outputOperator.substring(1, outputOperator.length-1), cEdge:edges});
+ }
+ findAllOutputOperators(cEdge._children, outputOperatorsList, edges, patterns);
+ });
+}
+function findAllOperatorsVertex(cEdge, operatorIdList, edgesOp) {
+ cEdge.forEach(cChild => {
+ let operatorId = cChild["OperatorId:"];
+ if(operatorId) {
+ let operatorObj = {};
+ operatorObj[operatorId]= cChild["_operator"];
+ operatorIdList.push(operatorObj);
+ edgesOp.push(cChild);
+ if(edgesOp.length > 4) {
+ edgesOp.pop();
+ }
+ }
+ findAllOperatorsVertex(cChild._children, operatorIdList, edgesOp);
+ });
+}
+function findTheOperatorIDChains(vertices, operatorIdList, edgesOp) {
+ vertices.forEach(cChild => {
+ let subOperatorList = [];
+ edgesOp = [];
+ findAllOperatorsVertex(cChild._children, subOperatorList, edgesOp);
+ operatorIdList.push({subOperatorList:subOperatorList, edgesOp:edgesOp});
+ });
+}
+
+function findPatternParent(edges, patternArray) {
+ let newVertex;
+ edges._children.forEach(cChild => {
+ cChild._children.forEach(cSubChild => {
+ if(cSubChild && cSubChild["OperatorId:"] === patternArray[0]["OperatorId:"]){
+ if(cChild._children.length>1){
+ cChild._children = [cChild._children[0]];
+ cChild["OutputOperators:"] = patternArray[1]["OutputOperators:"];
+ newVertex = Object.assign(patternArray[2], {
+ "_operator":"Build Bloom Filter",
+ "_children":[],
+ "_groups": [
+ ...patternArray[0].groups||[doCloneAndOmit(patternArray[0], ['_groups'])],
+ ...patternArray[1].groups||[doCloneAndOmit(patternArray[1], ['_groups'])],
+ ...patternArray[2].groups||[doCloneAndOmit(patternArray[2], ['_groups'])],
+ ...patternArray[3].groups||[doCloneAndOmit(patternArray[3], ['_groups'])]]
+ });
+ }
+ } else if(cSubChild && cSubChild["OperatorId:"] === patternArray[3]["OperatorId:"]){
+ cChild._children = newVertex ? [newVertex]:[];
+ } else {
+ findPatternParent(cChild, patternArray);
+ }
+ });
+
+ });
+}
+
+function findTheOperatorIndex(vertices, patternArray) {
+ vertices.forEach(cChild => {
+ findPatternParent(cChild, patternArray);
+ });
+}
+
+function setTheNewVertex(vertices, patternArray) {
+ findTheOperatorIndex(vertices, patternArray);
+}
+
+function isPatternExists(outputOperators, operatorIds, vertices){
+ let patternArray = [outputOperators.cEdge[2], outputOperators.cEdge[3], operatorIds.edgesOp[0], operatorIds.edgesOp[1]];
+ setTheNewVertex(vertices, patternArray);
+}
+
+function findPatterns(outputOperatorsList, operatorIdList, pattern, patterns, vertices) {
+ patterns.forEach(item => {
+ operatorIdList.forEach(operatorDetails => {
+ operatorDetails.edgesOp.forEach(op => {
+ if(op["OperatorId:"] === item.outputOperator) {
+ pattern.push(op["OperatorId:"]);
+ isPatternExists(item, operatorDetails, vertices);
+ }
+ });
+ });
+ });
+
+}
+
// DANGER: impure function
// DANGER: breaks if there is a many-one / one-many connection
export function getAdjustedVerticesAndEdges(vertices, edges) {
+ let outputOperatorsList = [], operatorIdList = [], pattern = [], edges_opOperator = [], edges_operatorId = [], patterns = [];
+ findAllOutputOperators(vertices, outputOperatorsList, edges_opOperator, patterns);
+ findTheOperatorIDChains(vertices, operatorIdList, edges_operatorId);
+ findPatterns(outputOperatorsList, operatorIdList, pattern, patterns, vertices);
+
vertices
.filter(cVertex => ['Select Operator', 'HASHTABLEDUMMY', 'File Output Operator'].indexOf(getFirstOperatorOf(cVertex)._operator) >= 0)
.map(cVertex => edges.filter(cEdge => cEdge._target === cVertex._vertex))
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f5d5507/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js
index ac592ec..57fff1e 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js
+++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js
@@ -405,21 +405,6 @@ function getConnections(vertices, edges) {
});
});
- // iterate over edges to build connections
- edges.forEach(cEdge => {
- // get source uuid from source vertex
- const sourceVertex = vertices.find(cVertex => cVertex._vertex === cEdge._source);
- const sourceOperator = getLastOperatorOf(sourceVertex);
- // get target uuid from target vertex
- const targetVertex = vertices.find(cVertex => cVertex._vertex === cEdge._target);
- const targetOperator = findVertexAsInputInNode(targetVertex, cEdge._source) || getFirstOperatorOf(targetVertex);
- // push connection
- connections.push({
- _source: sourceOperator,
- _target: targetOperator,
- });
- });
-
// iterate over vertices to find dynamic partitioning event operator
// - build connection from dpp to tablescan of target vertex
vertices.forEach(cVertex => {
@@ -440,6 +425,38 @@ function getConnections(vertices, edges) {
});
});
+ // iterate over edges to build connections
+ edges.forEach(cEdge => {
+ // get source uuid from source vertex
+ const sourceVertex = vertices.find(cVertex => cVertex._vertex === cEdge._source);
+ // get target uuid from target vertex
+ const targetVertex = vertices.find(cVertex => cVertex._vertex === cEdge._target);
+
+ const sourceOperator = getLastOperatorOf(sourceVertex);
+ const targetOperator = findVertexAsInputInNode(targetVertex, cEdge._source) || getFirstOperatorOf(targetVertex);
+
+ let sourceOperatorList = [], targetOperatorList = [], srcNode = {}, targNode = {}, sourceVertexDef = undefined, targetVertexDef = undefined;
+ findAllOperatorsInSourceVertex(sourceVertex, sourceOperatorList, srcNode);
+ findAllOperatorsInTargetVertex(targetVertex, targetOperatorList, targNode);
+
+ if(sourceOperatorList.length && targetOperatorList.length) {
+ sourceOperatorList.forEach(function(item){
+ if(targetOperatorList.indexOf(item)>-1) {
+ sourceVertexDef = srcNode[item];
+ targetVertexDef = targNode[item];
+ console.log(sourceVertexDef, targetVertexDef)
+ }
+ });
+ }
+
+ connections.push({
+ _source: sourceVertexDef?sourceVertexDef:sourceOperator,
+ _target: targetVertexDef?targetVertexDef:targetOperator,
+ });
+ });
+
+
+
return connections;
}
@@ -476,6 +493,30 @@ function findVertexAsInputInNode(node, vertexId) {
return false;
}
+function findAllOperatorsInTargetVertex(node, resultsAggregator, targetNode) {
+ let outputOperator = node["OperatorId:"];
+ if(outputOperator) {
+ resultsAggregator.push(outputOperator);
+ targetNode[outputOperator] = node;
+ }
+ node._children.forEach(cChild => findAllOperatorsInTargetVertex(cChild, resultsAggregator, targetNode));
+ if(!node._children) {
+ return resultsAggregator;
+ }
+}
+
+function findAllOperatorsInSourceVertex(node, resultsAggregator, srcNode) {
+ let outputOperator = node["OutputOperators:"];
+ if(outputOperator) {
+ resultsAggregator.push(outputOperator.substring(1, outputOperator.length-1));
+ srcNode[outputOperator.substring(1, outputOperator.length-1)] = node;
+ }
+ node._children.forEach(cChild => findAllOperatorsInSourceVertex(cChild, resultsAggregator, srcNode));
+ if(!node._children) {
+ return resultsAggregator;
+ }
+}
+
function getLastOperatorOf(vertex) {
let operator = vertex._children[0];
while(operator._children.length > 0) {