You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by pa...@apache.org on 2017/02/28 09:38:25 UTC
ambari git commit: AMBARI-20215. Select operators should not appear
before fetch (pallavkul)
Repository: ambari
Updated Branches:
refs/heads/trunk 560b27e90 -> 924749be0
AMBARI-20215. Select operators should not appear before fetch (pallavkul)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/924749be
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/924749be
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/924749be
Branch: refs/heads/trunk
Commit: 924749be0670ffedad0dcdab00166aa80958c141
Parents: 560b27e
Author: pallavkul <pa...@gmail.com>
Authored: Tue Feb 28 15:07:29 2017 +0530
Committer: pallavkul <pa...@gmail.com>
Committed: Tue Feb 28 15:08:16 2017 +0530
----------------------------------------------------------------------
.../ui/app/utils/hive-explainer/processor.js | 67 ++++++++++++++++++++
.../ui/app/utils/hive-explainer/renderer.js | 2 +-
.../ui/app/utils/hive-explainer/transformer.js | 33 ++++++----
3 files changed, 87 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/924749be/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js
index a252498..48706cb 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js
+++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/processor.js
@@ -260,3 +260,70 @@ export function getEdgesWithCorrectedUnion(edges) {
});
}
+
+// DANGER: impure function
+// DANGER: breaks if there is a many-one / one-many connection
+export function getAdjustedVerticesAndEdges(vertices, edges) {
+
+ vertices
+ .filter(cVertex => ['Select Operator', 'HASHTABLEDUMMY', 'File Output Operator'].indexOf(getFirstOperatorOf(cVertex)._operator) >= 0)
+ .map(cVertex => edges.filter(cEdge => cEdge._target === cVertex._vertex))
+ .forEach(cEdges => {
+ const source = vertices.find(cVertex => cEdges.some(tcEdge => cVertex._vertex === tcEdge._source));
+ const target = vertices.find(cVertex => cEdges.some(tcEdge => cVertex._vertex === tcEdge._target));
+
+ const operatorLastOfSource = getLastOperatorOf(source);
+ const operatorFirstOfTarget = getFirstOperatorOf(target);
+
+ operatorLastOfSource._groups = [
+ ...(operatorLastOfSource._groups || [doCloneAndOmit(operatorLastOfSource, ['_groups'])]),
+ ...(operatorFirstOfTarget._groups || [doCloneAndOmit(operatorFirstOfTarget, ['_groups'])]),
+ ];
+ target._children = operatorFirstOfTarget._children;
+
+ target._isGroupedWith = source._vertex;
+ });
+
+ // cleanup
+ const adjustedVertices = vertices.filter(cVertex => cVertex._children.length > 0);
+ const cleanedVertices = vertices.filter(cVertex => cVertex._children.length === 0);
+ console.log(cleanedVertices);
+
+ const adjustedEdges =
+ edges
+ .reduce((accumulator, cEdge) => {
+ const cleanedAtSourceVertex = cleanedVertices.find(cVertex => cEdge._source === cVertex._vertex);
+ const cleanedAtTargetVertex = cleanedVertices.find(cVertex => cEdge._target === cVertex._vertex);
+ if(cleanedAtSourceVertex) {
+ // do not add edge back
+ // add new edge instead
+ accumulator.push(Object.assign({}, cEdge, {
+ _source: cleanedAtSourceVertex._isGroupedWith,
+ _target: cEdge._target
+ }));
+ } else if(cleanedAtTargetVertex) {
+ // do not add edge back
+ } else {
+ accumulator.push(cEdge);
+ }
+ return accumulator;
+ }, []);
+
+ return ({
+ adjustedVertices,
+ adjustedEdges
+ });
+}
+
+
+function getLastOperatorOf(vertex) {
+ let operator = vertex._children[0];
+ while(operator._children.length > 0) {
+ operator = operator._children[0];
+ }
+ return operator;
+}
+
+function getFirstOperatorOf(vertex) {
+ return vertex._children[0];
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/924749be/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer.js
index 196a514..f69b1d4 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer.js
+++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/renderer.js
@@ -18,7 +18,7 @@
export default function doRender(data, selector, onRequestDetail) {
- const width = '1200', height = '960';
+ const width = '1600', height = '800';
d3.select(selector).select('*').remove();
const svg =
http://git-wip-us.apache.org/repos/asf/ambari/blob/924749be/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js
index 01d6000..e6cf3f3 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js
+++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer/transformer.js
@@ -17,37 +17,42 @@
*/
import doEnhance from './enhancer';
-import {getProcessedVertices, getEdgesWithCorrectedUnion} from './processor';
+import {getProcessedVertices, getEdgesWithCorrectedUnion, getAdjustedVerticesAndEdges} from './processor';
export default function doTransform(data) {
const plan = getTezPlan(data);
const fetch = getFetchPlan(data);
- const vertices = [
+ let vertices = [
...getVertices(plan),
getFetchVertex(fetch)
];
- const tezEdges = getEdges(plan, vertices);
- const edgesWithCorrectedUnion = getEdgesWithCorrectedUnion(tezEdges);
- const edges = getEdgesWithFetch(edgesWithCorrectedUnion, vertices);
+ let edges = getEdges(plan, vertices);
+ edges = getEdgesWithCorrectedUnion(edges);
+ edges = getEdgesWithFetch(edges, vertices);
- const enhancedVertices = doEnhance(vertices);
+ vertices = doEnhance(vertices);
- const processedVertices = getProcessedVertices(enhancedVertices, edges);
+ vertices = getProcessedVertices(vertices, edges);
- const verticesWithIndexOfChildren = getVerticesWithIndexOfChildren(processedVertices);
+ const {adjustedVertices, adjustedEdges} = getAdjustedVerticesAndEdges(vertices, edges);
+ vertices = adjustedVertices;
+ edges = adjustedEdges;
- const tree = getVertexTree(edges);
- const connections = getConnections(verticesWithIndexOfChildren, edges);
- const treeWithOffsetY = getTreeWithOffsetAndHeight(tree, verticesWithIndexOfChildren, connections);
+ vertices = getVerticesWithIndexOfChildren(vertices);
- const nodes = getNodes(verticesWithIndexOfChildren);
+ let tree = getVertexTree(edges);
+
+ const connections = getConnections(vertices, edges);
+ tree = getTreeWithOffsetAndHeight(tree, vertices, connections);
+
+ const nodes = getNodes(vertices);
return ({
- vertices: verticesWithIndexOfChildren,
+ vertices,
edges,
- tree: treeWithOffsetY,
+ tree,
nodes,
connections,
});