You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by nw...@apache.org on 2019/07/04 02:32:31 UTC
[incubator-heron] branch master updated: Spout source extra links
(#3306)
This is an automated email from the ASF dual-hosted git repository.
nwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new cbbcd27 Spout source extra links (#3306)
cbbcd27 is described below
commit cbbcd27f5f78c0c70c78c7da7ba049cd950131cf
Author: Xiaoyao Qian <xq...@twitter.com>
AuthorDate: Wed Jul 3 19:32:25 2019 -0700
Spout source extra links (#3306)
* tracker changes
* UI change
* more condition check
* fix style
---
.../config/src/yaml/tracker/heron_tracker.yaml | 14 ++++++++
heron/tools/tracker/src/python/config.py | 34 ++++++++++++++----
.../src/python/handlers/logicalplanhandler.py | 1 +
heron/tools/tracker/src/python/tracker.py | 40 ++++++++++++++++++----
heron/tools/ui/resources/static/js/topologies.js | 32 ++++++++++++++---
5 files changed, 103 insertions(+), 18 deletions(-)
diff --git a/heron/tools/config/src/yaml/tracker/heron_tracker.yaml b/heron/tools/config/src/yaml/tracker/heron_tracker.yaml
index 1d8cf5d..15f87c8 100644
--- a/heron/tools/config/src/yaml/tracker/heron_tracker.yaml
+++ b/heron/tools/config/src/yaml/tracker/heron_tracker.yaml
@@ -69,3 +69,17 @@ statemgrs:
# formatter: "http://127.0.0.1/viz/${ENVIRON}-${CLUSTER}-${TOPOLOGY}"
# - name: "Alerts"
# formatter: "http://127.0.0.1/alerts/${ENVIRON}-${CLUSTER}-${TOPOLOGY}"
+#
+# spout.extra.links:
+# - spout.type: "kafka"
+# extra.links:
+# - name: "Viz"
+# formatter: "http://127.0.0.1/kafka/viz/${ENVIRON}-${CLUSTER}-${TOPOLOGY}-${SPOUT_NAME}-${SPOUT_SOURCE}"
+# - name: "Alerts"
+# formatter: "http://127.0.0.1/kafka/alerts/${ENVIRON}-${CLUSTER}-${TOPOLOGY}-${SPOUT_NAME}-${SPOUT_SOURCE}"
+# - spout.type: "default"
+# extra.links:
+# - name: "Viz"
+# formatter: "http://127.0.0.1/default/viz/${ENVIRON}-${CLUSTER}-${TOPOLOGY}-${SPOUT_NAME}-${SPOUT_SOURCE}"
+# - name: "Alerts"
+# formatter: "http://127.0.0.1/default/alerts/${ENVIRON}-${CLUSTER}-${TOPOLOGY}-${SPOUT_NAME}-${SPOUT_SOURCE}"
diff --git a/heron/tools/tracker/src/python/config.py b/heron/tools/tracker/src/python/config.py
index 8da884f..6dbfc3d 100644
--- a/heron/tools/tracker/src/python/config.py
+++ b/heron/tools/tracker/src/python/config.py
@@ -26,7 +26,9 @@ STATEMGRS_KEY = "statemgrs"
EXTRA_LINKS_KEY = "extra.links"
EXTRA_LINK_NAME_KEY = "name"
EXTRA_LINK_FORMATTER_KEY = "formatter"
-
+EXTRA_LINK_URL_KEY = "url"
+SPOUT_EXTRA_LINKS_KEY = "spout.extra.links"
+SPOUT_TYPE_KEY = "spout.type"
class Config(object):
"""
@@ -38,9 +40,11 @@ class Config(object):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.extra_links = []
+ self.spout_extra_links = {}
self.load_configs()
+ # pylint: disable=line-too-long
def load_configs(self):
"""load config files"""
self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY])
@@ -48,6 +52,10 @@ class Config(object):
for extra_link in self.configs[EXTRA_LINKS_KEY]:
self.extra_links.append(self.validate_extra_link(extra_link))
+ if SPOUT_EXTRA_LINKS_KEY in self.configs:
+ for extra_link in self.configs[SPOUT_EXTRA_LINKS_KEY]:
+ self.spout_extra_links[extra_link[SPOUT_TYPE_KEY]] = [self.validate_extra_link(link) for link in extra_link[EXTRA_LINKS_KEY]]
+
def validate_extra_link(self, extra_link):
"""validate extra link"""
if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link:
@@ -69,6 +77,8 @@ class Config(object):
"${TOPOLOGY}": "topology",
"${ROLE}": "role",
"${USER}": "user",
+ "${SPOUT_NAME}": "spout_name",
+ "${SPOUT_SOURCE}": "spout_source",
}
dummy_formatted_url = url_format
for key, value in valid_parameters.items():
@@ -81,19 +91,29 @@ class Config(object):
# No error is thrown, so the format is valid.
return url_format
- def get_formatted_url(self, execution_state, formatter):
+ def get_formatted_url(self, formatter, execution_state, **additional):
"""
@param execution_state: The python dict representing JSON execution_state
+ @param additional: additional kwargs to interpolate
@return Formatted viz url
"""
# Create the parameters based on execution state
valid_parameters = {
- "${CLUSTER}": execution_state["cluster"],
- "${ENVIRON}": execution_state["environ"],
- "${TOPOLOGY}": execution_state["jobname"],
- "${ROLE}": execution_state["role"],
- "${USER}": execution_state["submission_user"],
+ "${CLUSTER}": execution_state.get("cluster",
+ additional.get("cluster", "${CLUSTER}")),
+ "${ENVIRON}": execution_state.get("environ",
+ additional.get("environ", "${ENVIRON}")),
+ "${TOPOLOGY}": execution_state.get("jobname",
+ additional.get("jobname", "${TOPOLOGY}")),
+ "${ROLE}": execution_state.get("role",
+ additional.get("role", "${ROLE}")),
+ "${USER}": execution_state.get("submission_user",
+ additional.get("submission_user", "${USER}")),
+ "${SPOUT_NAME}": execution_state.get("spout.name",
+ additional.get("spout.name", "${SPOUT_NAME}")),
+ "${SPOUT_SOURCE}": execution_state.get("spout.source",
+ additional.get("spout.source", "${SPOUT_SOURCE}")),
}
formatted_url = formatter
diff --git a/heron/tools/tracker/src/python/handlers/logicalplanhandler.py b/heron/tools/tracker/src/python/handlers/logicalplanhandler.py
index ff751a9..186cf70 100644
--- a/heron/tools/tracker/src/python/handlers/logicalplanhandler.py
+++ b/heron/tools/tracker/src/python/handlers/logicalplanhandler.py
@@ -63,6 +63,7 @@ class LogicalPlanHandler(BaseHandler):
outputs=value["outputs"],
spout_type=value["type"],
spout_source=value["source"],
+ extra_links=value["extra_links"],
)
bolts_map = dict()
diff --git a/heron/tools/tracker/src/python/tracker.py b/heron/tools/tracker/src/python/tracker.py
index d8914bb..77be6a2 100644
--- a/heron/tools/tracker/src/python/tracker.py
+++ b/heron/tools/tracker/src/python/tracker.py
@@ -29,7 +29,7 @@ from functools import partial
from heron.common.src.python.utils.log import Log
from heron.proto import topology_pb2
from heron.statemgrs.src.python import statemanagerfactory
-from heron.tools.tracker.src.python.config import EXTRA_LINK_FORMATTER_KEY
+from heron.tools.tracker.src.python.config import EXTRA_LINK_FORMATTER_KEY, EXTRA_LINK_URL_KEY
from heron.tools.tracker.src.python.topology import Topology
from heron.tools.tracker.src.python import javaobj
from heron.tools.tracker.src.python import pyutils
@@ -282,8 +282,8 @@ class Tracker(object):
for extra_link in self.config.extra_links:
link = extra_link.copy()
- link["url"] = self.config.get_formatted_url(executionState,
- link[EXTRA_LINK_FORMATTER_KEY])
+ link[EXTRA_LINK_URL_KEY] = self.config.get_formatted_url(link[EXTRA_LINK_FORMATTER_KEY],
+ executionState)
executionState["extra_links"].append(link)
return executionState
@@ -308,8 +308,8 @@ class Tracker(object):
for extra_link in self.config.extra_links:
link = extra_link.copy()
- link["url"] = self.config.get_formatted_url(metadata,
- link[EXTRA_LINK_FORMATTER_KEY])
+ link[EXTRA_LINK_URL_KEY] = self.config.get_formatted_url(link[EXTRA_LINK_FORMATTER_KEY],
+ metadata)
metadata["extra_links"].append(link)
return metadata
@@ -375,6 +375,7 @@ class Tracker(object):
return tmasterLocation
+ # pylint: disable=too-many-locals
def extract_logical_plan(self, topology):
"""
Returns the representation of logical plan that will
@@ -385,6 +386,25 @@ class Tracker(object):
"bolts": {},
}
+ # Pre-render component extra links with general params
+ execution_state = topology.execution_state
+ executionState = {
+ "cluster": execution_state.cluster,
+ "environ": execution_state.environ,
+ "role": execution_state.role,
+ "jobname": topology.name,
+ "submission_user": execution_state.submission_user,
+ }
+
+ spout_extra_links = {}
+ for spout_type, extra_links in self.config.spout_extra_links.items():
+ spout_extra_links[spout_type] = []
+ for extra_link in extra_links:
+ link = extra_link.copy()
+ link[EXTRA_LINK_URL_KEY] = self.config.get_formatted_url(link[EXTRA_LINK_FORMATTER_KEY],
+ executionState)
+ spout_extra_links[spout_type].append(link)
+
# Add spouts.
for spout in topology.spouts():
spoutName = spout.comp.name
@@ -404,8 +424,16 @@ class Tracker(object):
"type": spoutType,
"source": spoutSource,
"version": spoutVersion,
- "outputs": []
+ "outputs": [],
+ "extra_links": [],
}
+
+ for extra_link in spout_extra_links.get(spoutType, []):
+ extra_link[EXTRA_LINK_URL_KEY] = self.config.get_formatted_url(
+ extra_link[EXTRA_LINK_URL_KEY],
+ spoutPlan["config"], **{"spout.name": spoutName})
+ spoutPlan["extra_links"].append(extra_link)
+
for outputStream in list(spout.outputs):
spoutPlan["outputs"].append({
"stream_name": outputStream.stream.id
diff --git a/heron/tools/ui/resources/static/js/topologies.js b/heron/tools/ui/resources/static/js/topologies.js
index 2db0798..9fa9f23 100644
--- a/heron/tools/ui/resources/static/js/topologies.js
+++ b/heron/tools/ui/resources/static/js/topologies.js
@@ -8,9 +8,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -263,7 +263,7 @@ var AllMetrics = React.createClass({
fetchLplan: function () {
if (this.state.lplan === undefined) {
- var fetch_url = this.props.baseUrl +
+ var fetch_url = this.props.baseUrl +
"/topologies/" +
this.props.cluster + "/" +
this.props.environ + "/" +
@@ -545,6 +545,7 @@ var AllMetrics = React.createClass({
cluster: this.props.cluster,
environ: this.props.environ,
metrics: this.state.metrics,
+ lplan: this.state.lplan,
pplan: this.state.pplan,
instance: this.props.instance,
};
@@ -710,8 +711,13 @@ var ComponentCounters = React.createClass({
headings.push.apply(headings, timeRanges);
var rows = [];
+ var extraLinks = [];
if (this.props.info.comp_name) {
rows = this.getComponentMetricsRows();
+ var spoutDetail = this.props.info.lplan.spouts[this.props.info.comp_name];
+ if (spoutDetail) {
+ extraLinks = spoutDetail.extra_links;
+ }
} else {
rows = this.getTopologyMetricsRows();
}
@@ -720,7 +726,24 @@ var ComponentCounters = React.createClass({
<div>
<div className="widget-header">
<div className="title">
- <h4>{title}</h4>
+ <h4 style={{
+ "display": "inline-block",
+ "float": "left",
+ "margin-right": "10px"
+ }}>{title}</h4>
+ <div style={{
+ "padding-top": "10px",
+ "padding-bottom": "10px",
+ }}>
+ {extraLinks.map(function (extraLink) {
+ return <a id={extraLink['name']}
+ className="btn btn-primary btn-xs"
+ href={extraLink['url']}
+ target="_blank"
+ style={{"margin-right": "5px"}}>{extraLink['name']}
+ </a>
+ })}
+ </div>
</div>
</div>
<table className="table table-striped table-hover no-margin">
@@ -1037,4 +1060,3 @@ var InstanceCounters = React.createClass({
);
}
});
-