You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by ea...@apache.org on 2017/10/23 21:47:39 UTC
[37/45] qpid-dispatch git commit: DISPATCH-834 Added ability to
deploy to multiple machines
DISPATCH-834 Added ability to deploy to multiple machines
Project: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/repo
Commit: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/commit/91df5a67
Tree: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/tree/91df5a67
Diff: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/diff/91df5a67
Branch: refs/heads/master
Commit: 91df5a6754b1c5778cf758c47bb0b4d271f9734d
Parents: f163d38
Author: Ernest Allen <ea...@redhat.com>
Authored: Wed Oct 11 08:30:39 2017 -0400
Committer: Ernest Allen <ea...@redhat.com>
Committed: Wed Oct 11 08:30:39 2017 -0400
----------------------------------------------------------------------
console/config/config.py | 189 ++++++++++++++++++++++--------
console/config/css/mock.css | 10 ++
console/config/html/qdrTopology.html | 30 ++++-
console/config/js/qdrTopology.js | 151 ++++++++++++++++++++----
console/config/mock/section.py | 5 +
5 files changed, 311 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/91df5a67/console/config/config.py
----------------------------------------------------------------------
diff --git a/console/config/config.py b/console/config/config.py
index 193b4ad..d17bc34 100755
--- a/console/config/config.py
+++ b/console/config/config.py
@@ -29,8 +29,12 @@ import SimpleHTTPServer
import SocketServer
import json
import cStringIO
+import yaml
+import threading
+import subprocess
import pdb
+from distutils.spawn import find_executable
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
@@ -38,7 +42,7 @@ def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
get_class = lambda x: globals()[x]
sectionKeys = {"log": "module", "sslProfile": "name", "connector": "port", "listener": "port"}
-# borrowed from qpid-dispatch/python/qpid_dispatch_internal/management/config.py
+# modified from qpid-dispatch/python/qpid_dispatch_internal/management/config.py
def _parse(lines):
"""Parse config file format into a section list"""
begin = re.compile(r'([\w-]+)[ \t]*{') # WORD {
@@ -50,7 +54,10 @@ def _parse(lines):
"""Do substitutions to make line json-friendly"""
line = line.strip()
if line.startswith("#"):
- return ""
+ if line.startswith("#deploy_host:"):
+ line = line[1:]
+ else:
+ return ""
# 'pattern:' is a special snowflake. It allows '#' characters in
# its value, so they cannot be treated as comment delimiters
if line.split(':')[0].strip().lower() == "pattern":
@@ -92,7 +99,9 @@ class Manager(object):
def __init__(self, topology, verbose):
self.topology = topology
self.verbose = verbose
- self.base = "topologies/"
+ self.topo_base = "topologies/"
+ self.deploy_base = "deployments/"
+ self.state = None
def operation(self, op, request):
m = op.replace("-", "_")
@@ -105,6 +114,85 @@ class Manager(object):
print "Got request " + op
return method(request)
+ def ANSIBLE_INSTALLED(self, request):
+ if self.verbose:
+ print "Ansible is", "installed" if find_executable("ansible") else "not installed"
+ return "installed" if find_executable("ansible") else ""
+
+ # if the node has listeners, and one of them has an http:'true'
+ def has_console(self, node):
+ #n = False
+ #return node.get('listeners') and any([n or h.get('http') for l, h in node.get('listeners').iteritems()])
+
+ listeners = node.get('listeners')
+ if listeners:
+ for k, listener in listeners.iteritems():
+ if listener.get('http'):
+ return True
+
+ return False
+
+ def DEPLOY(self, request):
+ nodes = request["nodes"]
+ topology = request["topology"]
+
+ self.PUBLISH(request, deploy=True)
+
+ inventory = {'deploy_routers':
+ {'vars': {'topology': topology},
+ 'hosts': {}
+ }
+ }
+ hosts = inventory['deploy_routers']['hosts']
+
+ #pdb.set_trace()
+ for node in nodes:
+ if node['cls'] == 'router':
+ host = node['host']
+ if not host in hosts:
+ hosts[host] = {'nodes': [], 'create_console': False}
+ # if any of the nodes for this host has a console, set create_console for this host to true
+ hosts[host]['create_console'] = (hosts[host]['create_console'] or self.has_console(node))
+ hosts[host]['nodes'].append(node['name'])
+ # local hosts need to be marked as such
+ if host in ('0.0.0.0', 'localhost', '127.0.0.1'):
+ hosts[host]['ansible_connection'] = 'local'
+
+ with open(self.deploy_base + 'inventory.yml', 'w') as n:
+ yaml.safe_dump(inventory, n, default_flow_style=False)
+
+ # start ansible-playbook in separate thread and callback when done
+ def popenCallback(callback, args):
+ def popen(callback, args):
+ # send all output to deploy.txt so we can send it to the console in DEPLOY_STATUS
+ with open('deploy.txt', 'w') as fout:
+ proc = subprocess.Popen(args, stdout=fout, stderr=fout)
+ proc.wait()
+ callback()
+ return
+ thread = threading.Thread(target=popen, args=(callback, args))
+ thread.start()
+
+ def ansible_done():
+ if self.verbose:
+ print "-------------- DEPLOYMENT DONE ----------------"
+ self.state = "DONE"
+
+ self.state = "DEPLOYING"
+ popenCallback(ansible_done, ['ansible-playbook', self.deploy_base + 'install_dispatch.yaml', '-i', self.deploy_base + 'inventory.yml'])
+
+ return "deployment started"
+
+ def DEPLOY_STATUS(self, request):
+ with open('deploy.txt', 'r') as fin:
+ content = fin.readlines()
+
+ # remove leading blank line
+ if len(content) > 1 and content[0] == '\n':
+ content.pop(0)
+
+ return [''.join(content), self.state]
+
def GET_LOG(self, request):
return []
@@ -118,7 +206,7 @@ class Manager(object):
nodes = []
links = []
- dc = DirectoryConfigs('./' + self.base + topology + '/')
+ dc = DirectoryConfigs('./' + self.topo_base + topology + '/')
configs = dc.configs
port_map = []
@@ -126,6 +214,8 @@ class Manager(object):
port_map.append({'connectors': [], 'listeners': []})
node = {}
for sect in configs[file]:
+ # remove notes to self
+ host = sect[1].pop('deploy_host', None)
section = dc.asSection(sect)
if section:
if section.type == "router":
@@ -133,15 +223,11 @@ class Manager(object):
node["nodeType"] = unicode("inter-router")
node["name"] = section.entries["id"]
node["key"] = "amqp:/_topo/0/" + node["name"] + "/$management"
+ if host:
+ node['host'] = host
nodes.append(node)
elif section.type in sectionKeys:
- # look for a host in a listener
- if section.type == 'listener':
- host = section.entries.get('host')
- if host and 'host' not in node:
- node['host'] = host
-
role = section.entries.get('role')
if role == 'inter-router':
# we are processing an inter-router listener or connector: so create a link
@@ -171,50 +257,20 @@ class Manager(object):
return unicode(self.topology)
def GET_TOPOLOGY_LIST(self, request):
- return [unicode(f) for f in os.listdir(self.base) if os.path.isdir(self.base + f)]
+ return [unicode(f) for f in os.listdir(self.topo_base) if os.path.isdir(self.topo_base + f)]
def SWITCH(self, request):
self.topology = request["topology"]
- tdir = './' + self.base + self.topology + '/'
+ tdir = './' + self.topo_base + self.topology + '/'
if not os.path.exists(tdir):
os.makedirs(tdir)
return self.LOAD(request)
- def FIND_DIR(self, request):
- dir = request['relativeDir']
- files = request['fileList']
- # find a directory with this name that contains these files
-
-
def SHOW_CONFIG(self, request):
nodeIndex = request['nodeIndex']
return self.PUBLISH(request, nodeIndex)
- def PUBLISH(self, request, nodeIndex=None):
- nodes = request["nodes"]
- links = request["links"]
- topology = request["topology"]
- settings = request["settings"]
- http_port = settings.get('http_port', 5675)
- listen_port = settings.get('internal_port', 2000)
- default_host = settings.get('default_host', '0.0.0.0')
-
- if nodeIndex and nodeIndex >= len(nodes):
- return "Node index out of range"
-
- if self.verbose:
- if nodeIndex is None:
- print("PUBLISHing to " + topology)
- else:
- print("Creating config for " + topology + " node " + nodes[nodeIndex]['name'])
-
- if nodeIndex is None:
- # remove all .conf files from the output dir. they will be recreated below possibly under new names
- for f in glob(self.base + topology + "/*.conf"):
- if self.verbose:
- print "Removing", f
- os.remove(f)
-
+ def _connect_(self, links, nodes, default_host, listen_port):
for link in links:
s = nodes[link['source']]
t = nodes[link['target']]
@@ -239,6 +295,36 @@ class Manager(object):
t['conns'].append({"port": lport, "host": lhost})
t['conn_to'].append(s['name'])
+ def PUBLISH(self, request, nodeIndex=None, deploy=False):
+ nodes = request["nodes"]
+ links = request["links"]
+ topology = request["topology"]
+ settings = request["settings"]
+ http_port = settings.get('http_port', 5675)
+ listen_port = settings.get('internal_port', 2000)
+ default_host = settings.get('default_host', '0.0.0.0')
+
+ if nodeIndex and nodeIndex >= len(nodes):
+ return "Node index out of range"
+
+ if self.verbose:
+ if nodeIndex is not None:
+ print("Creating config for " + topology + " node " + nodes[nodeIndex]['name'])
+ elif deploy:
+ print("DEPLOYing to " + topology)
+ else:
+ print("PUBLISHing to " + topology)
+
+ if nodeIndex is None:
+ # remove all .conf files from the output dir. they will be recreated below possibly under new names
+ for f in glob(self.topo_base + topology + "/*.conf"):
+ if self.verbose:
+ print "Removing", f
+ os.remove(f)
+
+ # establish connections and listeners for each node based on links
+ self._connect_(links, nodes, default_host, listen_port)
+
# now process all the routers
for node in nodes:
if node['nodeType'] == 'inter-router':
@@ -246,10 +332,10 @@ class Manager(object):
print "------------- processing node", node["name"], "---------------"
nname = node["name"]
- if nodeIndex is None:
- config_fp = open(self.base + topology + "/" + nname + ".conf", "w+")
- else:
+ if nodeIndex is not None:
config_fp = cStringIO.StringIO()
+ else:
+ config_fp = open(self.topo_base + topology + "/" + nname + ".conf", "w+")
# add a router section in the config file
r = RouterSection(**node)
@@ -258,6 +344,8 @@ class Manager(object):
else:
r.setEntry('mode', 'interior')
r.setEntry('id', node['name'])
+ if nodeIndex is None:
+ r.setEntry('deploy_host', node.get('host', ''))
config_fp.write(str(r) + "\n")
# write other sections
@@ -269,10 +357,15 @@ class Manager(object):
c = get_class(cname)
if sectionKey == "listener" and o['port'] != 'amqp' and int(o['port']) == http_port:
config_fp.write("\n# Listener for a console\n")
+ if deploy:
+ o['httpRoot'] = '/usr/local/share/qpid-dispatch/stand-alone'
+ if node.get('host') == o.get('host'):
+ o['host'] = '0.0.0.0'
config_fp.write(str(c(**o)) + "\n")
if 'listener' in node:
- lhost = node.get('host', default_host)
+ # always listen on localhost
+ lhost = "0.0.0.0"
listenerSection = ListenerSection(node['listener'], **{'host': lhost, 'role': 'inter-router'})
if 'listen_from' in node and len(node['listen_from']) > 0:
config_fp.write("\n# listener for connectors from " + ', '.join(node['listen_from']) + "\n")
@@ -282,6 +375,8 @@ class Manager(object):
for idx, conns in enumerate(node['conns']):
conn_port = conns['port']
conn_host = conns['host']
+ if node.get('host') == conn_host:
+ conn_host = "0.0.0.0"
connectorSection = ConnectorSection(conn_port, **{'host': conn_host, 'role': 'inter-router'})
if 'conn_to' in node and len(node['conn_to']) > idx:
config_fp.write("\n# connect to " + node['conn_to'][idx] + "\n")
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/91df5a67/console/config/css/mock.css
----------------------------------------------------------------------
diff --git a/console/config/css/mock.css b/console/config/css/mock.css
index 307ee8c..6ae1156 100644
--- a/console/config/css/mock.css
+++ b/console/config/css/mock.css
@@ -125,4 +125,14 @@ div.boolean label {
.alert {
max-width: 20em;
+}
+
+input.router-host {
+ width: 45em;
+}
+
+pre.tail {
+ max-height: 15em;
+ min-height: 15em;
+ overflow-y: scroll;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/91df5a67/console/config/html/qdrTopology.html
----------------------------------------------------------------------
diff --git a/console/config/html/qdrTopology.html b/console/config/html/qdrTopology.html
index 5dc9d5b..7b6f794 100644
--- a/console/config/html/qdrTopology.html
+++ b/console/config/html/qdrTopology.html
@@ -20,13 +20,14 @@ under the License.
<div id="buttonBar" class="navbar-primary">
Current topology
<select ng-model="mockTopologyDir" ng-options="item for item in mockTopologies"></select>
- <button class="btn btn-primary" type="button" ng-click="Publish()">Publish</button>
- <button class="btn btn-primary" type="button" ng-click="Clear()">Clear</button>
- <button class="btn btn-primary pull-right" type="button" ng-click="doSettings()">Settings</button>
- <button class="btn btn-primary pull-right" type="button" ng-click="showNewDlg()">New Topology</button>
+ <button class="btn btn-primary" type="button" ng-click="Publish()" title="Save this topology">Publish</button>
+ <button class="btn btn-primary" type="button" ng-click="Clear()" title="Remove all routers">Clear</button>
+ <button class="btn btn-primary" type="button" ng-click="Deploy()" ng-disabled="!canDeploy()" ng-if="ansible" title="Deploy this topology">Deploy</button>
+ <button class="btn btn-primary pull-right" type="button" ng-click="doSettings()" title="Show global settings">Settings</button>
+ <button class="btn btn-primary pull-right" type="button" ng-click="showNewDlg()" title="Enter a new topology name">New Topology</button>
<div class="selected-node pull-right">
<button class="btn btn-primary" type="button" ng-click="addAnotherNode(true)"><b class="plus caret"></b> Add new router</button>
- <button id="action_button" class="btn btn-primary" type="button" ng-disabled="!selected_node" ng-click="showActions($event)">Actions <b class="down caret"></b></button> on selected router
+ <button id="action_button" class="btn btn-primary" type="button" ng-disabled="!selected_node" ng-click="showActions($event)" title="Show actions on selected router">Actions <b class="down caret"></b></button>
</div>
</div>
<div id="topology"><!-- d3 toplogy here --></div>
@@ -210,7 +211,7 @@ under the License.
</div>
<label for="host" class="entity-description">Enter a machine name or IP address</label>
<fieldset>
- <input type="text" name="host" id="host" ng-model="host" ng-required="true" class="ui-widget-content ui-corner-all"/>
+ <input type="text" name="host" id="host" ng-model="host" ng-required="true" class="router-host ui-widget-content ui-corner-all"/>
</fieldset>
</div>
@@ -220,3 +221,20 @@ under the License.
</div>
</form>
</script>
+
+<script type="text/ng-template" id="deploy-template.html">
+ <form novalidate>
+ <div class="modal-header">
+ <h3 class="modal-title">{{state}}</h3>
+ </div>
+ <div class="modal-body">
+ <pre id="deploy_status" class="tail">{{status}}</pre>
+ <div ng-hide="polling" id="message">Deployed.
+ <div ng-show="hasConsole()">Browse to <a ng-href="{{address}}" target="_blank">here</a> to manage the router.</div>
+ </div>
+ </div>
+ <div class="modal-footer">
+ <button class="btn btn-warning" type="button" ng-click="cancel()">Close</button>
+ </div>
+ </form>
+</script>
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/91df5a67/console/config/js/qdrTopology.js
----------------------------------------------------------------------
diff --git a/console/config/js/qdrTopology.js b/console/config/js/qdrTopology.js
index e8b7df8..421191b 100644
--- a/console/config/js/qdrTopology.js
+++ b/console/config/js/qdrTopology.js
@@ -33,9 +33,26 @@ var QDR = (function(QDR) {
var sections = ['log', 'connector', 'sslProfile', 'listener']
$scope.Publish = function () {
- doPublish()
+ doOperation("PUBLISH", function (response) {
+ Core.notification('info', $scope.mockTopologyDir + " published");
+ QDR.log.info("published " + $scope.mockTopologyDir)
+ })
+ }
+ $scope.Deploy = function () {
+ // send the deploy command
+ doOperation("DEPLOY", function (response) {
+ QDR.log.info("deployment " + $scope.mockTopologyDir + " started")
+ })
+ // show the deploy status dialog
+ doDeployDialog()
+ }
+
+ $scope.showConfig = function (node) {
+ doOperation("SHOW-CONFIG", function (response) {
+ doShowConfigDialog(response)
+ }, {nodeIndex: node.index})
}
- var doPublish = function (nodeIndex, callback) {
+ var doOperation = function (operation, callback, extraProps) {
var l = []
links.forEach( function (link) {
if (link.source.nodeType === 'inter-router' && link.target.nodeType === 'inter-router')
@@ -44,27 +61,17 @@ var QDR = (function(QDR) {
cls: link.cls})
})
var props = {nodes: nodes, links: l, topology: $scope.mockTopologyDir, settings: settings}
- if (angular.isDefined(nodeIndex)) {
- op = "SHOW-CONFIG"
- props.nodeIndex = nodeIndex
- } else {
- op = "PUBLISH"
- }
- QDRService.sendMethod(op, props, function (response) {
- if (!angular.isDefined(nodeIndex)) {
- Core.notification('info', props.topology + " published");
- QDR.log.info("published " + $scope.mockTopologyDir)
- } else {
+ if (extraProps)
+ Object.assign(props, props, extraProps)
+ QDRService.sendMethod(operation, props, function (response) {
+ if (callback)
callback(response)
- }
- })
- }
- $scope.showConfig = function (node) {
- doPublish(node.index, function (response) {
- doShowConfigDialog(response)
})
}
+ $scope.canDeploy = function () {
+ return nodes.length > 0
+ }
$scope.$watch('mockTopologyDir', function(newVal, oldVal) {
if (oldVal != newVal) {
switchTopology(newVal)
@@ -127,7 +134,9 @@ var QDR = (function(QDR) {
animate = true
QDR.log.info("switched to " + topology)
initForceGraph()
- Core.notification('info', "switched to " + props.topology);
+ $timeout( function () {
+ Core.notification('info', "switched to " + props.topology);
+ })
})
}
@@ -137,7 +146,9 @@ var QDR = (function(QDR) {
$scope.selected_node = null
resetMouseVars()
force.nodes(nodes).links(links).start();
- restart();
+ $timeout( function () {
+ restart();
+ })
}
$scope.delNode = function (node, skipinit) {
@@ -1370,6 +1381,10 @@ var QDR = (function(QDR) {
$scope.mockTopologies = []
$scope.mockTopologyDir = ""
+ $scope.ansible = false
+ QDRService.sendMethod("ANSIBLE-INSTALLED", {}, function (response) {
+ $scope.ansible = (response !== "")
+ })
QDRService.sendMethod("GET-TOPOLOGY-LIST", {}, function (response) {
$scope.mockTopologies = response.sort()
QDRService.sendMethod("GET-TOPOLOGY", {}, function (response) {
@@ -1453,6 +1468,48 @@ var QDR = (function(QDR) {
});
})
};
+ function doDeployDialog() {
+ var host = undefined
+ var port = undefined
+ for (var i=0; i<nodes.length; i++) {
+ var node = nodes[i]
+ if (node.listeners) {
+ for (var l in node.listeners) {
+ var listener = node.listeners[l]
+ if (listener.http) {
+ host = node.host
+ port = listener.port
+ }
+ }
+ }
+ }
+ var d = $uibModal.open({
+ dialogClass: "modal dlg-large",
+ backdrop: true,
+ keyboard: true,
+ backdropClick: true,
+ controller: 'QDR.DeployDialogController',
+ templateUrl: 'deploy-template.html',
+ resolve: {
+ dir: function () {
+ return $scope.mockTopologyDir
+ },
+ http_host: function () {
+ return host
+ },
+ http_port: function () {
+ return port
+ }
+ }
+ });
+ $timeout(function () {
+ d.result.then(function(result) {
+ if (result) {
+ }
+ });
+ })
+ }
+
function doSettingsDialog(opts) {
var d = $uibModal.open({
dialogClass: "modal dlg-large",
@@ -1674,6 +1731,58 @@ var QDR = (function(QDR) {
})
+ QDR.module.controller("QDR.DeployDialogController", function($scope, $uibModalInstance, QDRService, $timeout, $sce, dir, http_host, http_port) {
+ // setup polling to get deployment status
+ $scope.polling = true
+ $scope.state = "Deploying"
+ $scope.address = ""
+ var pollTimer = null
+ function doPoll() {
+ QDRService.sendMethod("DEPLOY-STATUS", {config: dir}, function (response) {
+ if (response[1] === 'DONE') {
+ $scope.polling = false
+ $scope.state = "Deploy Completed"
+ Core.notification('info', dir + " deployed");
+ if (http_host && http_port) {
+ $scope.address = $sce.trustAsHtml("http://" + http_host + ":" + http_port + "/#!/topology")
+ }
+ }
+ $timeout(function () {
+ $scope.status = response[0]
+ scrollToEnd()
+ if (response[1] === 'DONE') {
+ }
+ if ($scope.polling) (
+ pollTimer = setTimeout(doPoll, 1000)
+ )
+ })
+ })
+ }
+ pollTimer = setTimeout(doPoll, 1000)
+ $scope.hasConsole = function () {
+ return http_host && http_port
+ }
+
+ function scrollTopTween(scrollTop) {
+ return function() {
+ var i = d3.interpolateNumber(this.scrollTop, scrollTop);
+ return function(t) { this.scrollTop = i(t); };
+ }
+ }
+ var scrollToEnd = function () {
+ var scrollheight = d3.select("#deploy_status").property("scrollHeight");
+
+ d3.select('#deploy_status')
+ .transition().duration(1000)
+ .tween("uniquetweenname", scrollTopTween(scrollheight));
+ }
+ $scope.cancel = function () {
+ polling = false
+ clearTimeout(pollTimer)
+ $uibModalInstance.close()
+ }
+
+ })
return QDR;
}(QDR || {}));
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/91df5a67/console/config/mock/section.py
----------------------------------------------------------------------
diff --git a/console/config/mock/section.py b/console/config/mock/section.py
index 79d0601..9df97aa 100644
--- a/console/config/mock/section.py
+++ b/console/config/mock/section.py
@@ -20,6 +20,7 @@
import json
import re
from schema import Schema
+import pdb
class ConfigSection(object):
def __init__(self, type, defaults, ignore, opts):
@@ -57,6 +58,10 @@ class RouterSection(ConfigSection):
super(RouterSection, self).__init__("router", RouterSection.defaults, RouterSection.ignore, kwargs)
self.setEntry("id", id)
+ def __repr__(self):
+ s = super(RouterSection, self).__repr__()
+ return s.replace('deploy_host', '#deploy_host', 1)
+
class ListenerSection(ConfigSection):
defaults = {"role": "normal",
"host": "0.0.0.0",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org