You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/07/10 01:30:21 UTC
ambari git commit: AMBARI-12367. Automate creating multiple
ambari-agents using Docker on GCE (Pengcheng Xu via alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk ba21f8988 -> 6eb0e6b16
AMBARI-12367. Automate creating multiple ambari-agents using Docker on GCE (Pengcheng Xu via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6eb0e6b1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6eb0e6b1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6eb0e6b1
Branch: refs/heads/trunk
Commit: 6eb0e6b16632b4ddf7f5397e4adce00852306c4b
Parents: ba21f89
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Thu Jul 9 16:29:45 2015 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Thu Jul 9 16:30:09 2015 -0700
----------------------------------------------------------------------
contrib/agent-simulator/Docker/Dockerfile | 39 +++++
contrib/agent-simulator/Docker/__init__.py | 17 ++
.../Docker/ambari_agent_start.sh | 18 ++
.../agent-simulator/Docker/launcher_agent.py | 70 ++++++++
.../Linux/CentOS7/docker_install.sh | 20 +++
.../Linux/CentOS7/weave_install.sh | 18 ++
.../Linux/Ubuntu12/docker_install.sh | 17 ++
.../Linux/Ubuntu12/weave_install.sh | 19 ++
contrib/agent-simulator/cluster.py | 175 +++++++++++++++++++
contrib/agent-simulator/config.py | 36 ++++
contrib/agent-simulator/config/config.ini | 49 ++++++
contrib/agent-simulator/docker.py | 27 +++
contrib/agent-simulator/launcher_cluster.py | 136 ++++++++++++++
contrib/agent-simulator/launcher_docker.py | 41 +++++
contrib/agent-simulator/server_setup.sh | 42 +++++
contrib/agent-simulator/tips.txt | 18 ++
contrib/agent-simulator/vm.py | 81 +++++++++
17 files changed, 823 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/Docker/Dockerfile
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Docker/Dockerfile b/contrib/agent-simulator/Docker/Dockerfile
new file mode 100644
index 0000000..d22435a
--- /dev/null
+++ b/contrib/agent-simulator/Docker/Dockerfile
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Set the bas image to CentOS 6.x
+FROM centos:7
+
+# Author
+MAINTAINER Pengcheng_Xu
+
+# Copy the files into Docker: Agent package file
+ADD agent.rpm /agent.rpm
+# Copy the files into Docker: launcher_agent.py
+ADD launcher_agent.py /launcher_agent.py
+# Copy the files into Docker: ambari_agent_start.sh
+ADD ambari_agent_start.sh /ambari_agent_start.sh
+# Copy hostname file into Docker: hosts
+ADD hosts /hosts
+
+RUN chmod 755 /ambari_agent_start.sh
+
+# Install ambari-agent
+# RUN yum install -y /agent.rpm
+RUN yum install -y wget
+RUN wget -O /etc/yum.repos.d/ambari.repo http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos7/2.x/latest/2.1.0/ambaribn.repo
+RUN yum install -y epel-release
+RUN yum install -y ambari-agent
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/Docker/__init__.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Docker/__init__.py b/contrib/agent-simulator/Docker/__init__.py
new file mode 100644
index 0000000..62aeca1
--- /dev/null
+++ b/contrib/agent-simulator/Docker/__init__.py
@@ -0,0 +1,17 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/Docker/ambari_agent_start.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Docker/ambari_agent_start.sh b/contrib/agent-simulator/Docker/ambari_agent_start.sh
new file mode 100644
index 0000000..4ba2361
--- /dev/null
+++ b/contrib/agent-simulator/Docker/ambari_agent_start.sh
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#!/bin/bash
+ambari-agent start
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/Docker/launcher_agent.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Docker/launcher_agent.py b/contrib/agent-simulator/Docker/launcher_agent.py
new file mode 100644
index 0000000..5ff008d
--- /dev/null
+++ b/contrib/agent-simulator/Docker/launcher_agent.py
@@ -0,0 +1,70 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import sys
+import subprocess
+import time
+
+def replace_conf(server_ip):
+ f = open("/etc/ambari-agent/conf/ambari-agent.ini")
+ lines = f.readlines()
+ f.close()
+
+ f = open("/etc/ambari-agent/conf/ambari-agent.ini", "w+")
+ for line in lines:
+ line = line.replace("hostname=localhost", "hostname=" + server_ip)
+ f.write(line)
+ f.close()
+
+def run_ambari_agent():
+ # command = ["sudo", "ambari-agent", "start"]
+ # subprocess.call(command)
+ subprocess.call("./ambari_agent_start.sh")
+
+# add all the hostnames of other containers to /etc/hosts
+def add_hostnames():
+ etc_hosts = open("/etc/hosts", "a")
+ etc_hosts.write("\n")
+
+ docker_hosts = open("/hosts")
+ for line in docker_hosts.readlines():
+ etc_hosts.write(line)
+ docker_hosts.close()
+
+ etc_hosts.close()
+
+def remove_default_hostname(hostname):
+ etc_hosts = open("/etc/hosts")
+ all_resolution = etc_hosts.readlines()
+ etc_hosts.close()
+
+ etc_hosts = open("/etc/hosts", "w")
+ for line in all_resolution:
+ if hostname not in line:
+ etc_hosts.write(line)
+ else:
+ etc_hosts.write("#")
+ etc_hosts.write(line)
+ etc_hosts.close()
+
+ambari_server_ip = sys.argv[1]
+my_hostname = sys.argv[2]
+replace_conf(ambari_server_ip)
+remove_default_hostname(my_hostname)
+add_hostnames()
+run_ambari_agent()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/Linux/CentOS7/docker_install.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Linux/CentOS7/docker_install.sh b/contrib/agent-simulator/Linux/CentOS7/docker_install.sh
new file mode 100644
index 0000000..033c2b9
--- /dev/null
+++ b/contrib/agent-simulator/Linux/CentOS7/docker_install.sh
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+sudo curl -O -sSL https://get.docker.com/rpm/1.7.0/centos-7/RPMS/x86_64/docker-engine-1.7.0-1.el7.centos.x86_64.rpm
+sudo yum localinstall -y --nogpgcheck docker-engine-1.7.0-1.el7.centos.x86_64.rpm
+sudo yum update -y device-mapper
+sudo service docker start
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/Linux/CentOS7/weave_install.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Linux/CentOS7/weave_install.sh b/contrib/agent-simulator/Linux/CentOS7/weave_install.sh
new file mode 100644
index 0000000..6788184
--- /dev/null
+++ b/contrib/agent-simulator/Linux/CentOS7/weave_install.sh
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+sudo curl -L git.io/weave -o /usr/bin/weave
+sudo chmod a+x /usr/bin/weave
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/Linux/Ubuntu12/docker_install.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Linux/Ubuntu12/docker_install.sh b/contrib/agent-simulator/Linux/Ubuntu12/docker_install.sh
new file mode 100644
index 0000000..220f1a4
--- /dev/null
+++ b/contrib/agent-simulator/Linux/Ubuntu12/docker_install.sh
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+wget -qO- https://get.docker.com/ | sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/Linux/Ubuntu12/weave_install.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/Linux/Ubuntu12/weave_install.sh b/contrib/agent-simulator/Linux/Ubuntu12/weave_install.sh
new file mode 100644
index 0000000..332488d
--- /dev/null
+++ b/contrib/agent-simulator/Linux/Ubuntu12/weave_install.sh
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+sudo apt-get install -y curl
+sudo curl -L git.io/weave -o /usr/bin/weave
+sudo chmod a+x /usr/bin/weave
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/cluster.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/cluster.py b/contrib/agent-simulator/cluster.py
new file mode 100644
index 0000000..55c3c44
--- /dev/null
+++ b/contrib/agent-simulator/cluster.py
@@ -0,0 +1,175 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import subprocess
+import time
+from config import Config
+from docker import Docker
+from vm import VM
+
+
+
+class Cluster:
+ def __init__(self):
+ self.cluster_name = ""
+ self.VMs_num = 0
+ self.VM_list = []
+
+ # read cluster info from a file
+ def load_cluster_info(self, filename):
+ file = open(filename)
+
+ self.cluster_name = file.next().split()[1]
+ self.VMs_num = int(file.next().split()[1])
+ for VM_index in range(0, self.VMs_num):
+ vm = VM(file.next().split()[1])
+ docker_num = int(file.next().split()[1])
+ for Docker_index in range(0, docker_num):
+ line = file.next()
+ IP = line.split()[0].split("/")[0]
+ mask = line.split()[0].split("/")[1]
+ hostname = line.split()[1]
+ docker = Docker(IP, mask, hostname)
+ vm.add_docker(docker)
+ self.VM_list.append(vm)
+
+ file.close()
+
+ def __extract_VM_IP__(self, GCE_info_file_name):
+ f = open(GCE_info_file_name)
+ lines = f.readlines()
+ f.close()
+
+ ip_list = []
+ for line in lines:
+ tokens = line.split()
+ ip_list.append(tokens[1])
+ return ip_list[1:]
+
+ # request a new cluster
+ def request_GCE_cluster(self, vms_num, docker_num, cluster_name):
+ # reload configuration file
+ config = Config()
+ config.load()
+ # request cluster
+ gce_key = config.ATTRIBUTES["GCE_controller_key_file"]
+ gce_login = config.ATTRIBUTES["GCE_controller_user"] + "@" + config.ATTRIBUTES["GCE_controller_IP"]
+ gce_up_cmd = "gce up " + cluster_name + " " + str(vms_num) + " " + config.ATTRIBUTES["GCE_VM_type"] + \
+ " " + config.ATTRIBUTES["GCE_VM_OS"]
+ subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-i", gce_key, gce_login, gce_up_cmd])
+
+ print "cluster launched successufully, wait 5 seconds for cluster info ... ..."
+ time.sleep(5)
+
+ # request cluster info
+ gce_info_output_file = open(config.ATTRIBUTES["GCE_info_output"], "w")
+ gce_info_cmd = "gce info " + cluster_name
+ subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-i", gce_key, gce_login, gce_info_cmd], \
+ stdout=gce_info_output_file)
+ gce_info_output_file.close()
+ print "cluster info is saved to file " + config.ATTRIBUTES["GCE_info_output"]
+
+ # prepare all attributes of the cluster, write to a file
+ VM_IP_list = self.__extract_VM_IP__(config.ATTRIBUTES["GCE_info_output"])
+ self.generate_cluster_info(VM_IP_list, cluster_name, docker_num)
+ self.overwrite_to_file(config.ATTRIBUTES["cluster_info_file"])
+ # server need this file to resolve the host names of the agents
+ self.export_hostnames(config.ATTRIBUTES["Docker_hostname_info"])
+
+ # save info to file
+ def overwrite_to_file(self, filename):
+ file = open(filename, "w")
+ file.write("cluster_name: " + self.cluster_name + "\n")
+ file.write("VMs_num: " + str(self.VMs_num) + "\n")
+
+ for vm in self.VM_list:
+ file.write("\t\t")
+ file.write("VM_IP: " + vm.external_ip + "\n")
+ file.write("\t\t")
+ file.write("Docker_num: " + str(len(vm.docker_list)) + "\n")
+ for docker in vm.docker_list:
+ file.write("\t\t\t\t")
+ file.write(docker.IP + "/" + docker.mask + " " + docker.hostname + "\n")
+
+ file.close()
+
+ def __increase_IP__(self, base_IP, increase):
+ IP = [int(base_IP[0]), int(base_IP[1]), int(base_IP[2]), int(base_IP[3])]
+ IP[3] = IP[3] + increase
+ for index in reversed(range(0, 4)):
+ if IP[index] > 255:
+ IP[index - 1] = IP[index - 1] + IP[index] / 256
+ IP[index] = IP[index] % 256
+ return IP
+
+ # generate VM and docker info for this cluster
+ # set up parameter as this info
+ def generate_cluster_info(self, VM_IP_list, cluster_name, docker_num):
+ config = Config()
+ config.load()
+ Docker_IP_base = config.ATTRIBUTES["Docker_IP_base"].split(".")
+ Docker_IP_mask = config.ATTRIBUTES["Docker_IP_mask"]
+
+ VM_index = 0
+ for VM_IP in VM_IP_list:
+ vm = VM(VM_IP)
+
+ for Docker_index in range(0, docker_num):
+ total_Docker_index = VM_index * docker_num + Docker_index
+ docker_IP = self.__increase_IP__(Docker_IP_base, total_Docker_index)
+ docker_IP_str = str(docker_IP[0]) + "." + str(docker_IP[1]) + "." + \
+ str(docker_IP[2]) + "." + str(docker_IP[3])
+ docker_hostname = cluster_name + "-" + str(VM_index) + "-" + str(Docker_index)
+ docker = Docker(docker_IP_str, str(Docker_IP_mask), docker_hostname)
+ # print docker
+ vm.add_docker(docker)
+ VM_index = VM_index + 1
+ self.VM_list.append(vm)
+
+ self.VMs_num = len(VM_IP_list)
+ self.cluster_name = cluster_name
+
+ # run all dockers for all the VMs in the cluster
+ # upload necessary file to each machine in cluster, run launcher_docker.py in each machine with parameter
+ def run_docker_on_cluster(self, server_external_IP, server_Weave_IP):
+ config = Config()
+ config.load()
+
+ for vm in self.VM_list:
+ # upload necessary file to each machine in cluster
+ VM_external_IP = vm.external_ip
+ VM_directory = "root@" + VM_external_IP + ":" + config.ATTRIBUTES["VM_code_directory"]
+ VM_key = config.ATTRIBUTES["GCE_VM_key_file"]
+ subprocess.call(["scp", "-o", "StrictHostKeyChecking=no", "-i", VM_key, "-r", ".", VM_directory])
+
+ # run launcher_docker.py in each machine with parameters
+ subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-t", "-i", VM_key, \
+ "root@" + VM_external_IP, \
+ "cd " + config.ATTRIBUTES["VM_code_directory"] + "; python launcher_docker.py" + \
+ " " + VM_external_IP + " " + server_Weave_IP + " " + server_external_IP])
+
+ # export host names to a file
+ def export_hostnames(self, filename):
+ hostname_file = open(filename, "w")
+ for vm in self.VM_list:
+ for docker in vm.docker_list:
+ hostname_file.write(docker.IP)
+ hostname_file.write(" ")
+ hostname_file.write(docker.hostname)
+ hostname_file.write("\n")
+ hostname_file.close()
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/config.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/config.py b/contrib/agent-simulator/config.py
new file mode 100644
index 0000000..20cc8de
--- /dev/null
+++ b/contrib/agent-simulator/config.py
@@ -0,0 +1,36 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import ConfigParser
+
+class Config:
+ ATTRIBUTES = {}
+
+ @staticmethod
+ def load():
+ config = ConfigParser.RawConfigParser()
+ # keep file case sensitive
+ config.optionxform = str
+ config.read("config/config.ini")
+ for section in config.sections():
+ for key in config.options(section):
+ Config.ATTRIBUTES[key] = config.get(section, key)
+
+
+# Config.load()
+# print Config.ATTRIBUTES
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/config/config.ini
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/config/config.ini b/contrib/agent-simulator/config/config.ini
new file mode 100644
index 0000000..a3afa33
--- /dev/null
+++ b/contrib/agent-simulator/config/config.ini
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This configuration file is case SENSITIVE
+# For more information about this configuration file, check https://docs.python.org/2/library/configparser.html
+
+[GCE]
+GCE_controller_IP = 104.196.89.197
+GCE_controller_user = root
+GCE_controller_key_file = ~/gce-key
+GCE_VM_type = --xlarge
+# only support centos 7
+GCE_VM_OS = --centos7
+# the result of command "gce info clustername" on the GCE controller
+GCE_info_output = output/gce_info_output.txt
+# the same as the GCE controller for GCE
+GCE_VM_key_file = ~/gce-key
+
+
+[Cluster]
+cluster_info_file = config/cluster.txt
+VM_code_directory = /simulator-script
+VM_hostname_file = /etc/hosts
+Docker_hostname_info = config/hosts.txt
+
+[Weave]
+# Docker_IP_base is the starting point of IP address. with mask 16, ideally:
+# the IP of docker will be 192.168.[1-255].[1-255]/16
+# if there are more dockers, which means there is not enough IPs within this mask for all dockers,
+# the IP will continue to increase to be 192.169.*.*, dockers might not be able to talk to each other,
+# because the mask is 16
+Docker_IP_base = 192.168.1.2
+Docker_IP_mask = 16
+
+
+[Docker]
+Docker_image_name = ambari/agent-sim
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/docker.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/docker.py b/contrib/agent-simulator/docker.py
new file mode 100644
index 0000000..e397c4b
--- /dev/null
+++ b/contrib/agent-simulator/docker.py
@@ -0,0 +1,27 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+class Docker:
+ def __init__(self, IP, mask, hostname):
+ self.IP = IP
+ self.mask = mask
+ self.hostname = hostname
+
+ def __str__(self):
+ return str(self.IP) + "/" + str(self.mask) + " " + self.hostname
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/launcher_cluster.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/launcher_cluster.py b/contrib/agent-simulator/launcher_cluster.py
new file mode 100644
index 0000000..0a7cbe0
--- /dev/null
+++ b/contrib/agent-simulator/launcher_cluster.py
@@ -0,0 +1,136 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+import time
+import sys
+from cluster import Cluster
+from config import Config
+
+def all(argv):
+ if len(argv) < 7:
+ print_help()
+ exit(1)
+
+ cluster_name = argv[2]
+ VMs_num = int(argv[3])
+ Docker_num_each_VM = int(argv[4])
+ server_Weave_IP = argv[5]
+ server_external_IP = argv[6]
+
+ cluster = Cluster()
+ cluster.request_GCE_cluster(VMs_num, Docker_num_each_VM, cluster_name)
+ print "wait 50 seconds for the cluster to boot ... ..."
+ time.sleep(50)
+ cluster.run_docker_on_cluster(server_external_IP, server_Weave_IP)
+
+def request_cluster(argv):
+ if len(argv) < 5:
+ print_help()
+ exit(1)
+
+ cluster_name = argv[2]
+ VMs_num = int(argv[3])
+ Docker_num_each_VM = int(argv[4])
+
+ cluster = Cluster()
+ cluster.request_GCE_cluster(VMs_num, Docker_num_each_VM, cluster_name)
+ print "Before run Docker on the Cluster, wait at least 50 seconds for the cluster to boot"
+
+def run_cluster(argv):
+ if len(argv) < 4:
+ print_help()
+ exit(1)
+
+ server_Weave_IP = argv[2]
+ server_external_IP = argv[3]
+
+ config = Config
+ config.load()
+
+ cluster = Cluster()
+ cluster.load_cluster_info(config.ATTRIBUTES["cluster_info_file"])
+ cluster.run_docker_on_cluster(server_external_IP, server_Weave_IP)
+
+def terminate():
+ print "Log into GCE controller to terminate your cluster manually"
+
+def print_help():
+ print "usage:"
+ print
+
+ print "all", " ", "request a GCE cluster and run Docker containers with Ambari-agent in all VMs"
+ print "\t\t", "<the name of the cluster>"
+ print "\t\t", "<number of VMs>"
+ print "\t\t", "<number of dockers each VMs>"
+ print "\t\t", "<Weave IP of Ambari-server>"
+ print "\t\t", "<external IP of Ambari-server>"
+ print
+
+ print "request", " ", "request a cluster from GCE, generate the configuration for the cluster"
+ print "\t\t", "<the name of the cluster>"
+ print "\t\t", "<number of VMs>"
+ print "\t\t", "<number of dockers each VMs>"
+ print
+
+ print "run", " ", "run Docker containers with Ambari-agent in all VMs of the cluster"
+ print "\t\t", "<Weave IP of Ambari-server>"
+ print "\t\t", "<external IP of Ambari-server>"
+ print
+
+ print "terminate", " ", "terminate the cluster"
+ print
+
+ print "help", " ", "help info"
+ print
+
+ print "more options in configuration file config/config.ini"
+ print "see more instructions in tips.txt"
+ print
+
+def main(argv):
+ # the first argument is the python file name
+ if len(argv) < 2:
+ print_help()
+ exit(1)
+
+ command = argv[1]
+ if command == "all":
+ all(argv)
+
+ elif command == "request":
+ request_cluster(argv)
+
+ elif command == "run":
+ run_cluster(argv)
+
+ elif command == "terminate":
+ terminate()
+
+ elif command == "help":
+ print_help()
+
+ else:
+ print_help()
+
+main(sys.argv)
+
+# python launcher_cluster.py all test-cluster 2 3 192.168.10.10 104.196.84.248
+
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/launcher_docker.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/launcher_docker.py b/contrib/agent-simulator/launcher_docker.py
new file mode 100644
index 0000000..48e8fac
--- /dev/null
+++ b/contrib/agent-simulator/launcher_docker.py
@@ -0,0 +1,41 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+import sys
+from config import Config
+from cluster import Cluster
+
+def get_VM(VM_IP, cluster):
+ for vm in cluster.VM_list:
+ if vm.external_ip == VM_IP:
+ return vm
+
+config = Config()
+config.load()
+
+cluster = Cluster()
+cluster.load_cluster_info(config.ATTRIBUTES["cluster_info_file"])
+
+my_external_IP = sys.argv[1]
+server_weave_IP = sys.argv[2]
+server_external_IP = sys.argv[3]
+
+vm = get_VM(my_external_IP, cluster)
+vm.run_docker(server_weave_IP, server_external_IP, cluster)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/server_setup.sh
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/server_setup.sh b/contrib/agent-simulator/server_setup.sh
new file mode 100644
index 0000000..3b499aa
--- /dev/null
+++ b/contrib/agent-simulator/server_setup.sh
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+# run this script with root
+# $1 <Weave internal IP with mask>
+# $2 <hostname files with all agents>
+
+if [ $# -lt 2 ]; then
+ echo "usage: ./sever_setup.sh <Weave internal IP with mask> <hostname file with all agents>"
+ echo "example: ./server_setup.sh 192.168.10.10/16 /user/simulator-script/hosts.txt"
+ echo "note: the hostname file is generated automatically when you request a cluster"
+ exit 1
+fi
+
+# install weave
+chmod 755 ./Linux/CentOS7/weave_install.sh
+./Linux/CentOS7/weave_install.sh
+
+# reset weave
+weave reset
+
+# launch weave
+weave launch
+
+# expose IP
+weave expose $1
+
+# add hosname of all agents
+cat $2 >> /etc/hosts3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/tips.txt
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/tips.txt b/contrib/agent-simulator/tips.txt
new file mode 100644
index 0000000..207177d
--- /dev/null
+++ b/contrib/agent-simulator/tips.txt
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+1. Ambari-agent and Ambari-server have to be the same version to successfully register. The command used to install Ambari-agent is in the Dockerfile
+2. Use CTRL + P, then CTRL + Q to exit Docker container without terminate the container.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6eb0e6b1/contrib/agent-simulator/vm.py
----------------------------------------------------------------------
diff --git a/contrib/agent-simulator/vm.py b/contrib/agent-simulator/vm.py
new file mode 100644
index 0000000..90de4de
--- /dev/null
+++ b/contrib/agent-simulator/vm.py
@@ -0,0 +1,81 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+import subprocess
+from config import Config
+
+class VM:
+ def __init__(self, external_ip):
+ self.external_ip = external_ip
+ self.docker_list = []
+
+ def add_docker(self, docker):
+ self.docker_list.append(docker)
+
+ # install Weave on this VM
+ def __centos7_weave_install__(self):
+ subprocess.call(["sudo", "chmod", "755", "Linux/CentOS7/weave_install.sh"])
+ subprocess.call("./Linux/CentOS7/weave_install.sh")
+
+ # launch Weave, make this VM connect with other VM
+ def __set_weave_network__(self, VMs_external_IP_list, server_external_ip):
+ # add other VMs and the ambari-server to set up connections
+ command = ["sudo", "weave", "launch"]
+ command.extend(VMs_external_IP_list)
+ command.append(server_external_ip)
+ subprocess.call(command)
+
+ # install Docker on this VM
+ def __centos7_docker_install__(self):
+ subprocess.call(["sudo", "chmod", "755", "Linux/CentOS7/docker_install.sh"])
+ subprocess.call("./Linux/CentOS7/docker_install.sh")
+
+ # build docker image
+ def __build_docker_image__(self, image_name):
+ subprocess.call(["sudo", "docker", "build", "-t", image_name, "Docker/"])
+
+ # launch Docker containers, issue the script to install, configure and launch Agent inside Docker.
+ def __launch_containers__(self, docker_image, server_weave_ip):
+ # print docker_ip_list
+ for docker in self.docker_list:
+ docker_IP = docker.IP
+ docker_mask = docker.mask
+ docker_hostname = docker.hostname
+
+ cmd = "python /launcher_agent.py " + server_weave_ip + " " + docker_hostname + "; /bin/bash"
+ command = ["sudo", "weave", "run", docker_IP + "/" + docker_mask, "-d", "-it", "-h", docker_hostname, \
+ docker_image, "bash", "-c", cmd]
+ print command
+ subprocess.call(command)
+
+ def run_docker(self, server_weave_IP, server_external_IP, cluster):
+ config = Config()
+ config.load()
+
+ VMs_IP_list = []
+ for vm in cluster.VM_list:
+ VMs_IP_list.append(vm.external_ip)
+
+ cluster.export_hostnames("./Docker/hosts")
+
+ self.__centos7_docker_install__()
+ self.__centos7_weave_install__()
+ self.__build_docker_image__(config.ATTRIBUTES["Docker_image_name"])
+ self.__set_weave_network__(VMs_IP_list, server_external_IP)
+ self.__launch_containers__(config.ATTRIBUTES["Docker_image_name"], server_weave_IP)
\ No newline at end of file