You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2019/01/10 11:57:16 UTC
[ambari] 01/04: AMBARI-25095. deploy-gce-perf-cluster.py fails
after upgrade on gce controller (aonishuk)
This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 7c3275fde332585b5b3ce7c93afec7a84177498a
Author: Andrew Onishuk <ao...@hortonworks.com>
AuthorDate: Thu Jan 10 08:20:26 2019 +0200
AMBARI-25095. deploy-gce-perf-cluster.py fails after upgrade on gce controller (aonishuk)
---
agent.sh | 12 ++++++
contrib/utils/perf/deploy-gce-perf-cluster.py | 11 ++---
contrib/utils/perf/deploy-gce-perf-cluster.py.rej | 27 ++++++++++++
server.sh | 52 +++++++++++++++++++++++
4 files changed, 97 insertions(+), 5 deletions(-)
diff --git a/agent.sh b/agent.sh
new file mode 100644
index 0000000..f1966f9
--- /dev/null
+++ b/agent.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+yum install wget -y
+wget -O /etc/yum.repos.d/ambari.repo http://10.240.0.30/ambari.repo
+yum clean all; yum install krb5-workstation git ambari-agent -y
+mkdir /home ; cd /home; git clone https://github.com/apache/ambari.git ; cd ambari ; git checkout branch-2.5
+cp -r /home/ambari/ambari-server/src/main/resources/stacks/PERF /var/lib/ambari-agent/cache/stacks/PERF
+sed -i -f /var/lib/ambari-agent/cache/stacks/PERF/PythonExecutor.sed /usr/lib/ambari-agent/lib/ambari_agent/PythonExecutor.py
+sed -i -f /var/lib/ambari-agent/cache/stacks/PERF/check_host.sed /var/lib/ambari-agent/cache/custom_actions/scripts/check_host.py
+sed -i -e 's/hostname=localhost/hostname=perf-server-test-perf-1.c.pramod-thangali.internal/g' /etc/ambari-agent/conf/ambari-agent.ini
+sed -i -e 's/agent]/agent]\nhostname_script=foo\npublic_hostname_script=foo\n/1' /etc/ambari-agent/conf/ambari-agent.ini
+wget http://10.240.0.30/agent-multiplier.py ; python /home/ambari/ambari-agent/conf/unix/agent-multiplier.py start
+exit 0
\ No newline at end of file
diff --git a/contrib/utils/perf/deploy-gce-perf-cluster.py b/contrib/utils/perf/deploy-gce-perf-cluster.py
index a1259eb..e34b07c 100644
--- a/contrib/utils/perf/deploy-gce-perf-cluster.py
+++ b/contrib/utils/perf/deploy-gce-perf-cluster.py
@@ -475,22 +475,23 @@ def get_vms_list(args):
def __get_vms_list_from_name(args, cluster_name):
"""
- Method to parse "gce fqdn {cluster-name}" command output and get hosts and ips pairs for every host in cluster
+ Method to parse "gce info {cluster-name}" command output and get hosts and ips pairs for every host in cluster
:param args: Command line args
:return: Mapping of VM host name to ip.
"""
- gce_fqdb_cmd = '/opt/gce-utils/gce fqdn {0}'.format(cluster_name)
+ gce_fqdb_cmd = '/opt/gce-utils/gce info {0}'.format(cluster_name)
out = execute_command(args, args.controller, gce_fqdb_cmd, "Failed to get VMs list!", "-tt")
lines = out.split('\n')
#print "LINES=" + str(lines)
if lines[0].startswith("Using profile") and not lines[1].strip():
result = {}
- for s in lines[2:]: # Ignore non-meaningful lines
+ for s in lines[4:]: # Ignore non-meaningful lines
if not s:
continue
- match = re.match(r'^([\d\.]*)\s+([\w\.-]*)\s+([\w\.-]*)\s+$', s, re.M)
+
+ match = re.match(r'^ [^ ]+ ([\w\.-]*)\s+([\d\.]*).*$', s, re.M)
if match:
- result[match.group(2)] = match.group(1)
+ result[match.group(1)] = match.group(2)
else:
raise Exception('Cannot parse "{0}"'.format(s))
return result
diff --git a/contrib/utils/perf/deploy-gce-perf-cluster.py.rej b/contrib/utils/perf/deploy-gce-perf-cluster.py.rej
new file mode 100644
index 0000000..06eca6e
--- /dev/null
+++ b/contrib/utils/perf/deploy-gce-perf-cluster.py.rej
@@ -0,0 +1,27 @@
+--- contrib/utils/perf/deploy-gce-perf-cluster.py
++++ contrib/utils/perf/deploy-gce-perf-cluster.py
+@@ -28,7 +28,7 @@ import re
+ import socket
+
+ cluster_prefix = "perf"
+-ambari_repo_file_url = "http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos6/2.x/updates/2.5.0.0/ambaribn.repo"
++ambari_repo_file_url = "http://10.240.0.30/ambari.repo"
+
+ public_hostname_script = "foo"
+ hostname_script = "foo"
+@@ -397,13 +394,11 @@ def create_agent_script(server_host_name):
+ # TODO, instead of cloning Ambari repo on each VM, do it on the server once and distribute to all of the agents.
+ contents = "#!/bin/bash\n" + \
+ "wget -O /etc/yum.repos.d/ambari.repo {0}\n".format(ambari_repo_file_url) + \
+- "yum clean all; yum install krb5-workstation git ambari-agent -y\n" + \
+- "mkdir /home ; cd /home; git clone https://github.com/apache/ambari.git ; cd ambari ; git checkout branch-2.5\n" + \
+- "cp -r /home/ambari/ambari-server/src/main/resources/stacks/PERF /var/lib/ambari-agent/cache/stacks/PERF\n" + \
++ "yum clean all; yum install krb5-workstation ambari-agent -y\n" + \
+ "sed -i -f /var/lib/ambari-agent/cache/stacks/PERF/PythonExecutor.sed /usr/lib/python2.6/site-packages/ambari_agent/PythonExecutor.py\n" + \
+ "sed -i -e 's/hostname=localhost/hostname={0}/g' /etc/ambari-agent/conf/ambari-agent.ini\n".format(server_host_name) + \
+ "sed -i -e 's/agent]/agent]\\nhostname_script={0}\\npublic_hostname_script={1}\\n/1' /etc/ambari-agent/conf/ambari-agent.ini\n".format(hostname_script, public_hostname_script) + \
+- "python /home/ambari/ambari-agent/conf/unix/agent-multiplier.py start\n" + \
++ "wget http://10.240.0.30/agent-multiplier.py ; python agent-multiplier.py start\n" + \
+ "exit 0"
+
+ with open("agent.sh", "w") as f:
diff --git a/server.sh b/server.sh
new file mode 100644
index 0000000..880ab44
--- /dev/null
+++ b/server.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+yum install wget -y
+wget -O /etc/yum.repos.d/ambari.repo http://10.240.0.30/ambari.repo
+yum clean all; yum install ambari-server -y
+sed -i -f /home/ambari/ambari-server/src/main/resources/stacks/PERF/install_packages.sed /var/lib/ambari-server/resources/custom_actions/scripts/install_packages.py
+sed -i -f /home/ambari/ambari-server/src/main/resources/stacks/PERF/install_packages.sed /var/lib/ambari-agent/cache/custom_actions/scripts/install_packages.py
+
+
+cd /; wget http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.40/mysql-connector-java-5.1.40.jar;
+mkdir /usr/share/java; chmod 777 /usr/share/java;cp mysql-connector-java-5.1.40.jar /usr/share/java/; chmod 777 /usr/share/java/mysql-connector-java-5.1.40.jar;
+ln -s /usr/share/java/mysql-connector-java-5.1.40.jar /usr/share/java/mysql-connector-java.jar;
+cd /etc/yum.repos.d/; wget http://repo.mysql.com/mysql-community-release-el6-5.noarch.rpm; rpm -ivh mysql-community-release-el6-5.noarch.rpm;yum clean all; yum install mysql-server -y
+sed -i -e 's/mysqld]/mysqld]\nmax_allowed_packet=1024M\njoin_buffer_size=512M\nsort_buffer_size=128M\nread_rnd_buffer_size=128M\ninnodb_buffer_pool_size=16G\ninnodb_file_io_threads=16\ninnodb_thread_concurrency=32\nkey_buffer_size=16G\nquery_cache_limit=16M\nquery_cache_size=512M\nthread_cache_size=128\ninnodb_log_buffer_size=512M/1' /etc/my.cnf
+service mysqld start
+mysql -uroot -e "CREATE DATABASE ambari;"
+mysql -uroot -e "SOURCE /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql;" ambari
+mysql -uroot -e "CREATE USER 'ambari'@'%' IDENTIFIED BY 'bigdata';"
+mysql -uroot -e "GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'%%';"
+mysql -uroot -e "CREATE USER 'ambari'@'localhost' IDENTIFIED BY 'bigdata';"
+mysql -uroot -e "GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'localhost';"
+mysql -uroot -e "CREATE USER 'ambari'@'perf-server-test-perf-1.c.pramod-thangali.internal' IDENTIFIED BY 'bigdata';"
+mysql -uroot -e "GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'perf-server-test-perf-1.c.pramod-thangali.internal';"
+mysql -uroot -e "FLUSH PRIVILEGES;"
+
+
+ambari-server setup -s
+ambari-server setup --database mysql --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar --databasehost=localhost --databaseport=3306 --databasename=ambari --databaseusername=ambari --databasepassword=bigdata
+sed -i -e 's/=postgres/=mysql/g' /etc/ambari-server/conf/ambari.properties
+sed -i -e 's/server.persistence.type=local/server.persistence.type=remote/g' /etc/ambari-server/conf/ambari.properties
+sed -i -e 's/local.database.user=postgres//g' /etc/ambari-server/conf/ambari.properties
+sed -i -e 's/server.jdbc.postgres.schema=ambari//g' /etc/ambari-server/conf/ambari.properties
+sed -i -e 's/agent.threadpool.size.max=25/agent.threadpool.size.max=100/g' /etc/ambari-server/conf/ambari.properties
+sed -i -e 's/client.threadpool.size.max=25/client.threadpool.size.max=65/g' /etc/ambari-server/conf/ambari.properties
+sed -i -e 's/false/true/g' /var/lib/ambari-server/resources/stacks/PERF/1.0/metainfo.xml
+sed -i -e 's/false/true/g' /var/lib/ambari-server/resources/stacks/PERF/2.0/metainfo.xml
+sed -i -e 's/-Xmx2048m/-Xmx16384m/g' /var/lib/ambari-server/ambari-env.sh
+
+echo 'server.jdbc.driver=com.mysql.jdbc.Driver' >> /etc/ambari-server/conf/ambari.properties
+echo 'server.jdbc.rca.url=jdbc:mysql://perf-server-test-perf-1.c.pramod-thangali.internal:3306/ambari' >> /etc/ambari-server/conf/ambari.properties
+echo 'server.jdbc.rca.driver=com.mysql.jdbc.Driver' >> /etc/ambari-server/conf/ambari.properties
+echo 'server.jdbc.url=jdbc:mysql://perf-server-test-perf-1.c.pramod-thangali.internal:3306/ambari' >> /etc/ambari-server/conf/ambari.properties
+echo 'server.jdbc.port=3306' >> /etc/ambari-server/conf/ambari.properties
+echo 'server.jdbc.hostname=localhost' >> /etc/ambari-server/conf/ambari.properties
+echo 'server.jdbc.driver.path=/usr/share/java/mysql-connector-java.jar' >> /etc/ambari-server/conf/ambari.properties
+echo 'alerts.cache.enabled=true' >> /etc/ambari-server/conf/ambari.properties
+echo 'alerts.cache.size=100000' >> /etc/ambari-server/conf/ambari.properties
+echo 'alerts.execution.scheduler.maxThreads=4' >> /etc/ambari-server/conf/ambari.properties
+echo 'security.temporary.keystore.retention.minutes=180' >> /etc/ambari-server/conf/ambari.properties
+echo 'stack.hooks.folder=stacks/PERF/1.0/hooks' >> /etc/ambari-server/conf/ambari.properties
+
+ambari-server start --skip-database-check
+exit 0
\ No newline at end of file