You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by gu...@apache.org on 2022/07/12 10:14:05 UTC
[bigtop] branch master updated: BIGTOP-3728: Docker dev/test env for centos7 (#940)
This is an automated email from the ASF dual-hosted git repository.
guyuqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new c94af912 BIGTOP-3728: Docker dev/test env for centos7 (#940)
c94af912 is described below
commit c94af9123e3a5a56d4a02ba4881b4c7c11e88f29
Author: 吴治国 <ch...@startdt.com>
AuthorDate: Tue Jul 12 18:13:59 2022 +0800
BIGTOP-3728: Docker dev/test env for centos7 (#940)
* BIGTOP-3728: Docker dev/test env for centos7
---
.../dev-support/docker/centos7/Dockerfile | 29 ++++++
.../dev-support/docker/centos7/build-containers.sh | 104 +++++++++++++++++++++
.../dev-support/docker/centos7/build-image.sh | 22 +++++
.../dev-support/docker/centos7/clear-containers.sh | 28 ++++++
.../dev-support/docker/centos7/krb5.conf | 41 ++++++++
.../dev-support/docker/centos7/sync-scripts.sh | 30 ++++++
6 files changed, 254 insertions(+)
diff --git a/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/Dockerfile b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/Dockerfile
new file mode 100755
index 00000000..960a1298
--- /dev/null
+++ b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/Dockerfile
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM centos:7
+
+RUN yum -y install sudo wget openssh-clients openssh-server vim mariadb mariadb-server java-1.8.0-openjdk* net-tools krb5-server krb5-libs krb5-workstation
+RUN wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.47/mysql-connector-java-5.1.47.jar -O /usr/share/java/mysql-connector-java.jar
+
+RUN /bin/sed -i 's,# StrictHostKeyChecking ask,StrictHostKeyChecking no,g' /etc/ssh/ssh_config
+
+RUN ssh-keygen -f "/root/.ssh/id_rsa" -N ""
+
+RUN wget https://downloads.apache.org/bigtop/bigtop-3.1.0/repos/centos-7/bigtop.repo -O /etc/yum.repos.d/bigtop.repo \
+ && yum -y install ambari-server ambari-agent \
+ && rm -f /etc/yum.repos.d/bigtop.repo
+
+EXPOSE 1-65535
diff --git a/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/build-containers.sh b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/build-containers.sh
new file mode 100755
index 00000000..f948e804
--- /dev/null
+++ b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/build-containers.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo -e "\033[32mCreating network ambari\033[0m"
+docker network create --driver bridge ambari
+
+echo -e "\033[32mCreating container ambari-server\033[0m"
+docker run -d -p 3306:3306 -p 5005:5005 -p 8080:8080 --name ambari-server --hostname ambari-server --network ambari --privileged -e "container=docker" -v /sys/fs/cgroup:/sys/fs/cgroup:ro ambari:2.7.5 /usr/sbin/init
+SERVER_PUB_KEY=`docker exec ambari-server /bin/cat /root/.ssh/id_rsa.pub`
+docker exec ambari-server bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys"
+docker exec ambari-server /bin/systemctl enable sshd
+docker exec ambari-server /bin/systemctl start sshd
+
+echo -e "\033[32mSetting up mariadb-server\033[0m"
+docker exec ambari-server /bin/systemctl enable mariadb
+docker exec ambari-server /bin/systemctl start mariadb
+docker exec ambari-server bash -c "mysql -e \"UPDATE mysql.user SET Password = PASSWORD('root') WHERE User = 'root'\""
+docker exec ambari-server bash -c "mysql -e \"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'root' WITH GRANT OPTION\""
+docker exec ambari-server bash -c "mysql -e \"DROP USER ''@'localhost'\""
+docker exec ambari-server bash -c "mysql -e \"DROP USER ''@'ambari-server'\""
+docker exec ambari-server bash -c "mysql -e \"DROP DATABASE test\""
+docker exec ambari-server bash -c "mysql -e \"CREATE DATABASE ambari\""
+docker exec ambari-server bash -c "mysql --database=ambari -e \"source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql\""
+docker exec ambari-server bash -c "mysql -e \"FLUSH PRIVILEGES\""
+
+echo -e "\033[32mSetting up ambari-server\033[0m"
+docker exec ambari-server bash -c "ambari-server setup --java-home=/usr/lib/jvm/java --database=mysql --databasehost=localhost --databaseport=3306 --databasename=ambari --databaseusername=root --databasepassword=root -s"
+
+echo -e "\033[32mCreating container ambari-agent-01\033[0m"
+docker run -d --name ambari-agent-01 --hostname ambari-agent-01 --network ambari --privileged -e "container=docker" -v /sys/fs/cgroup:/sys/fs/cgroup:ro ambari:2.7.5 /usr/sbin/init
+docker exec ambari-agent-01 bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys"
+docker exec ambari-agent-01 /bin/systemctl enable sshd
+docker exec ambari-agent-01 /bin/systemctl start sshd
+
+echo -e "\033[32mCreating container ambari-agent-02\033[0m"
+docker run -d --name ambari-agent-02 --hostname ambari-agent-02 --network ambari --privileged -e "container=docker" -v /sys/fs/cgroup:/sys/fs/cgroup:ro ambari:2.7.5 /usr/sbin/init
+docker exec ambari-agent-02 bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys"
+docker exec ambari-agent-02 /bin/systemctl enable sshd
+docker exec ambari-agent-02 /bin/systemctl start sshd
+
+echo -e "\033[32mConfiguring hosts file\033[0m"
+AMBARI_SERVER_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ambari-server`
+AMBARI_AGENT_01_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ambari-agent-01`
+AMBARI_AGENT_02_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ambari-agent-02`
+docker exec ambari-server bash -c "echo '$AMBARI_AGENT_01_IP ambari-agent-01' >> /etc/hosts"
+docker exec ambari-server bash -c "echo '$AMBARI_AGENT_02_IP ambari-agent-02' >> /etc/hosts"
+docker exec ambari-agent-01 bash -c "echo '$AMBARI_SERVER_IP ambari-server' >> /etc/hosts"
+docker exec ambari-agent-01 bash -c "echo '$AMBARI_AGENT_02_IP ambari-agent-02' >> /etc/hosts"
+docker exec ambari-agent-02 bash -c "echo '$AMBARI_SERVER_IP ambari-server' >> /etc/hosts"
+docker exec ambari-agent-02 bash -c "echo '$AMBARI_AGENT_01_IP ambari-agent-01' >> /etc/hosts"
+
+
+echo -e "\033[32mConfiguring Kerberos\033[0m"
+docker cp ./krb5.conf ambari-server:/etc/krb5.conf
+docker cp ./krb5.conf ambari-agent-01:/etc/krb5.conf
+docker cp ./krb5.conf ambari-agent-02:/etc/krb5.conf
+docker exec ambari-server bash -c "echo -e \"admin\nadmin\" | kdb5_util create -s -r EXAMPLE.COM"
+docker exec ambari-server bash -c "echo -e \"admin\nadmin\" | kadmin.local -q \"addprinc admin/admin\""
+docker exec ambari-server bash -c "systemctl start krb5kdc"
+docker exec ambari-server bash -c "systemctl enable krb5kdc"
+docker exec ambari-server bash -c "systemctl start kadmin"
+docker exec ambari-server bash -c "systemctl enable kadmin"
+# KDC HOST: ambari-server
+# REALM NAME: EXAMPLE.COM
+# ADMIN PRINCIPAL: admin/admin@EXAMPLE.COM
+# ADMIN PASSWORD: admin
+
+echo -e "\033[32mInstalling Bigtop Ambari Mpack\033[0m"
+ARCH=`uname -m`
+basearch=
+if [ ${ARCH} == "aarch64" ]; then
+ basearch="-aarch64"
+elif [ ${ARCH} == "ppc64le" ]; then
+ basearch="-ppc64le"
+fi
+
+cd ../../../../../../../
+docker run --rm -v `pwd`:/ws --workdir /ws bigtop/slaves:trunk-centos-7${basearch} bash -c '. /etc/profile.d/bigtop.sh; ./gradlew bigtop-utils-clean bigtop-ambari-mpack-clean bigtop-utils-pkg bigtop-ambari-mpack-pkg'
+docker cp output/bigtop-ambari-mpack/noarch/bigtop-ambari-mpack-2.7.5.0-*.rpm ambari-server:/
+docker cp output/bigtop-utils/noarch/bigtop-utils-*.rpm ambari-server:/
+cd -
+
+docker exec ambari-server bash -c "rpm -ivh -f /bigtop-utils-*.rpm; rpm -ivh -f /bigtop-ambari-mpack-2.7.5.0-1.el7.noarch.rpm"
+docker exec ambari-server bash -c "cp /usr/lib/bigtop-ambari-mpack/bgtp-ambari-mpack-1.0.0.0-SNAPSHOT-bgtp-ambari-mpack.tar.gz /"
+
+docker exec ambari-server bash -c "ambari-server install-mpack --mpack=/bgtp-ambari-mpack-1.0.0.0-SNAPSHOT-bgtp-ambari-mpack.tar.gz"
+docker exec ambari-server bash -c "ambari-server restart --debug"
+
+echo -e "\033[32mPrint Ambari Server RSA Private Key\033[0m"
+docker exec ambari-server bash -c "cat ~/.ssh/id_rsa"
\ No newline at end of file
diff --git a/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/build-image.sh b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/build-image.sh
new file mode 100755
index 00000000..d93caa56
--- /dev/null
+++ b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/build-image.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo -e "\033[32mRemoving image ambari:2.7.5\033[0m"
+docker rmi ambari:2.7.5
+
+echo -e "\033[32mBuilding image ambari:2.7.5\033[0m"
+docker build -t ambari:2.7.5 .
diff --git a/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/clear-containers.sh b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/clear-containers.sh
new file mode 100755
index 00000000..63be0962
--- /dev/null
+++ b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/clear-containers.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo -e "\033[32mRemoving container ambari-server\033[0m"
+docker rm -f ambari-server
+
+echo -e "\033[32mRemoving container ambari-agent-01\033[0m"
+docker rm -f ambari-agent-01
+
+echo -e "\033[32mRemoving container ambari-agent-02\033[0m"
+docker rm -f ambari-agent-02
+
+echo -e "\033[32mRemoving network ambari\033[0m"
+docker network rm ambari
\ No newline at end of file
diff --git a/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/krb5.conf b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/krb5.conf
new file mode 100644
index 00000000..85d2f264
--- /dev/null
+++ b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/krb5.conf
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+includedir /etc/krb5.conf.d/
+
+[logging]
+ default = FILE:/var/log/krb5libs.log
+ kdc = FILE:/var/log/krb5kdc.log
+ admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ pkinit_anchors = FILE:/etc/pki/tls/certs/ca-bundle.crt
+ default_realm = EXAMPLE.COM
+ default_ccache_name = KEYRING:persistent:%{uid}
+
+[realms]
+EXAMPLE.COM = {
+ kdc = ambari-server
+ admin_server = ambari-server
+}
+
+[domain_realm]
+.example.com = EXAMPLE.COM
+example.com = EXAMPLE.COM
\ No newline at end of file
diff --git a/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/sync-scripts.sh b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/sync-scripts.sh
new file mode 100755
index 00000000..ebc5f73d
--- /dev/null
+++ b/bigtop-packages/src/common/bigtop-ambari-mpack/dev-support/docker/centos7/sync-scripts.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo -e "\033[32mSynchronizing script to ambari-server\033[0m"
+docker exec ambari-server bash -c "mkdir -p /var/lib/ambari-agent/cache/stacks/BGTP/1.0/services/"
+docker cp ../../../bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/ ambari-server:/var/lib/ambari-agent/cache/stacks/BGTP/1.0/
+
+echo -e "\033[32mSynchronizing script to ambari-agent-01\033[0m"
+docker exec ambari-agent-01 bash -c "mkdir -p /var/lib/ambari-agent/cache/stacks/BGTP/1.0/services/"
+docker cp ../../../bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/ ambari-agent-01:/var/lib/ambari-agent/cache/stacks/BGTP/1.0/
+
+echo -e "\033[32mSynchronizing script to ambari-agent-02\033[0m"
+docker exec ambari-agent-02 bash -c "mkdir -p /var/lib/ambari-agent/cache/stacks/BGTP/1.0/services/"
+docker cp ../../../bgtp-ambari-mpack/src/main/resources/stacks/BGTP/1.0/services/ ambari-agent-02:/var/lib/ambari-agent/cache/stacks/BGTP/1.0/
+
+echo -e "\033[32mDone!\033[0m"
\ No newline at end of file