You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2016/08/09 18:27:06 UTC
[03/11] bigtop git commit: BIGTOP-2505: Support systemd containers,
fix exec (-e) flag
BIGTOP-2505: Support systemd containers, fix exec (-e) flag
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/40a52870
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/40a52870
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/40a52870
Branch: refs/heads/BIGTOP-2253
Commit: 40a52870b5a4694ae3af9cf59734a8b2d35675ec
Parents: 6591e4b
Author: Olaf Flebbe <of...@oflebbe.de>
Authored: Sat Jul 23 12:34:12 2016 +0200
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue Aug 9 20:26:21 2016 +0200
----------------------------------------------------------------------
provisioner/docker/config_centos7.yaml | 32 +++++++++++++++++
provisioner/docker/docker-compose.yml | 2 +-
provisioner/docker/docker-hadoop.sh | 10 +++---
provisioner/utils/setup-env-centos.sh | 36 -------------------
provisioner/utils/setup-env-debian.sh | 34 ------------------
provisioner/utils/smoke-tests.sh | 56 -----------------------------
6 files changed, 38 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/docker/config_centos7.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_centos7.yaml b/provisioner/docker/config_centos7.yaml
new file mode 100644
index 0000000..3610a8f
--- /dev/null
+++ b/provisioner/docker/config_centos7.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+ memory_size: "4096"
+ image: "bigtop/deploy:centos-7"
+
+boot2docker:
+ memory_size: "4096"
+ number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/7/x86_64"
+distro: centos
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "java-1.7.0-openjdk-devel.x86_64"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/docker/docker-compose.yml
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml
index 95831d1..d71b247 100644
--- a/provisioner/docker/docker-compose.yml
+++ b/provisioner/docker/docker-compose.yml
@@ -1,6 +1,6 @@
bigtop:
image: ${DOCKER_IMAGE}
- command: sleep infinity
+ command: /sbin/init
domainname: bigtop.apache.org
privileged: true
volumes:
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 89ac6bb..4b1c0df 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -41,8 +41,8 @@ create() {
PROVISION_ID=`cat .provision_id`
# Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
mkdir -p config/hieradata 2> /dev/null
- cat /dev/null > ./config/hiera.yaml
- cat /dev/null > ./config/hosts
+ echo > ./config/hiera.yaml
+ echo > ./config/hosts
export DOCKER_IMAGE=$(get-yaml-config docker image)
# Startup instances
@@ -81,8 +81,8 @@ generate-hosts() {
generate-config() {
echo "Bigtop Puppet configurations are shared between instances, and can be modified under config/hieradata"
- cat $BIGTOP_PUPPET_DIR/hiera.yaml > ./config/hiera.yaml
- yes | cp -vr $BIGTOP_PUPPET_DIR/hieradata ./config/
+ cat $BIGTOP_PUPPET_DIR/hiera.yaml >> ./config/hiera.yaml
+ cp -vfr $BIGTOP_PUPPET_DIR/hieradata ./config/
cat > ./config/hieradata/site.yaml << EOF
bigtop::hadoop_head_node: $1
hadoop::hadoop_storage_dirs: [/data/1, /data/2]
@@ -185,7 +185,7 @@ if [ -e .provision_id ]; then
PROVISION_ID=`cat .provision_id`
fi
if [ -n "$PROVISION_ID" ]; then
- NODES=`docker-compose -p $PROVISION_ID ps -q`
+ NODES=(`docker-compose -p $PROVISION_ID ps -q`)
fi
while [ $# -gt 0 ]; do
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/utils/setup-env-centos.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh
deleted file mode 100755
index bb25d28..0000000
--- a/provisioner/utils/setup-env-centos.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-enable_local_repo=${1:-false}
-
-bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
-
-# Setup rng-tools to improve virtual machine entropy performance.
-# The poor entropy performance will cause kerberos provisioning failed.
-yum -y install rng-tools
-sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd
-service rngd start
-
-if [ $enable_local_repo == "true" ]; then
- echo "Enabling local yum."
- yum -y install yum-utils
- sudo echo "gpgcheck=0" >> /etc/yum.conf
- sudo yum-config-manager --add-repo file:///bigtop-home/output
-else
- echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..."
-fi
-
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/utils/setup-env-debian.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh
deleted file mode 100755
index c5768d9..0000000
--- a/provisioner/utils/setup-env-debian.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-enable_local_repo=${1:-false}
-
-bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
-
-# Setup rng-tools to improve virtual machine entropy performance.
-# The poor entropy performance will cause kerberos provisioning failed.
-apt-get -y install rng-tools
-sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools
-service rng-tools start
-
-if [ $enable_local_repo == "true" ]; then
- echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list
- apt-get update
-else
- echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..."
-fi
-
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/utils/smoke-tests.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
deleted file mode 100755
index 8dac31c..0000000
--- a/provisioner/utils/smoke-tests.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HCFS_USER="hdfs"
-SMOKE_TESTS=${1:-mapreduce,pig}
-
-# Autodetect JAVA_HOME
-if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
- . /usr/lib/bigtop-utils/bigtop-detect-javahome
-else
- >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n"
- exit 2
-fi
-
-echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
-
-export HADOOP_CONF_DIR=/etc/hadoop/conf/
-export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
-export HIVE_HOME=/usr/lib/hive/
-export PIG_HOME=/usr/lib/pig/
-export FLUME_HOME=/usr/lib/flume/
-export SQOOP_HOME=/usr/lib/sqoop/
-export HIVE_CONF_DIR=/etc/hive/conf/
-export MAHOUT_HOME="/usr/lib/mahout"
-
-su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root'
-su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant'
-su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root'
-
-if [ -f /etc/debian_version ] ; then
- apt-get -y install pig hive flume mahout sqoop
-else
- yum install -y pig hive flume mahout sqoop
-fi
-ALL_SMOKE_TASKS=""
-for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do
- ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test"
-done
-cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info
-# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same
-# permissions as used for smoke-tests execution
-rm -rf buildSrc/build/test-results/binary
-