You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/20 11:26:06 UTC
[08/50] [abbrv] git commit: SLIDER-318 rm slider-assembly module that
isn't used
SLIDER-318 rm slider-assembly module that isn't used
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/52f1f6fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/52f1f6fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/52f1f6fb
Branch: refs/heads/master
Commit: 52f1f6fbe041a8bc604802e9eaaed2376b0e8ed5
Parents: 564462e
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 13:56:07 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 13:56:07 2014 +0100
----------------------------------------------------------------------
pom.xml | 1 -
slider-assembly/src/main/bash/README.md | 102 +++++++++++
slider-assembly/src/main/bash/slider-client.xml | 83 +++++++++
slider-assembly/src/main/bash/slider_destroy | 64 +++++++
slider-assembly/src/main/bash/slider_setup | 173 +++++++++++++++++++
slider-assembly/src/main/bash/slider_setup.conf | 60 +++++++
slider-install/README.md | 102 -----------
slider-install/pom.xml | 111 ------------
slider-install/src/main/bash/slider-client.xml | 83 ---------
slider-install/src/main/bash/slider_destroy | 64 -------
slider-install/src/main/bash/slider_setup | 173 -------------------
slider-install/src/main/bash/slider_setup.conf | 60 -------
12 files changed, 482 insertions(+), 594 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ed823c4..0816479 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,6 @@
<module>slider-providers/hbase/hbase-funtests</module>
<module>slider-providers/accumulo/slider-accumulo-provider</module>
<module>slider-providers/accumulo/accumulo-funtests</module>
- <module>slider-install</module>
</modules>
<licenses>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/README.md
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/README.md b/slider-assembly/src/main/bash/README.md
new file mode 100644
index 0000000..a4b7b08
--- /dev/null
+++ b/slider-assembly/src/main/bash/README.md
@@ -0,0 +1,102 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+slider_setup
+============
+
+Tools for installing, starting, and destroying HBase, Accumulo, and Storm slider apps on YARN.
+
+**WARNING: This is intended for POC/sandbox testing, may not be idempotent so DO NOT use on an existing Production cluster!!!**
+
+Setup
+-----
+1. Clone the repo
+2. Set the necessary cluster variables in `slider_setup.conf`, it shouldn't be necessary to change any other values but thoe ones below
+
+
+ # Zookeeper nodes
+ ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
+
+ # Resource Manager address (yarn.resourcemanager.address)
+ RM_ADDRESS="rm1:8050"
+
+ # Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
+ RM_SCHED_ADDRESS="rm1:8030"
+
+ # Default FS (fs.defaultFS)
+ DEFAULT_FS="hdfs://nn1:8020"
+
+Running
+-------
+* slider_setup is the main script and handles the following
+ 1. Pulls down slider and extracts the contents to the SLIDER_INST_DIR
+ 2. Modifies slider-client.xml with cluster related info
+ 3. Pulls down the slider enabled version of the specified product
+ 4. Creates necessary directories and copies required files to HDFS
+ 5. For HBase, creates the app dir in HDFS
+ 6. Submits the slider base application to the YARN cluster
+
+* The following args are required
+ * -f - The path to the slider_setup.conf that has been modified with cluster info
+ * -p - The product to run (hbase, accumulo, or storm are all that are supported at this time)
+ * -w - The number of "worker" nodes. This has different meaning depending on product.
+ * HBase - number of region servers
+ * Accumulo - number of tablet servers
+ * Storm - number of supervisors
+ * -n - The name of the app, this will be the display name in the resource manager and is used by the teardown process
+
+* HBase Example:
+
+
+ ./slider_setup -f slider_setup.conf -p hbase -w 5 -n hbase-slider
+
+* Accumulo Example:
+
+
+ ./slider_setup -f slider_setup.conf -p accumulo -w 3 -n accumulo-slider
+
+* Storm Example:
+
+
+ ./slider_setup -f slider_setup.conf -p storm -w 3 -n storm-slider
+
+Tear Down
+---------
+
+* slider_destroy will do the following
+ 1. Freeze the slider application based on provided name
+ 2. Destory the slider application based on provided name
+
+* The following args are required
+ * `-f` - The path to the `slider_setup.conf` that has been modified with cluster info
+ * `-n` - The name of the app, this was provided to the slider_setup tool
+
+* HBase Example:
+
+
+ ./slider_destroy -f slider_setup.conf -n hbase-slider
+
+* Accumulo Example:
+
+
+ ./slider_destroy -f slider_setup.conf -n accumulo-slider
+
+* Storm Example:
+
+
+ ./slider_destroy -f slider_setup.conf -n storm-slider
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider-client.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider-client.xml b/slider-assembly/src/main/bash/slider-client.xml
new file mode 100644
index 0000000..f7060c1
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider-client.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Properties set here are picked up in the client.
+ They are not passed to the AM -though the filesystem
+ binding details (URL And principal) are added to the
+ hbase-site.xml file when a cluster is created.
+-->
+<configuration>
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+
+
+ <property>
+ <name>slider.yarn.queue</name>
+ <value>default</value>
+ <description>YARN queue for the Application Master</description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>@@RM_ADDRESS@@</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>@@RM_SCHED_ADDRESS@@</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>@@DEFAULT_FS@@</value>
+ </property>
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>@@YARN_CP@@</value>
+ </property>
+
+ <property>
+ <name>slider.zookeeper.quorum</name>
+ <value>@@ZK_QUORUM@@</value>
+ </property>
+
+<!--
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>yarn/master@MINICLUSTER</value>
+ </property>
+
+ <property>
+ <name>slider.security.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>hdfs/master@MINICLUSTER</value>
+ </property>
+-->
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider_destroy
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider_destroy b/slider-assembly/src/main/bash/slider_destroy
new file mode 100755
index 0000000..9039751
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider_destroy
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Base Vars
+#
+SCRIPT_NAME=`basename $0`
+SCRIPT_DIR=`cd $(dirname $0) && pwd`
+
+#
+# Functions
+#
+usage() {
+ echo "Usage: $SCRIPT_NAME -f </path/to/config> -n <app name>"
+ exit 1
+}
+
+#
+# Parse cmd line args
+#
+while getopts "f:n:" opt; do
+ case "$opt" in
+ f) config=$OPTARG;;
+ n) app_name=$OPTARG;;
+ *) usage;;
+ esac
+done
+shift $((OPTIND-1))
+
+if [ -z "$config" ] || [ -z "$app_name" ]; then
+ usage
+fi
+
+#
+# Source the config
+#
+source $config
+
+#
+# Main
+#
+echo -e "\n## Freezing app $app_name"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider freeze $app_name --manager $RM_ADDRESS || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Destroying app $app_name"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider destroy $app_name --manager $RM_ADDRESS || exit 1
+echo "SUCCESS"
+
+exit 0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider_setup
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider_setup b/slider-assembly/src/main/bash/slider_setup
new file mode 100755
index 0000000..2c15c95
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider_setup
@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#
+# Base Vars
+#
+SCRIPT_NAME=`basename $0`
+SCRIPT_DIR=`cd $(dirname $0) && pwd`
+
+#
+# Functions
+#
+usage() {
+ echo "Usage: $SCRIPT_NAME -f </path/to/config> -p <product (hbase or storm)> -w <worker count> -n <app name>"
+ exit 1
+}
+
+#
+# Parse cmd line args
+#
+while getopts "f:p:w:n:" opt; do
+ case "$opt" in
+ f) config=$OPTARG;;
+ p) product=$OPTARG;;
+ w) worker_cnt=$OPTARG;;
+ n) app_name=$OPTARG;;
+ *) usage;;
+ esac
+done
+shift $((OPTIND-1))
+
+if [ -z "$config" ] || [ -z "$product" ] || [ -z "$worker_cnt" ] || [ -z "$app_name" ]; then
+ usage
+fi
+
+# Only support know products for now
+if [ $product != "hbase" ] && [ $product != "storm" ] && [ $product != "accumulo"]; then
+ echo "ERROR: Only HBase, Storm, and Accumulo are currently supported"
+ usage
+fi
+
+#
+# Source the config
+#
+source $config
+
+#
+# Product URLs
+#
+if [ $product = "hbase" ]; then
+ app_url=$HBASE_APP_URL
+elif [ $product = "storm" ]; then
+ app_url=$STORM_APP_URL
+elif [ $product = "accumulo" ]; then
+ app_url=$ACCUMULO_APP_URL
+fi
+
+#
+# Main
+#
+echo -e "\n## Creating slider install dir: $SLIDER_INST_DIR"
+mkdir -p $SLIDER_INST_DIR || exit 1
+chown yarn:hadoop $SLIDER_INST_DIR || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Downloading slider from: $SLIDER_URL"
+if [ -f /tmp/${SLIDER_VER}*tar ]; then
+ rm -f /tmp/${SLIDER_VER}*tar
+fi
+cd /tmp && wget $SLIDER_URL || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Extracting slider to $SLIDER_INST_DIR"
+tar -xf /tmp/${SLIDER_VER}*tar --strip-components=1 -C $SLIDER_INST_DIR || exit 1
+chown -R yarn:hadoop $SLIDER_INST_DIR || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Setting conf values"
+(cd $SCRIPT_DIR && cp slider-client.xml $SLIDER_INST_DIR/conf) || exit 1
+sed -i 's|@@RM_ADDRESS@@|'$RM_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@RM_SCHED_ADDRESS@@|'$RM_SCHED_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@DEFAULT_FS@@|'$DEFAULT_FS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@YARN_CP@@|'$YARN_CP'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@ZK_QUORUM@@|'$ZK_QUORUM'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Setting PATH to include the JDK bin: $JDK_BIN"
+export PATH=$PATH:$JDK_BIN
+echo "SUCCESS"
+
+echo -e "\n## Checking version of Hadoop slider was compiled against"
+hadoop_compiled_ver=`$SLIDER_INST_DIR/bin/slider version | grep "Compiled against Hadoop" | awk '{print $NF}'`
+if [ "$hadoop_compiled_ver" != "2.4.0" ]; then
+ echo "ERROR: Compiled against Hadoop version $hadoop_compiled_ver instead of 2.4.0" && exit 1
+else
+ echo "Compiled against Hadoop version: $hadoop_compiled_ver"
+fi
+echo "SUCCESS"
+
+echo -e "\n## Setting up HDFS directories for slider"
+sudo -u hdfs hdfs dfs -mkdir -p /slider || exit 1
+sudo -u hdfs hdfs dfs -chown yarn:hdfs /slider || exit 1
+sudo -u hdfs hdfs dfs -mkdir -p /user/yarn || exit 1
+sudo -u hdfs hdfs dfs -chown yarn:hdfs /user/yarn || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Loading the Slider agent"
+sudo -u yarn hdfs dfs -mkdir -p /slider/agent/conf || exit 1
+sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/agent/* /slider/agent
+echo "SUCCESS"
+
+echo -e "\n## Downloading $product to $SLIDER_INST_DIR/apps/$product"
+sudo -u yarn mkdir -p $SLIDER_INST_DIR/apps/$product || exit 1
+(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn wget $app_url) || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Extracting $product in $SLIDER_INST_DIR/apps/$product"
+(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn unzip -o $product*zip) || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Adding $product to HDFS slider dir"
+sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/apps/$product/$product*zip /slider
+echo "SUCCESS"
+
+echo -e "\n## Setting number of workers in $SLIDER_INST_DIR/apps/$product/resources.json"
+if [ $product = "hbase" ]; then
+ component="HBASE_REGIONSERVER"
+elif [ $product = "storm" ]; then
+ component="SUPERVISOR"
+elif [ $product = "accumulo" ]; then
+ component="ACCUMULO_TSERVER"
+fi
+# Update the resource.json file with worker_cnt
+python << END
+import json
+with open("$SLIDER_INST_DIR/apps/$product/resources.json", "r+") as f:
+ data = json.load(f)
+ data["components"]["$component"]["yarn.component.instances"] = "$worker_cnt"
+
+with open("$SLIDER_INST_DIR/apps/$product/resources.json", "w+") as f:
+ f.write(json.dumps(data, sort_keys = False, indent = 4))
+END
+echo "SUCCESS"
+
+# Handle HBase HDFS dir needs
+if [ $product = "hbase" ]; then
+ echo -e "\n## Creating hbase HDFS dir /apps/hbase"
+ sudo -u hdfs hdfs dfs -mkdir -p /apps/hbase || exit 1
+ sudo -u hdfs hdfs dfs -chown yarn:hdfs /apps/hbase || exit 1
+ echo "SUCCESS"
+fi
+
+echo -e "\n##Starting app $product with $worker_cnt workers via slider"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider create $app_name \
+ --image $DEFAULT_FS/slider/agent/slider-agent.tar.gz \
+ --template $SLIDER_INST_DIR/apps/$product/appConfig.json \
+ --resources $SLIDER_INST_DIR/apps/$product/resources.json || exit 1
+echo "SUCCESS"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider_setup.conf
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider_setup.conf b/slider-assembly/src/main/bash/slider_setup.conf
new file mode 100644
index 0000000..fed3f4c
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider_setup.conf
@@ -0,0 +1,60 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Change values below to match the cluster being deployed on
+#
+
+# Zookeeper nodes
+ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
+
+# Resource Manager address (yarn.resourcemanager.address)
+RM_ADDRESS="rm1:8050"
+
+# Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
+RM_SCHED_ADDRESS="rm1:8030"
+
+# Default FS (fs.defaultFS)
+DEFAULT_FS="hdfs://nn1:8020"
+
+
+#
+# Should not be necessary to change anything below except on new version releases
+#
+
+# Slider version
+SLIDER_VER="slider-0.30"
+
+# Slider URL
+SLIDER_URL="http://public-repo-1.hortonworks.com/slider/0.30/slider-0.30-all.tar"
+
+# HBase Slider App URL
+HBASE_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/hbase_v096.zip"
+
+# Storm Slider App URL
+STORM_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/storm_v091.zip"
+
+# Accumulo Slider App URL
+ACCUMULO_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/accumulo_v151.zip"
+
+# Where to install Slider
+SLIDER_INST_DIR="/usr/lib/slider"
+
+# JDK bin to add to PATH
+JDK_BIN="/usr/jdk64/jdk1.7.0_45/bin"
+
+# Yarn container classpath (yarn.application.classpath)
+YARN_CP="/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*"
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/README.md
----------------------------------------------------------------------
diff --git a/slider-install/README.md b/slider-install/README.md
deleted file mode 100644
index a4b7b08..0000000
--- a/slider-install/README.md
+++ /dev/null
@@ -1,102 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-
-slider_setup
-============
-
-Tools for installing, starting, and destroying HBase, Accumulo, and Storm slider apps on YARN.
-
-**WARNING: This is intended for POC/sandbox testing, may not be idempotent so DO NOT use on an existing Production cluster!!!**
-
-Setup
------
-1. Clone the repo
-2. Set the necessary cluster variables in `slider_setup.conf`, it shouldn't be necessary to change any other values but thoe ones below
-
-
- # Zookeeper nodes
- ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
-
- # Resource Manager address (yarn.resourcemanager.address)
- RM_ADDRESS="rm1:8050"
-
- # Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
- RM_SCHED_ADDRESS="rm1:8030"
-
- # Default FS (fs.defaultFS)
- DEFAULT_FS="hdfs://nn1:8020"
-
-Running
--------
-* slider_setup is the main script and handles the following
- 1. Pulls down slider and extracts the contents to the SLIDER_INST_DIR
- 2. Modifies slider-client.xml with cluster related info
- 3. Pulls down the slider enabled version of the specified product
- 4. Creates necessary directories and copies required files to HDFS
- 5. For HBase, creates the app dir in HDFS
- 6. Submits the slider base application to the YARN cluster
-
-* The following args are required
- * -f - The path to the slider_setup.conf that has been modified with cluster info
- * -p - The product to run (hbase, accumulo, or storm are all that are supported at this time)
- * -w - The number of "worker" nodes. This has different meaning depending on product.
- * HBase - number of region servers
- * Accumulo - number of tablet servers
- * Storm - number of supervisors
- * -n - The name of the app, this will be the display name in the resource manager and is used by the teardown process
-
-* HBase Example:
-
-
- ./slider_setup -f slider_setup.conf -p hbase -w 5 -n hbase-slider
-
-* Accumulo Example:
-
-
- ./slider_setup -f slider_setup.conf -p accumulo -w 3 -n accumulo-slider
-
-* Storm Example:
-
-
- ./slider_setup -f slider_setup.conf -p storm -w 3 -n storm-slider
-
-Tear Down
----------
-
-* slider_destroy will do the following
- 1. Freeze the slider application based on provided name
- 2. Destory the slider application based on provided name
-
-* The following args are required
- * `-f` - The path to the `slider_setup.conf` that has been modified with cluster info
- * `-n` - The name of the app, this was provided to the slider_setup tool
-
-* HBase Example:
-
-
- ./slider_destroy -f slider_setup.conf -n hbase-slider
-
-* Accumulo Example:
-
-
- ./slider_destroy -f slider_setup.conf -n accumulo-slider
-
-* Storm Example:
-
-
- ./slider_destroy -f slider_setup.conf -n storm-slider
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/pom.xml
----------------------------------------------------------------------
diff --git a/slider-install/pom.xml b/slider-install/pom.xml
deleted file mode 100644
index 8dd2ec6..0000000
--- a/slider-install/pom.xml
+++ /dev/null
@@ -1,111 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>slider-install</artifactId>
- <name>Slider Install</name>
- <packaging>pom</packaging>
- <description>
-
- Builds any RPMs and other install packaging for Slider.
- This is downstream of the slider assembly and the app packages, so
- that anything from these can be installed in the RPM.
- </description>
- <parent>
- <groupId>org.apache.slider</groupId>
- <artifactId>slider</artifactId>
- <version>0.41.0-incubating-SNAPSHOT</version>
- </parent>
-
- <properties>
- <basedir>/usr/local/slider</basedir>
- <confdir>${basedir}/conf</confdir>
- <bindir>${basedir}/bin</bindir>
- </properties>
-
- <build>
- <plugins>
- <!--read in a build.properties file if defined-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>properties-maven-plugin</artifactId>
- <version>${maven.properties.version}</version>
- <executions>
- <execution>
- <phase>initialize</phase>
- <goals>
- <goal>read-project-properties</goal>
- </goals>
- <configuration>
- <quiet>true</quiet>
- <files>
- <file>build.properties</file>
- <file>../build.properties</file>
- </files>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
-
- </build>
-
- <reporting>
- <plugins>
-
-
-
- </plugins>
- </reporting>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.slider</groupId>
- <artifactId>slider-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <!--
- needed to order the build and ensure the agent tar is found
- the test scope ensures that it isn't copied into the lib dir
- -->
- <dependency>
- <groupId>org.apache.slider</groupId>
- <artifactId>slider-agent</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>tar.gz</type>
- </dependency>
-
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
-
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </dependency>
-
- </dependencies>
-
-
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider-client.xml
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider-client.xml b/slider-install/src/main/bash/slider-client.xml
deleted file mode 100644
index f7060c1..0000000
--- a/slider-install/src/main/bash/slider-client.xml
+++ /dev/null
@@ -1,83 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
- Properties set here are picked up in the client.
- They are not passed to the AM -though the filesystem
- binding details (URL And principal) are added to the
- hbase-site.xml file when a cluster is created.
--->
-<configuration>
-
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
-
-
- <property>
- <name>slider.yarn.queue</name>
- <value>default</value>
- <description>YARN queue for the Application Master</description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.address</name>
- <value>@@RM_ADDRESS@@</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.address</name>
- <value>@@RM_SCHED_ADDRESS@@</value>
- </property>
-
- <property>
- <name>fs.defaultFS</name>
- <value>@@DEFAULT_FS@@</value>
- </property>
-
- <property>
- <name>yarn.application.classpath</name>
- <value>@@YARN_CP@@</value>
- </property>
-
- <property>
- <name>slider.zookeeper.quorum</name>
- <value>@@ZK_QUORUM@@</value>
- </property>
-
-<!--
- <property>
- <name>yarn.resourcemanager.principal</name>
- <value>yarn/master@MINICLUSTER</value>
- </property>
-
- <property>
- <name>slider.security.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.kerberos.principal</name>
- <value>hdfs/master@MINICLUSTER</value>
- </property>
--->
-
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider_destroy
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider_destroy b/slider-install/src/main/bash/slider_destroy
deleted file mode 100755
index 9039751..0000000
--- a/slider-install/src/main/bash/slider_destroy
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# Base Vars
-#
-SCRIPT_NAME=`basename $0`
-SCRIPT_DIR=`cd $(dirname $0) && pwd`
-
-#
-# Functions
-#
-usage() {
- echo "Usage: $SCRIPT_NAME -f </path/to/config> -n <app name>"
- exit 1
-}
-
-#
-# Parse cmd line args
-#
-while getopts "f:n:" opt; do
- case "$opt" in
- f) config=$OPTARG;;
- n) app_name=$OPTARG;;
- *) usage;;
- esac
-done
-shift $((OPTIND-1))
-
-if [ -z "$config" ] || [ -z "$app_name" ]; then
- usage
-fi
-
-#
-# Source the config
-#
-source $config
-
-#
-# Main
-#
-echo -e "\n## Freezing app $app_name"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider freeze $app_name --manager $RM_ADDRESS || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Destroying app $app_name"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider destroy $app_name --manager $RM_ADDRESS || exit 1
-echo "SUCCESS"
-
-exit 0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider_setup
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider_setup b/slider-install/src/main/bash/slider_setup
deleted file mode 100755
index 2c15c95..0000000
--- a/slider-install/src/main/bash/slider_setup
+++ /dev/null
@@ -1,173 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-#
-# Base Vars
-#
-SCRIPT_NAME=`basename $0`
-SCRIPT_DIR=`cd $(dirname $0) && pwd`
-
-#
-# Functions
-#
-usage() {
- echo "Usage: $SCRIPT_NAME -f </path/to/config> -p <product (hbase or storm)> -w <worker count> -n <app name>"
- exit 1
-}
-
-#
-# Parse cmd line args
-#
-while getopts "f:p:w:n:" opt; do
- case "$opt" in
- f) config=$OPTARG;;
- p) product=$OPTARG;;
- w) worker_cnt=$OPTARG;;
- n) app_name=$OPTARG;;
- *) usage;;
- esac
-done
-shift $((OPTIND-1))
-
-if [ -z "$config" ] || [ -z "$product" ] || [ -z "$worker_cnt" ] || [ -z "$app_name" ]; then
- usage
-fi
-
-# Only support know products for now
-if [ $product != "hbase" ] && [ $product != "storm" ] && [ $product != "accumulo"]; then
- echo "ERROR: Only HBase, Storm, and Accumulo are currently supported"
- usage
-fi
-
-#
-# Source the config
-#
-source $config
-
-#
-# Product URLs
-#
-if [ $product = "hbase" ]; then
- app_url=$HBASE_APP_URL
-elif [ $product = "storm" ]; then
- app_url=$STORM_APP_URL
-elif [ $product = "accumulo" ]; then
- app_url=$ACCUMULO_APP_URL
-fi
-
-#
-# Main
-#
-echo -e "\n## Creating slider install dir: $SLIDER_INST_DIR"
-mkdir -p $SLIDER_INST_DIR || exit 1
-chown yarn:hadoop $SLIDER_INST_DIR || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Downloading slider from: $SLIDER_URL"
-if [ -f /tmp/${SLIDER_VER}*tar ]; then
- rm -f /tmp/${SLIDER_VER}*tar
-fi
-cd /tmp && wget $SLIDER_URL || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Extracting slider to $SLIDER_INST_DIR"
-tar -xf /tmp/${SLIDER_VER}*tar --strip-components=1 -C $SLIDER_INST_DIR || exit 1
-chown -R yarn:hadoop $SLIDER_INST_DIR || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Setting conf values"
-(cd $SCRIPT_DIR && cp slider-client.xml $SLIDER_INST_DIR/conf) || exit 1
-sed -i 's|@@RM_ADDRESS@@|'$RM_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@RM_SCHED_ADDRESS@@|'$RM_SCHED_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@DEFAULT_FS@@|'$DEFAULT_FS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@YARN_CP@@|'$YARN_CP'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@ZK_QUORUM@@|'$ZK_QUORUM'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Setting PATH to include the JDK bin: $JDK_BIN"
-export PATH=$PATH:$JDK_BIN
-echo "SUCCESS"
-
-echo -e "\n## Checking version of Hadoop slider was compiled against"
-hadoop_compiled_ver=`$SLIDER_INST_DIR/bin/slider version | grep "Compiled against Hadoop" | awk '{print $NF}'`
-if [ "$hadoop_compiled_ver" != "2.4.0" ]; then
- echo "ERROR: Compiled against Hadoop version $hadoop_compiled_ver instead of 2.4.0" && exit 1
-else
- echo "Compiled against Hadoop version: $hadoop_compiled_ver"
-fi
-echo "SUCCESS"
-
-echo -e "\n## Setting up HDFS directories for slider"
-sudo -u hdfs hdfs dfs -mkdir -p /slider || exit 1
-sudo -u hdfs hdfs dfs -chown yarn:hdfs /slider || exit 1
-sudo -u hdfs hdfs dfs -mkdir -p /user/yarn || exit 1
-sudo -u hdfs hdfs dfs -chown yarn:hdfs /user/yarn || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Loading the Slider agent"
-sudo -u yarn hdfs dfs -mkdir -p /slider/agent/conf || exit 1
-sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/agent/* /slider/agent
-echo "SUCCESS"
-
-echo -e "\n## Downloading $product to $SLIDER_INST_DIR/apps/$product"
-sudo -u yarn mkdir -p $SLIDER_INST_DIR/apps/$product || exit 1
-(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn wget $app_url) || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Extracting $product in $SLIDER_INST_DIR/apps/$product"
-(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn unzip -o $product*zip) || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Adding $product to HDFS slider dir"
-sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/apps/$product/$product*zip /slider
-echo "SUCCESS"
-
-echo -e "\n## Setting number of workers in $SLIDER_INST_DIR/apps/$product/resources.json"
-if [ $product = "hbase" ]; then
- component="HBASE_REGIONSERVER"
-elif [ $product = "storm" ]; then
- component="SUPERVISOR"
-elif [ $product = "accumulo" ]; then
- component="ACCUMULO_TSERVER"
-fi
-# Update the resource.json file with worker_cnt
-python << END
-import json
-with open("$SLIDER_INST_DIR/apps/$product/resources.json", "r+") as f:
- data = json.load(f)
- data["components"]["$component"]["yarn.component.instances"] = "$worker_cnt"
-
-with open("$SLIDER_INST_DIR/apps/$product/resources.json", "w+") as f:
- f.write(json.dumps(data, sort_keys = False, indent = 4))
-END
-echo "SUCCESS"
-
-# Handle HBase HDFS dir needs
-if [ $product = "hbase" ]; then
- echo -e "\n## Creating hbase HDFS dir /apps/hbase"
- sudo -u hdfs hdfs dfs -mkdir -p /apps/hbase || exit 1
- sudo -u hdfs hdfs dfs -chown yarn:hdfs /apps/hbase || exit 1
- echo "SUCCESS"
-fi
-
-echo -e "\n##Starting app $product with $worker_cnt workers via slider"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider create $app_name \
- --image $DEFAULT_FS/slider/agent/slider-agent.tar.gz \
- --template $SLIDER_INST_DIR/apps/$product/appConfig.json \
- --resources $SLIDER_INST_DIR/apps/$product/resources.json || exit 1
-echo "SUCCESS"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider_setup.conf
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider_setup.conf b/slider-install/src/main/bash/slider_setup.conf
deleted file mode 100644
index fed3f4c..0000000
--- a/slider-install/src/main/bash/slider_setup.conf
+++ /dev/null
@@ -1,60 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Change values below to match the cluster being deployed on
-#
-
-# Zookeeper nodes
-ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
-
-# Resource Manager address (yarn.resourcemanager.address)
-RM_ADDRESS="rm1:8050"
-
-# Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
-RM_SCHED_ADDRESS="rm1:8030"
-
-# Default FS (fs.defaultFS)
-DEFAULT_FS="hdfs://nn1:8020"
-
-
-#
-# Should not be necessary to change anything below except on new version releases
-#
-
-# Slider version
-SLIDER_VER="slider-0.30"
-
-# Slider URL
-SLIDER_URL="http://public-repo-1.hortonworks.com/slider/0.30/slider-0.30-all.tar"
-
-# HBase Slider App URL
-HBASE_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/hbase_v096.zip"
-
-# Storm Slider App URL
-STORM_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/storm_v091.zip"
-
-# Accumulo Slider App URL
-ACCUMULO_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/accumulo_v151.zip"
-
-# Where to install Slider
-SLIDER_INST_DIR="/usr/lib/slider"
-
-# JDK bin to add to PATH
-JDK_BIN="/usr/jdk64/jdk1.7.0_45/bin"
-
-# Yarn container classpath (yarn.application.classpath)
-YARN_CP="/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*"
-