You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2015/11/30 11:11:50 UTC
[08/20] incubator-eagle git commit: [EAGLE-53] Loaded eagle binary
package from relative path instead of external link and add eagle-docker in
project root path
[EAGLE-53] Loaded eagle binary package from relative path instead of external link and add eagle-docker in project root path
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/43bae921
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/43bae921
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/43bae921
Branch: refs/heads/master
Commit: 43bae921711c2db6c505d0b4bca444ed7dd871d5
Parents: dc5b5ab
Author: Hao Chen <ha...@apache.org>
Authored: Sun Nov 29 10:50:31 2015 -0700
Committer: Hao Chen <ha...@apache.org>
Committed: Sun Nov 29 10:50:31 2015 -0700
----------------------------------------------------------------------
.gitignore | 2 +-
eagle-docker | 20 ++
eagle-external/eagle-docker/Dockerfile | 21 +-
eagle-external/eagle-docker/bin/eagle-docker.sh | 271 +++++++++++++++++++
eagle-external/eagle-docker/bin/eagle-lib.sh | 162 +++++++++++
eagle-external/eagle-docker/resource/.gitignore | 1 -
.../eagle-docker/resource/deploy-eagle.sh | 6 +-
.../eagle-docker/resource/eagle-multinode.json | 2 +-
.../eagle-docker/resource/eagle-singlenode.json | 2 +-
9 files changed, 475 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/43bae921/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 46796c9..e40eaef 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,6 +36,7 @@ hs_err_pid*
target/
target/*
*/target/*
+**/target/*
pom.xml.tag
pom.xml.releaseBackup
pom.xml.versionsBackup
@@ -43,7 +44,6 @@ pom.xml.next
release.properties
dependency-reduced-pom.xml
buildNumber.properties
-**/bin/*
eagle-security/eagle-security-mltraining/src/test/resources/outputDir/
eagle-security/eagle-security-userprofile/src/test/resources/models/
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/43bae921/eagle-docker
----------------------------------------------------------------------
diff --git a/eagle-docker b/eagle-docker
new file mode 100755
index 0000000..f581845
--- /dev/null
+++ b/eagle-docker
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTICE: This script is developed and maintained by Apache Eagle community under Apache Softwarw Foundation but not from official Docker product or community.
+
+$(dirname $0)/eagle-external/eagle-docker/bin/eagle-docker.sh $@
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/43bae921/eagle-external/eagle-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/Dockerfile b/eagle-external/eagle-docker/Dockerfile
index 0dd8a83..f468ed7 100644
--- a/eagle-external/eagle-docker/Dockerfile
+++ b/eagle-external/eagle-docker/Dockerfile
@@ -13,19 +13,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
+# NOTICE: This docker image is built based sequenceiq/ambari which is open sourced under github: https://github.com/sequenceiq/docker-ambari
FROM sequenceiq/ambari:1.7.0
MAINTAINER dev@eagle.incubator.apache.org
-ENV EAGLE_DOWNLOAD_LINK http://66.211.190.194/eagle-0.1.0.tar.gz
-
-RUN curl -sL $EAGLE_DOWNLOAD_LINK | tar -xz -C /usr/hdp/current
-RUN cd /usr/hdp/current && ln -s ./eagle-0.1.0 eagle
+# Eagle environment
ENV EAGLE_HOME=/usr/hdp/current/eagle
+# Install Eagle Package
+#
+# [Option #1] Download eagle package from external link
+# ENV EAGLE_DOWNLOAD_LINK http://66.211.190.194/eagle-0.1.0.tar.gz
+# RUN curl -sL $EAGLE_DOWNLOAD_LINK | tar -xz -C /usr/hdp/current
+# RUN cd /usr/hdp/current && ln -s ./eagle-0.1.0 eagle
+
+# [Option #2] Load eagle package from locally built,
+# In such way, should build with `eagle-docker.sh buid` instead of executing native
+# `docker build` directly, to prepare the eagle package into build/eagle
+ADD target/eagle-current /usr/hdp/current/eagle
+
+# Load External Packages
RUN yum install -y httpd ganglia ganglia-gmetad ganglia-gmond ganglia-web nagios kafka zookeeper storm hbase tez hadoop snappy snappy-devel hadoop-libhdfs ambari-log4j hive hive-hcatalog hive-webhcat webhcat-tar-hive webhcat-tar-pig mysql-connector-java mysql-server
+# Load Static Resources
ADD resource/serf /usr/local/serf
ADD resource/httpd /var/log/httpd
ADD resource/install-cluster.sh /tmp/
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/43bae921/eagle-external/eagle-docker/bin/eagle-docker.sh
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/bin/eagle-docker.sh b/eagle-external/eagle-docker/bin/eagle-docker.sh
new file mode 100755
index 0000000..494fa5c
--- /dev/null
+++ b/eagle-external/eagle-docker/bin/eagle-docker.sh
@@ -0,0 +1,271 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTICE: This script is developed and maintained by Apache Eagle community under Apache Softwarw Foundation but not from official Docker product or community.
+
+EAGLE_DOCKER_VERSION=0.3.0-snapshot
+EAGLE_DOCKER_NAME=apacheeagle/standalone
+
+NODE_NUM=1
+
+cd `dirname $0`/../
+source bin/eagle-lib.sh
+
+function check_env(){
+ which docker 1>/dev/null 2>&1
+
+ if [ $? != 0 ];then
+ echo "[ERROR] docker commnad is not found, please make sure install and configure Docker Engine properly, docker docs: http://docs.docker.com" 1>&2
+ exit 1
+ else
+ log=`docker images 2>&1`
+ if [ $? != 0 ];then
+ echo "[ERROR] $log"
+ exit 1
+ fi
+ fi
+}
+
+function usage() {
+ echo "Usage: ./eagle-docker [options] [command]"
+
+ echo ""
+ echo "Apache Eagle Docker Image : ${EAGLE_DOCKER_NAME}:${EAGLE_DOCKER_VERSION}"
+ echo ""
+ echo "Commands:"
+ echo " build Build eagle docker image"
+ echo " deploy Deploy eagle docker image"
+ echo " start Start eagle docker instance"
+ echo " stop Stop eagle docker instance"
+ echo " status List eagle docker image and instance status"
+ echo " clean Clean docker image and instance"
+ echo ""
+ echo "Options:"
+ echo " --node [number] Docker instances node number, default is 1"
+ echo " --help Display eagle docker image usage information"
+ echo ""
+ exit 0
+}
+
+function deploy(){
+ check_env
+ eagle-deploy-cluster $NODE_NUM
+}
+
+function build(){
+ check_env
+ # ==========================================
+ # Check Eagle Docker Image
+ # ==========================================
+ echo "[1/3] Validate Environment"
+ # echo "Checking eagle docker image ..."
+ docker images | grep $EAGLE_DOCKER_NAME 1>/dev/null 2>&1
+ if [ $? == 0 ];then
+ echo "Eagle docker image already exists:"
+ echo ""
+ docker images | grep $EAGLE_DOCKER_NAME
+ echo ""
+ echo "Rebuild it now [Y/N]? "
+ read rebuild
+ if [ $rebuild == "Y" ] || [ $rebuild == "y" ];then
+ echo "Clean eagle image"
+ clean
+ else
+ echo "Quit now"
+ exit 1
+ fi
+ else
+ echo "No existing eagle docker images, environment is ready"
+ fi
+
+ # ==========================================
+ # Build Eagle Binary Package
+ # ==========================================
+ echo "[2/3] Build Eagle Binary Package"
+ # echo "Checking eagle binary package"
+ cd ../../
+ ls eagle-assembly/target/eagle-*-bin.tar.gz 1>/dev/null 2>&1
+ if [ $? == 0 ];then
+ eagle_bin_pkg=`ls eagle-assembly/target/eagle-*-bin.tar.gz`
+ echo "Found eagle binary package at $eagle_bin_pkg"
+ else
+ echo "Eagle binary package is not found"
+ echo "Build eagle binary package now"
+ # ==========================================
+ # Build Eagle Binary Package with Maven
+ # ==========================================
+ echo ""
+ echo "Execute: mvn package -DskipTests "
+ mvn package -DskipTests
+ if [ $? == 0 ];then
+ echo "Built successfully existing with 0"
+ ls eagle-assembly/target/eagle-*-bin.tar.gz 1>/dev/null 2>&1
+ if [ $? == 0 ];then
+ eagle_bin_pkg=`ls eagle-assembly/target/eagle-*-bin.tar.gz`
+ echo ""
+ echo "[SUCCESS] Successfully build eagle binary package at $eagle_bin_pkg"
+ else
+ echo ""
+ echo "[FAILED] Built eagle binary package exiting with 0, but package is not found"
+ exit 1
+ fi
+ else
+ echo ""
+ echo "[FAILED] Failed to build eagle binary package, exit 1"
+ exit 1
+ fi
+ fi
+
+ # =====================================
+ # Build Eagle Docker Image
+ # =====================================
+ echo "[3/3] Build Eagle Docker Image: $EAGLE_DOCKER_NAME"
+ echo "Extracting $eagle_bin_pkg"
+ if [ -e eagle-external/eagle-docker/target ];then
+ rm -rf eagle-external/eagle-docker/target
+ fi
+ mkdir -p eagle-external/eagle-docker/target/eagle-current
+
+ out=`tar -xzf $eagle_bin_pkg -C eagle-external/eagle-docker/target/`
+ if [ $? != 0 ];then
+ echo "[ERROR] Failed to execute 'tar -xzf $eagle_bin_pkg -C eagle-external/eagle-docker/target/': $out" 1>&2
+ exit 1
+ fi
+ mv eagle-external/eagle-docker/target/eagle-*/* eagle-external/eagle-docker/target/eagle-current
+
+ echo "Execute: docker build -t $EAGLE_DOCKER_NAME ."
+ cd eagle-external/eagle-docker
+ docker build -t $EAGLE_DOCKER_NAME .
+
+ if [ $? == 0 ];then
+ echo ""
+ echo "[SUCCESS] Successfully built eagle docker image : $EAGLE_DOCKER_NAME"
+ else
+ echo ""
+ echo "[FAILED] Failed to build eagle docker image : $EAGLE_DOCKER_NAME"
+ fi
+}
+
+function start(){
+ check_env
+ docker ps -a | grep $EAGLE_DOCKER_NAME 1>/dev/null
+ if [ $? != 0 ];then
+ echo "No eagle docker instance found"
+ exit 1
+ fi
+ docker ps -a | grep $EAGLE_DOCKER_NAME | grep Exited 1>/dev/null
+ if [ $? != 0 ];then
+ echo "No exited eagle docker instance found, all should be running"
+ exit 1
+ fi
+ docker ps -a | grep $EAGLE_DOCKER_NAME | grep Exited | awk '{ print $1 }' | while read id; do echo "Starting $id";docker start $id 1>/dev/null;done
+
+ echo ""
+ echo "[Eagle Docker Instances (Up)]"
+ echo ""
+ docker ps | grep $EAGLE_DOCKER_NAME
+}
+
+function stop(){
+ check_env
+ docker ps | grep $EAGLE_DOCKER_NAME 1>/dev/null
+ if [ $? != 0 ];then
+ echo "No eagle docker instance is running"
+ exit 1
+ fi
+ docker ps | grep $EAGLE_DOCKER_NAME | awk '{ print $1 }' | while read id; do echo "Stopping $id";docker kill $id;done
+
+ echo ""
+ echo "[Eagle Docker Instances (Exited)]"
+ echo ""
+ docker ps -a| grep $EAGLE_DOCKER_NAME
+}
+
+function status(){
+ check_env
+ echo "[Eagle Docker Images]"
+ echo ""
+ docker images | grep $EAGLE_DOCKER_NAME
+ [[ $? == 0 ]] || echo "no images found for eagle"
+ echo ""
+
+ echo "[Eagle Docker Instances (All)]"
+ echo ""
+ docker ps -a | grep $EAGLE_DOCKER_NAME
+ [[ $? == 0 ]] || echo "no instances found eagle"
+ echo ""
+}
+
+
+function clean(){
+ check_env
+ echo "[1/4] Stop eagle instances"
+ # Call in multi-process to avoid exiting by stopping
+ echo `stop`
+
+ echo "[2/4] Remove eagle instances"
+ docker ps -a | grep $EAGLE_DOCKER_NAME |awk '{print $1}'| while read id; do echo "Removing instance $id";docker rm $id;done
+
+ echo "[3/4] Remove eagle images"
+ docker images | grep apacheeagle/standalone | awk '{print $3}' | while read id; do echo "Removing image $id"; docker rmi $id;done
+
+ echo "[4/4] Clean temp files"
+ [[ -e target/ ]] && rm -rf target/
+
+ echo "Cleaning Done."
+}
+
+
+case $1 in
+"--node")
+ if [ $# -lt 2 ]; then
+ usage
+ exit 1
+ fi
+ NODE_NUM=$2
+ shift 2
+ ;;
+"deploy")
+ deploy
+ exit
+ ;;
+"build")
+ build
+ exit
+ ;;
+"status")
+ status
+ exit
+ ;;
+"start")
+ start
+ exit
+ ;;
+"stop")
+ stop
+ exit
+ ;;
+"clean")
+ clean
+ exit
+ ;;
+*)
+ usage
+ exit 1
+ ;;
+esac
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/43bae921/eagle-external/eagle-docker/bin/eagle-lib.sh
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/bin/eagle-lib.sh b/eagle-external/eagle-docker/bin/eagle-lib.sh
new file mode 100755
index 0000000..e133f7c
--- /dev/null
+++ b/eagle-external/eagle-docker/bin/eagle-lib.sh
@@ -0,0 +1,162 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+: ${VERSION:=latest}
+: ${IMAGE:="apacheeagle/standalone:${VERSION}"}
+
+# : ${NODE_PREFIX:=docker}
+# : ${AMBARI_SERVER_NAME:=${NODE_PREFIX}-master}
+: ${AMBARI_SERVER_NAME:=master}
+: ${MYDOMAIN:=eagle.apache.org}
+: ${DOCKER_OPTS:="--dns 127.0.0.1 --entrypoint /usr/local/serf/bin/start-serf-agent.sh -e KEYCHAIN=$KEYCHAIN --env EAGLE_SERVER_HOST=${AMBARI_SERVER_NAME}.${MYDOMAIN}"}
+: ${CLUSTER_SIZE:=1}
+: ${DEBUG:=1}
+: ${SLEEP_TIME:=2}
+: ${DRY_RUN:=false}
+
+run-command() {
+ CMD="$@"
+ if [ "$DRY_RUN" == "false" ]; then
+ debug "$CMD"
+ "$@"
+ else
+ debug [DRY_RUN] "$CMD"
+ fi
+}
+
+amb-clean() {
+ unset NODE_PREFIX AMBARI_SERVER_NAME MYDOMAIN IMAGE DOCKER_OPTS DEBUG SLEEP_TIME AMBARI_SERVER_IP DRY_RUN
+}
+
+get-ambari-server-ip() {
+ AMBARI_SERVER_IP=$(get-host-ip ${AMBARI_SERVER_NAME})
+}
+
+get-host-ip() {
+ HOST=$1
+ docker inspect --format="{{.NetworkSettings.IPAddress}}" ${HOST}
+}
+
+amb-members() {
+ get-ambari-server-ip
+ serf members --rpc-addr $(docker inspect --format "{{.NetworkSettings.IPAddress}}" ${AMBARI_SERVER_NAME}):7373
+}
+
+amb-settings() {
+ cat <<EOF
+ NODE_PREFIX=$NODE_PREFIX
+ MYDOMAIN=$MYDOMAIN
+ CLUSTER_SIZE=$CLUSTER_SIZE
+ AMBARI_SERVER_NAME=$AMBARI_SERVER_NAME
+ IMAGE=$IMAGE
+ DOCKER_OPTS=$DOCKER_OPTS
+ AMBARI_SERVER_IP=$AMBARI_SERVER_IP
+ DRY_RUN=$DRY_RUN
+EOF
+}
+
+debug() {
+ [ $DEBUG -gt 0 ] && echo [DEBUG] "$@" 1>&2
+}
+
+docker-ps() {
+ #docker ps|sed "s/ \{3,\}/#/g"|cut -d '#' -f 1,2,7|sed "s/#/\t/g"
+ docker inspect --format="{{.Name}} {{.NetworkSettings.IPAddress}} {{.Config.Image}} {{.Config.Entrypoint}} {{.Config.Cmd}}" $(docker ps -q)
+}
+
+docker-psa() {
+ #docker ps|sed "s/ \{3,\}/#/g"|cut -d '#' -f 1,2,7|sed "s/#/\t/g"
+ docker inspect --format="{{.Name}} {{.NetworkSettings.IPAddress}} {{.Config.Image}} {{.Config.Entrypoint}} {{.Config.Cmd}}" $(docker ps -qa)
+}
+
+amb-start-cluster() {
+ local act_cluster_size=$1
+ : ${act_cluster_size:=$CLUSTER_SIZE}
+ echo starting an ambari cluster with: $act_cluster_size nodes
+
+ amb-start-first
+ [ $act_cluster_size -gt 1 ] && for i in $(seq $((act_cluster_size - 1))); do
+ amb-start-node $i
+ done
+}
+
+_amb_run_shell() {
+ COMMAND=$1
+ : ${COMMAND:? required}
+ get-ambari-server-ip
+ NODES=$(docker inspect --format="{{.Config.Image}} {{.Name}}" $(docker ps -q)|grep $IMAGE|grep $NODE_PREFIX|wc -l|xargs)
+ run-command docker run -it --rm -e EXPECTED_HOST_COUNT=$NODES -e BLUEPRINT=$BLUEPRINT --link ${AMBARI_SERVER_NAME}:ambariserver --entrypoint /bin/sh $IMAGE -c $COMMAND
+}
+
+amb-shell() {
+ _amb_run_shell /tmp/ambari-shell.sh
+}
+
+eagle-deploy-cluster() {
+ local act_cluster_size=$1
+ : ${act_cluster_size:=$CLUSTER_SIZE}
+
+ if [ $# -gt 1 ]; then
+ BLUEPRINT=$2
+ else
+ [ $act_cluster_size -gt 1 ] && BLUEPRINT=hdp-multinode-eagle || BLUEPRINT=hdp-singlenode-eagle
+ fi
+
+ : ${BLUEPRINT:?" required (hdp-singlenode-eagle / hdp-multinode-eagle)"}
+
+ amb-start-cluster $act_cluster_size
+ _amb_run_shell /tmp/install-cluster.sh
+}
+
+amb-start-first() {
+ run-command docker run -P -d $DOCKER_OPTS --name $AMBARI_SERVER_NAME -h $AMBARI_SERVER_NAME.$MYDOMAIN --privileged=true $IMAGE --tag ambari-server=true
+}
+
+amb-copy-to-hdfs() {
+ get-ambari-server-ip
+ FILE_PATH=${1:?"usage: <FILE_PATH> <NEW_FILE_NAME_ON_HDFS> <HDFS_PATH>"}
+ FILE_NAME=${2:?"usage: <FILE_PATH> <NEW_FILE_NAME_ON_HDFS> <HDFS_PATH>"}
+ DIR=${3:?"usage: <FILE_PATH> <NEW_FILE_NAME_ON_HDFS> <HDFS_PATH>"}
+ amb-create-hdfs-dir $DIR
+ DATANODE=$(curl -si -X PUT "http://$AMBARI_SERVER_IP:50070/webhdfs/v1$DIR/$FILE_NAME?user.name=hdfs&op=CREATE" |grep Location | sed "s/\..*//; s@.*http://@@")
+ DATANODE_IP=$(get-host-ip $DATANODE)
+ curl -T $FILE_PATH "http://$DATANODE_IP:50075/webhdfs/v1$DIR/$FILE_NAME?op=CREATE&user.name=hdfs&overwrite=true&namenoderpcaddress=$AMBARI_SERVER_IP:8020"
+}
+
+amb-create-hdfs-dir() {
+ get-ambari-server-ip
+ DIR=$1
+ curl -X PUT "http://$AMBARI_SERVER_IP:50070/webhdfs/v1$DIR?user.name=hdfs&op=MKDIRS" > /dev/null 2>&1
+}
+
+amb-scp-to-first() {
+ get-ambari-server-ip
+ FILE_PATH=${1:?"usage: <FILE_PATH> <DESTINATION_PATH>"}
+ DEST_PATH=${2:?"usage: <FILE_PATH> <DESTINATION_PATH>"}
+ scp $FILE_PATH root@$AMBARI_SERVER_IP:$DEST_PATH
+}
+
+amb-start-node() {
+ get-ambari-server-ip
+ : ${AMBARI_SERVER_IP:?"AMBARI_SERVER_IP is needed"}
+ NUMBER=${1:?"please give a <NUMBER> parameter it will be used as node<NUMBER>"}
+ if [ $# -eq 1 ] ;then
+ MORE_OPTIONS="-d"
+ else
+ shift
+ MORE_OPTIONS="$@"
+ fi
+ run-command docker run $MORE_OPTIONS -e SERF_JOIN_IP=$AMBARI_SERVER_IP $DOCKER_OPTS --name ${NODE_PREFIX}$NUMBER -h ${NODE_PREFIX}${NUMBER}.$MYDOMAIN $IMAGE --log-level debug
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/43bae921/eagle-external/eagle-docker/resource/.gitignore
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/resource/.gitignore b/eagle-external/eagle-docker/resource/.gitignore
deleted file mode 100644
index 335ec95..0000000
--- a/eagle-external/eagle-docker/resource/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.tar.gz
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/43bae921/eagle-external/eagle-docker/resource/deploy-eagle.sh
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/resource/deploy-eagle.sh b/eagle-external/eagle-docker/resource/deploy-eagle.sh
index 834474c..a1e5a19 100755
--- a/eagle-external/eagle-docker/resource/deploy-eagle.sh
+++ b/eagle-external/eagle-docker/resource/deploy-eagle.sh
@@ -71,11 +71,11 @@ ${EAGLE_HOME}/examples/sample-sensitivity-resource-create.sh
[ $? != 0 ] && exit 1
${EAGLE_HOME}/examples/sample-policy-create.sh
[ $? != 0 ] && exit 1
-storm jar $EAGLE_TOPOLOGY_JAR eagle.security.auditlog.HdfsAuditLogProcessorMain -D config.file=${EAGLE_HOME}/conf/sandbox-hdfsAuditLog-application.conf -D eagleProps.eagleService.host=$EAGLE_SERVICE_HOST
+storm jar $EAGLE_TOPOLOGY_JAR org.apache.eagle.security.auditlog.HdfsAuditLogProcessorMain -D config.file=${EAGLE_HOME}/conf/sandbox-hdfsAuditLog-application.conf -D eagleProps.eagleService.host=$EAGLE_SERVICE_HOST
[ $? != 0 ] && exit 1
-storm jar $EAGLE_TOPOLOGY_JAR eagle.security.hive.jobrunning.HiveJobRunningMonitoringMain -D config.file=${EAGLE_HOME}/conf/sandbox-hiveQueryLog-application.conf -D eagleProps.eagleService.host=$EAGLE_SERVICE_HOST
+storm jar $EAGLE_TOPOLOGY_JAR org.apache.eagle.security.hive.jobrunning.HiveJobRunningMonitoringMain -D config.file=${EAGLE_HOME}/conf/sandbox-hiveQueryLog-application.conf -D eagleProps.eagleService.host=$EAGLE_SERVICE_HOST
[ $? != 0 ] && exit 1
-storm jar $EAGLE_TOPOLOGY_JAR eagle.security.userprofile.UserProfileDetectionMain -D config.file=${EAGLE_HOME}/conf/sandbox-userprofile-topology.conf -D eagleProps.eagleService.host=$EAGLE_SERVICE_HOST
+storm jar $EAGLE_TOPOLOGY_JAR org.apache.eagle.security.userprofile.UserProfileDetectionMain -D config.file=${EAGLE_HOME}/conf/sandbox-userprofile-topology.conf -D eagleProps.eagleService.host=$EAGLE_SERVICE_HOST
[ $? != 0 ] && exit 1
# TODO: More eagle start
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/43bae921/eagle-external/eagle-docker/resource/eagle-multinode.json
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/resource/eagle-multinode.json b/eagle-external/eagle-docker/resource/eagle-multinode.json
index 1da2fb3..9e399ae 100644
--- a/eagle-external/eagle-docker/resource/eagle-multinode.json
+++ b/eagle-external/eagle-docker/resource/eagle-multinode.json
@@ -19,7 +19,7 @@
{
"hdfs-log4j": {
"properties": {
- "content" : "\r\n#\r\n# Licensed to the Apache Software Foundation (ASF) under one\r\n# or more contributor license agreements. See the NOTICE file\r\n# distributed with this work for additional information\r\n# regarding copyright ownership. The ASF licenses this file\r\n# to you under the Apache License, Version 2.0 (the\r\n# \"License\"); you may not use this file except in compliance\r\n# with the License. You may obtain a copy of the License at\r\n#\r\n# http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing,\r\n# software distributed under the License is distributed on an\r\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n# KIND, either express or implied. See the License for the\r\n# specific language governing permissions and limitations\r\n# under the License.\r\n#\r\n\r\n\r\n# Define some default values that can be overridden by system properties\r\n# To change daemon root logger use hadoo
p_root_logger in hadoop-env\r\nhadoop.root.logger=INFO,console\r\nhadoop.log.dir=.\r\nhadoop.log.file=hadoop.log\r\n\r\n\r\n# Define the root logger to the system property \"hadoop.root.logger\".\r\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\r\n\r\n# Logging Threshold\r\nlog4j.threshhold=ALL\r\n\r\n#\r\n# Daily Rolling File Appender\r\n#\r\n\r\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Rollver at midnight\r\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\r\n\r\n# 30-day backup\r\n#log4j.appender.DRFA.MaxBackupIndex=30\r\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\r\n\r\n# Pattern format: Date LogLevel LoggerName LogMessage\r\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n# Debugging Pattern format\r\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n#\r\n# console\r\n# Add \"console\" to rootlog
ger above if you want to use this\r\n#\r\n\r\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\r\nlog4j.appender.console.target=System.err\r\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\r\n\r\n#\r\n# TaskLog Appender\r\n#\r\n\r\n#Default values\r\nhadoop.tasklog.taskid=null\r\nhadoop.tasklog.iscleanup=false\r\nhadoop.tasklog.noKeepSplits=4\r\nhadoop.tasklog.totalLogFileSize=100\r\nhadoop.tasklog.purgeLogSplits=true\r\nhadoop.tasklog.logsRetainHours=12\r\n\r\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\r\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\r\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\r\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\r\n\r\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n\r\n#\r\n#Security audit appender\r\
n#\r\nhadoop.security.logger=INFO,console\r\nhadoop.security.log.maxfilesize=256MB\r\nhadoop.security.log.maxbackupindex=20\r\nlog4j.category.SecurityLogger=${hadoop.security.logger}\r\nhadoop.security.log.file=SecurityAuth.audit\r\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\r\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\r\n\r\n#\r\n# hdfs audit l
ogging\r\n#\r\nhdfs.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\r\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\r\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.KAFKA_HDFS_AUDIT=eagle.log4j.kafka.KafkaLog4jAppender\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BrokerList=eagle-server.apache.org:6667\r\nlog4j.appender.KAFKA_HDFS_AUDIT.KeyClass=eagle.log4j.kafka.hadoop.AuditLogKeyer\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout.ConversionPattern=%d{ISO8
601} %p %c{2}: %m%n\r\nlog4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1\r\nlog4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1\r\n\r\n#\r\n# mapred audit logging\r\n#\r\nmapred.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\r\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\r\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\n#\r\n# Rolling File Appender\r\n#\r\n\r\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Logfile size and and 30-day backups\r\nlog4j.appender.RFA.MaxFileSize=256MB\r\nlog4j.appender.RFA.MaxBackupInde
x=10\r\n\r\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n# Custom Logging levels\r\n\r\nhadoop.metrics.log.level=INFO\r\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\r\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\r\n\r\n# Jets3t library\r\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\r\n\r\n#\r\n# Null Appender\r\n# Trap security logger on the hadoop client side\r\n#\r\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\r\n\r\n#\r\n# Event Counter Appender\r\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\r\n#\r\nlog4j.appender.EventCounter=org.apache.hadoop.log.metrics.Ev
entCounter\r\n\r\n# Removes \"deprecated\" messages\r\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\r\n\r\n#\r\n# HDFS block state change log from block manager\r\n#\r\n# Uncomment the following to suppress normal block state change\r\n# messages from BlockManager in NameNode.\r\n#log4j.logger.BlockStateChange=WARN\r\n"
+ "content" : "\r\n#\r\n# Licensed to the Apache Software Foundation (ASF) under one\r\n# or more contributor license agreements. See the NOTICE file\r\n# distributed with this work for additional information\r\n# regarding copyright ownership. The ASF licenses this file\r\n# to you under the Apache License, Version 2.0 (the\r\n# \"License\"); you may not use this file except in compliance\r\n# with the License. You may obtain a copy of the License at\r\n#\r\n# http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing,\r\n# software distributed under the License is distributed on an\r\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n# KIND, either express or implied. See the License for the\r\n# specific language governing permissions and limitations\r\n# under the License.\r\n#\r\n\r\n\r\n# Define some default values that can be overridden by system properties\r\n# To change daemon root logger use hadoo
p_root_logger in hadoop-env\r\nhadoop.root.logger=INFO,console\r\nhadoop.log.dir=.\r\nhadoop.log.file=hadoop.log\r\n\r\n\r\n# Define the root logger to the system property \"hadoop.root.logger\".\r\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\r\n\r\n# Logging Threshold\r\nlog4j.threshhold=ALL\r\n\r\n#\r\n# Daily Rolling File Appender\r\n#\r\n\r\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Rollver at midnight\r\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\r\n\r\n# 30-day backup\r\n#log4j.appender.DRFA.MaxBackupIndex=30\r\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\r\n\r\n# Pattern format: Date LogLevel LoggerName LogMessage\r\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n# Debugging Pattern format\r\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n#\r\n# console\r\n# Add \"console\" to rootlog
ger above if you want to use this\r\n#\r\n\r\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\r\nlog4j.appender.console.target=System.err\r\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\r\n\r\n#\r\n# TaskLog Appender\r\n#\r\n\r\n#Default values\r\nhadoop.tasklog.taskid=null\r\nhadoop.tasklog.iscleanup=false\r\nhadoop.tasklog.noKeepSplits=4\r\nhadoop.tasklog.totalLogFileSize=100\r\nhadoop.tasklog.purgeLogSplits=true\r\nhadoop.tasklog.logsRetainHours=12\r\n\r\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\r\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\r\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\r\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\r\n\r\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n\r\n#\r\n#Security audit appender\r\
n#\r\nhadoop.security.logger=INFO,console\r\nhadoop.security.log.maxfilesize=256MB\r\nhadoop.security.log.maxbackupindex=20\r\nlog4j.category.SecurityLogger=${hadoop.security.logger}\r\nhadoop.security.log.file=SecurityAuth.audit\r\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\r\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\r\n\r\n#\r\n# hdfs audit l
ogging\r\n#\r\nhdfs.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\r\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\r\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BrokerList=master.eagle.apache.org:6667\r\nlog4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout.Con
versionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1\r\nlog4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1\r\n\r\n#\r\n# mapred audit logging\r\n#\r\nmapred.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\r\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\r\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\n#\r\n# Rolling File Appender\r\n#\r\n\r\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Logfile size and and 30-day backups\r\nlog4j.appender.RFA.MaxFileSize=256MB\r\nlog4j.appe
nder.RFA.MaxBackupIndex=10\r\n\r\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n# Custom Logging levels\r\n\r\nhadoop.metrics.log.level=INFO\r\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\r\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\r\n\r\n# Jets3t library\r\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\r\n\r\n#\r\n# Null Appender\r\n# Trap security logger on the hadoop client side\r\n#\r\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\r\n\r\n#\r\n# Event Counter Appender\r\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\r\n#\r\nlog4j.appender.EventCounter=org.apache
.hadoop.log.metrics.EventCounter\r\n\r\n# Removes \"deprecated\" messages\r\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\r\n\r\n#\r\n# HDFS block state change log from block manager\r\n#\r\n# Uncomment the following to suppress normal block state change\r\n# messages from BlockManager in NameNode.\r\n#log4j.logger.BlockStateChange=WARN\r\n"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/43bae921/eagle-external/eagle-docker/resource/eagle-singlenode.json
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/resource/eagle-singlenode.json b/eagle-external/eagle-docker/resource/eagle-singlenode.json
index 499b978..1073107 100644
--- a/eagle-external/eagle-docker/resource/eagle-singlenode.json
+++ b/eagle-external/eagle-docker/resource/eagle-singlenode.json
@@ -19,7 +19,7 @@
{
"hdfs-log4j": {
"properties": {
- "content" : "\r\n#\r\n# Licensed to the Apache Software Foundation (ASF) under one\r\n# or more contributor license agreements. See the NOTICE file\r\n# distributed with this work for additional information\r\n# regarding copyright ownership. The ASF licenses this file\r\n# to you under the Apache License, Version 2.0 (the\r\n# \"License\"); you may not use this file except in compliance\r\n# with the License. You may obtain a copy of the License at\r\n#\r\n# http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing,\r\n# software distributed under the License is distributed on an\r\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n# KIND, either express or implied. See the License for the\r\n# specific language governing permissions and limitations\r\n# under the License.\r\n#\r\n\r\n\r\n# Define some default values that can be overridden by system properties\r\n# To change daemon root logger use hadoo
p_root_logger in hadoop-env\r\nhadoop.root.logger=INFO,console\r\nhadoop.log.dir=.\r\nhadoop.log.file=hadoop.log\r\n\r\n\r\n# Define the root logger to the system property \"hadoop.root.logger\".\r\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\r\n\r\n# Logging Threshold\r\nlog4j.threshhold=ALL\r\n\r\n#\r\n# Daily Rolling File Appender\r\n#\r\n\r\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Rollver at midnight\r\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\r\n\r\n# 30-day backup\r\n#log4j.appender.DRFA.MaxBackupIndex=30\r\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\r\n\r\n# Pattern format: Date LogLevel LoggerName LogMessage\r\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n# Debugging Pattern format\r\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n#\r\n# console\r\n# Add \"console\" to rootlog
ger above if you want to use this\r\n#\r\n\r\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\r\nlog4j.appender.console.target=System.err\r\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\r\n\r\n#\r\n# TaskLog Appender\r\n#\r\n\r\n#Default values\r\nhadoop.tasklog.taskid=null\r\nhadoop.tasklog.iscleanup=false\r\nhadoop.tasklog.noKeepSplits=4\r\nhadoop.tasklog.totalLogFileSize=100\r\nhadoop.tasklog.purgeLogSplits=true\r\nhadoop.tasklog.logsRetainHours=12\r\n\r\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\r\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\r\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\r\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\r\n\r\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n\r\n#\r\n#Security audit appender\r\
n#\r\nhadoop.security.logger=INFO,console\r\nhadoop.security.log.maxfilesize=256MB\r\nhadoop.security.log.maxbackupindex=20\r\nlog4j.category.SecurityLogger=${hadoop.security.logger}\r\nhadoop.security.log.file=SecurityAuth.audit\r\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\r\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\r\n\r\n#\r\n# hdfs audit l
ogging\r\n#\r\nhdfs.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\r\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\r\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.KAFKA_HDFS_AUDIT=eagle.log4j.kafka.KafkaLog4jAppender\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BrokerList=eagle-server.apache.org:6667\r\nlog4j.appender.KAFKA_HDFS_AUDIT.KeyClass=eagle.log4j.kafka.hadoop.AuditLogKeyer\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout.ConversionPattern=%d{ISO8
601} %p %c{2}: %m%n\r\nlog4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1\r\nlog4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1\r\n\r\n#\r\n# mapred audit logging\r\n#\r\nmapred.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\r\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\r\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\n#\r\n# Rolling File Appender\r\n#\r\n\r\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Logfile size and and 30-day backups\r\nlog4j.appender.RFA.MaxFileSize=256MB\r\nlog4j.appender.RFA.MaxBackupInde
x=10\r\n\r\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n# Custom Logging levels\r\n\r\nhadoop.metrics.log.level=INFO\r\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\r\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\r\n\r\n# Jets3t library\r\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\r\n\r\n#\r\n# Null Appender\r\n# Trap security logger on the hadoop client side\r\n#\r\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\r\n\r\n#\r\n# Event Counter Appender\r\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\r\n#\r\nlog4j.appender.EventCounter=org.apache.hadoop.log.metrics.Ev
entCounter\r\n\r\n# Removes \"deprecated\" messages\r\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\r\n\r\n#\r\n# HDFS block state change log from block manager\r\n#\r\n# Uncomment the following to suppress normal block state change\r\n# messages from BlockManager in NameNode.\r\n#log4j.logger.BlockStateChange=WARN\r\n"
+ "content" : "\r\n#\r\n# Licensed to the Apache Software Foundation (ASF) under one\r\n# or more contributor license agreements. See the NOTICE file\r\n# distributed with this work for additional information\r\n# regarding copyright ownership. The ASF licenses this file\r\n# to you under the Apache License, Version 2.0 (the\r\n# \"License\"); you may not use this file except in compliance\r\n# with the License. You may obtain a copy of the License at\r\n#\r\n# http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing,\r\n# software distributed under the License is distributed on an\r\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n# KIND, either express or implied. See the License for the\r\n# specific language governing permissions and limitations\r\n# under the License.\r\n#\r\n\r\n\r\n# Define some default values that can be overridden by system properties\r\n# To change daemon root logger use hadoo
p_root_logger in hadoop-env\r\nhadoop.root.logger=INFO,console\r\nhadoop.log.dir=.\r\nhadoop.log.file=hadoop.log\r\n\r\n\r\n# Define the root logger to the system property \"hadoop.root.logger\".\r\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\r\n\r\n# Logging Threshold\r\nlog4j.threshhold=ALL\r\n\r\n#\r\n# Daily Rolling File Appender\r\n#\r\n\r\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Rollver at midnight\r\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\r\n\r\n# 30-day backup\r\n#log4j.appender.DRFA.MaxBackupIndex=30\r\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\r\n\r\n# Pattern format: Date LogLevel LoggerName LogMessage\r\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n# Debugging Pattern format\r\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n#\r\n# console\r\n# Add \"console\" to rootlog
ger above if you want to use this\r\n#\r\n\r\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\r\nlog4j.appender.console.target=System.err\r\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\r\n\r\n#\r\n# TaskLog Appender\r\n#\r\n\r\n#Default values\r\nhadoop.tasklog.taskid=null\r\nhadoop.tasklog.iscleanup=false\r\nhadoop.tasklog.noKeepSplits=4\r\nhadoop.tasklog.totalLogFileSize=100\r\nhadoop.tasklog.purgeLogSplits=true\r\nhadoop.tasklog.logsRetainHours=12\r\n\r\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\r\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\r\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\r\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\r\n\r\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n\r\n#\r\n#Security audit appender\r\
n#\r\nhadoop.security.logger=INFO,console\r\nhadoop.security.log.maxfilesize=256MB\r\nhadoop.security.log.maxbackupindex=20\r\nlog4j.category.SecurityLogger=${hadoop.security.logger}\r\nhadoop.security.log.file=SecurityAuth.audit\r\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\r\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\r\n\r\n#\r\n# hdfs audit l
ogging\r\n#\r\nhdfs.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\r\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\r\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BrokerList=master.eagle.apache.org:6667\r\nlog4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout.Con
versionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1\r\nlog4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1\r\n\r\n#\r\n# mapred audit logging\r\n#\r\nmapred.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\r\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\r\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\n#\r\n# Rolling File Appender\r\n#\r\n\r\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Logfile size and and 30-day backups\r\nlog4j.appender.RFA.MaxFileSize=256MB\r\nlog4j.appe
nder.RFA.MaxBackupIndex=10\r\n\r\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n# Custom Logging levels\r\n\r\nhadoop.metrics.log.level=INFO\r\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\r\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\r\n\r\n# Jets3t library\r\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\r\n\r\n#\r\n# Null Appender\r\n# Trap security logger on the hadoop client side\r\n#\r\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\r\n\r\n#\r\n# Event Counter Appender\r\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\r\n#\r\nlog4j.appender.EventCounter=org.apache
.hadoop.log.metrics.EventCounter\r\n\r\n# Removes \"deprecated\" messages\r\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\r\n\r\n#\r\n# HDFS block state change log from block manager\r\n#\r\n# Uncomment the following to suppress normal block state change\r\n# messages from BlockManager in NameNode.\r\n#log4j.logger.BlockStateChange=WARN\r\n"
}
}
}