You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2014/12/17 21:05:39 UTC
[07/37] ambari git commit: AMBARI-8695: Common Services: Refactor
HDP-2.0.6 HDFS, ZOOKEEPER services (Jayush Luniya)
http://git-wip-us.apache.org/repos/asf/ambari/blob/17b71553/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml
deleted file mode 100644
index 1d6618d..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml
+++ /dev/null
@@ -1,210 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <property>
- <name>hdfs_log_dir_prefix</name>
- <value>/var/log/hadoop</value>
- <description>Hadoop Log Dir Prefix</description>
- </property>
- <property>
- <name>hadoop_pid_dir_prefix</name>
- <value>/var/run/hadoop</value>
- <description>Hadoop PID Dir Prefix</description>
- </property>
- <property>
- <name>hadoop_root_logger</name>
- <value>INFO,RFA</value>
- <description>Hadoop Root Logger</description>
- </property>
- <property>
- <name>hadoop_heapsize</name>
- <value>1024</value>
- <description>Hadoop maximum Java heap size</description>
- </property>
- <property>
- <name>namenode_heapsize</name>
- <value>1024</value>
- <description>NameNode Java heap size</description>
- </property>
- <property>
- <name>namenode_opt_newsize</name>
- <value>200</value>
- <description>NameNode new generation size</description>
- </property>
- <property>
- <name>namenode_opt_maxnewsize</name>
- <value>200</value>
- <description>NameNode maximum new generation size</description>
- </property>
- <property>
- <name>namenode_opt_permsize</name>
- <value>128</value>
- <description>NameNode permanent generation size</description>
- </property>
- <property>
- <name>namenode_opt_maxpermsize</name>
- <value>256</value>
- <description>NameNode maximum permanent generation size</description>
- </property>
- <property>
- <name>dtnode_heapsize</name>
- <value>1024</value>
- <description>DataNode maximum Java heap size</description>
- </property>
- <property>
- <name>proxyuser_group</name>
- <value>users</value>
- <property-type>GROUP</property-type>
- <description>Proxy user group.</description>
- </property>
- <property>
- <name>hdfs_user</name>
- <value>hdfs</value>
- <property-type>USER</property-type>
- <description>User to run HDFS as</description>
- </property>
- <property>
- <name>dfs.datanode.data.dir.mount.file</name>
- <value>/etc/hadoop/conf/dfs_data_dir_mount.hist</value>
- <description>File path that contains the last known mount point for each data dir. This file is used to avoid creating a DFS data dir on the root drive (and filling it up) if a path was previously mounted on a drive.</description>
- </property>
-
- <!-- hadoop-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for hadoop-env.sh file</description>
- <value>
-# Set Hadoop-specific environment variables here.
-
-# The only required environment variable is JAVA_HOME. All others are
-# optional. When running a distributed configuration it is best to
-# set JAVA_HOME in this file, so that it is correctly defined on
-# remote nodes.
-
-# The java implementation to use. Required.
-export JAVA_HOME={{java_home}}
-export HADOOP_HOME_WARN_SUPPRESS=1
-
-# Hadoop home directory
-export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}
-
-# Hadoop Configuration Directory
-#TODO: if env var set that can cause problems
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}}
-
-{# this is different for HDP1 #}
-# Path to jsvc required by secure HDP 2.0 datanode
-export JSVC_HOME={{jsvc_path}}
-
-
-# The maximum amount of heap to use, in MB. Default is 1000.
-export HADOOP_HEAPSIZE="{{hadoop_heapsize}}"
-
-export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms{{namenode_heapsize}}"
-
-# Extra Java runtime options. Empty by default.
-export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}"
-
-# Command specific options appended to HADOOP_OPTS when specified
-export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}"
-HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
-
-HADOOP_TASKTRACKER_OPTS="-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}"
-HADOOP_DATANODE_OPTS="-Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}"
-HADOOP_BALANCER_OPTS="-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}"
-
-export HADOOP_SECONDARYNAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}"
-
-# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
-export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS"
-# On secure datanodes, user to run the datanode as after dropping privileges
-export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}
-
-# Extra ssh options. Empty by default.
-export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR"
-
-# Where log files are stored. $HADOOP_HOME/logs by default.
-export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER
-
-# History server logs
-export HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER
-
-# Where log files are stored in the secure data environment.
-export HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER
-
-# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.
-# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves
-
-# host:path where hadoop code should be rsync'd from. Unset by default.
-# export HADOOP_MASTER=master:/home/$USER/src/hadoop
-
-# Seconds to sleep between slave commands. Unset by default. This
-# can be useful in large clusters, where, e.g., slave rsyncs can
-# otherwise arrive faster than the master can service them.
-# export HADOOP_SLAVE_SLEEP=0.1
-
-# The directory where pid files are stored. /tmp by default.
-export HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER
-export HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER
-
-# History server pid
-export HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER
-
-YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY"
-
-# A string representing this instance of hadoop. $USER by default.
-export HADOOP_IDENT_STRING=$USER
-
-# The scheduling priority for daemon processes. See 'man nice'.
-
-# export HADOOP_NICENESS=10
-
-# Use libraries from standard classpath
-JAVA_JDBC_LIBS=""
-#Add libraries required by mysql connector
-for jarFile in `ls /usr/share/java/*mysql* 2>/dev/null`
-do
- JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
-done
-#Add libraries required by oracle connector
-for jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`
-do
- JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
-done
-#Add libraries required by nodemanager
-MAPREDUCE_LIBS={{mapreduce_libs_path}}
-export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}
-
-if [ -d "/usr/lib/tez" ]; then
- export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/*:/etc/tez/conf
-fi
-
-# Setting path to hdfs command line
-export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
-
-#Mostly required for hadoop 2.0
-export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64
- </value>
- </property>
-
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/17b71553/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-policy.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-policy.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-policy.xml
deleted file mode 100644
index 41bde16..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-policy.xml
+++ /dev/null
@@ -1,134 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration supports_final="true">
- <property>
- <name>security.client.protocol.acl</name>
- <value>*</value>
- <description>ACL for ClientProtocol, which is used by user code
- via the DistributedFileSystem.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.client.datanode.protocol.acl</name>
- <value>*</value>
- <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
- for block recovery.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.datanode.protocol.acl</name>
- <value>*</value>
- <description>ACL for DatanodeProtocol, which is used by datanodes to
- communicate with the namenode.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.inter.datanode.protocol.acl</name>
- <value>*</value>
- <description>ACL for InterDatanodeProtocol, the inter-datanode protocol
- for updating generation timestamp.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.namenode.protocol.acl</name>
- <value>*</value>
- <description>ACL for NamenodeProtocol, the protocol used by the secondary
- namenode to communicate with the namenode.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.inter.tracker.protocol.acl</name>
- <value>*</value>
- <description>ACL for InterTrackerProtocol, used by the tasktrackers to
- communicate with the jobtracker.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.job.client.protocol.acl</name>
- <value>*</value>
- <description>ACL for JobSubmissionProtocol, used by job clients to
- communciate with the jobtracker for job submission, querying job status etc.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.job.task.protocol.acl</name>
- <value>*</value>
- <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
- tasks to communicate with the parent tasktracker.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.admin.operations.protocol.acl</name>
- <value>hadoop</value>
- <description>ACL for AdminOperationsProtocol. Used for admin commands.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.refresh.usertogroups.mappings.protocol.acl</name>
- <value>hadoop</value>
- <description>ACL for RefreshUserMappingsProtocol. Used to refresh
- users mappings. The ACL is a comma-separated list of user and
- group names. The user and group list is separated by a blank. For
- e.g. "alice,bob users,wheel". A special value of "*" means all
- users are allowed.</description>
- </property>
-
-<property>
- <name>security.refresh.policy.protocol.acl</name>
- <value>hadoop</value>
- <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
- dfsadmin and mradmin commands to refresh the security policy in-effect.
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/17b71553/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-log4j.xml
deleted file mode 100644
index 08822eb..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-log4j.xml
+++ /dev/null
@@ -1,201 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration supports_final="false">
-
- <property>
- <name>content</name>
- <description>Custom log4j.properties</description>
- <value>
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-# Define some default values that can be overridden by system properties
-# To change daemon root logger use hadoop_root_logger in hadoop-env
-hadoop.root.logger=INFO,console
-hadoop.log.dir=.
-hadoop.log.file=hadoop.log
-
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hadoop.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=ALL
-
-#
-# Daily Rolling File Appender
-#
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#
-# TaskLog Appender
-#
-
-#Default values
-hadoop.tasklog.taskid=null
-hadoop.tasklog.iscleanup=false
-hadoop.tasklog.noKeepSplits=4
-hadoop.tasklog.totalLogFileSize=100
-hadoop.tasklog.purgeLogSplits=true
-hadoop.tasklog.logsRetainHours=12
-
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
-log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
-log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
-log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-#
-#Security audit appender
-#
-hadoop.security.logger=INFO,console
-hadoop.security.log.maxfilesize=256MB
-hadoop.security.log.maxbackupindex=20
-log4j.category.SecurityLogger=${hadoop.security.logger}
-hadoop.security.log.file=SecurityAuth.audit
-log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
-
-log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
-log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
-log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
-
-#
-# hdfs audit logging
-#
-hdfs.audit.logger=INFO,console
-log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
-log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
-log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
-log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd
-
-#
-# mapred audit logging
-#
-mapred.audit.logger=INFO,console
-log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
-log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
-log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
-log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd
-
-#
-# Rolling File Appender
-#
-
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-# Logfile size and and 30-day backups
-log4j.appender.RFA.MaxFileSize=256MB
-log4j.appender.RFA.MaxBackupIndex=10
-
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-# Custom Logging levels
-
-hadoop.metrics.log.level=INFO
-#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
-#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
-#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
-log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}
-
-# Jets3t library
-log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
-
-#
-# Null Appender
-# Trap security logger on the hadoop client side
-#
-log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
-
-# Removes "deprecated" messages
-log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN
- </value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/17b71553/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-site.xml
deleted file mode 100644
index e67ed9f..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hdfs-site.xml
+++ /dev/null
@@ -1,430 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration supports_final="true">
-
- <!-- file system properties -->
-
- <property>
- <name>dfs.namenode.name.dir</name>
- <!-- cluster variant -->
- <value>/hadoop/hdfs/namenode</value>
- <description>Determines where on the local filesystem the DFS name node
- should store the name table. If this is a comma-delimited list
- of directories then the name table is replicated in all of the
- directories, for redundancy. </description>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.support.append</name>
- <value>true</value>
- <description>to enable dfs append</description>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.webhdfs.enabled</name>
- <value>true</value>
- <description>Whether to enable WebHDFS feature</description>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.failed.volumes.tolerated</name>
- <value>0</value>
- <description> Number of failed disks a DataNode would tolerate before it stops offering service</description>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir</name>
- <value>/hadoop/hdfs/data</value>
- <description>Determines where on the local filesystem an DFS data node
- should store its blocks. If this is a comma-delimited
- list of directories, then data will be stored in all named
- directories, typically on different devices.
- Directories that do not exist are ignored.
- </description>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.hosts.exclude</name>
- <value>/etc/hadoop/conf/dfs.exclude</value>
- <description>Names a file that contains a list of hosts that are
- not permitted to connect to the namenode. The full pathname of the
- file must be specified. If the value is empty, no hosts are
- excluded.</description>
- </property>
-
- <!--
- <property>
- <name>dfs.hosts</name>
- <value>/etc/hadoop/conf/dfs.include</value>
- <description>Names a file that contains a list of hosts that are
- permitted to connect to the namenode. The full pathname of the file
- must be specified. If the value is empty, all hosts are
- permitted.</description>
- </property>
- -->
-
- <property>
- <name>dfs.namenode.checkpoint.dir</name>
- <value>/hadoop/hdfs/namesecondary</value>
- <description>Determines where on the local filesystem the DFS secondary
- name node should store the temporary images to merge.
- If this is a comma-delimited list of directories then the image is
- replicated in all of the directories for redundancy.
- </description>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.edits.dir</name>
- <value>${dfs.namenode.checkpoint.dir}</value>
- <description>Determines where on the local filesystem the DFS secondary
- name node should store the temporary edits to merge.
- If this is a comma-delimited list of directoires then teh edits is
- replicated in all of the directoires for redundancy.
- Default value is same as dfs.namenode.checkpoint.dir
- </description>
- </property>
-
-
- <property>
- <name>dfs.namenode.checkpoint.period</name>
- <value>21600</value>
- <description>The number of seconds between two periodic checkpoints.
- </description>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.txns</name>
- <value>1000000</value>
- <description>The Secondary NameNode or CheckpointNode will create a checkpoint
- of the namespace every 'dfs.namenode.checkpoint.txns' transactions,
- regardless of whether 'dfs.namenode.checkpoint.period' has expired.
- </description>
- </property>
-
- <property>
- <name>dfs.replication.max</name>
- <value>50</value>
- <description>Maximal block replication.
- </description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>3</value>
- <description>Default block replication.
- </description>
- </property>
-
- <property>
- <name>dfs.heartbeat.interval</name>
- <value>3</value>
- <description>Determines datanode heartbeat interval in seconds.</description>
- </property>
-
- <property>
- <name>dfs.heartbeat.interval</name>
- <value>3</value>
- <description>Determines datanode heartbeat interval in seconds.</description>
- </property>
-
- <property>
- <name>dfs.namenode.safemode.threshold-pct</name>
- <value>1.0f</value>
- <description>
- Specifies the percentage of blocks that should satisfy
- the minimal replication requirement defined by dfs.namenode.replication.min.
- Values less than or equal to 0 mean not to start in safe mode.
- Values greater than 1 will make safe mode permanent.
- </description>
- </property>
-
- <property>
- <name>dfs.datanode.balance.bandwidthPerSec</name>
- <value>6250000</value>
- <description>
- Specifies the maximum amount of bandwidth that each datanode
- can utilize for the balancing purpose in term of
- the number of bytes per second.
- </description>
- </property>
-
- <property>
- <name>dfs.https.port</name>
- <value>50470</value>
- <description>
- This property is used by HftpFileSystem.
- </description>
- </property>
-
- <property>
- <name>dfs.datanode.address</name>
- <value>0.0.0.0:50010</value>
- <description>
- The datanode server address and port for data transfer.
- </description>
- </property>
-
- <property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:50075</value>
- <description>
- The datanode http server address and port.
- </description>
- </property>
-
- <property>
- <name>dfs.datanode.https.address</name>
- <value>0.0.0.0:50475</value>
- <description>
- The datanode https server address and port.
- </description>
- </property>
-
- <property>
- <name>dfs.blocksize</name>
- <value>134217728</value>
- <description>The default block size for new files.</description>
- </property>
-
- <property>
- <name>dfs.namenode.http-address</name>
- <value>localhost:50070</value>
- <description>The name of the default file system. Either the
- literal string "local" or a host:port for NDFS.</description>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.du.reserved</name>
- <!-- cluster variant -->
- <value>1073741824</value>
- <description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
- </description>
- </property>
-
- <property>
- <name>dfs.datanode.ipc.address</name>
- <value>0.0.0.0:8010</value>
- <description>
- The datanode ipc server address and port.
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
-
- <property>
- <name>dfs.blockreport.initialDelay</name>
- <value>120</value>
- <description>Delay for first block report in seconds.</description>
- </property>
-
- <property>
- <name>dfs.namenode.handler.count</name>
- <value>40</value>
- <description>The number of server threads for the namenode.</description>
- </property>
-
- <property>
- <name>dfs.datanode.max.transfer.threads</name>
- <value>1024</value>
- <description>Specifies the maximum number of threads to use for transferring data in and out of the datanode.</description>
- </property>
-
- <!-- Permissions configuration -->
-
- <property>
- <name>fs.permissions.umask-mode</name>
- <value>022</value>
- <description>
- The octal umask used when creating files and directories.
- </description>
- </property>
-
- <property>
- <name>dfs.permissions.enabled</name>
- <value>true</value>
- <description>
- If "true", enable permission checking in HDFS.
- If "false", permission checking is turned off,
- but all other behavior is unchanged.
- Switching from one parameter value to the other does not change the mode,
- owner or group of files or directories.
- </description>
- </property>
-
- <property>
- <name>dfs.permissions.superusergroup</name>
- <value>hdfs</value>
- <description>The name of the group of super-users.</description>
- </property>
-
- <property>
- <name>dfs.namenode.handler.count</name>
- <value>100</value>
- <description>Added to grow Queue size so that more client connections are allowed</description>
- </property>
-
- <property>
- <name>dfs.block.access.token.enable</name>
- <value>true</value>
- <description>
- If "true", access tokens are used as capabilities for accessing datanodes.
- If "false", no access tokens are checked on accessing datanodes.
- </description>
- </property>
-
- <property>
- <!-- cluster variant -->
- <name>dfs.namenode.secondary.http-address</name>
- <value>localhost:50090</value>
- <description>Address of secondary namenode web server</description>
- </property>
-
-
- <property>
- <name>dfs.namenode.https-address</name>
- <value>localhost:50470</value>
- <description>The https address where namenode binds</description>
-
- </property>
-
- <property>
- <name>dfs.datanode.data.dir.perm</name>
- <value>750</value>
- <description>The permissions that should be there on dfs.datanode.data.dir
- directories. The datanode will not come up if the permissions are
- different on existing dfs.datanode.data.dir directories. If the directories
- don't exist, they will be created with this permission.</description>
- </property>
-
- <property>
- <name>dfs.namenode.accesstime.precision</name>
- <value>0</value>
- <description>The access time for HDFS file is precise upto this value.
- The default value is 1 hour. Setting a value of 0 disables
- access times for HDFS.
- </description>
- </property>
-
- <property>
- <name>dfs.cluster.administrators</name>
- <value> hdfs</value>
- <description>ACL for who all can view the default servlets in the HDFS</description>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.read.stale.datanode</name>
- <value>true</value>
- <description>
- Indicate whether or not to avoid reading from stale datanodes whose
- heartbeat messages have not been received by the namenode for more than a
- specified time interval.
- </description>
- </property>
- <property>
- <name>dfs.namenode.avoid.write.stale.datanode</name>
- <value>true</value>
- <description>
- Indicate whether or not to avoid writing to stale datanodes whose
- heartbeat messages have not been received by the namenode for more than a
- specified time interval.
- </description>
- </property>
- <property>
- <name>dfs.namenode.write.stale.datanode.ratio</name>
- <value>1.0f</value>
- <description>When the ratio of number stale datanodes to total datanodes marked is greater
- than this ratio, stop avoiding writing to stale nodes so as to prevent causing hotspots.
- </description>
- </property>
- <property>
- <name>dfs.namenode.stale.datanode.interval</name>
- <value>30000</value>
- <description>Datanode is stale after not getting a heartbeat in this interval in ms</description>
- </property>
-
- <property>
- <name>dfs.journalnode.http-address</name>
- <value>0.0.0.0:8480</value>
- <description>The address and port the JournalNode web UI listens on.
- If the port is 0 then the server will start on a free port. </description>
- </property>
-
- <property>
- <name>dfs.journalnode.edits.dir</name>
- <value>/grid/0/hdfs/journal</value>
- <description>The path where the JournalNode daemon will store its local state. </description>
- </property>
-
- <!-- HDFS Short-Circuit Local Reads -->
-
- <property>
- <name>dfs.client.read.shortcircuit</name>
- <value>true</value>
- <description>
- This configuration parameter turns on short-circuit local reads.
- </description>
- </property>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- <description>
- This is a path to a UNIX domain socket that will be used for communication between the DataNode and local HDFS clients.
- If the string "_PORT" is present in this path, it will be replaced by the TCP port of the DataNode.
- </description>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit.streams.cache.size</name>
- <value>4096</value>
- <description>
- The DFSClient maintains a cache of recently opened file descriptors. This
- parameter controls the size of that cache. Setting this higher will use
- more file descriptors, but potentially provide better performance on
- workloads involving lots of seeks.
- </description>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir.restore</name>
- <value>true</value>
- <description>Set to true to enable NameNode to attempt recovering a previously failed dfs.namenode.name.dir.
- When enabled, a recovery of any failed directory is attempted during checkpoint.</description>
- </property>
-
- <property>
- <name>dfs.http.policy</name>
- <value>HTTP_ONLY</value>
- <description>
- Decide if HTTPS(SSL) is supported on HDFS This configures the HTTP endpoint for HDFS daemons:
- The following values are supported: - HTTP_ONLY : Service is provided only on http - HTTPS_ONLY :
- Service is provided only on https - HTTP_AND_HTTPS : Service is provided both on http and https
- </description>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/17b71553/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/metainfo.xml
index 93504a4..a5d6862 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/metainfo.xml
@@ -20,206 +20,7 @@
<services>
<service>
<name>HDFS</name>
- <displayName>HDFS</displayName>
- <comment>Apache Hadoop Distributed File System</comment>
- <version>2.1.0.2.0</version>
-
- <components>
- <component>
- <name>NAMENODE</name>
- <displayName>NameNode</displayName>
- <category>MASTER</category>
- <cardinality>1-2</cardinality>
- <commandScript>
- <script>scripts/namenode.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>DECOMMISSION</name>
- <commandScript>
- <script>scripts/namenode.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </customCommand>
- <customCommand>
- <name>REBALANCEHDFS</name>
- <background>true</background>
- <commandScript>
- <script>scripts/namenode.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </customCommand>
- </customCommands>
- </component>
-
- <component>
- <name>DATANODE</name>
- <displayName>DataNode</displayName>
- <category>SLAVE</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/datanode.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
- <name>SECONDARY_NAMENODE</name>
- <displayName>SNameNode</displayName>
- <!-- TODO: cardinality is conditional on HA usage -->
- <cardinality>1</cardinality>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/snamenode.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
- <name>HDFS_CLIENT</name>
- <displayName>HDFS Client</displayName>
- <category>CLIENT</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/hdfs_client.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- <configFiles>
- <configFile>
- <type>xml</type>
- <fileName>hdfs-site.xml</fileName>
- <dictionaryName>hdfs-site</dictionaryName>
- </configFile>
- <configFile>
- <type>xml</type>
- <fileName>core-site.xml</fileName>
- <dictionaryName>core-site</dictionaryName>
- </configFile>
- <configFile>
- <type>env</type>
- <fileName>log4j.properties</fileName>
- <dictionaryName>hdfs-log4j,yarn-log4j</dictionaryName>
- </configFile>
- <configFile>
- <type>env</type>
- <fileName>hadoop-env.sh</fileName>
- <dictionaryName>hadoop-env</dictionaryName>
- </configFile>
- </configFiles>
- </component>
-
- <component>
- <name>JOURNALNODE</name>
- <displayName>JournalNode</displayName>
- <category>SLAVE</category>
- <cardinality>0+</cardinality>
- <commandScript>
- <script>scripts/journalnode.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
- <name>ZKFC</name>
- <displayName>ZKFailoverController</displayName>
- <category>SLAVE</category>
- <!-- TODO: cardinality is conditional on HA topology -->
- <cardinality>0+</cardinality>
- <commandScript>
- <script>scripts/zkfc_slave.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>hadoop</name>
- </package>
- <package>
- <name>hadoop-lzo</name>
- </package>
- </packages>
- </osSpecific>
-
- <osSpecific>
- <osFamily>redhat5,redhat6,suse11</osFamily>
- <packages>
- <package>
- <name>snappy</name>
- </package>
- <package>
- <name>snappy-devel</name>
- </package>
- <package>
- <name>lzo</name>
- </package>
- <package>
- <name>hadoop-lzo-native</name>
- </package>
- <package>
- <name>hadoop-libhdfs</name>
- </package>
- <package>
- <name>ambari-log4j</name>
- </package>
- </packages>
- </osSpecific>
-
- <osSpecific>
- <osFamily>ubuntu12</osFamily>
- <packages>
- <package>
- <name>libsnappy1</name>
- </package>
- <package>
- <name>libsnappy-dev</name>
- </package>
- <package>
- <name>liblzo2-2</name>
- </package>
- <package>
- <name>hadoop-hdfs</name>
- </package>
- <package>
- <name>libhdfs0</name>
- </package>
- <package>
- <name>libhdfs0-dev</name>
- </package>
- </packages>
- </osSpecific>
-
- </osSpecifics>
-
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>ZOOKEEPER</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>core-site</config-type>
- <config-type>hdfs-site</config-type>
- <config-type>hadoop-env</config-type>
- <config-type>hadoop-policy</config-type>
- <config-type>hdfs-log4j</config-type>
- </configuration-dependencies>
+ <extends>common-services/HDFS/2.1.0.2.0</extends>
</service>
</services>
</metainfo>