You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2013/12/18 14:14:09 UTC
git commit: AMBARI-4091. Replace XML files with new properties for
HDP1, HDP2 (Arsen Babych via aonishuk)
Updated Branches:
refs/heads/trunk 13710eb2f -> 4aec796a1
AMBARI-4091. Replace XML files with new properties for HDP1, HDP2 (Arsen
Babych via aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4aec796a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4aec796a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4aec796a
Branch: refs/heads/trunk
Commit: 4aec796a165c7d94e3354bc4718c0cb3da27ae4d
Parents: 13710eb
Author: Andrew Onischuk <ao...@hortonworks.com>
Authored: Wed Dec 18 05:07:02 2013 -0800
Committer: Andrew Onischuk <ao...@hortonworks.com>
Committed: Wed Dec 18 05:07:02 2013 -0800
----------------------------------------------------------------------
.../services/HBASE/configuration/hbase-site.xml | 2 +-
.../services/HDFS/configuration/global.xml | 2 +-
.../services/HDFS/configuration/hdfs-site.xml | 398 +++++++-------
.../services/HIVE/configuration/hive-site.xml | 14 +-
.../MAPREDUCE/configuration/mapred-site.xml | 524 ++++++++++---------
.../services/HDFS/configuration/global.xml | 2 +-
.../services/HDFS/configuration/hdfs-site.xml | 33 +-
.../services/HIVE/configuration/hive-site.xml | 7 +
.../services/OOZIE/package/scripts/params.py | 2 +-
.../SQOOP/package/scripts/service_check.py | 2 +-
.../YARN/configuration/capacity-scheduler.xml | 8 +
11 files changed, 536 insertions(+), 458 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HBASE/configuration/hbase-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HBASE/configuration/hbase-site.xml b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HBASE/configuration/hbase-site.xml
index 1ce5ccb..68904a1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HBASE/configuration/hbase-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HBASE/configuration/hbase-site.xml
@@ -22,7 +22,7 @@
<configuration>
<property>
<name>hbase.rootdir</name>
- <value></value>
+ <value>hdfs://localhost:8020/apps/hbase/data</value>
<description>The directory shared by region servers and into
which HBase persists. The URL should be 'fully-qualified'
to include the filesystem scheme. For example, to specify the
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HDFS/configuration/global.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HDFS/configuration/global.xml b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HDFS/configuration/global.xml
index f10b9f9..04d51db 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HDFS/configuration/global.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HDFS/configuration/global.xml
@@ -83,7 +83,7 @@
</property>
<property>
<name>namenode_opt_maxnewsize</name>
- <value>640</value>
+ <value>200</value>
<description>NameNode maximum new generation size</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HDFS/configuration/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HDFS/configuration/hdfs-site.xml
index 1a0b933..1fc6c59 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HDFS/configuration/hdfs-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HDFS/configuration/hdfs-site.xml
@@ -22,7 +22,7 @@
<configuration>
-<!-- file system properties -->
+ <!-- file system properties -->
<property>
<name>dfs.name.dir</name>
@@ -49,7 +49,7 @@
<final>true</final>
</property>
- <property>
+ <property>
<name>dfs.datanode.socket.write.timeout</name>
<value>0</value>
<description>DFS Client write socket timeout</description>
@@ -66,7 +66,7 @@
<name>dfs.block.local-path-access.user</name>
<value>hbase</value>
<description>the user who is allowed to perform short
- circuit reads.
+ circuit reads.
</description>
<final>true</final>
</property>
@@ -75,11 +75,11 @@
<name>dfs.data.dir</name>
<value>/hadoop/hdfs/data</value>
<description>Determines where on the local filesystem an DFS data node
- should store its blocks. If this is a comma-delimited
- list of directories, then data will be stored in all named
- directories, typically on different devices.
- Directories that do not exist are ignored.
- </description>
+ should store its blocks. If this is a comma-delimited
+ list of directories, then data will be stored in all named
+ directories, typically on different devices.
+ Directories that do not exist are ignored.
+ </description>
<final>true</final>
</property>
@@ -87,32 +87,32 @@
<name>dfs.hosts.exclude</name>
<value>/etc/hadoop/conf/dfs.exclude</value>
<description>Names a file that contains a list of hosts that are
- not permitted to connect to the namenode. The full pathname of the
- file must be specified. If the value is empty, no hosts are
- excluded.</description>
+ not permitted to connect to the namenode. The full pathname of the
+ file must be specified. If the value is empty, no hosts are
+ excluded.</description>
</property>
<property>
<name>dfs.hosts</name>
<value>/etc/hadoop/conf/dfs.include</value>
<description>Names a file that contains a list of hosts that are
- permitted to connect to the namenode. The full pathname of the file
- must be specified. If the value is empty, all hosts are
- permitted.</description>
+ permitted to connect to the namenode. The full pathname of the file
+ must be specified. If the value is empty, all hosts are
+ permitted.</description>
</property>
<property>
<name>dfs.replication.max</name>
<value>50</value>
<description>Maximal block replication.
- </description>
+ </description>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
<description>Default block replication.
- </description>
+ </description>
</property>
<property>
@@ -125,21 +125,21 @@
<name>dfs.safemode.threshold.pct</name>
<value>1.0f</value>
<description>
- Specifies the percentage of blocks that should satisfy
- the minimal replication requirement defined by dfs.replication.min.
- Values less than or equal to 0 mean not to start in safe mode.
- Values greater than 1 will make safe mode permanent.
- </description>
+ Specifies the percentage of blocks that should satisfy
+ the minimal replication requirement defined by dfs.replication.min.
+ Values less than or equal to 0 mean not to start in safe mode.
+ Values greater than 1 will make safe mode permanent.
+ </description>
</property>
<property>
<name>dfs.balance.bandwidthPerSec</name>
<value>6250000</value>
<description>
- Specifies the maximum amount of bandwidth that each datanode
- can utilize for the balancing purpose in term of
- the number of bytes per second.
- </description>
+ Specifies the maximum amount of bandwidth that each datanode
+ can utilize for the balancing purpose in term of
+ the number of bytes per second.
+ </description>
</property>
<property>
@@ -151,13 +151,35 @@
</property>
<property>
+ <name>ambari.dfs.datanode.port</name>
+ <value>50010</value>
+ <description>
+ The datanode port for data transfer. This property is effective only if referenced from dfs.datanode.address property.
+ </description>
+ </property>
+
+ <property>
<name>dfs.datanode.address</name>
- <value></value>
+ <value>0.0.0.0:${ambari.dfs.datanode.port}</value>
+ <description>
+ The datanode server address and port for data transfer.
+ </description>
+ </property>
+
+ <property>
+ <name>ambari.dfs.datanode.http.port</name>
+ <value>50075</value>
+ <description>
+ The datanode http port. This property is effective only if referenced from dfs.datanode.http.address property.
+ </description>
</property>
<property>
<name>dfs.datanode.http.address</name>
- <value></value>
+ <value>0.0.0.0:${ambari.dfs.datanode.http.port}</value>
+ <description>
+ The datanode http server address and port.
+ </description>
</property>
<property>
@@ -169,133 +191,133 @@
<property>
<name>dfs.http.address</name>
<value>localhost:50070</value>
-<description>The name of the default file system. Either the
-literal string "local" or a host:port for NDFS.</description>
-<final>true</final>
-</property>
-
-<property>
-<name>dfs.datanode.du.reserved</name>
-<!-- cluster variant -->
-<value>1073741824</value>
-<description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
-</description>
-</property>
-
-<property>
-<name>dfs.datanode.ipc.address</name>
-<value>0.0.0.0:8010</value>
-<description>
-The datanode ipc server address and port.
-If the port is 0 then the server will start on a free port.
-</description>
-</property>
-
-<property>
-<name>dfs.blockreport.initialDelay</name>
-<value>120</value>
-<description>Delay for first block report in seconds.</description>
-</property>
-
-<property>
-<name>dfs.datanode.du.pct</name>
-<value>0.85f</value>
-<description>When calculating remaining space, only use this percentage of the real available space
-</description>
-</property>
-
-<property>
-<name>dfs.namenode.handler.count</name>
-<value>40</value>
-<description>The number of server threads for the namenode.</description>
-</property>
-
-<property>
-<name>dfs.datanode.max.xcievers</name>
-<value>4096</value>
-<description>PRIVATE CONFIG VARIABLE</description>
-</property>
-
-<!-- Permissions configuration -->
-
-<property>
-<name>dfs.umaskmode</name>
-<value>077</value>
-<description>
-The octal umask used when creating files and directories.
-</description>
-</property>
-
-<property>
-<name>dfs.web.ugi</name>
-<!-- cluster variant -->
-<value>gopher,gopher</value>
-<description>The user account used by the web interface.
-Syntax: USERNAME,GROUP1,GROUP2, ...
-</description>
-</property>
-
-<property>
-<name>dfs.permissions</name>
-<value>true</value>
-<description>
-If "true", enable permission checking in HDFS.
-If "false", permission checking is turned off,
-but all other behavior is unchanged.
-Switching from one parameter value to the other does not change the mode,
-owner or group of files or directories.
-</description>
-</property>
-
-<property>
-<name>dfs.permissions.supergroup</name>
-<value>hdfs</value>
-<description>The name of the group of super-users.</description>
-</property>
-
-<property>
-<name>dfs.namenode.handler.count</name>
-<value>100</value>
-<description>Added to grow Queue size so that more client connections are allowed</description>
-</property>
-
-<property>
-<name>ipc.server.max.response.size</name>
-<value>5242880</value>
-</property>
-<property>
-<name>dfs.block.access.token.enable</name>
-<value>true</value>
-<description>
-If "true", access tokens are used as capabilities for accessing datanodes.
-If "false", no access tokens are checked on accessing datanodes.
-</description>
-</property>
-
-<property>
-<name>dfs.namenode.kerberos.principal</name>
-<value></value>
-<description>
-Kerberos principal name for the NameNode
-</description>
-</property>
-
-<property>
-<name>dfs.secondary.namenode.kerberos.principal</name>
-<value></value>
+ <description>The name of the default file system. Either the
+ literal string "local" or a host:port for NDFS.</description>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>dfs.datanode.du.reserved</name>
+ <!-- cluster variant -->
+ <value>1073741824</value>
+ <description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.ipc.address</name>
+ <value>0.0.0.0:8010</value>
<description>
- Kerberos principal name for the secondary NameNode.
+ The datanode ipc server address and port.
+ If the port is 0 then the server will start on a free port.
</description>
</property>
+ <property>
+ <name>dfs.blockreport.initialDelay</name>
+ <value>120</value>
+ <description>Delay for first block report in seconds.</description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.du.pct</name>
+ <value>0.85f</value>
+ <description>When calculating remaining space, only use this percentage of the real available space
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>40</value>
+ <description>The number of server threads for the namenode.</description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.max.xcievers</name>
+ <value>4096</value>
+ <description>PRIVATE CONFIG VARIABLE</description>
+ </property>
+
+ <!-- Permissions configuration -->
+
+ <property>
+ <name>dfs.umaskmode</name>
+ <value>077</value>
+ <description>
+ The octal umask used when creating files and directories.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.web.ugi</name>
+ <!-- cluster variant -->
+ <value>gopher,gopher</value>
+ <description>The user account used by the web interface.
+ Syntax: USERNAME,GROUP1,GROUP2, ...
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.permissions</name>
+ <value>true</value>
+ <description>
+ If "true", enable permission checking in HDFS.
+ If "false", permission checking is turned off,
+ but all other behavior is unchanged.
+ Switching from one parameter value to the other does not change the mode,
+ owner or group of files or directories.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.permissions.supergroup</name>
+ <value>hdfs</value>
+ <description>The name of the group of super-users.</description>
+ </property>
+
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>100</value>
+ <description>Added to grow Queue size so that more client connections are allowed</description>
+ </property>
+
+ <property>
+ <name>ipc.server.max.response.size</name>
+ <value>5242880</value>
+ </property>
+ <property>
+ <name>dfs.block.access.token.enable</name>
+ <value>true</value>
+ <description>
+ If "true", access tokens are used as capabilities for accessing datanodes.
+ If "false", no access tokens are checked on accessing datanodes.
+ </description>
+ </property>
-<!--
- This is KRB DOMAIN specific. The FQDN of the namenode has to be mentioned.
--->
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value></value>
+ <description>
+ Kerberos principal name for the NameNode
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.secondary.namenode.kerberos.principal</name>
+ <value></value>
+ <description>
+ Kerberos principal name for the secondary NameNode.
+ </description>
+ </property>
+
+
+ <!--
+ This is KRB DOMAIN specific. The FQDN of the namenode has to be mentioned.
+ -->
<property>
<name>dfs.namenode.kerberos.https.principal</name>
<value></value>
- <description>The Kerberos principal for the host that the NameNode runs on.</description>
+ <description>The Kerberos principal for the host that the NameNode runs on.</description>
</property>
@@ -341,84 +363,84 @@ Kerberos principal name for the NameNode
<property>
<name>dfs.datanode.kerberos.principal</name>
<value></value>
- <description>
- The Kerberos principal that the DataNode runs as. "_HOST" is replaced by the real host name.
+ <description>
+ The Kerberos principal that the DataNode runs as. "_HOST" is replaced by the real host name.
</description>
</property>
<property>
<name>dfs.namenode.keytab.file</name>
<value></value>
- <description>
- Combined keytab file containing the namenode service and host principals.
+ <description>
+ Combined keytab file containing the namenode service and host principals.
</description>
</property>
<property>
<name>dfs.secondary.namenode.keytab.file</name>
<value></value>
- <description>
- Combined keytab file containing the namenode service and host principals.
+ <description>
+ Combined keytab file containing the namenode service and host principals.
</description>
</property>
<property>
<name>dfs.datanode.keytab.file</name>
<value></value>
- <description>
- The filename of the keytab file for the DataNode.
+ <description>
+ The filename of the keytab file for the DataNode.
</description>
</property>
<property>
<name>dfs.https.port</name>
<value>50470</value>
- <description>The https port where namenode binds</description>
+ <description>The https port where namenode binds</description>
</property>
<property>
<name>dfs.https.address</name>
<value>localhost:50470</value>
- <description>The https address where namenode binds</description>
+ <description>The https address where namenode binds</description>
</property>
<property>
<name>dfs.datanode.data.dir.perm</name>
<value>750</value>
-<description>The permissions that should be there on dfs.data.dir
-directories. The datanode will not come up if the permissions are
-different on existing dfs.data.dir directories. If the directories
-don't exist, they will be created with this permission.</description>
- </property>
-
- <property>
- <name>dfs.access.time.precision</name>
- <value>0</value>
- <description>The access time for HDFS file is precise upto this value.
- The default value is 1 hour. Setting a value of 0 disables
- access times for HDFS.
- </description>
-</property>
-
-<property>
- <name>dfs.cluster.administrators</name>
- <value> hdfs</value>
- <description>ACL for who all can view the default servlets in the HDFS</description>
-</property>
-
-<property>
- <name>ipc.server.read.threadpool.size</name>
- <value>5</value>
- <description></description>
-</property>
-
-<property>
- <name>dfs.datanode.failed.volumes.tolerated</name>
- <value>0</value>
- <description>Number of failed disks datanode would tolerate</description>
-</property>
+ <description>The permissions that should be there on dfs.data.dir
+ directories. The datanode will not come up if the permissions are
+ different on existing dfs.data.dir directories. If the directories
+ don't exist, they will be created with this permission.</description>
+ </property>
+
+ <property>
+ <name>dfs.access.time.precision</name>
+ <value>0</value>
+ <description>The access time for HDFS file is precise upto this value.
+ The default value is 1 hour. Setting a value of 0 disables
+ access times for HDFS.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cluster.administrators</name>
+ <value> hdfs</value>
+ <description>ACL for who all can view the default servlets in the HDFS</description>
+ </property>
+
+ <property>
+ <name>ipc.server.read.threadpool.size</name>
+ <value>5</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>dfs.datanode.failed.volumes.tolerated</name>
+ <value>0</value>
+ <description>Number of failed disks datanode would tolerate</description>
+ </property>
<property>
<name>dfs.namenode.avoid.read.stale.datanode</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HIVE/configuration/hive-site.xml
index 9d0b168..29ed54e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/HIVE/configuration/hive-site.xml
@@ -19,6 +19,12 @@ limitations under the License.
<configuration>
<property>
+ <name>ambari.hive.db.schema.name</name>
+ <value>hive</value>
+ <description>Database name used as the Hive Metastore</description>
+ </property>
+
+ <property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc</value>
<description>JDBC connect string for a JDBC metastore</description>
@@ -52,21 +58,21 @@ limitations under the License.
<name>hive.metastore.sasl.enabled</name>
<value></value>
<description>If true, the metastore thrift interface will be secured with SASL.
- Clients must authenticate with Kerberos.</description>
+ Clients must authenticate with Kerberos.</description>
</property>
<property>
<name>hive.metastore.kerberos.keytab.file</name>
<value></value>
<description>The path to the Kerberos Keytab file containing the metastore
- thrift server's service principal.</description>
+ thrift server's service principal.</description>
</property>
<property>
<name>hive.metastore.kerberos.principal</name>
<value></value>
<description>The service principal for the metastore thrift server. The special
- string _HOST will be replaced automatically with the correct host name.</description>
+ string _HOST will be replaced automatically with the correct host name.</description>
</property>
<property>
@@ -109,7 +115,7 @@ limitations under the License.
<name>hive.security.authorization.manager</name>
<value>org.apache.hcatalog.security.HdfsAuthorizationProvider</value>
<description>the hive client authorization manager class name.
- The user defined authorization class should implement interface org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider. </description>
+ The user defined authorization class should implement interface org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider. </description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/MAPREDUCE/configuration/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/MAPREDUCE/configuration/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/MAPREDUCE/configuration/mapred-site.xml
index 567c1e1..4a174f0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3._/services/MAPREDUCE/configuration/mapred-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3._/services/MAPREDUCE/configuration/mapred-site.xml
@@ -22,7 +22,7 @@
<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
-<!-- i/o properties -->
+ <!-- i/o properties -->
<property>
<name>io.sort.mb</name>
@@ -50,25 +50,25 @@
<description>No description</description>
</property>
-<!-- map/reduce properties -->
+ <!-- map/reduce properties -->
-<property>
- <name>mapred.tasktracker.tasks.sleeptime-before-sigkill</name>
- <value>250</value>
- <description>Normally, this is the amount of time before killing
- processes, and the recommended-default is 5.000 seconds - a value of
- 5000 here. In this case, we are using it solely to blast tasks before
- killing them, and killing them very quickly (1/4 second) to guarantee
- that we do not leave VMs around for later jobs.
- </description>
-</property>
+ <property>
+ <name>mapred.tasktracker.tasks.sleeptime-before-sigkill</name>
+ <value>250</value>
+ <description>Normally, this is the amount of time before killing
+ processes, and the recommended-default is 5.000 seconds - a value of
+ 5000 here. In this case, we are using it solely to blast tasks before
+ killing them, and killing them very quickly (1/4 second) to guarantee
+ that we do not leave VMs around for later jobs.
+ </description>
+ </property>
<property>
<name>mapred.job.tracker.handler.count</name>
<value>50</value>
<description>
- The number of server threads for the JobTracker. This should be roughly
- 4% of the number of tasktracker nodes.
+ The number of server threads for the JobTracker. This should be roughly
+ 4% of the number of tasktracker nodes.
</description>
</property>
@@ -104,8 +104,8 @@
</property>
<property>
- <name>mapreduce.cluster.administrators</name>
- <value> hadoop</value>
+ <name>mapreduce.cluster.administrators</name>
+ <value> hadoop</value>
</property>
<property>
@@ -135,14 +135,14 @@
<name>mapred.map.tasks.speculative.execution</name>
<value>false</value>
<description>If true, then multiple instances of some map tasks
- may be executed in parallel.</description>
+ may be executed in parallel.</description>
</property>
<property>
<name>mapred.reduce.tasks.speculative.execution</name>
<value>false</value>
<description>If true, then multiple instances of some reduce tasks
- may be executed in parallel.</description>
+ may be executed in parallel.</description>
</property>
<property>
@@ -154,29 +154,29 @@
<name>mapred.inmem.merge.threshold</name>
<value>1000</value>
<description>The threshold, in terms of the number of files
- for the in-memory merge process. When we accumulate threshold number of files
- we initiate the in-memory merge and spill to disk. A value of 0 or less than
- 0 indicates we want to DON'T have any threshold and instead depend only on
- the ramfs's memory consumption to trigger the merge.
- </description>
+ for the in-memory merge process. When we accumulate threshold number of files
+ we initiate the in-memory merge and spill to disk. A value of 0 or less than
+ 0 indicates we want to DON'T have any threshold and instead depend only on
+ the ramfs's memory consumption to trigger the merge.
+ </description>
</property>
<property>
<name>mapred.job.shuffle.merge.percent</name>
<value>0.66</value>
<description>The usage threshold at which an in-memory merge will be
- initiated, expressed as a percentage of the total memory allocated to
- storing in-memory map outputs, as defined by
- mapred.job.shuffle.input.buffer.percent.
- </description>
+ initiated, expressed as a percentage of the total memory allocated to
+ storing in-memory map outputs, as defined by
+ mapred.job.shuffle.input.buffer.percent.
+ </description>
</property>
<property>
<name>mapred.job.shuffle.input.buffer.percent</name>
<value>0.7</value>
<description>The percentage of memory to be allocated from the maximum heap
- size to storing map outputs during the shuffle.
- </description>
+ size to storing map outputs during the shuffle.
+ </description>
</property>
<property>
@@ -187,18 +187,18 @@
</description>
</property>
-<property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>If the job outputs are to compressed as SequenceFiles, how should
- they be compressed? Should be one of NONE, RECORD or BLOCK.
- </description>
-</property>
+ <property>
+ <name>mapred.output.compression.type</name>
+ <value>BLOCK</value>
+ <description>If the job outputs are to compressed as SequenceFiles, how should
+ they be compressed? Should be one of NONE, RECORD or BLOCK.
+ </description>
+ </property>
<property>
<name>mapred.jobtracker.completeuserjobs.maximum</name>
- <value>5</value>
+ <value>0</value>
</property>
<property>
@@ -210,7 +210,7 @@
<name>mapred.jobtracker.restart.recover</name>
<value>false</value>
<description>"true" to enable (job) recovery upon restart,
- "false" to start afresh
+ "false" to start afresh
</description>
</property>
@@ -218,20 +218,20 @@
<name>mapred.job.reduce.input.buffer.percent</name>
<value>0.0</value>
<description>The percentage of memory- relative to the maximum heap size- to
- retain map outputs during the reduce. When the shuffle is concluded, any
- remaining map outputs in memory must consume less than this threshold before
- the reduce can begin.
- </description>
+ retain map outputs during the reduce. When the shuffle is concluded, any
+ remaining map outputs in memory must consume less than this threshold before
+ the reduce can begin.
+ </description>
</property>
- <property>
- <name>mapreduce.reduce.input.limit</name>
- <value>10737418240</value>
- <description>The limit on the input size of the reduce. (This value
- is 10 Gb.) If the estimated input size of the reduce is greater than
- this value, job is failed. A value of -1 means that there is no limit
- set. </description>
-</property>
+ <property>
+ <name>mapreduce.reduce.input.limit</name>
+ <value>10737418240</value>
+ <description>The limit on the input size of the reduce. (This value
+ is 10 Gb.) If the estimated input size of the reduce is greater than
+ this value, job is failed. A value of -1 means that there is no limit
+ set. </description>
+ </property>
<!-- copied from kryptonite configuration -->
@@ -245,9 +245,9 @@
<name>mapred.task.timeout</name>
<value>600000</value>
<description>The number of milliseconds before a task will be
- terminated if it neither reads an input, writes an output, nor
- updates its status string.
- </description>
+ terminated if it neither reads an input, writes an output, nor
+ updates its status string.
+ </description>
</property>
<property>
@@ -259,9 +259,9 @@
<property>
<name>mapred.task.tracker.task-controller</name>
<value>org.apache.hadoop.mapred.DefaultTaskController</value>
- <description>
- TaskController which is used to launch and manage task execution.
- </description>
+ <description>
+ TaskController which is used to launch and manage task execution.
+ </description>
</property>
<property>
@@ -270,10 +270,16 @@
</property>
<property>
- <name>mapred.child.java.opts</name>
- <value></value>
+ <name>ambari.mapred.child.java.opts.memory</name>
+ <value>768</value>
- <description>No description</description>
+ <description>Java options Memory for the TaskTracker child processes</description>
+ </property>
+
+ <property>
+ <name>mapred.child.java.opts</name>
+ <value>-server -Xmx${ambari.mapred.child.java.opts.memory}m -Djava.net.preferIPv4Stack=true</value>
+ <description>Java options for the TaskTracker child processes</description>
</property>
<property>
@@ -288,7 +294,7 @@
<name>mapred.cluster.reduce.memory.mb</name>
<value>2048</value>
<description>
- The virtual memory size of a single Reduce slot in the MapReduce framework
+ The virtual memory size of a single Reduce slot in the MapReduce framework
</description>
</property>
@@ -324,147 +330,147 @@
</description>
</property>
-<property>
- <name>mapred.hosts</name>
- <value>/etc/hadoop/conf/mapred.include</value>
- <description>
- Names a file that contains the list of nodes that may
- connect to the jobtracker. If the value is empty, all hosts are
- permitted.
- </description>
-</property>
-
-<property>
- <name>mapred.hosts.exclude</name>
- <value>/etc/hadoop/conf/mapred.exclude</value>
- <description>
- Names a file that contains the list of hosts that
- should be excluded by the jobtracker. If the value is empty, no
- hosts are excluded.
- </description>
-</property>
-
-<property>
- <name>mapred.max.tracker.blacklists</name>
- <value>16</value>
- <description>
- if node is reported blacklisted by 16 successful jobs within timeout-window, it will be graylisted
- </description>
-</property>
-
-<property>
- <name>mapred.healthChecker.script.path</name>
- <value>file:////mapred/jobstatus</value>
- <description>
- Directory path to view job status
- </description>
-</property>
-
-<property>
- <name>mapred.healthChecker.interval</name>
- <value>135000</value>
-</property>
-
-<property>
- <name>mapred.healthChecker.script.timeout</name>
- <value>60000</value>
-</property>
-
-<property>
- <name>mapred.job.tracker.persist.jobstatus.active</name>
- <value>false</value>
- <description>Indicates if persistency of job status information is
- active or not.
- </description>
-</property>
-
-<property>
- <name>mapred.job.tracker.persist.jobstatus.hours</name>
- <value>1</value>
- <description>The number of hours job status information is persisted in DFS.
- The job status information will be available after it drops of the memory
- queue and between jobtracker restarts. With a zero value the job status
- information is not persisted at all in DFS.
- </description>
-</property>
-
-<property>
- <name>mapred.job.tracker.persist.jobstatus.dir</name>
- <value>/etc/hadoop/conf/health_check</value>
- <description>The directory where the job status information is persisted
- in a file system to be available after it drops of the memory queue and
- between jobtracker restarts.
- </description>
-</property>
-
-<property>
- <name>mapred.jobtracker.retirejob.check</name>
- <value>10000</value>
-</property>
-
-<property>
- <name>mapred.jobtracker.retirejob.interval</name>
- <value>21600000</value>
-</property>
-
-<property>
- <name>mapred.job.tracker.history.completed.location</name>
- <value>/mapred/history/done</value>
- <description>No description</description>
-</property>
-
-<property>
- <name>mapred.task.maxvmem</name>
- <value></value>
- <final>true</final>
- <description>No description</description>
-</property>
-
-<property>
- <name>mapred.jobtracker.maxtasks.per.job</name>
- <value>-1</value>
- <final>true</final>
- <description>The maximum number of tasks for a single job.
- A value of -1 indicates that there is no maximum. </description>
-</property>
-
-<property>
- <name>mapreduce.fileoutputcommitter.marksuccessfuljobs</name>
- <value>false</value>
-</property>
-
-<property>
- <name>mapred.userlog.retain.hours</name>
- <value>24</value>
- <description>
- The maximum time, in hours, for which the user-logs are to be retained after the job completion.
- </description>
-</property>
-
-<property>
- <name>mapred.job.reuse.jvm.num.tasks</name>
- <value>1</value>
- <description>
- How many tasks to run per jvm. If set to -1, there is no limit
- </description>
- <final>true</final>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.kerberos.principal</name>
- <value></value>
- <description>
+ <property>
+ <name>mapred.hosts</name>
+ <value>/etc/hadoop/conf/mapred.include</value>
+ <description>
+ Names a file that contains the list of nodes that may
+ connect to the jobtracker. If the value is empty, all hosts are
+ permitted.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.hosts.exclude</name>
+ <value>/etc/hadoop/conf/mapred.exclude</value>
+ <description>
+ Names a file that contains the list of hosts that
+ should be excluded by the jobtracker. If the value is empty, no
+ hosts are excluded.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.max.tracker.blacklists</name>
+ <value>16</value>
+ <description>
+ if node is reported blacklisted by 16 successful jobs within timeout-window, it will be graylisted
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.healthChecker.script.path</name>
+ <value>file:////mapred/jobstatus</value>
+ <description>
+ Directory path to view job status
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.healthChecker.interval</name>
+ <value>135000</value>
+ </property>
+
+ <property>
+ <name>mapred.healthChecker.script.timeout</name>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <name>mapred.job.tracker.persist.jobstatus.active</name>
+ <value>false</value>
+ <description>Indicates if persistency of job status information is
+ active or not.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.job.tracker.persist.jobstatus.hours</name>
+ <value>1</value>
+ <description>The number of hours job status information is persisted in DFS.
+ The job status information will be available after it drops of the memory
+ queue and between jobtracker restarts. With a zero value the job status
+ information is not persisted at all in DFS.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.job.tracker.persist.jobstatus.dir</name>
+ <value>/etc/hadoop/conf/health_check</value>
+ <description>The directory where the job status information is persisted
+ in a file system to be available after it drops of the memory queue and
+ between jobtracker restarts.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.jobtracker.retirejob.check</name>
+ <value>10000</value>
+ </property>
+
+ <property>
+ <name>mapred.jobtracker.retirejob.interval</name>
+ <value>21600000</value>
+ </property>
+
+ <property>
+ <name>mapred.job.tracker.history.completed.location</name>
+ <value>/mapred/history/done</value>
+ <description>No description</description>
+ </property>
+
+ <property>
+ <name>mapred.task.maxvmem</name>
+ <value></value>
+ <final>true</final>
+ <description>No description</description>
+ </property>
+
+ <property>
+ <name>mapred.jobtracker.maxtasks.per.job</name>
+ <value>-1</value>
+ <final>true</final>
+ <description>The maximum number of tasks for a single job.
+ A value of -1 indicates that there is no maximum. </description>
+ </property>
+
+ <property>
+ <name>mapreduce.fileoutputcommitter.marksuccessfuljobs</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>mapred.userlog.retain.hours</name>
+ <value>24</value>
+ <description>
+ The maximum time, in hours, for which the user-logs are to be retained after the job completion.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.job.reuse.jvm.num.tasks</name>
+ <value>1</value>
+ <description>
+ How many tasks to run per jvm. If set to -1, there is no limit
+ </description>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>mapreduce.jobtracker.kerberos.principal</name>
+ <value></value>
+ <description>
JT user name key.
- </description>
-</property>
+ </description>
+ </property>
-<property>
- <name>mapreduce.tasktracker.kerberos.principal</name>
- <value></value>
- <description>
- tt user name key. "_HOST" is replaced by the host name of the task tracker.
- </description>
-</property>
+ <property>
+ <name>mapreduce.tasktracker.kerberos.principal</name>
+ <value></value>
+ <description>
+ tt user name key. "_HOST" is replaced by the host name of the task tracker.
+ </description>
+ </property>
<property>
@@ -474,54 +480,54 @@
</property>
- <property>
- <name>mapreduce.jobtracker.keytab.file</name>
- <value></value>
- <description>
- The keytab for the jobtracker principal.
- </description>
+ <property>
+ <name>mapreduce.jobtracker.keytab.file</name>
+ <value></value>
+ <description>
+ The keytab for the jobtracker principal.
+ </description>
-</property>
+ </property>
- <property>
- <name>mapreduce.tasktracker.keytab.file</name>
- <value></value>
+ <property>
+ <name>mapreduce.tasktracker.keytab.file</name>
+ <value></value>
<description>The filename of the keytab for the task tracker</description>
- </property>
+ </property>
- <property>
- <name>mapred.task.tracker.http.address</name>
- <value></value>
- <description>Http address for task tracker.</description>
- </property>
+ <property>
+ <name>mapred.task.tracker.http.address</name>
+ <value></value>
+ <description>Http address for task tracker.</description>
+ </property>
- <property>
- <name>mapreduce.jobtracker.staging.root.dir</name>
- <value>/user</value>
- <description>The Path prefix for where the staging directories should be placed. The next level is always the user's
- name. It is a path in the default file system.</description>
- </property>
+ <property>
+ <name>mapreduce.jobtracker.staging.root.dir</name>
+ <value>/user</value>
+ <description>The Path prefix for where the staging directories should be placed. The next level is always the user's
+ name. It is a path in the default file system.</description>
+ </property>
- <property>
- <name>mapreduce.tasktracker.group</name>
- <value>hadoop</value>
- <description>The group that the task controller uses for accessing the task controller. The mapred user must be a member and users should *not* be members.</description>
+ <property>
+ <name>mapreduce.tasktracker.group</name>
+ <value>hadoop</value>
+ <description>The group that the task controller uses for accessing the task controller. The mapred user must be a member and users should *not* be members.</description>
- </property>
+ </property>
<property>
<name>mapreduce.jobtracker.split.metainfo.maxsize</name>
<value>50000000</value>
<final>true</final>
- <description>If the size of the split metainfo file is larger than this, the JobTracker will fail the job during
- initialize.
- </description>
+ <description>If the size of the split metainfo file is larger than this, the JobTracker will fail the job during
+ initialize.
+ </description>
</property>
<property>
<name>mapreduce.history.server.embedded</name>
<value>false</value>
<description>Should job history server be embedded within Job tracker
-process</description>
+ process</description>
<final>true</final>
</property>
@@ -536,38 +542,38 @@ process</description>
<property>
<name>mapreduce.jobhistory.kerberos.principal</name>
<!-- cluster variant -->
- <value></value>
+ <value></value>
<description>Job history user name key. (must map to same user as JT
-user)</description>
+ user)</description>
</property>
- <property>
- <name>mapreduce.jobhistory.keytab.file</name>
+ <property>
+ <name>mapreduce.jobhistory.keytab.file</name>
<!-- cluster variant -->
- <value></value>
- <description>The keytab for the job history server principal.</description>
- </property>
-
-<property>
- <name>mapred.jobtracker.blacklist.fault-timeout-window</name>
- <value>180</value>
- <description>
- 3-hour sliding window (value is in minutes)
- </description>
-</property>
-
-<property>
- <name>mapred.jobtracker.blacklist.fault-bucket-width</name>
- <value>15</value>
- <description>
- 15-minute bucket size (value is in minutes)
- </description>
-</property>
-
-<property>
- <name>mapred.queue.names</name>
- <value>default</value>
- <description> Comma separated list of queues configured for this jobtracker.</description>
-</property>
+ <value></value>
+ <description>The keytab for the job history server principal.</description>
+ </property>
+
+ <property>
+ <name>mapred.jobtracker.blacklist.fault-timeout-window</name>
+ <value>180</value>
+ <description>
+ 3-hour sliding window (value is in minutes)
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.jobtracker.blacklist.fault-bucket-width</name>
+ <value>15</value>
+ <description>
+ 15-minute bucket size (value is in minutes)
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.queue.names</name>
+ <value>default</value>
+ <description> Comma separated list of queues configured for this jobtracker.</description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HDFS/configuration/global.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HDFS/configuration/global.xml b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HDFS/configuration/global.xml
index 59b68ac..49d66bb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HDFS/configuration/global.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HDFS/configuration/global.xml
@@ -83,7 +83,7 @@
</property>
<property>
<name>namenode_opt_maxnewsize</name>
- <value>640</value>
+ <value>200</value>
<description>NameNode maximum new generation size</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HDFS/configuration/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HDFS/configuration/hdfs-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HDFS/configuration/hdfs-site.xml
index bd3b12a..fc292e6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HDFS/configuration/hdfs-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HDFS/configuration/hdfs-site.xml
@@ -181,13 +181,35 @@
</property>
<property>
+ <name>ambari.dfs.datanode.port</name>
+ <value>50010</value>
+ <description>
+ The datanode port for data transfer. This property is effective only if referenced from dfs.datanode.address property.
+ </description>
+ </property>
+
+ <property>
<name>dfs.datanode.address</name>
- <value>0.0.0.0:50010</value>
+ <value>0.0.0.0:${ambari.dfs.datanode.port}</value>
+ <description>
+ The datanode server address and port for data transfer.
+ </description>
+ </property>
+
+ <property>
+ <name>ambari.dfs.datanode.http.port</name>
+ <value>50075</value>
+ <description>
+ The datanode http port. This property is effective only if referenced from dfs.datanode.http.address property.
+ </description>
</property>
<property>
<name>dfs.datanode.http.address</name>
- <value>0.0.0.0:50075</value>
+ <value>0.0.0.0:${ambari.dfs.datanode.http.port}</value>
+ <description>
+ The datanode http server address and port.
+ </description>
</property>
<property>
@@ -481,4 +503,11 @@ don't exist, they will be created with this permission.</description>
</description>
</property>
+ <property>
+ <name>dfs.namenode.name.dir.restore</name>
+ <value>true</value>
+ <description>Set to true to enable NameNode to attempt recovering a previously failed dfs.namenode.name.dir.
+ When enabled, a recovery of any failed directory is attempted during checkpoint.</description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HIVE/configuration/hive-site.xml
index 7fb8969..b3ed5f5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/HIVE/configuration/hive-site.xml
@@ -18,6 +18,13 @@ limitations under the License.
-->
<configuration>
+
+ <property>
+ <name>ambari.hive.db.schema.name</name>
+ <value>hive</value>
+ <description>Database name used as the Hive Metastore</description>
+ </property>
+
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc</value>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/OOZIE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/OOZIE/package/scripts/params.py
index 1e4609d..32ffd15 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/OOZIE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/OOZIE/package/scripts/params.py
@@ -28,7 +28,7 @@ kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin",
oozie_service_keytab = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.keytab.file']
oozie_principal = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal']
smokeuser_keytab = config['configurations']['global']['smokeuser_keytab']
-oozie_keytab = config['configurations']['global']['keytab_path']
+oozie_keytab = config['configurations']['global']['oozie_keytab']
oracle_driver_jar_name = "ojdbc6.jar"
java_share_dir = "/usr/share/java"
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py
index c97d554..c42501a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py
@@ -27,7 +27,7 @@ class SqoopServiceCheck(Script):
import params
env.set_params(params)
if params.security_enabled:
- Execute(format("{kinit_path_local} -kt {smoke_user_keytab} {smoke_test_user}"))
+ Execute(format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}"))
Execute("sqoop version",
user = params.smokeuser,
logoutput = True
http://git-wip-us.apache.org/repos/asf/ambari/blob/4aec796a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/YARN/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/YARN/configuration/capacity-scheduler.xml
index ccfb779..4a19779 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/YARN/configuration/capacity-scheduler.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/YARN/configuration/capacity-scheduler.xml
@@ -116,5 +116,13 @@
No description
</description>
</property>
+ <property>
+ <name>yarn.scheduler.capacity.node-locality-delay</name>
+ <value>40</value>
+ <description>
+ No description
+ </description>
+ </property>
+
</configuration>