You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ab...@apache.org on 2015/02/06 00:20:26 UTC
[1/2] ambari git commit: AMBARI-9496 - Falcon and Pig service checks
fails on Windows with Hadoop 2.2
Repository: ambari
Updated Branches:
refs/heads/trunk c6f4d069d -> 50ee8e029
AMBARI-9496 - Falcon and Pig service checks fails on Windows with Hadoop 2.2
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/de026eb5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/de026eb5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/de026eb5
Branch: refs/heads/trunk
Commit: de026eb5ece3459f45956a4b4ad85d383111973d
Parents: c6f4d06
Author: Artem Baranchuk <ab...@hortonworks.con>
Authored: Thu Feb 5 18:07:48 2015 +0200
Committer: Artem Baranchuk <ab...@hortonworks.con>
Committed: Fri Feb 6 01:10:28 2015 +0200
----------------------------------------------------------------------
.../after-INSTALL/templates/Run-SmokeTests.ps1 | 1 +
.../configuration/falcon-startup.properties.xml | 3 +-
.../YARN/configuration-mapred/mapred-site.xml | 58 ++++++++------------
3 files changed, 26 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/de026eb5/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/after-INSTALL/templates/Run-SmokeTests.ps1
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/after-INSTALL/templates/Run-SmokeTests.ps1 b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/after-INSTALL/templates/Run-SmokeTests.ps1
index a5c3631..5647ff9 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/after-INSTALL/templates/Run-SmokeTests.ps1
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/after-INSTALL/templates/Run-SmokeTests.ps1
@@ -114,6 +114,7 @@ function Run-PigSmokeTest
$ts=Get-TimeStamp
Write-Host "Pig smoke test - wordcount using hadoop.cmd file"
$pigscript = Join-Path $ENV:TMP "script-$ts.pig"
+ Invoke-HadoopCmd "dfs -mkdir -p /user/hadoop/"
Invoke-HadoopCmd "dfs -copyFromLocal $ENV:HADOOP_HOME\bin\hadoop.cmd hadoop-$ts"
Add-Content $pigscript "A = load 'hadoop-$ts' using PigStorage(' ');"
Add-Content $pigscript "B = foreach A generate `$0 as id;"
http://git-wip-us.apache.org/repos/asf/ambari/blob/de026eb5/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
index 7651c07..1e0659f 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
@@ -33,8 +33,7 @@
org.apache.falcon.entity.store.ConfigurationStore,\
org.apache.falcon.rerun.service.RetryService,\
org.apache.falcon.rerun.service.LateRunService,\
- org.apache.falcon.service.LogCleanupService,\
- org.apache.falcon.metadata.MetadataMappingService
+ org.apache.falcon.service.LogCleanupService
</value>
<description>Falcon Services</description>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/de026eb5/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/YARN/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/YARN/configuration-mapred/mapred-site.xml
index d2b7ee1..8a1cc24 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/YARN/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/YARN/configuration-mapred/mapred-site.xml
@@ -24,87 +24,77 @@
<property>
<name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
- <description>
- Additional execution environment entries for map and reduce task processes.
- This is not an additive property. You must preserve the original value if
- you want your map and reduce tasks to have access to native libraries (compression, etc)
- </description>
+ <deleted>true</deleted>
+ <value/>
+ <description/>
</property>
<property>
<name>mapreduce.application.classpath</name>
- <value>$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
+ <value>
+ C:\hdp\hadoop\etc\hadoop;C:\hdp\hadoop\etc\hadoop;C:\hdp\hadoop\share\hadoop\common\lib\*;C:\hdp\hadoop\share\hadoop\common\*;C:\hdp\hadoop\share\hadoop\hdfs;C:\hdp\hadoop\share\hadoop\hdfs\lib\*;C:\hdp\hadoop\share\hadoop\hdfs\*;C:\hdp\hadoop\share\hadoop\yarn\lib\*;C:\hdp\hadoop\share\hadoop\yarn\*;C:\hdp\hadoop\share\hadoop\mapreduce\lib\*;C:\hdp\hadoop\share\hadoop\mapreduce\*;C:\hdp\hadoop\share\hadoop\yarn\*;C:\hdp\hadoop\share\hadoop\mapreduce\lib\*;C:\hdp\hadoop\modules\*
+ </value>
<description>
- CLASSPATH for MR applications. A comma-separated list of CLASSPATH
- entries.
+ CLASSPATH for MR applications. A comma-separated list of CLASSPATH entries.
</description>
</property>
<property>
<name>mapreduce.application.framework.path</name>
- <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
- <description></description>
+ <deleted>true</deleted>
+ <value/>
+ <description/>
</property>
<property>
<name>yarn.app.mapreduce.am.admin-command-opts</name>
- <value>-Dhdp.version=${hdp.version}</value>
- <description>
- Java opts for the MR App Master processes.
- The following symbol, if present, will be interpolated: @taskid@ is replaced
- by current TaskID. Any other occurrences of '@' will go unchanged.
- For example, to enable verbose gc logging to a file named for the taskid in
- /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
- -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
-
- Usage of -Djava.library.path can cause programs to no longer function if
- hadoop native libraries are used. These values should instead be set as part
- of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
- mapreduce.reduce.env config settings.
- </description>
+ <deleted>true</deleted>
+ <value/>
+ <description/>
</property>
<property>
<name>mapreduce.admin.map.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- <description></description>
+ <deleted>true</deleted>
+ <value/>
+ <description/>
</property>
<property>
<name>mapreduce.admin.reduce.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- <description></description>
+ <deleted>true</deleted>
+ <value/>
+ <description/>
</property>
<property>
<name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
<value>1</value>
- <description></description>
+ <description/>
</property>
<property>
<name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
<value>1000</value>
- <description></description>
+ <description/>
</property>
<property>
<name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
<value>30000</value>
- <description></description>
+ <description/>
</property>
<property>
<name>mapreduce.job.emit-timeline-data</name>
<value>false</value>
- <description></description>
+ <description/>
</property>
<property>
<name>mapreduce.jobhistory.bind-host</name>
<value>0.0.0.0</value>
- <description></description>
+ <description/>
</property>
</configuration>
[2/2] ambari git commit: AMBARI-35 - Monarch: preupload oozie files
not executed (Ivan Kozlov via abaranchuk)
Posted by ab...@apache.org.
AMBARI-35 - Monarch: preupload oozie files not executed (Ivan Kozlov via abaranchuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/50ee8e02
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/50ee8e02
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/50ee8e02
Branch: refs/heads/trunk
Commit: 50ee8e02993e3d34ea46dc81dffbbf2c02348332
Parents: de026eb
Author: Artem Baranchuk <ab...@hortonworks.con>
Authored: Fri Feb 6 01:19:20 2015 +0200
Committer: Artem Baranchuk <ab...@hortonworks.con>
Committed: Fri Feb 6 01:19:20 2015 +0200
----------------------------------------------------------------------
.../src/main/resources/scripts/Ambaripreupload.py | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/50ee8e02/ambari-server/src/main/resources/scripts/Ambaripreupload.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
index 7a6bf67..89e0742 100644
--- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py
+++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
@@ -63,6 +63,8 @@ class params():
hadoop_conf_dir = "/etc/hadoop/conf"
user_group = "hadoop"
security_enabled = False
+ oozie_user = "oozie"
+ execute_path = "/usr/hdp/current/hadoop-client/bin"
params = params()
@@ -255,8 +257,17 @@ with Environment() as env:
oozie_cmd = format("{put_shared_lib_to_hdfs_cmd} ; hadoop --config {hadoop_conf_dir} dfs -chmod -R 755 {oozie_hdfs_user_dir}/share")
not_if_command = format("{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls /user/oozie/share | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'")
-
- #Execute(format("hadoop --config {hadoop_conf_dir} dfs -mkdir {
+ HdfsDirectory(format("{oozie_hdfs_user_dir}/share"),
+ action="create",
+ owner=oozie_user,
+ mode=0555,
+ conf_dir=params.hadoop_conf_dir,
+ hdfs_user=params.hdfs_user,
+ )
+
+ Execute( oozie_cmd, user = params.oozie_user, not_if = not_if_command,
+ path = params.execute_path )
+
copy_tarballs_to_hdfs("/usr/hdp/current/hadoop-client/mapreduce.tar.gz", "wasb:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs("/usr/hdp/current/tez-client/lib/tez.tar.gz", "wasb:///hdp/apps/{{ hdp_stack_version }}/tez/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs("/usr/hdp/current/hive-client/hive.tar.gz", "wasb:///hdp/apps/{{ hdp_stack_version }}/hive/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)