You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ab...@apache.org on 2015/02/06 00:20:26 UTC

[1/2] ambari git commit: AMBARI-9496 - Falcon and Pig service checks fails on Windows with Hadoop 2.2

Repository: ambari
Updated Branches:
  refs/heads/trunk c6f4d069d -> 50ee8e029


AMBARI-9496 - Falcon and Pig service checks fails on Windows with Hadoop 2.2


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/de026eb5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/de026eb5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/de026eb5

Branch: refs/heads/trunk
Commit: de026eb5ece3459f45956a4b4ad85d383111973d
Parents: c6f4d06
Author: Artem Baranchuk <ab...@hortonworks.con>
Authored: Thu Feb 5 18:07:48 2015 +0200
Committer: Artem Baranchuk <ab...@hortonworks.con>
Committed: Fri Feb 6 01:10:28 2015 +0200

----------------------------------------------------------------------
 .../after-INSTALL/templates/Run-SmokeTests.ps1  |  1 +
 .../configuration/falcon-startup.properties.xml |  3 +-
 .../YARN/configuration-mapred/mapred-site.xml   | 58 ++++++++------------
 3 files changed, 26 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/de026eb5/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/after-INSTALL/templates/Run-SmokeTests.ps1
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/after-INSTALL/templates/Run-SmokeTests.ps1 b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/after-INSTALL/templates/Run-SmokeTests.ps1
index a5c3631..5647ff9 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/after-INSTALL/templates/Run-SmokeTests.ps1
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/hooks/after-INSTALL/templates/Run-SmokeTests.ps1
@@ -114,6 +114,7 @@ function Run-PigSmokeTest
     $ts=Get-TimeStamp
     Write-Host "Pig smoke test - wordcount using hadoop.cmd file"
     $pigscript = Join-Path $ENV:TMP "script-$ts.pig"
+    Invoke-HadoopCmd "dfs -mkdir -p /user/hadoop/"
     Invoke-HadoopCmd "dfs -copyFromLocal $ENV:HADOOP_HOME\bin\hadoop.cmd hadoop-$ts"
     Add-Content $pigscript "A = load 'hadoop-$ts' using PigStorage(' ');"
     Add-Content $pigscript "B = foreach A generate `$0 as id;"

http://git-wip-us.apache.org/repos/asf/ambari/blob/de026eb5/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
index 7651c07..1e0659f 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
@@ -33,8 +33,7 @@
       org.apache.falcon.entity.store.ConfigurationStore,\
       org.apache.falcon.rerun.service.RetryService,\
       org.apache.falcon.rerun.service.LateRunService,\
-      org.apache.falcon.service.LogCleanupService,\
-      org.apache.falcon.metadata.MetadataMappingService
+      org.apache.falcon.service.LogCleanupService
     </value>
     <description>Falcon Services</description>
   </property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/de026eb5/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/YARN/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/YARN/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/YARN/configuration-mapred/mapred-site.xml
index d2b7ee1..8a1cc24 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/YARN/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/YARN/configuration-mapred/mapred-site.xml
@@ -24,87 +24,77 @@
 
   <property>
     <name>mapreduce.admin.user.env</name>
-    <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
-    <description>
-      Additional execution environment entries for map and reduce task processes.
-      This is not an additive property. You must preserve the original value if
-      you want your map and reduce tasks to have access to native libraries (compression, etc)
-    </description>
+    <deleted>true</deleted>
+    <value/>
+    <description/>
   </property>
 
   <property>
     <name>mapreduce.application.classpath</name>
-    <value>$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
+    <value>
+      C:\hdp\hadoop\etc\hadoop;C:\hdp\hadoop\etc\hadoop;C:\hdp\hadoop\share\hadoop\common\lib\*;C:\hdp\hadoop\share\hadoop\common\*;C:\hdp\hadoop\share\hadoop\hdfs;C:\hdp\hadoop\share\hadoop\hdfs\lib\*;C:\hdp\hadoop\share\hadoop\hdfs\*;C:\hdp\hadoop\share\hadoop\yarn\lib\*;C:\hdp\hadoop\share\hadoop\yarn\*;C:\hdp\hadoop\share\hadoop\mapreduce\lib\*;C:\hdp\hadoop\share\hadoop\mapreduce\*;C:\hdp\hadoop\share\hadoop\yarn\*;C:\hdp\hadoop\share\hadoop\mapreduce\lib\*;C:\hdp\hadoop\modules\*
+    </value>
     <description>
-      CLASSPATH for MR applications. A comma-separated list of CLASSPATH
-      entries.
+      CLASSPATH for MR applications. A comma-separated list of CLASSPATH entries.
     </description>
   </property>
 
   <property>
     <name>mapreduce.application.framework.path</name>
-    <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
-    <description></description>
+    <deleted>true</deleted>
+    <value/>
+    <description/>
   </property>
 
   <property>
     <name>yarn.app.mapreduce.am.admin-command-opts</name>
-    <value>-Dhdp.version=${hdp.version}</value>
-    <description>
-      Java opts for the MR App Master processes.
-      The following symbol, if present, will be interpolated: @taskid@ is replaced
-      by current TaskID. Any other occurrences of '@' will go unchanged.
-      For example, to enable verbose gc logging to a file named for the taskid in
-      /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
-      -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
-
-      Usage of -Djava.library.path can cause programs to no longer function if
-      hadoop native libraries are used. These values should instead be set as part
-      of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
-      mapreduce.reduce.env config settings.
-    </description>
+    <deleted>true</deleted>
+    <value/>
+    <description/>
   </property>
 
   <property>
     <name>mapreduce.admin.map.child.java.opts</name>
-    <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
-    <description></description>
+    <deleted>true</deleted>
+    <value/>
+    <description/>
   </property>
 
   <property>
     <name>mapreduce.admin.reduce.child.java.opts</name>
-    <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
-    <description></description>
+    <deleted>true</deleted>
+    <value/>
+    <description/>
   </property>
 
   <property>
     <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
     <value>1</value>
-    <description></description>
+    <description/>
   </property>
 
   <property>
     <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
     <value>1000</value>
-    <description></description>
+    <description/>
   </property>
 
   <property>
     <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
     <value>30000</value>
-    <description></description>
+    <description/>
   </property>
 
   <property>
     <name>mapreduce.job.emit-timeline-data</name>
     <value>false</value>
-    <description></description>
+    <description/>
   </property>
 
   <property>
     <name>mapreduce.jobhistory.bind-host</name>
     <value>0.0.0.0</value>
-    <description></description>
+    <description/>
   </property>
 
 </configuration>


[2/2] ambari git commit: AMBARI-35 - Monarch: preupload oozie files not executed (Ivan Kozlov via abaranchuk)

Posted by ab...@apache.org.
AMBARI-35 - Monarch: preupload oozie files not executed (Ivan Kozlov via abaranchuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/50ee8e02
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/50ee8e02
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/50ee8e02

Branch: refs/heads/trunk
Commit: 50ee8e02993e3d34ea46dc81dffbbf2c02348332
Parents: de026eb
Author: Artem Baranchuk <ab...@hortonworks.con>
Authored: Fri Feb 6 01:19:20 2015 +0200
Committer: Artem Baranchuk <ab...@hortonworks.con>
Committed: Fri Feb 6 01:19:20 2015 +0200

----------------------------------------------------------------------
 .../src/main/resources/scripts/Ambaripreupload.py    | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/50ee8e02/ambari-server/src/main/resources/scripts/Ambaripreupload.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
index 7a6bf67..89e0742 100644
--- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py
+++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
@@ -63,6 +63,8 @@ class params():
   hadoop_conf_dir = "/etc/hadoop/conf"
   user_group = "hadoop"
   security_enabled = False
+  oozie_user = "oozie"
+  execute_path = "/usr/hdp/current/hadoop-client/bin"
  
 params = params()
  
@@ -255,8 +257,17 @@ with Environment() as env:
  
   oozie_cmd = format("{put_shared_lib_to_hdfs_cmd} ; hadoop --config {hadoop_conf_dir} dfs -chmod -R 755 {oozie_hdfs_user_dir}/share")
   not_if_command = format("{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls /user/oozie/share | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'")
- 
-  #Execute(format("hadoop --config {hadoop_conf_dir} dfs -mkdir {
+  HdfsDirectory(format("{oozie_hdfs_user_dir}/share"),
+                action="create",
+                owner=oozie_user,
+                mode=0555,
+                conf_dir=params.hadoop_conf_dir,
+                hdfs_user=params.hdfs_user,
+                )
+
+  Execute( oozie_cmd, user = params.oozie_user, not_if = not_if_command,
+           path = params.execute_path )
+
   copy_tarballs_to_hdfs("/usr/hdp/current/hadoop-client/mapreduce.tar.gz", "wasb:///hdp/apps/{{ hdp_stack_version }}/mapreduce/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
   copy_tarballs_to_hdfs("/usr/hdp/current/tez-client/lib/tez.tar.gz", "wasb:///hdp/apps/{{ hdp_stack_version }}/tez/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
   copy_tarballs_to_hdfs("/usr/hdp/current/hive-client/hive.tar.gz", "wasb:///hdp/apps/{{ hdp_stack_version }}/hive/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)