You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2017/01/31 03:25:35 UTC
ambari git commit: AMBARI-19702. Switch Pig default engine to tez in
HDP 2.6 (Daniel Dai via alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk 0b157069a -> 2f06c1cef
AMBARI-19702. Switch Pig default engine to tez in HDP 2.6 (Daniel Dai via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2f06c1ce
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2f06c1ce
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2f06c1ce
Branch: refs/heads/trunk
Commit: 2f06c1cefac30e0a53d631cf2a2a6aef64a89e0c
Parents: 0b15706
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Mon Jan 30 19:26:18 2017 -0800
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Mon Jan 30 19:26:18 2017 -0800
----------------------------------------------------------------------
.../stacks/HDP/2.3/upgrades/config-upgrade.xml | 11 +++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 5 ++
.../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 3 +
.../stacks/HDP/2.4/upgrades/config-upgrade.xml | 11 +++
.../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 5 ++
.../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 3 +
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 11 +++
.../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 5 ++
.../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 3 +
.../PIG/configuration/pig-properties.xml | 93 ++++++++++++++++++++
10 files changed, 150 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index 1f3b5ad..3ea660d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -123,6 +123,17 @@
</component>
</service>
+ <service name="PIG">
+ <component name="PIG">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez">
+ <type>pig-properties</type>
+ <replace key="exectype" find="exectype=mapreduce" replace-with="exectype=tez" />
+ </definition>
+ </changes>
+ </component>
+ </service>
+
<service name="HIVE">
<component name="HIVE_SERVER">
<changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
index a9b1480..6153a8c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
@@ -359,6 +359,11 @@
<task xsi:type="configure" id="hdp_2_5_0_0_add_sqoop_atlas_security_configs" />
</execute-stage>
+ <!-- PIG -->
+ <execute-stage service="PIG" component="PIG" title="Apply config changes for Pig">
+ <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez" />
+ </execute-stage>
+
<!-- HIVE -->
<execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for WebHCat Server">
<task xsi:type="configure" id="hdp_2_5_0_0_webhcat_server_update_configuration_paths"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
index a65b32c..daf249c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
@@ -831,6 +831,9 @@
<service name="PIG">
<component name="PIG">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez"/>
+ </pre-upgrade>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index 7179099..8c8997d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -57,6 +57,17 @@
</component>
</service>
+ <service name="PIG">
+ <component name="PIG">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez">
+ <type>pig-properties</type>
+ <replace key="exectype" find="exectype=mapreduce" replace-with="exectype=tez" />
+ </definition>
+ </changes>
+ </component>
+ </service>
+
<service name="HIVE">
<component name="HIVE_SERVER">
<changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
index 7b9b062..f473071 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
@@ -408,6 +408,11 @@
<task xsi:type="configure" id="hdp_2_5_0_0_add_sqoop_atlas_security_configs" />
</execute-stage>
+ <!-- PIG -->
+ <execute-stage service="PIG" component="PIG" title="Apply config changes for Pig">
+ <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez" />
+ </execute-stage>
+
<!-- HIVE -->
<execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hive_audit_db"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
index 781149c..2be1ad7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
@@ -834,6 +834,9 @@
<service name="PIG">
<component name="PIG">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez"/>
+ </pre-upgrade>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 157e0e9..2aca9b8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -278,6 +278,17 @@
</component>
</service>
+ <service name="PIG">
+ <component name="PIG">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez">
+ <type>pig-properties</type>
+ <replace key="exectype" find="exectype=mapreduce" replace-with="exectype=tez" />
+ </definition>
+ </changes>
+ </component>
+ </service>
+
<service name="HIVE">
<component name="HIVE_SERVER">
<changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index 8092506..1adf75f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -413,6 +413,11 @@
</task>
</execute-stage>
+ <!-- PIG -->
+ <execute-stage service="PIG" component="PIG" title="Apply config changes for Pig">
+ <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez" />
+ </execute-stage>
+
<!--HIVE-->
<execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties">
<task xsi:type="configure" id="hive_log4j_parameterize">
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index a739551..6287789 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -734,6 +734,9 @@
<service name="PIG">
<component name="PIG">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_6_0_0_pig_use_tez"/>
+ </pre-upgrade>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2f06c1ce/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/configuration/pig-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/configuration/pig-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/configuration/pig-properties.xml
new file mode 100644
index 0000000..0f3ebdd
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/PIG/configuration/pig-properties.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <display-name>pig-properties template</display-name>
+ <description>Describe all the Pig agent configurations</description>
+ <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.
+# see bin/pig -help
+
+# brief logging (no timestamps)
+brief=false
+
+# debug level, INFO is default
+debug=INFO
+
+# verbose print all log messages to screen (default to print only INFO and above to screen)
+verbose=false
+
+# exectype local|mapreduce|tez, mapreduce is default
+exectype=tez
+
+# Enable insertion of information about script into hadoop job conf
+pig.script.info.enabled=true
+
+# Do not spill temp files smaller than this size (bytes)
+pig.spill.size.threshold=5000000
+
+# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+# This should help reduce the number of files being spilled.
+pig.spill.gc.activation.size=40000000
+
+# the following two parameters are to help estimate the reducer number
+pig.exec.reducers.bytes.per.reducer=1000000000
+pig.exec.reducers.max=999
+
+# Temporary location to store the intermediate data.
+pig.temp.dir=/tmp/
+
+# Threshold for merging FRJoin fragment files
+pig.files.concatenation.threshold=100
+pig.optimistic.files.concatenation=false;
+
+pig.disable.counter=false
+
+hcat.bin=/usr/bin/hcat
+
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <empty-value-valid>true</empty-value-valid>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>