You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2014/05/12 19:16:08 UTC
git commit: AMBARI-5720. pig.properties should set
pig.location.check.strict to false. (Jonathan Hurley via swagle)
Repository: ambari
Updated Branches:
refs/heads/trunk 7e395b9bb -> b7dd31447
AMBARI-5720. pig.properties should set pig.location.check.strict to false. (Jonathan Hurley via swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b7dd3144
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b7dd3144
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b7dd3144
Branch: refs/heads/trunk
Commit: b7dd314479888fe88c6483e875e78396ecb933fe
Parents: 7e395b9
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Mon May 12 10:15:55 2014 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Mon May 12 10:15:55 2014 -0700
----------------------------------------------------------------------
.../PIG/configuration/pig-properties.xml | 89 +++++++++++++++++++
.../services/PIG/configuration/pig.properties | 52 -----------
.../stacks/HDP/2.0.6/services/PIG/metainfo.xml | 1 +
.../services/PIG/package/scripts/params.py | 11 ++-
.../2.0.6/services/PIG/package/scripts/pig.py | 9 +-
.../PIG/package/templates/pig.properties.j2 | 73 ----------------
.../PIG/configuration/pig-properties.xml | 91 ++++++++++++++++++++
.../python/stacks/2.0.6/PIG/test_pig_client.py | 18 ++--
.../python/stacks/2.0.6/configs/default.json | 8 +-
.../python/stacks/2.0.6/configs/secured.json | 3 +
10 files changed, 221 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml
new file mode 100644
index 0000000..185d3b0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+ <property>
+ <name>pig-content</name>
+ <description>Describe all the Pig agent configurations</description>
+ <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.
+# see bin/pig -help
+
+# brief logging (no timestamps)
+brief=false
+
+# debug level, INFO is default
+debug=INFO
+
+# verbose print all log messages to screen (default to print only INFO and above to screen)
+verbose=false
+
+# exectype local|mapreduce, mapreduce is default
+exectype=mapreduce
+
+# Enable insertion of information about script into hadoop job conf
+pig.script.info.enabled=true
+
+# Do not spill temp files smaller than this size (bytes)
+pig.spill.size.threshold=5000000
+
+# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+# This should help reduce the number of files being spilled.
+pig.spill.gc.activation.size=40000000
+
+# the following two parameters are to help estimate the reducer number
+pig.exec.reducers.bytes.per.reducer=1000000000
+pig.exec.reducers.max=999
+
+# Temporary location to store the intermediate data.
+pig.temp.dir=/tmp/
+
+# Threshold for merging FRJoin fragment files
+pig.files.concatenation.threshold=100
+pig.optimistic.files.concatenation=false;
+
+pig.disable.counter=false
+
+hcat.bin=/usr/bin/hcat
+
+ </value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig.properties
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig.properties b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig.properties
deleted file mode 100644
index 01000b5..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig.properties
+++ /dev/null
@@ -1,52 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.
-# see bin/pig -help
-
-# brief logging (no timestamps)
-brief=false
-
-#debug level, INFO is default
-debug=INFO
-
-#verbose print all log messages to screen (default to print only INFO and above to screen)
-verbose=false
-
-#exectype local|mapreduce, mapreduce is default
-exectype=mapreduce
-
-#Enable insertion of information about script into hadoop job conf
-pig.script.info.enabled=true
-
-#Do not spill temp files smaller than this size (bytes)
-pig.spill.size.threshold=5000000
-#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
-#This should help reduce the number of files being spilled.
-pig.spill.gc.activation.size=40000000
-
-#the following two parameters are to help estimate the reducer number
-pig.exec.reducers.bytes.per.reducer=1000000000
-pig.exec.reducers.max=999
-
-#Temporary location to store the intermediate data.
-pig.temp.dir=/tmp/
-
-#Threshold for merging FRJoin fragment files
-pig.files.concatenation.threshold=100
-pig.optimistic.files.concatenation=false;
-
-pig.disable.counter=false
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml
index 7db411b..c8a3cf4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml
@@ -55,6 +55,7 @@
<configuration-dependencies>
<config-type>global</config-type>
<config-type>pig-log4j</config-type>
+ <config-type>pig-properties</config-type>
</configuration-dependencies>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
index c44acfa..747687d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
@@ -38,7 +38,16 @@ kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "
java64_home = config['hostLevelParams']['java_home']
hadoop_home = "/usr"
-#log4j.properties
+# pig.properties - if not in the JSON command, then we need to esnure some
+# basic properties are set; this is a safety mechanism
+if (('pig-properties' in config['configurations']) and ('pig-content' in config['configurations']['pig-properties'])):
+ pig_properties = config['configurations']['pig-properties']['pig-content']
+else:
+ pig_properties = {}
+ pig_properties['hcat.bin'] = '/usr/bin/hcat'
+ pig_properties['pig.location.check.strict'] = 'false'
+
+# log4j.properties
if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])):
log4j_props = config['configurations']['pig-log4j']['content']
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py
index 329a0bc..629f7cb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py
@@ -30,7 +30,14 @@ def pig():
group = params.user_group
)
- pig_TemplateConfig( ['pig-env.sh','pig.properties'])
+ pig_TemplateConfig( ['pig-env.sh'])
+
+ # pig_properties is always set to a default even if it's not in the payload
+ PropertiesFile(format("{params.pig_conf_dir}/pig.properties"),
+ mode = 0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ properties=params.pig_properties)
if (params.log4j_props != None):
File(format("{params.pig_conf_dir}/log4j.properties"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/templates/pig.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/templates/pig.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/templates/pig.properties.j2
deleted file mode 100644
index 90a32b1..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/templates/pig.properties.j2
+++ /dev/null
@@ -1,73 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-#
-
-# Pig configuration file. All values can be overwritten by command line arguments.
-
-# log4jconf log4j configuration file
-# log4jconf=./conf/log4j.properties
-
-# a file that contains pig script
-#file=
-
-# load jarfile, colon separated
-#jar=
-
-#verbose print all log messages to screen (default to print only INFO and above to screen)
-#verbose=true
-
-#exectype local|mapreduce, mapreduce is default
-#exectype=local
-
-#pig.logfile=
-
-#Do not spill temp files smaller than this size (bytes)
-#pig.spill.size.threshold=5000000
-#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
-#This should help reduce the number of files being spilled.
-#pig.spill.gc.activation.size=40000000
-
-#the following two parameters are to help estimate the reducer number
-#pig.exec.reducers.bytes.per.reducer=1000000000
-#pig.exec.reducers.max=999
-
-#Use this option only when your Pig job will otherwise die because of
-#using more counter than hadoop configured limit
-#pig.disable.counter=true
-hcat.bin=/usr/bin/hcat
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/main/resources/stacks/HDP/2.1/services/PIG/configuration/pig-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/PIG/configuration/pig-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/PIG/configuration/pig-properties.xml
new file mode 100644
index 0000000..97d42bb
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/PIG/configuration/pig-properties.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+
+ <property>
+ <name>pig-content</name>
+ <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.
+# see bin/pig -help
+
+# brief logging (no timestamps)
+brief=false
+
+# debug level, INFO is default
+debug=INFO
+
+# verbose print all log messages to screen (default to print only INFO and above to screen)
+verbose=false
+
+# exectype local|mapreduce, mapreduce is default
+exectype=mapreduce
+
+# Enable insertion of information about script into hadoop job conf
+pig.script.info.enabled=true
+
+# Do not spill temp files smaller than this size (bytes)
+pig.spill.size.threshold=5000000
+
+# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+# This should help reduce the number of files being spilled.
+pig.spill.gc.activation.size=40000000
+
+# the following two parameters are to help estimate the reducer number
+pig.exec.reducers.bytes.per.reducer=1000000000
+pig.exec.reducers.max=999
+
+# Temporary location to store the intermediate data.
+pig.temp.dir=/tmp/
+
+# Threshold for merging FRJoin fragment files
+pig.files.concatenation.threshold=100
+pig.optimistic.files.concatenation=false;
+
+pig.disable.counter=false
+
+# Avoid pig failures when multiple jobs write to the same location
+pig.location.check.strict=false
+
+hcat.bin=/usr/bin/hcat
+
+ </value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
index 5ff3b4f..87b8e0c 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
@@ -31,13 +31,16 @@ class TestPigClient(RMFTestCase):
self.assertResourceCalled('Directory', '/etc/pig/conf',
owner = 'hdfs',
- group = 'hadoop',
+ group = 'hadoop'
)
self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig-env.sh',
- owner = 'hdfs',
+ owner = 'hdfs'
)
- self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
+ self.assertResourceCalled('PropertiesFile', '/etc/pig/conf/pig.properties',
owner = 'hdfs',
+ group = 'hadoop',
+ mode = 0644,
+ properties = 'pigproperties\nline2'
)
self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
owner = 'hdfs',
@@ -56,13 +59,16 @@ class TestPigClient(RMFTestCase):
self.assertResourceCalled('Directory', '/etc/pig/conf',
owner = 'hdfs',
- group = 'hadoop',
+ group = 'hadoop'
)
self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig-env.sh',
- owner = 'hdfs',
+ owner = 'hdfs'
)
- self.assertResourceCalled('TemplateConfig', '/etc/pig/conf/pig.properties',
+ self.assertResourceCalled('PropertiesFile', '/etc/pig/conf/pig.properties',
owner = 'hdfs',
+ group = 'hadoop',
+ mode = 0644,
+ properties = 'pigproperties\nline2'
)
self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
owner = 'hdfs',
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index 3561b4b..dc61567 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -512,6 +512,9 @@
"pig-log4j": {
"content": "log4jproperties\nline2"
},
+ "pig-properties": {
+ "pig-content": "pigproperties\nline2"
+ },
"oozie-log4j": {
"content": "log4jproperties\nline2"
},
@@ -579,7 +582,10 @@
},
"pig-log4j": {
"tag": "version1"
- }
+ },
+ "pig-properties": {
+ "tag": "version1"
+ }
},
"commandId": "7-1",
"clusterHostInfo": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7dd3144/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
index ffa1c55..82b9855 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
@@ -636,6 +636,9 @@
"pig-log4j": {
"content": "log4jproperties\nline2"
},
+ "pig-properties": {
+ "pig-content": "pigproperties\nline2"
+ },
"oozie-log4j": {
"content": "log4jproperties\nline2"
}