You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by sh...@apache.org on 2015/05/22 07:42:27 UTC
oozie git commit: OOZIE-1963 Create a Hive Server 2 example
(qwertymaniac via shwethags)
Repository: oozie
Updated Branches:
refs/heads/master 1fd6a6b41 -> 3c8c2016e
OOZIE-1963 Create a Hive Server 2 example (qwertymaniac via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3c8c2016
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3c8c2016
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3c8c2016
Branch: refs/heads/master
Commit: 3c8c2016ece188e1a68209c6fb2640f747756c2c
Parents: 1fd6a6b
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Fri May 22 11:12:18 2015 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Fri May 22 11:12:18 2015 +0530
----------------------------------------------------------------------
examples/src/main/apps/hive2/README | 14 +++++
examples/src/main/apps/hive2/job.properties | 26 +++++++++
.../src/main/apps/hive2/job.properties.security | 27 +++++++++
examples/src/main/apps/hive2/script.q | 19 ++++++
examples/src/main/apps/hive2/workflow.xml | 49 ++++++++++++++++
.../src/main/apps/hive2/workflow.xml.security | 61 ++++++++++++++++++++
release-log.txt | 1 +
7 files changed, 197 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/README
----------------------------------------------------------------------
diff --git a/examples/src/main/apps/hive2/README b/examples/src/main/apps/hive2/README
new file mode 100644
index 0000000..4558e17
--- /dev/null
+++ b/examples/src/main/apps/hive2/README
@@ -0,0 +1,14 @@
+Running the HiveServer2 example
+
+1. Setup and run a HiveServer2 (hive --service hiveserver2) to run on localhost, port 10000,
+ prior to running the example.
+2. When running in a secure cluster, follow below
+ a. The files job.properties.security and workflow.xml.security can be
+ used and modified as needed. Please refer to those two files and change the jdbcURL
+ and jdbcPrincipal as necessary.
+ b. Make sure oozie-site.xml under /etc/oozie/conf has the following property
+ specified.
+ <property>
+ <name>oozie.credentials.credentialclasses</name>
+ <value>hive2=org.apache.oozie.action.hadoop.Hive2Credentials</value>
+ </property>
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/job.properties
----------------------------------------------------------------------
diff --git a/examples/src/main/apps/hive2/job.properties b/examples/src/main/apps/hive2/job.properties
new file mode 100644
index 0000000..2640402
--- /dev/null
+++ b/examples/src/main/apps/hive2/job.properties
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+nameNode=hdfs://localhost:8020
+jobTracker=localhost:8021
+queueName=default
+jdbcURL=jdbc:hive2://localhost:10000/default
+examplesRoot=examples
+
+oozie.use.system.libpath=true
+
+oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/hive2
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/job.properties.security
----------------------------------------------------------------------
diff --git a/examples/src/main/apps/hive2/job.properties.security b/examples/src/main/apps/hive2/job.properties.security
new file mode 100644
index 0000000..ac980cd
--- /dev/null
+++ b/examples/src/main/apps/hive2/job.properties.security
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+nameNode=hdfs://localhost:8020
+jobTracker=localhost:8021
+queueName=default
+jdbcURL=jdbc:hive2://localhost:10000/default
+jdbcPrincipal=hive/localhost@EXAMPLE.COM
+examplesRoot=examples
+
+oozie.use.system.libpath=true
+
+oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/hive2
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/script.q
----------------------------------------------------------------------
diff --git a/examples/src/main/apps/hive2/script.q b/examples/src/main/apps/hive2/script.q
new file mode 100644
index 0000000..3abc757
--- /dev/null
+++ b/examples/src/main/apps/hive2/script.q
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+CREATE EXTERNAL TABLE test (a INT) STORED AS TEXTFILE LOCATION '${INPUT}';
+INSERT OVERWRITE DIRECTORY '${OUTPUT}' SELECT * FROM test;
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/workflow.xml
----------------------------------------------------------------------
diff --git a/examples/src/main/apps/hive2/workflow.xml b/examples/src/main/apps/hive2/workflow.xml
new file mode 100644
index 0000000..5f1ccc4
--- /dev/null
+++ b/examples/src/main/apps/hive2/workflow.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.5" name="hive2-wf">
+ <start to="hive2-node"/>
+
+ <action name="hive2-node">
+ <hive2 xmlns="uri:oozie:hive2-action:0.1">
+ <job-tracker>${jobTracker}</job-tracker>
+ <name-node>${nameNode}</name-node>
+ <prepare>
+ <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/hive2"/>
+ <mkdir path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data"/>
+ </prepare>
+ <configuration>
+ <property>
+ <name>mapred.job.queue.name</name>
+ <value>${queueName}</value>
+ </property>
+ </configuration>
+ <jdbc-url>${jdbcURL}</jdbc-url>
+ <script>script.q</script>
+ <param>INPUT=/user/${wf:user()}/${examplesRoot}/input-data/table</param>
+ <param>OUTPUT=/user/${wf:user()}/${examplesRoot}/output-data/hive2</param>
+ </hive2>
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+
+ <kill name="fail">
+ <message>Hive2 (Beeline) action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+ </kill>
+ <end name="end"/>
+</workflow-app>
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/examples/src/main/apps/hive2/workflow.xml.security
----------------------------------------------------------------------
diff --git a/examples/src/main/apps/hive2/workflow.xml.security b/examples/src/main/apps/hive2/workflow.xml.security
new file mode 100644
index 0000000..42ea4cf
--- /dev/null
+++ b/examples/src/main/apps/hive2/workflow.xml.security
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.5" name="hive2-wf">
+ <credentials>
+ <credential name="hs2-creds" type="hive2">
+ <property>
+ <name>hive2.server.principal</name>
+ <value>${jdbcPrincipal}</value>
+ </property>
+ <property>
+ <name>hive2.jdbc.url</name>
+ <value>${jdbcURL}</value>
+ </property>
+ </credential>
+ </credentials>
+ <start to="hive2-node"/>
+
+ <action name="hive2-node">
+ <hive2 xmlns="uri:oozie:hive2-action:0.1" cred="hs2-creds">
+ <job-tracker>${jobTracker}</job-tracker>
+ <name-node>${nameNode}</name-node>
+ <prepare>
+ <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/hive2"/>
+ <mkdir path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data"/>
+ </prepare>
+ <configuration>
+ <property>
+ <name>mapred.job.queue.name</name>
+ <value>${queueName}</value>
+ </property>
+ </configuration>
+ <jdbc-url>${jdbcURL}</jdbc-url>
+ <script>script.q</script>
+ <param>INPUT=/user/${wf:user()}/${examplesRoot}/input-data/table</param>
+ <param>OUTPUT=/user/${wf:user()}/${examplesRoot}/output-data/hive2</param>
+ </hive2>
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+
+ <kill name="fail">
+ <message>Hive2 (Beeline) action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+ </kill>
+ <end name="end"/>
+</workflow-app>
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c8c2016/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 23a8601..32eea5c 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -2,6 +2,7 @@
-- Oozie 4.2.0 release (unreleased)
+OOZIE-1963 Create a Hive Server 2 example (qwertymaniac via shwethags)
OOZIE-1993 Rerun fails during join in certain condition (shwethags)
OOZIE-2236 Need to package hive-hcatalog-server-extensions.jar in the hcatalog sharelib (venkatnrangan via bzhang)
OOZIE-2232 Oozie should invalidate bulk write command when "-filter" is missing (venkatnrangan via bzhang)