You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2014/06/01 08:09:54 UTC
[1/2] git commit: OOZIE-1724 Make it easier to specify the HCat
hive-site.xml for the Oozie Server (rkanter)
Repository: oozie
Updated Branches:
refs/heads/master 3637874e9 -> b6769f4dd
OOZIE-1724 Make it easier to specify the HCat hive-site.xml for the Oozie Server (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/83af85c1
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/83af85c1
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/83af85c1
Branch: refs/heads/master
Commit: 83af85c19133b2a9d5a482ca2f61949c8327d348
Parents: 3637874
Author: Robert Kanter <rk...@cloudera.com>
Authored: Sat May 31 23:00:54 2014 -0700
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Sat May 31 23:00:54 2014 -0700
----------------------------------------------------------------------
.../apache/oozie/dependency/HCatURIHandler.java | 4 ++
.../oozie/service/HCatAccessorService.java | 70 ++++++++++++++++++++
.../hadoop/TestLauncherHCatURIHandler.java | 2 +
.../apache/oozie/coord/TestHCatELFunctions.java | 2 +
.../oozie/dependency/TestHCatURIHandler.java | 2 +
.../oozie/service/TestHCatAccessorService.java | 51 +++++++++++++-
docs/src/site/twiki/AG_Install.twiki | 11 ++-
release-log.txt | 1 +
8 files changed, 139 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java b/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java
index fecd783..da78105 100644
--- a/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java
+++ b/core/src/main/java/org/apache/oozie/dependency/HCatURIHandler.java
@@ -177,6 +177,10 @@ public class HCatURIHandler implements URIHandler {
}
private HCatClient getHCatClient(URI uri, Configuration conf, String user) throws HCatAccessorException {
+ HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
+ if (hcatService.getHCatConf() != null) {
+ conf = hcatService.getHCatConf();
+ }
final HiveConf hiveConf = new HiveConf(conf, this.getClass());
String serverURI = getMetastoreConnectURI(uri);
if (!serverURI.equals("")) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java b/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java
index a4b23de..ebef79b 100644
--- a/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java
+++ b/core/src/main/java/org/apache/oozie/service/HCatAccessorService.java
@@ -17,6 +17,10 @@
*/
package org.apache.oozie.service;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
@@ -27,16 +31,22 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.ErrorCode;
import org.apache.oozie.dependency.hcat.HCatMessageHandler;
import org.apache.oozie.jms.JMSConnectionInfo;
import org.apache.oozie.util.HCatURI;
import org.apache.oozie.util.MappingRule;
+import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XLog;
public class HCatAccessorService implements Service {
public static final String CONF_PREFIX = Service.CONF_PREFIX + "HCatAccessorService.";
public static final String JMS_CONNECTIONS_PROPERTIES = CONF_PREFIX + "jmsconnections";
+ public static final String HCAT_CONFIGURATION = CONF_PREFIX + "hcat.configuration";
private static XLog LOG;
private static String DELIMITER = "#";
@@ -44,6 +54,7 @@ public class HCatAccessorService implements Service {
private JMSAccessorService jmsService;
private List<MappingRule> mappingRules;
private JMSConnectionInfo defaultJMSConnInfo;
+ private Configuration hcatConf;
/**
* Map of publisher(host:port) to JMS connection info
*/
@@ -66,6 +77,65 @@ public class HCatAccessorService implements Service {
this.nonJMSPublishers = new HashSet<String>();
this.publisherJMSConnInfoMap = new HashMap<String, JMSConnectionInfo>();
this.registeredTopicsMap = new HashMap<String, String>();
+ try {
+ loadHCatConf(services);
+ } catch(IOException ioe) {
+ throw new ServiceException(ErrorCode.E0100, HCatAccessorService.class.getName(), "An exception occured while attempting"
+ + "to load the HCat Configuration", ioe);
+ }
+ }
+
+ private void loadHCatConf(Services services) throws IOException {
+ String path = conf.get(HCAT_CONFIGURATION);
+ if (path != null) {
+ if (path.startsWith("hdfs")) {
+ Path p = new Path(path);
+ HadoopAccessorService has = services.get(HadoopAccessorService.class);
+ try {
+ FileSystem fs = has.createFileSystem(
+ System.getProperty("user.name"), p.toUri(), has.createJobConf(p.toUri().getAuthority()));
+ if (fs.exists(p)) {
+ FSDataInputStream is = null;
+ try {
+ is = fs.open(p);
+ hcatConf = new XConfiguration(is);
+ } finally {
+ if (is != null) {
+ is.close();
+ }
+ }
+ LOG.info("Loaded HCat Configuration: " + path);
+ } else {
+ LOG.warn("HCat Configuration could not be found at [" + path + "]");
+ }
+ } catch (HadoopAccessorException hae) {
+ throw new IOException(hae);
+ }
+ } else {
+ File f = new File(path);
+ if (f.exists()) {
+ InputStream is = null;
+ try {
+ is = new FileInputStream(f);
+ hcatConf = new XConfiguration(is);
+ } finally {
+ if (is != null) {
+ is.close();
+ }
+ }
+ LOG.info("Loaded HCat Configuration: " + path);
+ } else {
+ LOG.warn("HCat Configuration could not be found at [" + path + "]");
+ }
+ }
+ }
+ else {
+ LOG.info("HCat Configuration not specified");
+ }
+ }
+
+ public Configuration getHCatConf() {
+ return hcatConf;
}
private void initializeMappingRules() {
http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java
index 47d766d..7a8c00b 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherHCatURIHandler.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.oozie.dependency.FSURIHandler;
import org.apache.oozie.dependency.HCatURIHandler;
import org.apache.oozie.dependency.URIHandler;
+import org.apache.oozie.service.HCatAccessorService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.test.XHCatTestCase;
@@ -43,6 +44,7 @@ public class TestLauncherHCatURIHandler extends XHCatTestCase {
services = new Services();
services.getConf().set(URIHandlerService.URI_HANDLERS,
FSURIHandler.class.getName() + "," + HCatURIHandler.class.getName());
+ services.setService(HCatAccessorService.class);
services.init();
conf = createJobConf();
uriService = Services.get().get(URIHandlerService.class);
http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java b/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java
index a689cb1..f63b682 100644
--- a/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java
+++ b/core/src/test/java/org/apache/oozie/coord/TestHCatELFunctions.java
@@ -27,6 +27,7 @@ import org.apache.oozie.client.OozieClient;
import org.apache.oozie.dependency.FSURIHandler;
import org.apache.oozie.dependency.HCatURIHandler;
import org.apache.oozie.service.ELService;
+import org.apache.oozie.service.HCatAccessorService;
import org.apache.oozie.service.LiteWorkflowStoreService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.URIHandlerService;
@@ -52,6 +53,7 @@ public class TestHCatELFunctions extends XHCatTestCase {
services = new Services();
services.getConf().set(URIHandlerService.URI_HANDLERS,
FSURIHandler.class.getName() + "," + HCatURIHandler.class.getName());
+ services.setService(HCatAccessorService.class);
services.init();
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java b/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java
index a23ed11..a2383c6 100644
--- a/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java
+++ b/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java
@@ -20,6 +20,7 @@ package org.apache.oozie.dependency;
import java.net.URI;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.oozie.service.HCatAccessorService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.test.XHCatTestCase;
@@ -39,6 +40,7 @@ public class TestHCatURIHandler extends XHCatTestCase {
services = new Services();
services.getConf().set(URIHandlerService.URI_HANDLERS,
FSURIHandler.class.getName() + "," + HCatURIHandler.class.getName());
+ services.setService(HCatAccessorService.class);
services.init();
conf = createJobConf();
uriService = Services.get().get(URIHandlerService.class);
http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/core/src/test/java/org/apache/oozie/service/TestHCatAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestHCatAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHCatAccessorService.java
index 978c72c..43ac3db 100644
--- a/core/src/test/java/org/apache/oozie/service/TestHCatAccessorService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestHCatAccessorService.java
@@ -17,14 +17,18 @@
*/
package org.apache.oozie.service;
+import java.io.File;
+import java.io.FileOutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
import org.apache.oozie.jms.JMSConnectionInfo;
-import org.apache.oozie.test.XTestCase;
+import org.apache.oozie.test.XHCatTestCase;
import org.junit.Test;
-public class TestHCatAccessorService extends XTestCase {
+public class TestHCatAccessorService extends XHCatTestCase {
private Services services;
@Override
@@ -36,7 +40,9 @@ public class TestHCatAccessorService extends XTestCase {
@Override
protected void tearDown() throws Exception {
- services.destroy();
+ if (services != null) {
+ services.destroy();
+ }
super.tearDown();
}
@@ -100,4 +106,43 @@ public class TestHCatAccessorService extends XTestCase {
connInfo.getJNDIPropertiesString());
}
+ @Test
+ public void testGetHCatConfLocal() throws Exception {
+ File hcatConfFile = new File(getTestCaseConfDir(), "hive-site.xml");
+ assertFalse(hcatConfFile.exists());
+ assertNull(services.get(HCatAccessorService.class).getHCatConf());
+
+ Configuration hcatConf = new Configuration(false);
+ hcatConf.set("A", "a");
+ hcatConf.writeXml(new FileOutputStream(hcatConfFile));
+ assertTrue(hcatConfFile.exists());
+ services.destroy();
+ services = super.setupServicesForHCatalog();
+ Configuration conf = services.getConf();
+ conf.set("oozie.service.HCatAccessorService.hcat.configuration", hcatConfFile.getAbsolutePath());
+ services.init();
+ Configuration hcatConfLoaded = services.get(HCatAccessorService.class).getHCatConf();
+ assertEquals("a", hcatConfLoaded.get("A"));
+ }
+
+ @Test
+ public void testGetHCatConfHDFS() throws Exception {
+ Path hcatConfPath = new Path(getFsTestCaseDir(), "hive-site.xml");
+ assertFalse(getFileSystem().exists(hcatConfPath));
+ assertNull(services.get(HCatAccessorService.class).getHCatConf());
+
+ Configuration hcatConf = new Configuration(false);
+ hcatConf.set("A", "a");
+ FSDataOutputStream out = getFileSystem().create(hcatConfPath);
+ hcatConf.writeXml(out);
+ out.close();
+ assertTrue(getFileSystem().exists(hcatConfPath));
+ services.destroy();
+ services = super.setupServicesForHCatalog();
+ Configuration conf = services.getConf();
+ conf.set("oozie.service.HCatAccessorService.hcat.configuration", hcatConfPath.toUri().toString());
+ services.init();
+ Configuration hcatConfLoaded = services.get(HCatAccessorService.class).getHCatConf();
+ assertEquals("a", hcatConfLoaded.get("A"));
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/docs/src/site/twiki/AG_Install.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki
index e343d7e..89f7407 100644
--- a/docs/src/site/twiki/AG_Install.twiki
+++ b/docs/src/site/twiki/AG_Install.twiki
@@ -422,7 +422,16 @@ the Oozie server to enable Oozie to work with HCatalog.
*Adding HCatalog jars to Oozie war:*
For Oozie server to talk to HCatalog server, HCatalog and hive jars need to be in the server classpath.
-hive-site.xml which has the configuration to talk to the HCatalog server also needs to be in the classpath.
+hive-site.xml which has the configuration to talk to the HCatalog server also needs to be in the classpath or specified by the
+following configuration property in oozie-site.xml:
+<verbatim>
+ <property>
+ <name>oozie.service.HCatAccessorService.hcat.configuration</name>
+ <value>/local/filesystem/path/to/hive-site.xml</value>
+ </property>
+</verbatim>
+The hive-site.xml can also be placed in a location on HDFS and the above property can have a value
+of =hdfs://HOST:PORT/path/to/hive-site.xml= to point there instead of the local file system.
The oozie-[version]-hcataloglibs.tar.gz in the oozie distribution bundles the required hcatalog and hive jars that
needs to be placed in the Oozie server classpath. If using a version of HCatalog bundled in
http://git-wip-us.apache.org/repos/asf/oozie/blob/83af85c1/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index fbd7bea..e7eaa1e 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1724 Make it easier to specify the HCat hive-site.xml for the Oozie Server (rkanter)
OOZIE-1812 Bundle status is always in RUNNING if one of the action status is in PREP (puru via rohini)
OOZIE-1848 Pig actions fail due to missing joda-time jar from pig sharelib (bzhang)
OOZIE-1319 "LAST_ONLY" in execution control for coordinator job still runs all the actions (rkanter)
[2/2] git commit: OOZIE-1828 Introduce counters JobStatus terminal
states metrics (rkanter)
Posted by rk...@apache.org.
OOZIE-1828 Introduce counters JobStatus terminal states metrics (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/b6769f4d
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/b6769f4d
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/b6769f4d
Branch: refs/heads/master
Commit: b6769f4ddac8f983c1695f565ec320437a79ebd2
Parents: 83af85c
Author: Robert Kanter <rk...@cloudera.com>
Authored: Sat May 31 23:02:50 2014 -0700
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Sat May 31 23:02:50 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/oozie/command/wf/ActionXCommand.java | 4 +---
.../main/java/org/apache/oozie/command/wf/SignalXCommand.java | 3 +--
.../main/java/org/apache/oozie/command/wf/WorkflowXCommand.java | 5 +++++
release-log.txt | 1 +
4 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/b6769f4d/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java
index e38540f..0c4c548 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java
@@ -58,8 +58,6 @@ import org.apache.oozie.workflow.lite.LiteWorkflowInstance;
public abstract class ActionXCommand<T> extends WorkflowXCommand<Void> {
private static final String INSTRUMENTATION_GROUP = "action.executors";
- protected static final String INSTR_FAILED_JOBS_COUNTER = "failed";
-
protected static final String RECOVERY_ID_SEPARATOR = "@";
public ActionXCommand(String name, String type, int priority) {
@@ -197,7 +195,7 @@ public abstract class ActionXCommand<T> extends WorkflowXCommand<Void> {
action.resetPending();
queue(new NotificationXCommand(workflow, action));
queue(new KillXCommand(workflow.getId()));
- InstrumentUtils.incrJobCounter(INSTR_FAILED_JOBS_COUNTER, 1, getInstrumentation());
+ InstrumentUtils.incrJobCounter(INSTR_FAILED_JOBS_COUNTER_NAME, 1, getInstrumentation());
}
catch (WorkflowException ex) {
throw new CommandException(ex);
http://git-wip-us.apache.org/repos/asf/oozie/blob/b6769f4d/core/src/main/java/org/apache/oozie/command/wf/SignalXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SignalXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SignalXCommand.java
index 5c457f8..7968973 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SignalXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SignalXCommand.java
@@ -73,8 +73,6 @@ import org.apache.oozie.client.OozieClient;
@SuppressWarnings("deprecation")
public class SignalXCommand extends WorkflowXCommand<Void> {
- protected static final String INSTR_SUCCEEDED_JOBS_COUNTER_NAME = "succeeded";
-
private JPAService jpaService = null;
private String jobId;
private String actionId;
@@ -282,6 +280,7 @@ public class SignalXCommand extends WorkflowXCommand<Void> {
boolean isUserRetry = false;
ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(wfJob, wfAction, isRetry,
isUserRetry);
+ InstrumentUtils.incrJobCounter(INSTR_KILLED_JOBS_COUNTER_NAME, 1, getInstrumentation());
try {
String tmpNodeConf = nodeDef.getConf();
String actionConf = context.getELEvaluator().evaluate(tmpNodeConf, String.class);
http://git-wip-us.apache.org/repos/asf/oozie/blob/b6769f4d/core/src/main/java/org/apache/oozie/command/wf/WorkflowXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/WorkflowXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/WorkflowXCommand.java
index 0eee6b1..8995fd2 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/WorkflowXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/WorkflowXCommand.java
@@ -32,6 +32,11 @@ import org.apache.oozie.event.WorkflowJobEvent;
* @param <T>
*/
public abstract class WorkflowXCommand<T> extends XCommand<T> {
+
+ protected static final String INSTR_SUCCEEDED_JOBS_COUNTER_NAME = "succeeded";
+ protected static final String INSTR_KILLED_JOBS_COUNTER_NAME = "killed";
+ protected static final String INSTR_FAILED_JOBS_COUNTER_NAME = "failed";
+
/**
* Base class constructor for workflow commands.
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/b6769f4d/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index e7eaa1e..05894cf 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1828 Introduce counters JobStatus terminal states metrics (rkanter)
OOZIE-1724 Make it easier to specify the HCat hive-site.xml for the Oozie Server (rkanter)
OOZIE-1812 Bundle status is always in RUNNING if one of the action status is in PREP (puru via rohini)
OOZIE-1848 Pig actions fail due to missing joda-time jar from pig sharelib (bzhang)