You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/03/22 11:23:10 UTC
[05/50] [abbrv] oozie git commit: Temp: addressing CodeReview
comments (JobConf mostly)
Temp: addressing CodeReview comments (JobConf mostly)
Change-Id: I5c9c173f894d62aa68452bbe56bbced1b9b6e8fa
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/74f4afb2
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/74f4afb2
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/74f4afb2
Branch: refs/heads/oya
Commit: 74f4afb223da3ce367304f6e115b4f5c9455a8b9
Parents: 4c237f9
Author: Peter Bacsko <pb...@cloudera.com>
Authored: Tue Jan 17 18:21:05 2017 +0100
Committer: Peter Bacsko <pb...@cloudera.com>
Committed: Tue Jan 17 18:21:05 2017 +0100
----------------------------------------------------------------------
.../org/apache/oozie/client/XOozieClient.java | 2 +-
.../java/org/apache/oozie/WorkflowJobBean.java | 2 +-
.../apache/oozie/action/hadoop/Credentials.java | 38 ------
.../action/hadoop/CredentialsProvider.java | 92 +++----------
.../hadoop/CredentialsProviderFactory.java | 102 +++++++++++++++
.../action/hadoop/HCatCredentialHelper.java | 19 +--
.../oozie/action/hadoop/HCatCredentials.java | 14 +-
.../oozie/action/hadoop/HbaseCredentials.java | 22 ++--
.../oozie/action/hadoop/Hive2Credentials.java | 13 +-
.../oozie/action/hadoop/HiveActionExecutor.java | 5 +-
.../oozie/action/hadoop/JavaActionExecutor.java | 128 +++++++------------
.../action/hadoop/LauncherMapperHelper.java | 14 +-
.../action/hadoop/MapReduceActionExecutor.java | 15 ++-
.../oozie/action/hadoop/PigActionExecutor.java | 5 +-
.../action/hadoop/SparkActionExecutor.java | 5 +-
.../action/hadoop/SqoopActionExecutor.java | 4 +-
.../oozie/service/HadoopAccessorService.java | 6 +-
.../oozie/action/hadoop/CredentialForTest.java | 15 +--
.../oozie/action/hadoop/InsertTestToken.java | 14 +-
.../action/hadoop/TestHCatCredentials.java | 34 ++---
.../action/hadoop/TestJavaActionExecutor.java | 27 ++--
.../oozie/action/hadoop/TestOozieJobInfo.java | 3 +-
.../oozie/client/TestWorkflowXClient.java | 2 +-
.../command/wf/TestActionCheckXCommand.java | 7 +-
.../command/wf/TestActionStartXCommand.java | 4 +-
.../oozie/service/TestConfigurationService.java | 4 +-
.../oozie/service/TestRecoveryService.java | 2 +-
.../hadoop/TestMapReduceActionExecutor.java | 28 ++--
28 files changed, 300 insertions(+), 326 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/client/src/main/java/org/apache/oozie/client/XOozieClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/client/XOozieClient.java b/client/src/main/java/org/apache/oozie/client/XOozieClient.java
index a76f544..379819b 100644
--- a/client/src/main/java/org/apache/oozie/client/XOozieClient.java
+++ b/client/src/main/java/org/apache/oozie/client/XOozieClient.java
@@ -116,7 +116,7 @@ public class XOozieClient extends OozieClient {
private void validateHttpSubmitConf(Properties conf) {
String RM = conf.getProperty(XOozieClient.RM);
if (RM == null) {
- throw new RuntimeException("jobtracker is not specified in conf");
+ throw new RuntimeException("Resource manager is not specified in conf");
}
String NN = conf.getProperty(XOozieClient.NN);
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/WorkflowJobBean.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/WorkflowJobBean.java b/core/src/main/java/org/apache/oozie/WorkflowJobBean.java
index f2f79dc..2042063 100644
--- a/core/src/main/java/org/apache/oozie/WorkflowJobBean.java
+++ b/core/src/main/java/org/apache/oozie/WorkflowJobBean.java
@@ -604,7 +604,7 @@ public class WorkflowJobBean implements Writable, WorkflowJob, JsonBean {
@SuppressWarnings("unchecked")
public List<WorkflowAction> getActions() {
- return (List<WorkflowAction>)(List<?>) actions;
+ return (List) actions;
}
public void setActions(List<WorkflowActionBean> nodes) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java
deleted file mode 100644
index 728d626..0000000
--- a/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.oozie.action.ActionExecutor.Context;
-
-public abstract class Credentials {
-
- /**
- * This is the interface for all the Credentials implementation. Any new credential implementaion must implement
- * this function. This function should modify the jobconf which will be used further to pass the credentials to the
- * tasks while running it. Credentials properties and context is also provided by that user can get all the
- * necessary configuration.
- *
- * @param jobconf job conf where the token will insert into
- * @param props properties for getting credential token or certificate
- * @param context workflow context
- * @throws Exception thrown if failed
- */
- public abstract void addtoJobConf(JobConf jobconf, CredentialsProperties props, Context context) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java
index 9cc1c28..1801b2d 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java
@@ -18,85 +18,25 @@
package org.apache.oozie.action.hadoop;
-import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.oozie.action.ActionExecutor.Context;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.oozie.service.ConfigurationService;
-import org.apache.oozie.util.XLog;
+public interface CredentialsProvider {
-public class CredentialsProvider {
- Credentials cred;
- String type;
- public static final String CRED_KEY = "oozie.credentials.credentialclasses";
- private static final XLog LOG = XLog.getLog(CredentialsProvider.class);
-
- public CredentialsProvider(String type) {
- this.type = type;
- this.cred = null;
- LOG.debug("Credentials Provider is created for Type: " + type);
- }
-
- /**
- * Create Credential object
- *
- * @return Credential object
- * @throws Exception
- */
- public Credentials createCredentialObject() throws Exception {
- String type;
- String classname;
- for (String function : ConfigurationService.getStrings(CRED_KEY)) {
- function = Trim(function);
- LOG.debug("Creating Credential class for : " + function);
- String[] str = function.split("=");
- if (str.length > 0) {
- type = str[0];
- classname = str[1];
- if (classname != null) {
- LOG.debug("Creating Credential type : '" + type + "', class Name : '" + classname + "'");
- if (this.type.equalsIgnoreCase(str[0])) {
- Class<?> klass = null;
- try {
- klass = Thread.currentThread().getContextClassLoader().loadClass(classname);
- }
- catch (ClassNotFoundException ex) {
- LOG.warn("Exception while loading the class", ex);
- throw ex;
- }
-
- cred = (Credentials) ReflectionUtils.newInstance(klass, null);
- }
- }
- }
- }
- return cred;
- }
-
- /**
- * Relogs into Kerberos using the Keytab for the Oozie server user. This should be called before attempting to get delegation
- * tokens via {@link Credentials} implementations to ensure that the Kerberos credentials are current and won't expire too soon.
+ /**
+ * This is the interface for all the Credentials implementation. Any new credential implementation must implement
+ * this function. This function should modify the jobconf which will be used further to pass the credentials to the
+ * tasks while running it. Credentials properties and context is also provided by that user can get all the
+ * necessary configuration.
*
- * @throws IOException
- */
- public static void ensureKerberosLogin() throws IOException {
- LOG.debug("About to relogin from keytab");
- UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
- LOG.debug("Relogin from keytab successful");
- }
-
- /**
- * To trim string
+ * @param credentials the credentials object which is updated
+ * @param config launcher AM configuration
+ * @param props properties for getting credential token or certificate
+ * @param context workflow context
+ * @throws Exception thrown if failed
*
- * @param str
- * @return trim string
*/
- public String Trim(String str) {
- if (str != null) {
- str = str.replaceAll("\\n", "");
- str = str.replaceAll("\\t", "");
- str = str.trim();
- }
- return str;
- }
+ public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props, Context context)
+ throws Exception;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
new file mode 100644
index 0000000..fc907c4
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import java.io.IOException;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.oozie.service.ConfigurationService;
+import org.apache.oozie.util.XLog;
+
+public class CredentialsProviderFactory {
+ CredentialsProvider cred;
+ String type;
+ public static final String CRED_KEY = "oozie.credentials.credentialclasses";
+ private static final XLog LOG = XLog.getLog(CredentialsProviderFactory.class);
+
+ public CredentialsProviderFactory(String type) {
+ this.type = type;
+ this.cred = null;
+ LOG.debug("Credentials Provider is created for Type: " + type);
+ }
+
+ /**
+ * Create Credential object
+ *
+ * @return Credential object
+ * @throws Exception
+ */
+ public CredentialsProvider createCredentialsProvider() throws Exception {
+ String type;
+ String classname;
+ for (String function : ConfigurationService.getStrings(CRED_KEY)) {
+ function = Trim(function);
+ LOG.debug("Creating Credential class for : " + function);
+ String[] str = function.split("=");
+ if (str.length > 0) {
+ type = str[0];
+ classname = str[1];
+ if (classname != null) {
+ LOG.debug("Creating Credential type : '" + type + "', class Name : '" + classname + "'");
+ if (this.type.equalsIgnoreCase(str[0])) {
+ Class<?> klass = null;
+ try {
+ klass = Thread.currentThread().getContextClassLoader().loadClass(classname);
+ }
+ catch (ClassNotFoundException ex) {
+ LOG.warn("Exception while loading the class", ex);
+ throw ex;
+ }
+
+ cred = (CredentialsProvider) ReflectionUtils.newInstance(klass, null);
+ }
+ }
+ }
+ }
+ return cred;
+ }
+
+ /**
+ * Relogs into Kerberos using the Keytab for the Oozie server user. This should be called before attempting to get delegation
+ * tokens via {@link CredentialsProvider} implementations to ensure that the Kerberos credentials are current and won't expire too soon.
+ *
+ * @throws IOException
+ */
+ public static void ensureKerberosLogin() throws IOException {
+ LOG.debug("About to relogin from keytab");
+ UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
+ LOG.debug("Relogin from keytab successful");
+ }
+
+ /**
+ * To trim string
+ *
+ * @param str
+ * @return trim string
+ */
+ public String Trim(String str) {
+ if (str != null) {
+ str = str.replaceAll("\\n", "");
+ str = str.replaceAll("\\t", "");
+ str = str.trim();
+ }
+ return str;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java
index 4f12152..4d9d3fa 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java
@@ -18,10 +18,11 @@
package org.apache.oozie.action.hadoop;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -50,18 +51,18 @@ public class HCatCredentialHelper {
* @param server - Serevr URI for HCat server
* @throws Exception
*/
- public void set(JobConf launcherJobConf, String principal, String server) throws Exception {
+ public void set(Credentials credentials, Configuration launcherConfig, String principal, String server) throws Exception {
HCatClient client = null;
try {
- client = getHCatClient(launcherJobConf, principal, server);
+ client = getHCatClient(launcherConfig, principal, server);
XLog.getLog(getClass()).debug(
"HCatCredentialHelper: set: User name for which token will be asked from HCat: "
- + launcherJobConf.get(USER_NAME));
- String tokenStrForm = client.getDelegationToken(launcherJobConf.get(USER_NAME), UserGroupInformation
+ + launcherConfig.get(USER_NAME));
+ String tokenStrForm = client.getDelegationToken(launcherConfig.get(USER_NAME), UserGroupInformation
.getLoginUser().getShortUserName());
Token<DelegationTokenIdentifier> hcatToken = new Token<DelegationTokenIdentifier>();
hcatToken.decodeFromUrlString(tokenStrForm);
- launcherJobConf.getCredentials().addToken(new Text("HCat Token"), hcatToken);
+ credentials.addToken(new Text("HCat Token"), hcatToken);
XLog.getLog(getClass()).debug("Added the HCat token in job conf");
}
catch (Exception ex) {
@@ -77,13 +78,13 @@ public class HCatCredentialHelper {
/**
* Getting the HCat client.
- * @param launcherJobConf
+ * @param launcherConfig
* @param principal
* @param server
* @return HCatClient
* @throws HCatException
*/
- public HCatClient getHCatClient(JobConf launcherJobConf,
+ public HCatClient getHCatClient(Configuration launcherConfig,
String principal, String server) throws HCatException {
HiveConf hiveConf = null;
HCatClient hiveclient = null;
@@ -95,7 +96,7 @@ public class HCatCredentialHelper {
hiveConf.set(HIVE_METASTORE_KERBEROS_PRINCIPAL, principal);
hiveConf.set(HIVE_METASTORE_LOCAL, "false");
hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, server);
- String protection = launcherJobConf.get(HADOOP_RPC_PROTECTION,
+ String protection = launcherConfig.get(HADOOP_RPC_PROTECTION,
SaslRpcServer.QualityOfProtection.AUTHENTICATION.name()
.toLowerCase());
XLog.getLog(getClass()).debug("getHCatClient, setting rpc protection to " + protection);
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentials.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentials.java
index d7689a9..52abbf1 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentials.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentials.java
@@ -21,16 +21,13 @@ package org.apache.oozie.action.hadoop;
import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.security.Credentials;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.action.ActionExecutor.Context;
import org.apache.oozie.service.HCatAccessorService;
import org.apache.oozie.service.Services;
import org.apache.oozie.util.XLog;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* Credentials implementation to store in jobConf, HCat-specific properties such as Principal and Uri
* User specifies these credential properties along with the action configuration
@@ -39,7 +36,7 @@ import com.google.common.annotations.VisibleForTesting;
* User can extend the parent class to implement own class as well
* for handling custom token-based credentials and add to the above server property
*/
-public class HCatCredentials extends Credentials {
+public class HCatCredentials implements CredentialsProvider {
private static final String HCAT_METASTORE_PRINCIPAL = "hcat.metastore.principal";
private static final String HCAT_METASTORE_URI = "hcat.metastore.uri";
@@ -54,7 +51,8 @@ public class HCatCredentials extends Credentials {
* @see org.apache.oozie.action.hadoop.Credentials#addtoJobConf(org.apache.hadoop.mapred.JobConf, org.apache.oozie.action.hadoop.CredentialsProperties, org.apache.oozie.action.ActionExecutor.Context)
*/
@Override
- public void addtoJobConf(JobConf jobconf, CredentialsProperties props, Context context) throws Exception {
+ public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props,
+ Context context) throws Exception {
try {
String principal = getProperty(props.getProperties(), HCAT_METASTORE_PRINCIPAL, HIVE_METASTORE_PRINCIPAL);
@@ -69,7 +67,7 @@ public class HCatCredentials extends Credentials {
HCAT_METASTORE_URI + " is required to get hcat credential");
}
HCatCredentialHelper hcch = new HCatCredentialHelper();
- hcch.set(jobconf, principal, server);
+ hcch.set(credentials, config, principal, server);
}
catch (Exception e) {
XLog.getLog(getClass()).warn("Exception in addtoJobConf", e);
@@ -102,4 +100,6 @@ public class HCatCredentials extends Credentials {
}
return value;
}
+
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java
index 3a99b6a..327ba83 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
import org.apache.hadoop.hbase.security.token.TokenUtil;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.oozie.action.ActionExecutor.Context;
@@ -40,17 +40,16 @@ import org.apache.oozie.util.XLog;
* Oozie server should be configured to use this Credentials class by including it via property 'oozie.credentials.credentialclasses'
*
*/
-public class HbaseCredentials extends Credentials {
-
-
+public class HbaseCredentials implements CredentialsProvider {
/* (non-Javadoc)
* @see org.apache.oozie.action.hadoop.Credentials#addtoJobConf(org.apache.hadoop.mapred.JobConf, org.apache.oozie.action.hadoop.CredentialsProperties, org.apache.oozie.action.ActionExecutor.Context)
*/
@Override
- public void addtoJobConf(JobConf jobConf, CredentialsProperties props, Context context) throws Exception {
+ public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props,
+ Context context) throws Exception {
try {
- copyHbaseConfToJobConf(jobConf, props);
- obtainToken(jobConf, context);
+ copyHbaseConfToJobConf(config, props);
+ obtainToken(credentials, config, context);
}
catch (Exception e) {
XLog.getLog(getClass()).warn("Exception in receiving hbase credentials", e);
@@ -58,7 +57,7 @@ public class HbaseCredentials extends Credentials {
}
}
- void copyHbaseConfToJobConf(JobConf jobConf, CredentialsProperties props) {
+ void copyHbaseConfToJobConf(Configuration jobConf, CredentialsProperties props) {
// Create configuration using hbase-site.xml/hbase-default.xml
Configuration hbaseConf = new Configuration(false);
HBaseConfiguration.addHbaseResources(hbaseConf);
@@ -71,7 +70,8 @@ public class HbaseCredentials extends Credentials {
injectConf(hbaseConf, jobConf);
}
- private void obtainToken(final JobConf jobConf, Context context) throws IOException, InterruptedException {
+ private void obtainToken(Credentials credentials, final Configuration jobConf, Context context)
+ throws IOException, InterruptedException {
String user = context.getWorkflow().getUser();
UserGroupInformation ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
User u = User.create(ugi);
@@ -84,7 +84,7 @@ public class HbaseCredentials extends Credentials {
}
}
);
- jobConf.getCredentials().addToken(token.getService(), token);
+ credentials.addToken(token.getService(), token);
}
private void addPropsConf(CredentialsProperties props, Configuration destConf) {
@@ -102,4 +102,6 @@ public class HbaseCredentials extends Credentials {
}
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/Hive2Credentials.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2Credentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2Credentials.java
index bd2f1f7..0b495f7 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2Credentials.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2Credentials.java
@@ -20,9 +20,11 @@ package org.apache.oozie.action.hadoop;
import java.sql.Connection;
import java.sql.DriverManager;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hive.jdbc.HiveConnection;
import org.apache.oozie.ErrorCode;
@@ -37,14 +39,15 @@ import org.apache.oozie.util.XLog;
* User can extend the parent class to implement own class as well
* for handling custom token-based credentials and add to the above server property
*/
-public class Hive2Credentials extends Credentials {
+public class Hive2Credentials implements CredentialsProvider {
private static final String USER_NAME = "user.name";
private static final String HIVE2_JDBC_URL = "hive2.jdbc.url";
private static final String HIVE2_SERVER_PRINCIPAL = "hive2.server.principal";
@Override
- public void addtoJobConf(JobConf jobconf, CredentialsProperties props, Context context) throws Exception {
+ public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props,
+ Context context) throws Exception {
try {
// load the driver
Class.forName("org.apache.hive.jdbc.HiveDriver");
@@ -66,7 +69,7 @@ public class Hive2Credentials extends Credentials {
con = DriverManager.getConnection(url);
XLog.getLog(getClass()).debug("Connected successfully to " + url);
// get delegation token for the given proxy user
- tokenStr = ((HiveConnection)con).getDelegationToken(jobconf.get(USER_NAME), principal);
+ tokenStr = ((HiveConnection)con).getDelegationToken(config.get(USER_NAME), principal);
} finally {
if (con != null) {
con.close();
@@ -76,7 +79,7 @@ public class Hive2Credentials extends Credentials {
Token<DelegationTokenIdentifier> hive2Token = new Token<DelegationTokenIdentifier>();
hive2Token.decodeFromUrlString(tokenStr);
- jobconf.getCredentials().addToken(new Text("hive.server2.delegation.token"), hive2Token);
+ credentials.addToken(new Text("hive.server2.delegation.token"), hive2Token);
XLog.getLog(getClass()).debug("Added the Hive Server 2 token in job conf");
}
catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
index a2d4ca0..012d1ec 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
@@ -20,7 +20,6 @@ package org.apache.oozie.action.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.client.XOozieClient;
import org.apache.oozie.service.ConfigurationService;
@@ -138,10 +137,10 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor {
}
@Override
- protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) {
+ protected Configuration loadHadoopDefaultResources(Context context, Element actionXml) {
boolean loadDefaultResources = ConfigurationService
.getBoolean(HadoopAccessorService.ACTION_CONFS_LOAD_DEFAULT_RESOURCES);
- JobConf conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources);
+ Configuration conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources);
return conf;
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 0f3f816..eb59ff4 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -26,7 +26,6 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
-import java.security.PrivilegedExceptionAction;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
@@ -56,10 +55,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
-import org.apache.hadoop.mapred.JobID;
-import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.DiskChecker;
@@ -237,21 +233,20 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
- public JobConf createBaseHadoopConf(Context context, Element actionXml) {
+ public Configuration createBaseHadoopConf(Context context, Element actionXml) {
return createBaseHadoopConf(context, actionXml, true);
}
- protected JobConf createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) {
+ protected Configuration createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) {
Namespace ns = actionXml.getNamespace();
String jobTracker = actionXml.getChild("job-tracker", ns).getTextTrim();
String nameNode = actionXml.getChild("name-node", ns).getTextTrim();
- JobConf conf = null;
+ Configuration conf = null;
if (loadResources) {
conf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
}
else {
- conf = new JobConf(false);
- // conf.set(HadoopAccessorService.OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, "true");
+ conf = new Configuration(false);
}
conf.set(HADOOP_USER, context.getProtoActionConf().get(WorkflowAppService.HADOOP_USER));
@@ -268,7 +263,7 @@ public class JavaActionExecutor extends ActionExecutor {
return conf;
}
- protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) {
+ protected Configuration loadHadoopDefaultResources(Context context, Element actionXml) {
return createBaseHadoopConf(context, actionXml);
}
@@ -777,7 +772,7 @@ public class JavaActionExecutor extends ActionExecutor {
}
@SuppressWarnings("unchecked")
- JobConf createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml, Configuration actionConf)
+ Configuration createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml, Configuration actionConf)
throws ActionExecutorException {
try {
@@ -788,7 +783,7 @@ public class JavaActionExecutor extends ActionExecutor {
}
// launcher job configuration
- JobConf launcherJobConf = createBaseHadoopConf(context, actionXml);
+ Configuration launcherJobConf = createBaseHadoopConf(context, actionXml);
// cancel delegation token on a launcher job which stays alive till child job(s) finishes
// otherwise (in mapred action), doesn't cancel not to disturb running child job
launcherJobConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true);
@@ -823,15 +818,6 @@ public class JavaActionExecutor extends ActionExecutor {
}
setLibFilesArchives(context, actionXml, appPathRoot, launcherJobConf);
- String jobName = launcherJobConf.get(HADOOP_JOB_NAME);
- if (jobName == null || jobName.isEmpty()) {
- jobName = XLog.format(
- "oozie:launcher:T={0}:W={1}:A={2}:ID={3}", getType(),
- context.getWorkflow().getAppName(), action.getName(),
- context.getWorkflow().getId());
- launcherJobConf.setJobName(jobName);
- }
-
// Inject Oozie job information if enabled.
injectJobInfo(launcherJobConf, actionConf, context, action);
@@ -912,7 +898,7 @@ public class JavaActionExecutor extends ActionExecutor {
injectCallback(context, launcherConf);
}
- private void actionConfToLauncherConf(Configuration actionConf, JobConf launcherConf) {
+ private void actionConfToLauncherConf(Configuration actionConf, Configuration launcherConf) {
for (String name : SPECIAL_PROPERTIES) {
if (actionConf.get(name) != null && launcherConf.get("oozie.launcher." + name) == null) {
launcherConf.set(name, actionConf.get(name));
@@ -938,14 +924,6 @@ public class JavaActionExecutor extends ActionExecutor {
LOG.debug("Setting LibFilesArchives ");
setLibFilesArchives(context, actionXml, appPathRoot, actionConf);
- String jobName = actionConf.get(HADOOP_JOB_NAME);
- if (jobName == null || jobName.isEmpty()) {
- jobName = XLog.format("oozie:action:T={0}:W={1}:A={2}:ID={3}",
- getType(), context.getWorkflow().getAppName(),
- action.getName(), context.getWorkflow().getId());
- actionConf.set(HADOOP_JOB_NAME, jobName);
- }
-
injectActionCallback(context, actionConf);
if(actionConf.get(ACL_MODIFY_JOB) == null || actionConf.get(ACL_MODIFY_JOB).trim().equals("")) {
@@ -959,15 +937,17 @@ public class JavaActionExecutor extends ActionExecutor {
}
// Setting the credential properties in launcher conf
- JobConf credentialsConf = null;
+ Configuration credentialsConf = null;
+
HashMap<String, CredentialsProperties> credentialsProperties = setCredentialPropertyToActionConf(context,
action, actionConf);
+ Credentials credentials = null;
if (credentialsProperties != null) {
-
+ credentials = new Credentials();
// Adding if action need to set more credential tokens
- credentialsConf = new JobConf(false);
+ credentialsConf = new Configuration(false);
XConfiguration.copy(actionConf, credentialsConf);
- setCredentialTokens(credentialsConf, context, action, credentialsProperties);
+ setCredentialTokens(credentials, credentialsConf, context, action, credentialsProperties);
// insert conf to action conf from credentialsConf
for (Entry<String, String> entry : credentialsConf) {
@@ -976,7 +956,7 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
}
- JobConf launcherJobConf = createLauncherConf(actionFs, context, action, actionXml, actionConf);
+ Configuration launcherJobConf = createLauncherConf(actionFs, context, action, actionXml, actionConf);
String consoleUrl;
String launcherId = LauncherMapperHelper.getRecoveryId(launcherJobConf, context.getActionDir(), context
@@ -1010,21 +990,21 @@ public class JavaActionExecutor extends ActionExecutor {
// launcherJobConf.getCredentials().addToken(HadoopAccessorService.MR_TOKEN_ALIAS, mrdt);
// insert credentials tokens to launcher job conf if needed
- if (needInjectCredentials() && credentialsConf != null) {
- for (Token<? extends TokenIdentifier> tk : credentialsConf.getCredentials().getAllTokens()) {
+ if (credentialsConf != null) {
+ for (Token<? extends TokenIdentifier> tk :credentials.getAllTokens()) {
Text fauxAlias = new Text(tk.getKind() + "_" + tk.getService());
LOG.debug("ADDING TOKEN: " + fauxAlias);
- launcherJobConf.getCredentials().addToken(fauxAlias, tk);
+ credentials.addToken(fauxAlias, tk);
}
- if (credentialsConf.getCredentials().numberOfSecretKeys() > 0) {
+ if (credentials.numberOfSecretKeys() > 0) {
for (Entry<String, CredentialsProperties> entry : credentialsProperties.entrySet()) {
CredentialsProperties credProps = entry.getValue();
if (credProps != null) {
Text credName = new Text(credProps.getName());
- byte[] secKey = credentialsConf.getCredentials().getSecretKey(credName);
+ byte[] secKey = credentials.getSecretKey(credName);
if (secKey != null) {
LOG.debug("ADDING CREDENTIAL: " + credProps.getName());
- launcherJobConf.getCredentials().addSecretKey(credName, secKey);
+ credentials.addSecretKey(credName, secKey);
}
}
}
@@ -1039,7 +1019,7 @@ public class JavaActionExecutor extends ActionExecutor {
YarnClientApplication newApp = yarnClient.createApplication();
ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();
ApplicationSubmissionContext appContext =
- createAppSubmissionContext(appId, launcherJobConf, user, context, actionConf);
+ createAppSubmissionContext(appId, launcherJobConf, user, context, actionConf, action.getName(), credentials);
yarnClient.submitApplication(appContext);
launcherId = appId.toString();
@@ -1061,20 +1041,26 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
- private ApplicationSubmissionContext createAppSubmissionContext(ApplicationId appId, JobConf launcherJobConf, String user,
- Context context, Configuration actionConf)
+ private ApplicationSubmissionContext createAppSubmissionContext(ApplicationId appId, Configuration launcherJobConf,
+ String user, Context context, Configuration actionConf, String actionName,
+ Credentials credentials)
throws IOException, HadoopAccessorException, URISyntaxException {
ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
+ String jobName = XLog.format(
+ "oozie:launcher:T={0}:W={1}:A={2}:ID={3}", getType(),
+ context.getWorkflow().getAppName(), actionName,
+ context.getWorkflow().getId());
+
appContext.setApplicationId(appId);
- appContext.setApplicationName(launcherJobConf.getJobName());
+ appContext.setApplicationName(jobName);
appContext.setApplicationType("Oozie Launcher");
Priority pri = Records.newRecord(Priority.class);
int priority = 0; // TODO: OYA: Add a constant or a config
pri.setPriority(priority);
appContext.setPriority(pri);
- appContext.setQueue(launcherJobConf.getQueueName());
+ appContext.setQueue("default"); // TODO: will be possible to set in <launcher>
ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
// Set the resources to localize
@@ -1096,7 +1082,6 @@ public class JavaActionExecutor extends ActionExecutor {
// This adds the Hadoop jars to the classpath in the Launcher JVM
ClasspathUtils.setupClasspath(env, launcherJobConf);
- // FIXME: move this to specific places where it's actually needed - keeping it here for now
if (needToAddMRJars()) {
ClasspathUtils.addMapReduceToClasspath(env, launcherJobConf);
}
@@ -1132,9 +1117,11 @@ public class JavaActionExecutor extends ActionExecutor {
appContext.setAMContainerSpec(amContainer);
// Set tokens
- DataOutputBuffer dob = new DataOutputBuffer();
- launcherJobConf.getCredentials().writeTokenStorageToStream(dob);
- amContainer.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
+ if (credentials != null) {
+ DataOutputBuffer dob = new DataOutputBuffer();
+ credentials.writeTokenStorageToStream(dob);
+ amContainer.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
+ }
// Set Resources
// TODO: OYA: make resources allocated for the AM configurable and choose good defaults (memory MB, vcores)
@@ -1144,24 +1131,6 @@ public class JavaActionExecutor extends ActionExecutor {
return appContext;
}
- private boolean needInjectCredentials() {
- boolean methodExists = true;
-
- Class<?> klass;
- try {
- klass = Class.forName("org.apache.hadoop.mapred.JobConf");
- klass.getMethod("getCredentials");
- }
- catch (ClassNotFoundException ex) {
- methodExists = false;
- }
- catch (NoSuchMethodException ex) {
- methodExists = false;
- }
-
- return methodExists;
- }
-
protected HashMap<String, CredentialsProperties> setCredentialPropertyToActionConf(Context context,
WorkflowAction action, Configuration actionConf) throws Exception {
HashMap<String, CredentialsProperties> credPropertiesMap = null;
@@ -1199,20 +1168,20 @@ public class JavaActionExecutor extends ActionExecutor {
return credPropertiesMap;
}
- protected void setCredentialTokens(JobConf jobconf, Context context, WorkflowAction action,
+ protected void setCredentialTokens(Credentials credentials, Configuration jobconf, Context context, WorkflowAction action,
HashMap<String, CredentialsProperties> credPropertiesMap) throws Exception {
if (context != null && action != null && credPropertiesMap != null) {
// Make sure we're logged into Kerberos; if not, or near expiration, it will relogin
- CredentialsProvider.ensureKerberosLogin();
+ CredentialsProviderFactory.ensureKerberosLogin();
for (Entry<String, CredentialsProperties> entry : credPropertiesMap.entrySet()) {
String credName = entry.getKey();
CredentialsProperties credProps = entry.getValue();
if (credProps != null) {
- CredentialsProvider credProvider = new CredentialsProvider(credProps.getType());
- Credentials credentialObject = credProvider.createCredentialObject();
- if (credentialObject != null) {
- credentialObject.addtoJobConf(jobconf, credProps, context);
+ CredentialsProviderFactory tokenProviderFactory = new CredentialsProviderFactory(credProps.getType());
+ CredentialsProvider tokenProvider = tokenProviderFactory.createCredentialsProvider();
+ if (tokenProvider != null) {
+ tokenProvider.updateCredentials(credentials, jobconf, credProps, context);
LOG.debug("Retrieved Credential '" + credName + "' for action " + action.getId());
}
else {
@@ -1224,7 +1193,6 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
}
-
}
protected HashMap<String, CredentialsProperties> getActionCredentialsProperties(Context context,
@@ -1351,7 +1319,7 @@ public class JavaActionExecutor extends ActionExecutor {
* @return YarnClient
* @throws HadoopAccessorException
*/
- protected YarnClient createYarnClient(Context context, JobConf jobConf) throws HadoopAccessorException {
+ protected YarnClient createYarnClient(Context context, Configuration jobConf) throws HadoopAccessorException {
String user = context.getWorkflow().getUser();
return Services.get().get(HadoopAccessorService.class).createYarnClient(user, jobConf);
}
@@ -1391,7 +1359,7 @@ public class JavaActionExecutor extends ActionExecutor {
YarnClient yarnClient = null;
try {
Element actionXml = XmlUtils.parseXml(action.getConf());
- JobConf jobConf = createBaseHadoopConf(context, actionXml);
+ Configuration jobConf = createBaseHadoopConf(context, actionXml);
FileSystem actionFs = context.getAppFileSystem();
yarnClient = createYarnClient(context, jobConf);
FinalApplicationStatus appStatus = null;
@@ -1528,7 +1496,7 @@ public class JavaActionExecutor extends ActionExecutor {
try {
Element actionXml = XmlUtils.parseXml(action.getConf());
- JobConf jobConf = createBaseHadoopConf(context, actionXml);
+ Configuration jobConf = createBaseHadoopConf(context, actionXml);
yarnClient = createYarnClient(context, jobConf);
yarnClient.killApplication(ConverterUtils.toApplicationId(action.getExternalId()));
@@ -1641,7 +1609,7 @@ public class JavaActionExecutor extends ActionExecutor {
HadoopAccessorException, URISyntaxException {
}
- private void injectJobInfo(JobConf launcherJobConf, Configuration actionConf, Context context, WorkflowAction action) {
+ private void injectJobInfo(Configuration launcherJobConf, Configuration actionConf, Context context, WorkflowAction action) {
if (OozieJobInfo.isJobInfoEnabled()) {
try {
OozieJobInfo jobInfo = new OozieJobInfo(actionConf, context, action);
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
index e2a667e..8bd476e 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.oozie.client.OozieClient;
@@ -119,14 +118,9 @@ public class LauncherMapperHelper {
launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX, fsGlobMax);
}
- public static void setupLauncherInfo(JobConf launcherConf, String jobId, String actionId, Path actionDir,
+ public static void setupLauncherInfo(Configuration launcherConf, String jobId, String actionId, Path actionDir,
String recoveryId, Configuration actionConf, String prepareXML) throws IOException, HadoopAccessorException {
- launcherConf.setMapperClass(LauncherMapper.class);
- launcherConf.setSpeculativeExecution(false);
- launcherConf.setNumMapTasks(1);
- launcherConf.setNumReduceTasks(0);
-
launcherConf.set(LauncherMapper.OOZIE_JOB_ID, jobId);
launcherConf.set(LauncherMapper.OOZIE_ACTION_ID, actionId);
launcherConf.set(LauncherMapper.OOZIE_ACTION_DIR_PATH, actionDir.toString());
@@ -147,13 +141,9 @@ public class LauncherMapperHelper {
actionConf.setStrings("mapreduce.job.cache.files", purgedEntries.toArray(new String[purgedEntries.size()]));
launcherConf.setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true);
}
-
- launcherConf.setInputFormat(OozieLauncherInputFormat.class);
- launcherConf.setOutputFormat(OozieLauncherOutputFormat.class);
- launcherConf.setOutputCommitter(OozieLauncherOutputCommitter.class);
}
- public static void setupYarnRestartHandling(JobConf launcherJobConf, Configuration actionConf, String launcherTag,
+ public static void setupYarnRestartHandling(Configuration launcherJobConf, Configuration actionConf, String launcherTag,
long launcherTime)
throws NoSuchAlgorithmException {
launcherJobConf.setLong(LauncherMain.OOZIE_JOB_LAUNCH_TIME, launcherTime);
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
index ff89e48..918a269 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
@@ -209,8 +209,8 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
try {
if (action.getStatus() == WorkflowAction.Status.OK) {
Element actionXml = XmlUtils.parseXml(action.getConf());
- JobConf jobConf = createBaseHadoopConf(context, actionXml);
- jobClient = createJobClient(context, jobConf);
+ Configuration jobConf = createBaseHadoopConf(context, actionXml);
+ jobClient = createJobClient(context, new JobConf(jobConf));
RunningJob runningJob = jobClient.getJob(JobID.forName(action.getExternalChildIDs()));
if (runningJob == null) {
throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR002",
@@ -297,19 +297,20 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
}
@Override
- JobConf createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml,
+ Configuration createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml,
Configuration actionConf) throws ActionExecutorException {
// If the user is using a regular MapReduce job and specified an uber jar, we need to also set it for the launcher;
// so we override createLauncherConf to call super and then to set the uber jar if specified. At this point, checking that
// uber jars are enabled and resolving the uber jar path is already done by setupActionConf() when it parsed the actionConf
// argument and we can just look up the uber jar in the actionConf argument.
- JobConf launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf);
+ Configuration launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf);
Namespace ns = actionXml.getNamespace();
if (actionXml.getChild("streaming", ns) == null && actionXml.getChild("pipes", ns) == null) {
// Set for uber jar
String uberJar = actionConf.get(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR);
if (uberJar != null && uberJar.trim().length() > 0) {
- launcherJobConf.setJar(uberJar);
+ // TODO
+ // launcherJobConf.setJar(uberJar);
}
}
return launcherJobConf;
@@ -350,7 +351,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
@Override
public void check(Context context, WorkflowAction action) throws ActionExecutorException {
Map<String, String> actionData = Collections.emptyMap();
- JobConf jobConf = null;
+ Configuration jobConf = null;
try {
FileSystem actionFs = context.getAppFileSystem();
@@ -372,7 +373,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
boolean exception = false;
try {
- jobClient = createJobClient(context, jobConf);
+ jobClient = createJobClient(context, new JobConf(jobConf));
RunningJob runningJob = jobClient.getJob(JobID.forName(newId));
if (runningJob == null) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
index 347b034..17508eb 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
@@ -23,7 +23,6 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.XOozieClient;
@@ -129,10 +128,10 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor {
}
@Override
- protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) {
+ protected Configuration loadHadoopDefaultResources(Context context, Element actionXml) {
boolean loadDefaultResources = ConfigurationService
.getBoolean(HadoopAccessorService.ACTION_CONFS_LOAD_DEFAULT_RESOURCES);
- JobConf conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources);
+ Configuration conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources);
return conf;
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index c373758..b411c4c 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -21,7 +21,6 @@ package org.apache.oozie.action.hadoop;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.ConfigurationService;
@@ -99,10 +98,10 @@ public class SparkActionExecutor extends JavaActionExecutor {
}
@Override
- JobConf createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml,
+ Configuration createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml,
Configuration actionConf) throws ActionExecutorException {
- JobConf launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf);
+ Configuration launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf);
if (launcherJobConf.get("oozie.launcher." + TASK_USER_PRECEDENCE) == null) {
launcherJobConf.set(TASK_USER_PRECEDENCE, "true");
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
index 54a860e..7e2561d 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
@@ -130,8 +130,8 @@ public class SqoopActionExecutor extends JavaActionExecutor {
try {
if (action.getStatus() == WorkflowAction.Status.OK) {
Element actionXml = XmlUtils.parseXml(action.getConf());
- JobConf jobConf = createBaseHadoopConf(context, actionXml);
- jobClient = createJobClient(context, jobConf);
+ Configuration jobConf = createBaseHadoopConf(context, actionXml);
+ jobClient = createJobClient(context, new JobConf(jobConf));
// Cumulative counters for all Sqoop mapreduce jobs
Counters counters = null;
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
index 514f01e..3a36588 100644
--- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
+++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.oozie.ErrorCode;
-import org.apache.oozie.action.ActionExecutor;
import org.apache.oozie.action.hadoop.JavaActionExecutor;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.ParamChecker;
@@ -52,7 +51,6 @@ import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
-import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.URI;
@@ -534,6 +532,10 @@ public class HadoopAccessorService implements Service {
}
}
+ public JobClient createJobClient(String user, Configuration conf) throws HadoopAccessorException {
+ return createJobClient(user, new JobConf(conf));
+ }
+
/**
* Return a YarnClient created with the provided user and configuration.
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/action/hadoop/CredentialForTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/CredentialForTest.java b/core/src/test/java/org/apache/oozie/action/hadoop/CredentialForTest.java
index 0629891..23dd78a 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/CredentialForTest.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/CredentialForTest.java
@@ -20,20 +20,17 @@ package org.apache.oozie.action.hadoop;
import java.util.Map.Entry;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.action.ActionExecutor.Context;
-import org.apache.oozie.action.hadoop.Credentials;
import org.apache.oozie.action.hadoop.CredentialsProperties;
-
-
-@SuppressWarnings("deprecation")
-public class CredentialForTest extends Credentials {
+public class CredentialForTest implements CredentialsProvider {
@Override
- public void addtoJobConf(JobConf jobconf, CredentialsProperties props, Context context) throws Exception {
-
+ public void updateCredentials(Credentials credentials, Configuration config,
+ CredentialsProperties props, Context context) throws Exception {
String paramA = null;
String paramB = null;
for (Entry<String, String> parameter : props.getProperties().entrySet()) {
@@ -50,7 +47,7 @@ public class CredentialForTest extends Credentials {
throw new CredentialException(ErrorCode.E0510, "required parameters is null.");
}
- jobconf.set(props.getName(), "testcert");
+ config.set(props.getName(), "testcert");
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/action/hadoop/InsertTestToken.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/InsertTestToken.java b/core/src/test/java/org/apache/oozie/action/hadoop/InsertTestToken.java
index 9da8fbe..9558aa5 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/InsertTestToken.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/InsertTestToken.java
@@ -18,15 +18,15 @@
package org.apache.oozie.action.hadoop;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.oozie.action.ActionExecutor.Context;
import org.apache.oozie.util.XLog;
-
-public class InsertTestToken extends Credentials{
+public class InsertTestToken implements CredentialsProvider {
public static String DUMMY_SECRET_KEY = "DummySecretKey";
public InsertTestToken() {
}
@@ -34,14 +34,16 @@ public class InsertTestToken extends Credentials{
/* (non-Javadoc)
* @see org.apache.oozie.action.hadoop.Credentials#addtoJobConf(org.apache.hadoop.mapred.JobConf, org.apache.oozie.action.hadoop.CredentialsProperties, org.apache.oozie.action.ActionExecutor.Context)
*/
+
@Override
- public void addtoJobConf(JobConf jobconf, CredentialsProperties props, Context context) throws Exception {
+ public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props, Context context)
+ throws Exception {
try {
Token<DelegationTokenIdentifier> abctoken = new Token<DelegationTokenIdentifier>();
- jobconf.getCredentials().addToken(new Text("ABC Token"), abctoken);
+ credentials.addToken(new Text("ABC Token"), abctoken);
XLog.getLog(getClass()).debug("Added the ABC token in job conf");
- jobconf.getCredentials().addSecretKey(new Text(DUMMY_SECRET_KEY), DUMMY_SECRET_KEY.getBytes("UTF-8"));
+ credentials.addSecretKey(new Text(DUMMY_SECRET_KEY), DUMMY_SECRET_KEY.getBytes("UTF-8"));
XLog.getLog(getClass()).debug("Added the " + DUMMY_SECRET_KEY + " in job conf");
}
catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java
index e6d43ca..204245c 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java
@@ -34,6 +34,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.security.Credentials;
import org.apache.oozie.service.HCatAccessorService;
import org.apache.oozie.service.ServiceException;
import org.apache.oozie.service.Services;
@@ -130,17 +131,18 @@ public class TestHCatCredentials {
credProps.setProperties(new HashMap<String, String>());
HCatCredentials hcatCred = new HCatCredentials();
final JobConf jobConf = new JobConf(false);
+ Credentials credentials = new Credentials();
PowerMockito.doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
- JobConf jConf = (JobConf) args[0];
- jConf.set(HCAT_METASTORE_PRINCIPAL, (String) args[1]);
- jConf.set(HCAT_METASTORE_URI, (String) args[2]);
+ Configuration jConf = (Configuration) args[1];
+ jConf.set(HCAT_METASTORE_PRINCIPAL, (String) args[2]);
+ jConf.set(HCAT_METASTORE_URI, (String) args[3]);
return null;
}
- }).when(hcatCredHelper).set(jobConf, TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2);
- hcatCred.addtoJobConf(jobConf, credProps, null);
+ }).when(hcatCredHelper).set(credentials, jobConf, TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2);
+ hcatCred.updateCredentials(credentials, jobConf, credProps, null);
assertEquals(TEST_HIVE_METASTORE_PRINCIPAL2, jobConf.get(HCAT_METASTORE_PRINCIPAL));
assertEquals(TEST_HIVE_METASTORE_URI2, jobConf.get(HCAT_METASTORE_URI));
assertNull(jobConf.get(HIVE_METASTORE_PRINCIPAL));
@@ -155,19 +157,20 @@ public class TestHCatCredentials {
credProps.setProperties(new HashMap<String, String>());
HCatCredentials hcatCred = new HCatCredentials();
final JobConf jobConf = new JobConf(false);
+ Credentials credentials = new Credentials();
HCatCredentialHelper hcatCredHelper = Mockito.mock(HCatCredentialHelper.class);
PowerMockito.whenNew(HCatCredentialHelper.class).withNoArguments().thenReturn(hcatCredHelper);
PowerMockito.doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
- JobConf jConf = (JobConf) args[0];
- jConf.set(HIVE_METASTORE_PRINCIPAL, (String) args[1]);
- jConf.set(HIVE_METASTORE_URI, (String) args[2]);
+ Configuration jConf = (Configuration) args[1];
+ jConf.set(HIVE_METASTORE_PRINCIPAL, (String) args[2]);
+ jConf.set(HIVE_METASTORE_URI, (String) args[3]);
return null;
}
- }).when(hcatCredHelper).set(jobConf, TEST_HIVE_METASTORE_PRINCIPAL, TEST_HIVE_METASTORE_URI);
- hcatCred.addtoJobConf(jobConf, credProps, null);
+ }).when(hcatCredHelper).set(credentials, jobConf, TEST_HIVE_METASTORE_PRINCIPAL, TEST_HIVE_METASTORE_URI);
+ hcatCred.updateCredentials(credentials, jobConf, credProps, null);
assertEquals(TEST_HIVE_METASTORE_PRINCIPAL, jobConf.get(HIVE_METASTORE_PRINCIPAL));
assertEquals(TEST_HIVE_METASTORE_URI, jobConf.get(HIVE_METASTORE_URI));
assertNull(jobConf.get(HCAT_METASTORE_PRINCIPAL));
@@ -186,17 +189,18 @@ public class TestHCatCredentials {
credProps.setProperties(prop);
HCatCredentials hcatCred = new HCatCredentials();
final JobConf jobConf = new JobConf(false);
+ Credentials credentials = new Credentials();
PowerMockito.doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
- JobConf jConf = (JobConf) args[0];
- jConf.set(HCAT_METASTORE_PRINCIPAL, (String) args[1]);
- jConf.set(HCAT_METASTORE_URI, (String) args[2]);
+ JobConf jConf = (JobConf) args[1];
+ jConf.set(HCAT_METASTORE_PRINCIPAL, (String) args[2]);
+ jConf.set(HCAT_METASTORE_URI, (String) args[3]);
return null;
}
- }).when(hcatCredHelper).set(jobConf, TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2);
- hcatCred.addtoJobConf(jobConf, credProps, null);
+ }).when(hcatCredHelper).set(credentials, jobConf, TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2);
+ hcatCred.updateCredentials(credentials, jobConf, credProps, null);
assertEquals(TEST_HIVE_METASTORE_PRINCIPAL2, jobConf.get(HCAT_METASTORE_PRINCIPAL));
assertEquals(TEST_HIVE_METASTORE_URI2, jobConf.get(HCAT_METASTORE_URI));
assertNull(jobConf.get(HIVE_METASTORE_PRINCIPAL));
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index aa485ef..5996083 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -879,10 +880,11 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
// Try to load the token without it being defined in oozie-site; should get an exception
JobConf credentialsConf = new JobConf();
+ Credentials credentials = new Credentials();
Configuration launcherConf = ae.createBaseHadoopConf(context, actionXmlconf);
XConfiguration.copy(launcherConf, credentialsConf);
try {
- ae.setCredentialTokens(credentialsConf, context, action, credProperties);
+ ae.setCredentialTokens(credentials, credentialsConf, context, action, credProperties);
fail("Should have gotten an exception but did not");
}
catch (ActionExecutorException aee) {
@@ -896,13 +898,14 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
// Try to load the token after being defined in oozie-site; should work correctly
credentialsConf = new JobConf();
+ credentials = new Credentials();
launcherConf = ae.createBaseHadoopConf(context, actionXmlconf);
XConfiguration.copy(launcherConf, credentialsConf);
- ae.setCredentialTokens(credentialsConf, context, action, credProperties);
- Token<? extends TokenIdentifier> tk = credentialsConf.getCredentials().getToken(new Text("ABC Token"));
+ ae.setCredentialTokens(credentials, credentialsConf, context, action, credProperties);
+ Token<? extends TokenIdentifier> tk = credentials.getToken(new Text("ABC Token"));
assertNotNull(tk);
- byte[] secKey = credentialsConf.getCredentials().getSecretKey(new Text(InsertTestToken.DUMMY_SECRET_KEY));
+ byte[] secKey = credentials.getSecretKey(new Text(InsertTestToken.DUMMY_SECRET_KEY));
assertNotNull(secKey);
assertEquals(InsertTestToken.DUMMY_SECRET_KEY, new String(secKey, "UTF-8"));
}
@@ -938,8 +941,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
try {
// Setting the credential properties in launcher conf should fail
- HashMap<String, CredentialsProperties> credProperties = ae.setCredentialPropertyToActionConf(context, action,
- actionConf);
+ ae.setCredentialPropertyToActionConf(context, action, actionConf);
}
catch (ActionExecutorException e) {
assertEquals(e.getErrorCode(), "JA021");
@@ -1083,10 +1085,11 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
// Try to load the token without it being defined in oozie-site; should get an exception
JobConf credentialsConf = new JobConf();
+ Credentials credentials = new Credentials();
Configuration launcherConf = ae.createBaseHadoopConf(context, actionXmlconf);
XConfiguration.copy(launcherConf, credentialsConf);
- ae.setCredentialTokens(credentialsConf, context, action, credProperties);
- Token<? extends TokenIdentifier> tk = credentialsConf.getCredentials().getToken(new Text("ABC Token"));
+ ae.setCredentialTokens(credentials, credentialsConf, context, action, credProperties);
+ Token<? extends TokenIdentifier> tk = credentials.getToken(new Text("ABC Token"));
if (expectingTokens) {
assertNotNull(tk);
} else {
@@ -1523,7 +1526,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
getFileSystem().mkdirs(javaShareLibPath);
Services.get().setService(ShareLibService.class);
- JobConf conf = ae.createBaseHadoopConf(context, eActionXml);
+ Configuration conf = ae.createBaseHadoopConf(context, eActionXml);
// Despite systemLibPath is not fully qualified and the action refers to the
// second namenode the next line won't throw exception because default fs is used
ae.addShareLib(conf, new String[] { "java-action-executor" });
@@ -1547,7 +1550,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
Context context = createContext(actionXml, null);
Path appPath = new Path("localfs://namenode:port/mydir");
JavaActionExecutor ae = new JavaActionExecutor();
- JobConf conf = ae.createBaseHadoopConf(context, eActionXml);
+ Configuration conf = ae.createBaseHadoopConf(context, eActionXml);
Services.get().destroy();
setSystemProperty(HadoopAccessorService.SUPPORTED_FILESYSTEMS, "hdfs,viewfs");
new Services().init();
@@ -1868,10 +1871,10 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
action.setType(ae.getType());
Context context = new Context(wf, action);
- JobConf actionConf = new JobConf();
+ Configuration actionConf = new JobConf();
// Test when server side setting is not enabled
- JobConf launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf);
+ Configuration launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf);
assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); // disabled by default
ConfigurationService.set("oozie.action.launcher." + JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED, "true");
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/action/hadoop/TestOozieJobInfo.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestOozieJobInfo.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestOozieJobInfo.java
index 0925398..551adff 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestOozieJobInfo.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestOozieJobInfo.java
@@ -33,7 +33,6 @@ import java.util.regex.Matcher;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.oozie.BundleActionBean;
import org.apache.oozie.BundleJobBean;
import org.apache.oozie.CoordinatorJobBean;
@@ -154,7 +153,7 @@ public class TestOozieJobInfo extends XDataTestCase {
ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(wfbean, actionList.get(1), false,
false);
MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
- JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(actionList.get(1).getConf()));
+ Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(actionList.get(1).getConf()));
String user = conf.get("user.name");
FileSystem fs = getFileSystem();
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java b/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
index 633ef37..b4bce60 100644
--- a/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
+++ b/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java
@@ -152,7 +152,7 @@ public class TestWorkflowXClient extends DagServletTestCase {
fail("submit client without JT should throw exception");
}
catch (RuntimeException exception) {
- assertEquals("java.lang.RuntimeException: jobtracker is not specified in conf", exception.toString());
+ assertEquals("java.lang.RuntimeException: Resource manager is not specified in conf", exception.toString());
}
conf.setProperty(XOozieClient.RM, "localhost:9001");
try {
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
index 135c34a..4069a95 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java
@@ -23,6 +23,7 @@ import java.io.Writer;
import java.util.Date;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobClient;
@@ -257,7 +258,7 @@ public class TestActionCheckXCommand extends XDataTestCase {
ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job, action, false, false);
MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
- JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
+ Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
String user = conf.get("user.name");
JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
@@ -408,7 +409,7 @@ public class TestActionCheckXCommand extends XDataTestCase {
ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job2, action1, false, false);
WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd);
MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
- JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action2.getConf()));
+ Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action2.getConf()));
String user = conf.get("user.name");
JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
@@ -480,7 +481,7 @@ public class TestActionCheckXCommand extends XDataTestCase {
ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job0, action1, false, false);
MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
- JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
+ Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
String user = conf.get("user.name");
JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java
index 59cfdd2..6c3051b 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java
@@ -160,7 +160,7 @@ public class TestActionStartXCommand extends XDataTestCase {
ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job, action, false, false);
MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
- JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
+ Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
String launcherId = action.getExternalId();
@@ -227,7 +227,7 @@ public class TestActionStartXCommand extends XDataTestCase {
ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job, action, false, false);
MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
- JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
+ Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf()));
String user = conf.get("user.name");
String launcherId = action.getExternalId();
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
index 8d0e081..35a846a 100644
--- a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
@@ -19,7 +19,7 @@
package org.apache.oozie.service;
import org.apache.hadoop.conf.Configuration;
-import org.apache.oozie.action.hadoop.CredentialsProvider;
+import org.apache.oozie.action.hadoop.CredentialsProviderFactory;
import org.apache.oozie.action.hadoop.DistcpActionExecutor;
import org.apache.oozie.action.hadoop.JavaActionExecutor;
import org.apache.oozie.action.hadoop.LauncherMapper;
@@ -212,7 +212,7 @@ public class TestConfigurationService extends XTestCase {
assertEquals(false, ConfigurationService.getBoolean(HadoopAccessorService.KERBEROS_AUTH_ENABLED));
assertEquals(0, ConfigurationService.getStrings("no.defined").length);
- assertEquals(0, ConfigurationService.getStrings(CredentialsProvider.CRED_KEY).length);
+ assertEquals(0, ConfigurationService.getStrings(CredentialsProviderFactory.CRED_KEY).length);
assertEquals(1, ConfigurationService.getStrings(DistcpActionExecutor.CLASS_NAMES).length);
assertEquals("distcp=org.apache.hadoop.tools.DistCp",
ConfigurationService.getStrings(DistcpActionExecutor.CLASS_NAMES)[0]);
http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java b/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java
index a3270e9..c19d62a 100644
--- a/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java
@@ -246,7 +246,7 @@ public class TestRecoveryService extends XDataTestCase {
ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job1, action1, false, false);
MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
- JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
+ Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
String launcherId = action1.getExternalId();
waitUntilYarnAppDoneAndAssertSuccess(launcherId);