You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ba...@apache.org on 2016/05/10 21:50:07 UTC
falcon git commit: FALCON-1941 HiveDR fails with NN-HA enabled on
both the source and target clusters
Repository: falcon
Updated Branches:
refs/heads/master 1b8a93307 -> 8bd7a5c87
FALCON-1941 HiveDR fails with NN-HA enabled on both the source and target clusters
Fix for HiveDR to work NN-HA enabled clusters
Author: Venkat Ranganathan <ve...@hortonworks.com>
Reviewers: "Ying Zheng <yz...@hortonworks.com>, Balu Vellanki <ba...@apache.org>"
Closes #133 from vrangan/FALCON-1941
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/8bd7a5c8
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/8bd7a5c8
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/8bd7a5c8
Branch: refs/heads/master
Commit: 8bd7a5c87f9c80119471b1b29a72067e6650c20e
Parents: 1b8a933
Author: Venkat Ranganathan <ve...@hortonworks.com>
Authored: Tue May 10 14:50:02 2016 -0700
Committer: bvellanki <bv...@hortonworks.com>
Committed: Tue May 10 14:50:02 2016 -0700
----------------------------------------------------------------------
.../org/apache/falcon/hive/util/FileUtils.java | 4 +--
.../apache/falcon/hive/util/HiveDRUtils.java | 34 +++++++++++++++++---
2 files changed, 31 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/8bd7a5c8/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
index 6bd6319..483a79c 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
@@ -40,8 +40,8 @@ public final class FileUtils {
private FileUtils() {}
- public static Configuration getConfiguration(final String writeEP, final String nnKerberosPrincipal) {
- Configuration conf = new Configuration();
+ public static Configuration getConfiguration(final String writeEP, final String nnKerberosPrincipal) throws IOException {
+ Configuration conf = HiveDRUtils.getDefaultConf();
conf.set("fs.defaultFS", writeEP);
if (StringUtils.isNotEmpty(nnKerberosPrincipal)) {
conf.set("dfs.namenode.kerberos.principal", nnKerberosPrincipal);
http://git-wip-us.apache.org/repos/asf/falcon/blob/8bd7a5c8/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java
index dff0803..ddcc178 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/HiveDRUtils.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -56,6 +58,8 @@ public final class HiveDRUtils {
public static final String SEPARATOR = File.separator;
+ private static final Logger LOG = LoggerFactory.getLogger(HiveDRUtils.class);
+
private HiveDRUtils() {}
public static ReplicationType getReplicationType(List<String> sourceTables) {
@@ -65,12 +69,32 @@ public final class HiveDRUtils {
public static Configuration getDefaultConf() throws IOException {
Configuration conf = new Configuration();
- conf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml")));
- String delegationToken = getFilePathFromEnv("HADOOP_TOKEN_FILE_LOCATION");
- if (delegationToken != null) {
- conf.set("mapreduce.job.credentials.binary", delegationToken);
- conf.set("tez.credentials.path", delegationToken);
+ Path confPath = new Path("file:///", System.getProperty("oozie.action.conf.xml"));
+
+ final boolean actionConfExists = confPath.getFileSystem(conf).exists(confPath);
+ LOG.info("Oozie Action conf {} found ? {}", confPath, actionConfExists);
+ if (actionConfExists) {
+ LOG.info("Oozie Action conf found, adding path={}, conf={}", confPath, conf.toString());
+ conf.addResource(confPath);
}
+
+ String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
+ if (StringUtils.isNotBlank(tokenFile)) {
+ if (Shell.WINDOWS) {
+ if (tokenFile.charAt(0) == '"') {
+ tokenFile = tokenFile.substring(1);
+ }
+ if (tokenFile.charAt(tokenFile.length() - 1) == '"') {
+ tokenFile = tokenFile.substring(0, tokenFile.length() - 1);
+ }
+ }
+
+ conf.set("mapreduce.job.credentials.binary", tokenFile);
+ System.setProperty("mapreduce.job.credentials.binary", tokenFile);
+ conf.set("tez.credentials.path", tokenFile);
+ System.setProperty("tez.credentials.path", tokenFile);
+ }
+
return conf;
}