You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ba...@apache.org on 2016/07/27 20:22:53 UTC
falcon git commit: FALCON-2095 HiveDR fails in NN-HA config
Repository: falcon
Updated Branches:
refs/heads/master c588628aa -> ea0492dac
FALCON-2095 HiveDR fails in NN-HA config
Fix validated in a two cluster configuration with NN/RM HA setup
Author: Venkat Ranganathan <ve...@hortonworks.com>
Reviewers: "Balu Vellanki <ba...@apache.org>"
Closes #240 from vrangan/FALCON-2095
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/ea0492da
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/ea0492da
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/ea0492da
Branch: refs/heads/master
Commit: ea0492dacd3d0929751060bee26b1e5a61178431
Parents: c588628
Author: Venkat Ranganathan <ve...@hortonworks.com>
Authored: Wed Jul 27 13:22:40 2016 -0700
Committer: bvellanki <bv...@hortonworks.com>
Committed: Wed Jul 27 13:22:40 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/falcon/hive/HiveDRTool.java | 6 +++---
.../org/apache/falcon/hive/mapreduce/CopyReducer.java | 3 ++-
.../java/org/apache/falcon/hive/util/EventUtils.java | 6 +++---
.../java/org/apache/falcon/hive/util/FileUtils.java | 12 ++++++------
4 files changed, 14 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
index e45b0d8..2701e02 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/HiveDRTool.java
@@ -143,13 +143,13 @@ public class HiveDRTool extends Configured implements Tool {
LOG.info("srcStaginPath: {}", inputOptions.getSourceStagingPath());
LOG.info("tgtStaginPath: {}", inputOptions.getTargetStagingPath());
- Configuration sourceConf = FileUtils.getConfiguration(inputOptions.getSourceWriteEP(),
+ Configuration sourceConf = FileUtils.getConfiguration(getConf(), inputOptions.getSourceWriteEP(),
inputOptions.getSourceNNKerberosPrincipal());
sourceClusterFS = FileSystem.get(sourceConf);
- Configuration targetConf = FileUtils.getConfiguration(inputOptions.getTargetWriteEP(),
+ Configuration targetConf = FileUtils.getConfiguration(getConf(), inputOptions.getTargetWriteEP(),
inputOptions.getTargetNNKerberosPrincipal());
targetClusterFS = FileSystem.get(targetConf);
- jobConf = FileUtils.getConfiguration(inputOptions.getJobClusterWriteEP(),
+ jobConf = FileUtils.getConfiguration(getConf(), inputOptions.getJobClusterWriteEP(),
inputOptions.getJobClusterNNPrincipal());
jobFS = FileSystem.get(jobConf);
http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java
index f4bb31c..7c415c3 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/mapreduce/CopyReducer.java
@@ -48,7 +48,8 @@ public class CopyReducer extends Reducer<Text, Text, Text, Text> {
@Override
protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
- FileSystem fs= FileSystem.get(FileUtils.getConfiguration(
+ FileSystem fs= FileSystem.get(
+ FileUtils.getConfiguration(context.getConfiguration(),
conf.get(HiveDRArgs.TARGET_NN.getName()),
conf.get(HiveDRArgs.TARGET_NN_KERBEROS_PRINCIPAL.getName())));
hiveDRStore = new HiveDRStatusStore(fs);
http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
index 590a7e3..05b5f96 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/EventUtils.java
@@ -142,9 +142,9 @@ public class EventUtils {
LOG.info("Initializing staging directory");
sourceStagingUri = new Path(sourceNN, sourceStagingPath).toString();
targetStagingUri = new Path(targetNN, targetStagingPath).toString();
- sourceFileSystem = FileSystem.get(FileUtils.getConfiguration(sourceNN, sourceNNKerberosPrincipal));
- jobFileSystem = FileSystem.get(FileUtils.getConfiguration(jobNN, jobNNKerberosPrincipal));
- targetFileSystem = FileSystem.get(FileUtils.getConfiguration(targetNN, targetNNKerberosPrincipal));
+ sourceFileSystem = FileSystem.get(FileUtils.getConfiguration(conf, sourceNN, sourceNNKerberosPrincipal));
+ jobFileSystem = FileSystem.get(FileUtils.getConfiguration(conf, jobNN, jobNNKerberosPrincipal));
+ targetFileSystem = FileSystem.get(FileUtils.getConfiguration(conf, targetNN, targetNNKerberosPrincipal));
}
private String readEvents(Path eventFileName) throws IOException {
http://git-wip-us.apache.org/repos/asf/falcon/blob/ea0492da/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
index 8b5c865..dae4849 100644
--- a/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
+++ b/addons/hivedr/src/main/java/org/apache/falcon/hive/util/FileUtils.java
@@ -42,14 +42,14 @@ public final class FileUtils {
private FileUtils() {}
- public static Configuration getConfiguration(final String writeEP,
- final String nnKerberosPrincipal) throws IOException {
- Configuration conf = HiveDRUtils.getDefaultConf();
- conf.set("fs.defaultFS", writeEP);
+ public static Configuration getConfiguration(Configuration conf,
+ final String writeEP, final String nnKerberosPrincipal) throws IOException {
+ Configuration newConf = new Configuration(conf);
+ newConf.set("fs.defaultFS", writeEP);
if (StringUtils.isNotEmpty(nnKerberosPrincipal)) {
- conf.set("dfs.namenode.kerberos.principal", nnKerberosPrincipal);
+ newConf.set("dfs.namenode.kerberos.principal", nnKerberosPrincipal);
}
- return conf;
+ return newConf;
}
public static void validatePath(final FileSystem fileSystem, final Path basePath) throws IOException {