You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2019/09/26 04:10:58 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-887] Fix the
FileContext wrong fsUri issue.
This is an automated email from the ASF dual-hosted git repository.
hutran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new f24b5ea [GOBBLIN-887] Fix the FileContext wrong fsUri issue.
f24b5ea is described below
commit f24b5eaaa141e6b3447c2c7c6664b4a6a5e8028c
Author: Kuai Yu <ku...@linkedin.com>
AuthorDate: Wed Sep 25 21:10:51 2019 -0700
[GOBBLIN-887] Fix the FileContext wrong fsUri issue.
Closes #2743 from yukuai518/fc
---
.../src/main/java/org/apache/gobblin/writer/FsDataWriter.java | 4 +++-
.../src/main/java/org/apache/gobblin/util/WriterUtils.java | 11 ++++++++---
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/gobblin-core/src/main/java/org/apache/gobblin/writer/FsDataWriter.java b/gobblin-core/src/main/java/org/apache/gobblin/writer/FsDataWriter.java
index 23a885a..a408289 100644
--- a/gobblin-core/src/main/java/org/apache/gobblin/writer/FsDataWriter.java
+++ b/gobblin-core/src/main/java/org/apache/gobblin/writer/FsDataWriter.java
@@ -104,7 +104,9 @@ public abstract class FsDataWriter<D> implements DataWriter<D>, FinalState, Meta
// Add all job configuration properties so they are picked up by Hadoop
JobConfigurationUtils.putStateIntoConfiguration(properties, conf);
this.fs = WriterUtils.getWriterFS(properties, this.numBranches, this.branchId);
- this.fileContext = FileContext.getFileContext(conf);
+ this.fileContext = FileContext.getFileContext(
+ WriterUtils.getWriterFsUri(properties, this.numBranches, this.branchId),
+ conf);
// Initialize staging/output directory
Path writerStagingDir = this.writerAttemptIdOptional.isPresent() ? WriterUtils
diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/WriterUtils.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/WriterUtils.java
index 9a628d0..f7b8256 100644
--- a/gobblin-utility/src/main/java/org/apache/gobblin/util/WriterUtils.java
+++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/WriterUtils.java
@@ -24,6 +24,7 @@ import java.util.concurrent.ExecutionException;
import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileConstants;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -314,11 +315,15 @@ public class WriterUtils {
}
}
+ public static URI getWriterFsUri(State state, int numBranches, int branchId) {
+ return URI.create(state.getProp(
+ ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, numBranches, branchId),
+ ConfigurationKeys.LOCAL_FS_URI));
+ }
+
public static FileSystem getWriterFS(State state, int numBranches, int branchId)
throws IOException {
- URI uri = URI.create(state.getProp(
- ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, numBranches, branchId),
- ConfigurationKeys.LOCAL_FS_URI));
+ URI uri = getWriterFsUri(state, numBranches, branchId);
Configuration hadoopConf = getFsConfiguration(state);
if (state.getPropAsBoolean(ConfigurationKeys.SHOULD_FS_PROXY_AS_USER,