You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by ga...@apache.org on 2021/02/21 03:54:18 UTC
[hudi] branch master updated: [HUDI-1586] [Common Core] [Flink
Integration] Reduce the coupling of hadoop. (#2540)
This is an automated email from the ASF dual-hosted git repository.
garyli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 43a0776 [HUDI-1586] [Common Core] [Flink Integration] Reduce the coupling of hadoop. (#2540)
43a0776 is described below
commit 43a0776c7c88a5f7beac6c8853db7e341810635a
Author: ZhangChaoMing <72...@users.noreply.github.com>
AuthorDate: Sun Feb 21 11:54:04 2021 +0800
[HUDI-1586] [Common Core] [Flink Integration] Reduce the coupling of hadoop. (#2540)
Co-authored-by: zhangchaoming <zh...@360.com>
---
hudi-common/pom.xml | 2 +
.../java/org/apache/hudi/common/fs/FSUtils.java | 7 +---
hudi-flink/pom.xml | 43 ++--------------------
3 files changed, 8 insertions(+), 44 deletions(-)
diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml
index 2981155..41588ca 100644
--- a/hudi-common/pom.xml
+++ b/hudi-common/pom.xml
@@ -144,6 +144,7 @@
<artifactId>*</artifactId>
</exclusion>
</exclusions>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -154,6 +155,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index a2b3889..341e3f5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -80,9 +80,6 @@ public class FSUtils {
private static final PathFilter ALLOW_ALL_FILTER = file -> true;
public static Configuration prepareHadoopConf(Configuration conf) {
- conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
- conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
-
// look for all properties, prefixed to be picked up
for (Entry<String, String> prop : System.getenv().entrySet()) {
if (prop.getKey().startsWith(HOODIE_ENV_PROPS_PREFIX)) {
@@ -607,8 +604,8 @@ public class FSUtils {
* Helper to filter out paths under metadata folder when running fs.globStatus.
* @param fs File System
* @param globPath Glob Path
- * @return
- * @throws IOException
+ * @return the file status list of globPath exclude the meta folder
+ * @throws IOException when having trouble listing the path
*/
public static List<FileStatus> getGlobStatusExcludingMetaFolder(FileSystem fs, Path globPath) throws IOException {
FileStatus[] statuses = fs.globStatus(globPath);
diff --git a/hudi-flink/pom.xml b/hudi-flink/pom.xml
index 2a0f395..81a7849 100644
--- a/hudi-flink/pom.xml
+++ b/hudi-flink/pom.xml
@@ -153,39 +153,11 @@
<scope>provided</scope>
</dependency>
- <!-- Hadoop -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+ <!-- Parquet -->
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-avro</artifactId>
+ <scope>test</scope>
</dependency>
<!-- Avro -->
@@ -197,13 +169,6 @@
<scope>compile</scope>
</dependency>
- <!-- Parquet -->
- <dependency>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-avro</artifactId>
- <scope>compile</scope>
- </dependency>
-
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>