You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2019/11/27 05:18:45 UTC
[incubator-hudi] branch master updated: [HUDI-325] Fix Hive
partition error for updated HDFS Hudi table (#1001)
This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new d6e83e8 [HUDI-325] Fix Hive partition error for updated HDFS Hudi table (#1001)
d6e83e8 is described below
commit d6e83e8f49828940159cd34711cc88ee7b42dc1c
Author: wenningd <we...@gmail.com>
AuthorDate: Tue Nov 26 21:18:39 2019 -0800
[HUDI-325] Fix Hive partition error for updated HDFS Hudi table (#1001)
---
.../src/main/java/org/apache/hudi/common/util/FSUtils.java | 7 +++++++
.../src/main/java/org/apache/hudi/hive/HoodieHiveClient.java | 8 ++++++--
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
index 86e3e87..60d8538 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
@@ -531,6 +531,13 @@ public class FSUtils {
}
/**
+ * Get DFS full partition path (e.g. hdfs://ip-address:8020:/<absolute path>)
+ */
+ public static String getDFSFullPartitionPath(FileSystem fs, Path partitionPath) {
+ return fs.getUri() + partitionPath.toUri().getRawPath();
+ }
+
+ /**
* This is due to HUDI-140 GCS has a different behavior for detecting EOF during seek().
*
* @param inputStream FSDataInputStream
diff --git a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
index ed319ce..0b9e261 100644
--- a/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
+++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
@@ -51,6 +51,7 @@ import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.storage.StorageSchemes;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
@@ -194,7 +195,9 @@ public class HoodieHiveClient {
String alterTable = "ALTER TABLE " + syncConfig.tableName;
for (String partition : partitions) {
String partitionClause = getPartitionClause(partition);
- String fullPartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, partition).toString();
+ Path partitionPath = FSUtils.getPartitionPath(syncConfig.basePath, partition);
+ String fullPartitionPath = partitionPath.toUri().getScheme().equals(StorageSchemes.HDFS.getScheme())
+ ? FSUtils.getDFSFullPartitionPath(fs, partitionPath) : partitionPath.toString();
String changePartition =
alterTable + " PARTITION (" + partitionClause + ") SET LOCATION '" + fullPartitionPath + "'";
changePartitions.add(changePartition);
@@ -218,7 +221,8 @@ public class HoodieHiveClient {
List<PartitionEvent> events = Lists.newArrayList();
for (String storagePartition : partitionStoragePartitions) {
- String fullStoragePartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, storagePartition).toString();
+ Path storagePartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, storagePartition);
+ String fullStoragePartitionPath = Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
// Check if the partition values or if hdfs path is the same
List<String> storagePartitionValues = partitionValueExtractor.extractPartitionValuesInPath(storagePartition);
Collections.sort(storagePartitionValues);