You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ar...@apache.org on 2022/09/28 14:28:42 UTC
[impala] 03/03: IMPALA-11576: Fix for test_multiple_storage_locations on S3
This is an automated email from the ASF dual-hosted git repository.
arawat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 55194a9c83bba52c64d0d2e75d5ebd3d8c67b3e2
Author: Gergely Fürnstáhl <gf...@cloudera.com>
AuthorDate: Tue Sep 27 11:54:35 2022 +0200
IMPALA-11576: Fix for test_multiple_storage_locations on S3
Fixed absolute/relative path handling in filesystems not supporting
block location (e.g. S3).
Testing:
- test_multiple_storage_location passes on S3
Change-Id: I08badd96d060a2377c9a1eafa287a3adf8fa11db
Reviewed-on: http://gerrit.cloudera.org:8080/19045
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java | 5 ++---
.../main/java/org/apache/impala/catalog/FileMetadataLoader.java | 2 +-
fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java | 8 ++++----
3 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
index 5c25c0960..37b657b0c 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
@@ -537,10 +537,9 @@ public interface FeIcebergTable extends FeFsTable {
relPath = relUri.getPath();
}
-
if (!FileSystemUtil.supportsStorageIds(fs)) {
- return HdfsPartition.FileDescriptor.createWithNoBlocks(fileStatus,
- StringUtils.isNotEmpty(relPath) ? relPath : absPath);
+ return HdfsPartition.FileDescriptor.createWithNoBlocks(
+ fileStatus, relPath, absPath);
}
BlockLocation[] locations;
diff --git a/fe/src/main/java/org/apache/impala/catalog/FileMetadataLoader.java b/fe/src/main/java/org/apache/impala/catalog/FileMetadataLoader.java
index 272a24f0a..ee42fab49 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FileMetadataLoader.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FileMetadataLoader.java
@@ -259,7 +259,7 @@ public class FileMetadataLoader {
private FileDescriptor createFd(FileSystem fs, FileStatus fileStatus,
String relPath, Reference<Long> numUnknownDiskIds) throws IOException {
if (!FileSystemUtil.supportsStorageIds(fs)) {
- return FileDescriptor.createWithNoBlocks(fileStatus, relPath);
+ return FileDescriptor.createWithNoBlocks(fileStatus, relPath, null);
}
BlockLocation[] locations;
if (fileStatus instanceof LocatedFileStatus) {
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java b/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
index 88b7eec26..9d481d540 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
@@ -221,11 +221,11 @@ public class HdfsPartition extends CatalogObjectImpl
* Creates the file descriptor of a file represented by 'fileStatus' that
* resides in a filesystem that doesn't support the BlockLocation API (e.g. S3).
*/
- public static FileDescriptor createWithNoBlocks(FileStatus fileStatus,
- String relPath) {
+ public static FileDescriptor createWithNoBlocks(
+ FileStatus fileStatus, String relPath, String absPath) {
FlatBufferBuilder fbb = new FlatBufferBuilder(1);
- return new FileDescriptor(createFbFileDesc(fbb, fileStatus, relPath, null, false,
- null));
+ return new FileDescriptor(
+ createFbFileDesc(fbb, fileStatus, relPath, null, false, absPath));
}
/**