You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2020/06/17 23:53:42 UTC
[impala] branch master updated: IMPALA-9844: Ozone support for load
data inpath
This is an automated email from the ASF dual-hosted git repository.
stakiar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 23bb085 IMPALA-9844: Ozone support for load data inpath
23bb085 is described below
commit 23bb0859ccf027d46f4e8b079330f4bd73161f4c
Author: Sahil Takiar <ta...@gmail.com>
AuthorDate: Tue Jun 9 16:05:56 2020 -0700
IMPALA-9844: Ozone support for load data inpath
Add Ozone to the whitelist of URIs supported for 'load data
inpath' queries.
Removed a workaround in FileSystemUtil#isOzoneFileSystem that required
CDP jars by default. Now, we support only CDP and can require the CDP
Ozone jars.
Testing:
* Tested manually against a cluster with Ozone deployed
Change-Id: I60e8c9ebc6c5b2a29539fac9797cbda32e3cc257
Reviewed-on: http://gerrit.cloudera.org:8080/16055
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java | 9 ++++++---
fe/src/main/java/org/apache/impala/common/FileSystemUtil.java | 6 ++----
.../test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java | 2 +-
3 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java b/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
index 90ab829..b2daea8 100644
--- a/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
import org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem;
+import org.apache.hadoop.fs.ozone.OzoneFileSystem;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -149,9 +150,11 @@ public class LoadDataStmt extends StatementBase {
if (!(fs instanceof DistributedFileSystem) && !(fs instanceof S3AFileSystem) &&
!(fs instanceof AzureBlobFileSystem) &&
!(fs instanceof SecureAzureBlobFileSystem) &&
- !(fs instanceof AdlFileSystem)) {
- throw new AnalysisException(String.format("INPATH location '%s' " +
- "must point to an HDFS, S3A, ADL or ABFS filesystem.", sourceDataPath_));
+ !(fs instanceof AdlFileSystem) &&
+ !(fs instanceof OzoneFileSystem)) {
+ throw new AnalysisException(String.format("INPATH location '%s' "
+ + "must point to an HDFS, S3A, ADL, ABFS, or Ozone filesystem.",
+ sourceDataPath_));
}
if (!fs.exists(source)) {
throw new AnalysisException(String.format(
diff --git a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
index 0a7f484..38b1ddd 100644
--- a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
+++ b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
import org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem;
+import org.apache.hadoop.fs.ozone.OzoneFileSystem;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
@@ -394,10 +395,7 @@ public class FileSystemUtil {
* Returns true iff the filesystem is a OzoneFileSystem.
*/
public static boolean isOzoneFileSystem(FileSystem fs) {
- // TODO once CDP becomes the default build version, this check can directly use
- // org.apache.hadoop.fs.ozone.OzoneFileSystem, similar to the rest of the
- // is*FileSystem() methods.
- return fs.getUri().getScheme().equals("o3fs");
+ return fs instanceof OzoneFileSystem;
}
/**
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index 2e57b9f..06772ca 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -3380,7 +3380,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalysisError(String.format("load data inpath '%s' %s into table " +
"tpch.lineitem", "file:///test-warehouse/test.out", overwrite),
"INPATH location 'file:/test-warehouse/test.out' must point to an " +
- "HDFS, S3A, ADL or ABFS filesystem.");
+ "HDFS, S3A, ADL, ABFS, or Ozone filesystem.");
// File type / table type mismatch.
AnalyzesOk(String.format("load data inpath '%s' %s into table " +