You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bo...@apache.org on 2022/03/25 15:00:38 UTC
[impala] 01/03: IMPALA-11145: Block reads on JSON table until we support it
This is an automated email from the ASF dual-hosted git repository.
boroknagyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit dff5171b9879d3b7455c2deb41d9baf3207c716a
Author: pranav.lodha <pr...@cloudera.com>
AuthorDate: Tue Mar 15 14:41:57 2022 +0530
IMPALA-11145: Block reads on JSON table until we support it
Since, scanning JSON tables leads to a BE crash, it's been blocked at
FE using appropriate error message. An end-to-end test has also been
included to test the blocking and the error message.
Change-Id: Idcebf5891f8db52be442f66a8efb1724d1e545aa
Reviewed-on: http://gerrit.cloudera.org:8080/18323
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/planner/HdfsScanNode.java | 10 ++++++++++
tests/metadata/test_hms_integration.py | 23 ++++++++++++++++++++++
2 files changed, 33 insertions(+)
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index a0db795..9d33158 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -465,6 +465,16 @@ public class HdfsScanNode extends ScanNode {
Preconditions.checkNotNull(desc_);
Preconditions.checkNotNull(desc_.getTable());
+ // Since JSON file format is not yet supported, this block throws an
+ // exception. Once JSON file format will be supported, appropriate changes
+ // can be made under this block.
+ for (FeFsPartition part: partitions_) {
+ HdfsFileFormat format = part.getFileFormat();
+ if (format.equals(HdfsFileFormat.JSON)) {
+ throw new NotImplementedException("Scan of table " + desc_.getTableName() +
+ " in format 'JSON' is not supported.");
+ }
+ }
Column firstComplexTypedCol = null;
for (Column col: desc_.getTable().getColumns()) {
if (col.getType().isComplexType()) {
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index 1eab56f..100cdfe 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -129,6 +129,29 @@ class TestHmsIntegrationSanity(ImpalaTestSuite):
self.client.execute("DESCRIBE {0}.json_tbl"
.format(unique_database))
+ def test_json_file_unsupported(self, unique_database):
+ """
+ Since JSON file format is not yet supported,this function tests
+ the blocking logic of reading JSON tables.
+ """
+ self.client.execute("create table {0}.json_tbl(id int, name string, age int)"
+ " stored as jsonfile".format(unique_database))
+ self.run_stmt_in_hive("insert into {0}.json_tbl values(0,'Alice',10)"
+ .format(unique_database))
+ self.run_stmt_in_hive("insert into {0}.json_tbl values(1,'Bob',20)"
+ .format(unique_database))
+ self.run_stmt_in_hive("insert into {0}.json_tbl values(2,'Oracle',16)"
+ .format(unique_database))
+ self.client.execute("refresh {0}.json_tbl".format(unique_database))
+ self.client.execute("show files in {0}.json_tbl".format(unique_database))
+ try:
+ self.client.execute("select * from {0}.json_tbl".format(unique_database))
+ except Exception as e:
+ assert 'Scan of table {0}.json_tbl in format \'JSON\' is not supported.'\
+ .format(unique_database) in str(e)
+ else:
+ assert False
+
@SkipIfS3.hive
@SkipIfGCS.hive
@SkipIfCOS.hive