You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2020/10/14 22:48:31 UTC

[impala] branch master updated (481ea4a -> c7581b5)

This is an automated email from the ASF dual-hosted git repository.

joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.


    from 481ea4a  IMPALA-9815: Update URL for cdh-releases-rcs maven repo
     new 0fb2349  IMPALA-10055: Fix DCHECK hit on corrupt ORC file
     new 308d692  IMPALA-10113: Add feature flag for incremental metadata updates
     new 3d06757  IMPALA-10224: Add startup flag not to expose debug web url to clients
     new 13a78fc  IMPALA-10165: Implement Bucket and Truncate partition transforms for Iceberg tables
     new 0c0985a  IMPALA-10159: Supporting ORC file format for Iceberg table
     new c7581b5  IMPALA-10220: Fix negative value bug in RpcNetworkTime counter.

The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 be/src/common/global-flags.cc                      |   8 +
 be/src/exec/orc-column-readers.cc                  |  15 +-
 be/src/runtime/exec-env.cc                         |   3 +
 be/src/runtime/exec-env.h                          |   2 +-
 be/src/runtime/krpc-data-stream-sender.cc          |  14 +-
 be/src/service/impala-beeswax-server.cc            |   8 +-
 be/src/service/impala-hs2-server.cc                |   7 +-
 be/src/util/backend-gflag-util.cc                  |   3 +
 common/thrift/BackendGflags.thrift                 |   2 +
 common/thrift/CatalogObjects.thrift                |  11 +-
 fe/src/main/cup/sql-parser.cup                     |  17 ++-
 .../apache/impala/analysis/CreateTableStmt.java    |  14 +-
 .../impala/analysis/IcebergPartitionField.java     |  19 ++-
 .../impala/analysis/IcebergPartitionTransform.java |  88 +++++++++++
 .../impala/catalog/CatalogServiceCatalog.java      |  19 ++-
 .../org/apache/impala/catalog/FeIcebergTable.java  |  12 +-
 .../java/org/apache/impala/catalog/HdfsTable.java  |  15 +-
 .../org/apache/impala/catalog/IcebergTable.java    |   9 +-
 .../org/apache/impala/catalog/ImpaladCatalog.java  |   2 +-
 .../impala/catalog/PartitionMetaSummary.java       |   5 +
 .../org/apache/impala/service/BackendConfig.java   |   4 +
 .../java/org/apache/impala/util/IcebergUtil.java   | 168 +++++++++++++++++----
 .../apache/impala/analysis/AnalyzeStmtsTest.java   |  22 +++
 .../org/apache/impala/analysis/ParserTest.java     |   4 +
 shell/impala_client.py                             |  13 +-
 shell/impala_shell.py                              |  13 +-
 ...-0982a5d3-48c0-4dd0-ab87-d24190894251-00000.orc | Bin 0 -> 460 bytes
 ...-dc56d2c8-e285-428d-b81e-f3d07ec53c12-00000.orc | Bin 0 -> 460 bytes
 ...-d4a071f4-5d57-493e-8d21-8f82655087e4-00000.orc | Bin 0 -> 461 bytes
 ...-e9fa9638-8693-43dc-b1ba-2d8dc390b585-00000.orc | Bin 0 -> 460 bytes
 ...-4526ff61-8991-4645-b448-bf3535492111-00000.orc | Bin 0 -> 460 bytes
 ...-439376ff-3d17-4494-94ba-586fa00caadd-00000.orc | Bin 0 -> 461 bytes
 ...-fb7663c5-aade-4690-bdfc-2334d605a891-00000.orc | Bin 0 -> 474 bytes
 ...-fc53a761-1ee2-4ce4-9507-c9917917d2c9-00000.orc | Bin 0 -> 474 bytes
 ...-8887c762-b6a8-4e2c-b1e0-6069d6141917-00000.orc | Bin 0 -> 474 bytes
 ...-a85e02aa-ff28-4311-b996-a83f67a5e39b-00000.orc | Bin 0 -> 470 bytes
 ...-dea5a8ec-ea06-4688-8293-486a62563ef3-00000.orc | Bin 0 -> 474 bytes
 ...-d182fe7e-de19-4df8-ac19-27bbfe17b5bd-00000.orc | Bin 0 -> 474 bytes
 ...-d5df20b1-6ce6-486a-b084-ea55b7954431-00000.orc | Bin 0 -> 439 bytes
 ...-b30cc130-03a1-45e5-b927-d80545593f9a-00000.orc | Bin 0 -> 435 bytes
 ...-6476cc9e-9b5a-48d3-8cce-ea7ebb2d904f-00000.orc | Bin 0 -> 439 bytes
 ...-c2ab6dc5-3586-4b45-8a3c-860fdc71980d-00000.orc | Bin 0 -> 439 bytes
 ...-fd03c61f-8c73-4639-8974-1f6431a30bc5-00000.orc | Bin 0 -> 439 bytes
 ...-9baec6d2-8e31-4d19-8259-1a9e546aaea5-00000.orc | Bin 0 -> 439 bytes
 ...-433b1afa-9fee-4581-89fb-105483ebf1a2-00000.orc | Bin 0 -> 439 bytes
 ...-e0156a2d-4240-4c8e-9724-6ad3231cf178-00000.orc | Bin 0 -> 439 bytes
 .../02fb8198-e791-4d89-8afa-c75fb5443346-m0.avro   | Bin 0 -> 5018 bytes
 ...70-1-02fb8198-e791-4d89-8afa-c75fb5443346.avro} | Bin 2563 -> 2582 bytes
 .../metadata/v1.metadata.json                      |  27 +---
 .../metadata/v2.metadata.json                      |  45 ++----
 .../metadata/version-hint.text                     |   0
 .../functional/functional_schema_template.sql      |  12 ++
 .../datasets/functional/schema_constraints.csv     |   2 +
 .../queries/QueryTest/iceberg-create.test          |  85 ++++++++++-
 .../queries/QueryTest/iceberg-query.test           |  94 ++++++++++++
 .../queries/QueryTest/show-create-table.test       |  49 +++++-
 tests/custom_cluster/test_disable_features.py      |  14 ++
 tests/custom_cluster/test_local_catalog.py         |   3 +-
 tests/custom_cluster/test_web_pages.py             |  31 ++++
 59 files changed, 704 insertions(+), 155 deletions(-)
 create mode 100644 fe/src/main/java/org/apache/impala/analysis/IcebergPartitionTransform.java
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00004-4-0982a5d3-48c0-4dd0-ab87-d24190894251-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00014-14-dc56d2c8-e285-428d-b81e-f3d07ec53c12-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00015-15-d4a071f4-5d57-493e-8d21-8f82655087e4-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00019-19-e9fa9638-8693-43dc-b1ba-2d8dc390b585-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00020-20-4526ff61-8991-4645-b448-bf3535492111-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00028-28-439376ff-3d17-4494-94ba-586fa00caadd-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00003-3-fb7663c5-aade-4690-bdfc-2334d605a891-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00007-7-fc53a761-1ee2-4ce4-9507-c9917917d2c9-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00011-11-8887c762-b6a8-4e2c-b1e0-6069d6141917-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00012-12-a85e02aa-ff28-4311-b996-a83f67a5e39b-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00022-22-dea5a8ec-ea06-4688-8293-486a62563ef3-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00025-25-d182fe7e-de19-4df8-ac19-27bbfe17b5bd-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00001-1-d5df20b1-6ce6-486a-b084-ea55b7954431-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00006-6-b30cc130-03a1-45e5-b927-d80545593f9a-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00009-9-6476cc9e-9b5a-48d3-8cce-ea7ebb2d904f-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00017-17-c2ab6dc5-3586-4b45-8a3c-860fdc71980d-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00023-23-fd03c61f-8c73-4639-8974-1f6431a30bc5-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00027-27-9baec6d2-8e31-4d19-8259-1a9e546aaea5-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00030-30-433b1afa-9fee-4581-89fb-105483ebf1a2-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00031-31-e0156a2d-4240-4c8e-9724-6ad3231cf178-00000.orc
 create mode 100644 testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/02fb8198-e791-4d89-8afa-c75fb5443346-m0.avro
 copy testdata/data/iceberg_test/{iceberg_partitioned/metadata/snap-8270633197658268308-1-af797bab-2f2c-44df-a77b-d91c7198fe53.avro => hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/snap-3506237933060603670-1-02fb8198-e791-4d89-8afa-c75fb5443346.avro} (89%)
 copy testdata/data/iceberg_test/{iceberg_partitioned => hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc}/metadata/v1.metadata.json (61%)
 copy testdata/data/iceberg_test/hadoop_catalog/{hadoop_catalog_test/functional_parquet/hadoop_catalog_test => iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc}/metadata/v2.metadata.json (51%)
 copy testdata/data/iceberg_test/{iceberg_partitioned => hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc}/metadata/version-hint.text (100%)


[impala] 01/06: IMPALA-10055: Fix DCHECK hit on corrupt ORC file

Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 0fb234923c580ee39f3c296eebfc5062a71e8fb9
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Tue Oct 13 13:54:57 2020 +0200

    IMPALA-10055: Fix DCHECK hit on corrupt ORC file
    
    Our ORC scanner could hit a DCHECK on corrupt ORC files. In
    test_scanners_fuzz we randomly modify ORC files, so the this test
    might hit a DCHECK occasionally.
    
    I converted the DCHECK to a parse error. This way the fuzz test
    won't crash the Impala daemon.
    
    Testing:
    Unfortunately I don't have an ORC file on which we hit the DCHECK.
    So I manually changed the code to always raise this error and
    executed the fuzz test to see if it still succeeds.
    
    Change-Id: I18d9f56c3c37afd1a4898ee36f8cc2ddb5049972
    Reviewed-on: http://gerrit.cloudera.org:8080/16591
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/exec/orc-column-readers.cc | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/be/src/exec/orc-column-readers.cc b/be/src/exec/orc-column-readers.cc
index 2e6f664..138fa5a 100644
--- a/be/src/exec/orc-column-readers.cc
+++ b/be/src/exec/orc-column-readers.cc
@@ -418,9 +418,18 @@ Status OrcStructReader::TopLevelReadValueBatch(ScratchTupleBatch* scratch_batch,
   }
   int num_rows_read = scratch_batch->num_tuples - scratch_batch_idx;
   if (children_.empty()) {
-    DCHECK((scanner_->row_batches_need_validation_ &&
-            scanner_->scan_node_->IsZeroSlotTableScan()) ||
-            scanner_->acid_original_file_);
+    // We allow empty 'children_' for original files, because we might select the
+    // synthetic 'rowid' field which is not present in original files.
+    // We also allow empty 'children_' when we need to validate row batches of a zero slot
+    // scan. In that case 'children_' is empty and only 'row_validator_' owns an ORC
+    // vector batch (the write id batch).
+    bool valid_empty_children = scanner_->acid_original_file_ ||
+         (scanner_->row_batches_need_validation_ &&
+          scanner_->scan_node_->IsZeroSlotTableScan());
+    if (!valid_empty_children) {
+      return Status(Substitute("Parse error in possibly corrupt ORC file: '$0'",
+          scanner_->filename()));
+    }
     DCHECK_EQ(0, num_rows_read);
     num_rows_read = std::min(scratch_batch->capacity - scratch_batch->num_tuples,
                              NumElements() - row_idx_);


[impala] 03/06: IMPALA-10224: Add startup flag not to expose debug web url to clients

Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 3d067572dd401a08045de57fefa3d3131bea3c2b
Author: Attila Jeges <at...@cloudera.com>
AuthorDate: Tue Oct 6 17:52:58 2020 +0200

    IMPALA-10224: Add startup flag not to expose debug web url to clients
    
    This patch introduces a new startup flag
    --ping_expose_webserver_url (true by default) to control whether
    PingImpalaService, PingImpalaHS2Service RPC calls should expose
    the debug web url to the client or not.
    
    This is necessary as the debug web UI is not something that
    end-users will necessarily have access to.
    
    If the flag is set to false, the RPC calls will return an empty
    string instead of the real url signalling that the debug web ui
    is not available.
    
    Note that if the webserver is disabled (--enable_webserver flag
    is set to false) the RPC calls will behave the same and return an
    empty string for the url.
    
    Change-Id: I7ec3e92764d712b8fee63c1f45b038c31c184cfc
    Reviewed-on: http://gerrit.cloudera.org:8080/16573
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/runtime/exec-env.cc              |  3 +++
 be/src/runtime/exec-env.h               |  2 +-
 be/src/service/impala-beeswax-server.cc |  8 +++++++-
 be/src/service/impala-hs2-server.cc     |  7 ++++++-
 shell/impala_client.py                  | 13 +++++++------
 shell/impala_shell.py                   | 13 +++++++------
 tests/custom_cluster/test_web_pages.py  | 31 +++++++++++++++++++++++++++++++
 7 files changed, 62 insertions(+), 15 deletions(-)

diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc
index 3645504..689ec8f 100644
--- a/be/src/runtime/exec-env.cc
+++ b/be/src/runtime/exec-env.cc
@@ -81,6 +81,9 @@ using namespace strings;
 DEFINE_string(catalog_service_host, "localhost",
     "hostname where CatalogService is running");
 DEFINE_bool(enable_webserver, true, "If true, debug webserver is enabled");
+DEFINE_bool(ping_expose_webserver_url, true,
+    "If true, debug webserver url is exposed via PingImpalaService/PingImpalaHS2Service "
+    "RPC calls");
 DEFINE_string(state_store_host, "localhost",
     "hostname where StatestoreService is running");
 DEFINE_int32(state_store_subscriber_port, 23000,
diff --git a/be/src/runtime/exec-env.h b/be/src/runtime/exec-env.h
index 0d40ac0..e96a070 100644
--- a/be/src/runtime/exec-env.h
+++ b/be/src/runtime/exec-env.h
@@ -137,7 +137,7 @@ class ExecEnv {
   BufferPool* buffer_pool() { return buffer_pool_.get(); }
   SystemStateInfo* system_state_info() { return system_state_info_.get(); }
 
-  void set_enable_webserver(bool enable) { enable_webserver_ = enable; }
+  bool get_enable_webserver() const { return enable_webserver_; }
 
   ClusterMembershipMgr* cluster_membership_mgr() { return cluster_membership_mgr_.get(); }
   Scheduler* scheduler() { return scheduler_.get(); }
diff --git a/be/src/service/impala-beeswax-server.cc b/be/src/service/impala-beeswax-server.cc
index def746c..4d14903 100644
--- a/be/src/service/impala-beeswax-server.cc
+++ b/be/src/service/impala-beeswax-server.cc
@@ -49,6 +49,8 @@ using namespace beeswax;
     }                                                           \
   } while (false)
 
+DECLARE_bool(ping_expose_webserver_url);
+
 namespace impala {
 
 void ImpalaServer::query(beeswax::QueryHandle& beeswax_handle, const Query& query) {
@@ -473,7 +475,11 @@ void ImpalaServer::PingImpalaService(TPingImpalaServiceResp& return_val) {
 
   VLOG_RPC << "PingImpalaService()";
   return_val.version = GetVersionString(true);
-  return_val.webserver_address = ExecEnv::GetInstance()->webserver()->url();
+  if (ExecEnv::GetInstance()->get_enable_webserver() && FLAGS_ping_expose_webserver_url) {
+    return_val.webserver_address = ExecEnv::GetInstance()->webserver()->url();
+  } else {
+    return_val.webserver_address = "";
+  }
   VLOG_RPC << "PingImpalaService(): return_val=" << ThriftDebugString(return_val);
 }
 
diff --git a/be/src/service/impala-hs2-server.cc b/be/src/service/impala-hs2-server.cc
index 9ec92cf..43f01a1 100644
--- a/be/src/service/impala-hs2-server.cc
+++ b/be/src/service/impala-hs2-server.cc
@@ -95,6 +95,7 @@ DECLARE_string(hostname);
 DECLARE_int32(webserver_port);
 DECLARE_int32(idle_session_timeout);
 DECLARE_int32(disconnected_session_timeout);
+DECLARE_bool(ping_expose_webserver_url);
 
 namespace impala {
 
@@ -1200,7 +1201,11 @@ void ImpalaServer::PingImpalaHS2Service(TPingImpalaHS2ServiceResp& return_val,
   }
 
   return_val.__set_version(GetVersionString(true));
-  return_val.__set_webserver_address(ExecEnv::GetInstance()->webserver()->url());
+  if (ExecEnv::GetInstance()->get_enable_webserver() && FLAGS_ping_expose_webserver_url) {
+    return_val.__set_webserver_address(ExecEnv::GetInstance()->webserver()->url());
+  } else {
+    return_val.__set_webserver_address("");
+  }
   VLOG_RPC << "PingImpalaHS2Service(): return_val=" << ThriftDebugString(return_val);
 }
 }
diff --git a/shell/impala_client.py b/shell/impala_client.py
index 8ad86a4..684ef68 100755
--- a/shell/impala_client.py
+++ b/shell/impala_client.py
@@ -334,12 +334,13 @@ class ImpalaClient(object):
 
   def _append_retried_query_link(self, get_log_result):
     """Append the retried query link if the original query has been retried"""
-    query_id_search = re.search("Query has been retried using query id: (.*)\n",
-                                get_log_result)
-    if query_id_search and len(query_id_search.groups()) >= 1:
-      retried_query_id = query_id_search.group(1)
-      get_log_result += "Retried query link: %s" % \
-                        self.get_query_link(retried_query_id)
+    if self.webserver_address:
+      query_id_search = re.search("Query has been retried using query id: (.*)\n",
+                                  get_log_result)
+      if query_id_search and len(query_id_search.groups()) >= 1:
+        retried_query_id = query_id_search.group(1)
+        get_log_result += "Retried query link: %s" % \
+                          self.get_query_link(retried_query_id)
     return get_log_result
 
   def _get_http_transport(self, connect_timeout_ms):
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index 996f3b3..3d1d810 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -1133,12 +1133,13 @@ class ImpalaShell(cmd.Cmd, object):
       if summary.error_logs:
         for error_line in summary.error_logs:
           data += error_line + "\n"
-          query_id_search = re.search("Retrying query using query id: (.*)",
-                                      error_line)
-          if query_id_search and len(query_id_search.groups()) == 1:
-            retried_query_id = query_id_search.group(1)
-            data += "Retried query link: %s\n"\
-                    % self.imp_client.get_query_link(retried_query_id)
+          if self.webserver_address:
+            query_id_search = re.search("Retrying query using query id: (.*)",
+                                        error_line)
+            if query_id_search and len(query_id_search.groups()) == 1:
+              retried_query_id = query_id_search.group(1)
+              data += "Retried query link: %s\n"\
+                      % self.imp_client.get_query_link(retried_query_id)
 
       if summary.progress:
         progress = summary.progress
diff --git a/tests/custom_cluster/test_web_pages.py b/tests/custom_cluster/test_web_pages.py
index d2eeffb..7913398 100644
--- a/tests/custom_cluster/test_web_pages.py
+++ b/tests/custom_cluster/test_web_pages.py
@@ -23,6 +23,7 @@ import psutil
 import pytest
 
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.shell.util import run_impala_shell_cmd
 
 
 class TestWebPage(CustomClusterTestSuite):
@@ -138,3 +139,33 @@ class TestWebPage(CustomClusterTestSuite):
     response = requests.get("http://localhost:25000/queries?json")
     response_json = response.text
     assert expected in response_json, "No matching statement found in the queries site."
+
+  # Checks if 'messages' exists/does not exist in 'result_stderr' based on the value of
+  # 'should_exist'
+  def _validate_shell_messages(self, result_stderr, messages, should_exist=True):
+    for msg in messages:
+      if should_exist:
+        assert msg in result_stderr, result_stderr
+      else:
+        assert msg not in result_stderr, result_stderr
+
+  @pytest.mark.execute_serially
+  @CustomClusterTestSuite.with_args(
+      impalad_args="--ping_expose_webserver_url=false"
+  )
+  def test_webserver_url_not_exposed(self, vector):
+    if vector.get_value('table_format').file_format != 'text':
+      pytest.skip('runs only for text table_format')
+    # If webserver url is not exposed, debug web urls shouldn't be printed out.
+    shell_messages = ["Query submitted at: ", "(Coordinator: ",
+        "Query progress can be monitored at: "]
+    query_shell_arg = '--query=select * from functional.alltypes'
+    # hs2
+    results = run_impala_shell_cmd(vector, [query_shell_arg])
+    self._validate_shell_messages(results.stderr, shell_messages, should_exist=False)
+    # beeswax
+    results = run_impala_shell_cmd(vector, ['--protocol=beeswax', query_shell_arg])
+    self._validate_shell_messages(results.stderr, shell_messages, should_exist=False)
+    # Even though webserver url is not exposed, it is still accessible.
+    page = requests.get('http://localhost:25000')
+    assert page.status_code == requests.codes.ok


[impala] 04/06: IMPALA-10165: Implement Bucket and Truncate partition transforms for Iceberg tables

Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 13a78fc1b080baed16ac42888a54c60e9eb7a894
Author: Gabor Kaszab <ga...@cloudera.com>
AuthorDate: Tue Sep 22 14:53:41 2020 +0200

    IMPALA-10165: Implement Bucket and Truncate partition transforms for Iceberg tables
    
    This patch adds support for Iceberg Bucket and Truncate partition
    transforms. Both accept a parameter: number of buckets and width
    respectively.
    
    Usage:
    CREATE TABLE tbl_name (i int, p1 int, p2 timestamp)
    PARTITION BY SPEC (
      p1 BUCKET 10,
      p1 TRUNCATE 5
    ) STORED AS ICEBERG
    TBLPROPERTIES ('iceberg.catalog'='hadoop.tables');
    
    Testing:
      - Extended AnalyzerStmtsTest to cover creating partitioned Iceberg
        tables with the new partition transforms.
      - Extended ParserTest.
      - Extended iceberg-create.test to create Iceberg tables with the new
        partition transforms.
      - Extended show-create-table.test to check that the new partition
        transforms are displayed with their parameters in the SHOW CREATE
        TABLE output.
    
    Change-Id: Idc75cd23045b274885607c45886319f4f6da19de
    Reviewed-on: http://gerrit.cloudera.org:8080/16551
    Reviewed-by: Gabor Kaszab <ga...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 common/thrift/CatalogObjects.thrift                |  11 +-
 fe/src/main/cup/sql-parser.cup                     |  17 ++-
 .../apache/impala/analysis/CreateTableStmt.java    |  14 +-
 .../impala/analysis/IcebergPartitionField.java     |  19 ++-
 .../impala/analysis/IcebergPartitionTransform.java |  88 +++++++++++
 .../org/apache/impala/catalog/FeIcebergTable.java  |  12 +-
 .../org/apache/impala/catalog/IcebergTable.java    |   9 +-
 .../java/org/apache/impala/util/IcebergUtil.java   | 162 +++++++++++++++++----
 .../apache/impala/analysis/AnalyzeStmtsTest.java   |  22 +++
 .../org/apache/impala/analysis/ParserTest.java     |   4 +
 .../queries/QueryTest/iceberg-create.test          |  34 ++++-
 .../queries/QueryTest/show-create-table.test       |  27 +++-
 12 files changed, 352 insertions(+), 67 deletions(-)

diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift
index 457d0dd..acac4c3 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -128,7 +128,7 @@ enum TAccessLevel {
   WRITE_ONLY = 3
 }
 
-enum TIcebergPartitionTransform {
+enum TIcebergPartitionTransformType {
   IDENTITY = 0
   HOUR = 1
   DAY = 2
@@ -522,12 +522,19 @@ struct TKuduTable {
   4: required list<TKuduPartitionParam> partition_by
 }
 
+struct TIcebergPartitionTransform {
+  1: required TIcebergPartitionTransformType transform_type
+
+  // Parameter for BUCKET and TRUNCATE transforms.
+  2: optional i32 transform_param
+}
+
 struct TIcebergPartitionField {
   1: required i32 source_id
   2: required i32 field_id
   3: required string orig_field_name
   4: required string field_name
-  5: required TIcebergPartitionTransform field_type
+  5: required TIcebergPartitionTransform transform
 }
 
 struct TIcebergPartitionSpec {
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index 4bf208e..e85a55c 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -38,6 +38,7 @@ import org.apache.impala.analysis.TableSampleClause;
 import org.apache.impala.analysis.AlterTableAddDropRangePartitionStmt.Operation;
 import org.apache.impala.analysis.IcebergPartitionSpec;
 import org.apache.impala.analysis.IcebergPartitionField;
+import org.apache.impala.analysis.IcebergPartitionTransform;
 import org.apache.impala.catalog.ArrayType;
 import org.apache.impala.catalog.MapType;
 import org.apache.impala.catalog.RowFormat;
@@ -58,7 +59,6 @@ import org.apache.impala.thrift.TShowStatsOp;
 import org.apache.impala.thrift.TTablePropertyType;
 import org.apache.impala.thrift.TPrincipalType;
 import org.apache.impala.thrift.TSortingOrder;
-import org.apache.impala.thrift.TIcebergPartitionTransform;
 import org.apache.impala.service.BackendConfig;
 import org.apache.impala.util.IcebergUtil;
 import org.apache.impala.common.NotImplementedException;
@@ -495,7 +495,8 @@ nonterminal IcebergPartitionSpec iceberg_partition_spec_def;
 nonterminal List<IcebergPartitionField> iceberg_partition_field_list,
   iceberg_partition_field_defs;
 nonterminal IcebergPartitionField iceberg_partition_field_def;
-nonterminal TIcebergPartitionTransform iceberg_partition_transform;
+nonterminal IcebergPartitionTransform iceberg_partition_transform;
+nonterminal String iceberg_partition_transform_type;
 // Options for DDL commands - CREATE/DROP/ALTER
 nonterminal HdfsCachingOp cache_op_val, opt_cache_op_val;
 nonterminal BigDecimal opt_cache_op_replication;
@@ -1787,10 +1788,20 @@ iceberg_partition_field_def ::=
   ;
 
 iceberg_partition_transform ::=
-  IDENT:transfrom_type
+  iceberg_partition_transform_type:transfrom_type INTEGER_LITERAL:transform_param
+  {: RESULT = IcebergUtil.getPartitionTransform(transfrom_type,
+      transform_param.intValue()); :}
+  | IDENT:transfrom_type
   {: RESULT = IcebergUtil.getPartitionTransform(transfrom_type); :}
   ;
 
+iceberg_partition_transform_type ::=
+  IDENT:transform_type
+  {: RESULT = transform_type; :}
+  | KW_TRUNCATE
+  {: RESULT = "TRUNCATE"; :}
+;
+
 create_udf_stmt ::=
   KW_CREATE KW_FUNCTION if_not_exists_val:if_not_exists
   function_name:fn_name function_def_args:fn_args
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
index 990496c..ff6df71 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java
@@ -271,7 +271,7 @@ public class CreateTableStmt extends StatementBase {
     }
 
     if (getFileFormat() == THdfsFileFormat.ICEBERG) {
-      analyzeIcebergFormat();
+      analyzeIcebergFormat(analyzer);
     }
 
     // If lineage logging is enabled, compute minimal lineage graph.
@@ -565,7 +565,7 @@ public class CreateTableStmt extends StatementBase {
   /**
    * For iceberg file format, add related storage handler
    */
-  private void analyzeIcebergFormat() throws AnalysisException {
+  private void analyzeIcebergFormat(Analyzer analyzer) throws AnalysisException {
     // A managed table cannot have 'external.table.purge' property set
     if (!isExternal() && Boolean.parseBoolean(
         getTblProperties().get(IcebergTable.TBL_PROP_EXTERNAL_TABLE_PURGE))) {
@@ -585,7 +585,7 @@ public class CreateTableStmt extends StatementBase {
       }
 
       // Check partition columns for managed iceberg table
-      checkPartitionColumns();
+      checkPartitionColumns(analyzer);
     }
 
     String handler = getTblProperties().get(IcebergTable.KEY_STORAGE_HANDLER);
@@ -627,13 +627,17 @@ public class CreateTableStmt extends StatementBase {
   /**
    * For iceberg table, partition column must be from source column
    */
-  private void checkPartitionColumns() throws AnalysisException {
+  private void checkPartitionColumns(Analyzer analyzer) throws AnalysisException {
     // This check is unnecessary for iceberg table without partition spec
     List<IcebergPartitionSpec> specs = tableDef_.getIcebergPartitionSpecs();
     if (specs == null || specs.isEmpty()) return;
 
     // Iceberg table only has one partition spec now
-    List<IcebergPartitionField> fields = specs.get(0).getIcebergPartitionFields();
+    IcebergPartitionSpec spec = specs.get(0);
+    // Analyzes the partition spec and the underlying partition fields.
+    spec.analyze(analyzer);
+
+    List<IcebergPartitionField> fields = spec.getIcebergPartitionFields();
     Preconditions.checkState(fields != null && !fields.isEmpty());
     for (IcebergPartitionField field : fields) {
       String fieldName = field.getFieldName();
diff --git a/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionField.java b/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionField.java
index ef927e5..8ebd34c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionField.java
+++ b/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionField.java
@@ -19,7 +19,6 @@ package org.apache.impala.analysis;
 
 import org.apache.impala.common.AnalysisException;
 import org.apache.impala.thrift.TIcebergPartitionField;
-import org.apache.impala.thrift.TIcebergPartitionTransform;
 
 /**
  * Represents a PartitionField of iceberg
@@ -40,21 +39,21 @@ public class IcebergPartitionField extends StmtNode {
   // Holds the column name in the table.
   private String origFieldName_;
 
-  //Column partition type
-  private TIcebergPartitionTransform fieldType_;
+  // Partition transform type and transform param for this partition field.
+  private IcebergPartitionTransform transform_;
 
   public IcebergPartitionField(int sourceId, int fieldId, String origFieldName,
-      String fieldName, TIcebergPartitionTransform fieldType) {
+      String fieldName, IcebergPartitionTransform transform) {
     sourceId_ = sourceId;
     fieldId_ = fieldId;
     origFieldName_ = origFieldName;
     fieldName_ = fieldName;
-    fieldType_ = fieldType;
+    transform_ = transform;
   }
 
   // This constructor is called when creating a partitioned Iceberg table.
-  public IcebergPartitionField(String fieldName, TIcebergPartitionTransform fieldType) {
-    this(0, 0, fieldName, fieldName, fieldType);
+  public IcebergPartitionField(String fieldName, IcebergPartitionTransform transform) {
+    this(0, 0, fieldName, fieldName, transform);
   }
 
   public String getFieldName() {
@@ -67,7 +66,7 @@ public class IcebergPartitionField extends StmtNode {
 
   @Override
   public void analyze(Analyzer analyzer) throws AnalysisException {
-    return;
+    transform_.analyze(analyzer);
   }
 
   @Override
@@ -78,7 +77,7 @@ public class IcebergPartitionField extends StmtNode {
   @Override
   public String toSql(ToSqlOptions options) {
     StringBuilder builder = new StringBuilder();
-    builder.append(origFieldName_+ " " + fieldType_.toString());
+    builder.append(origFieldName_+ " " + transform_.toSql());
     return builder.toString();
   }
 
@@ -88,7 +87,7 @@ public class IcebergPartitionField extends StmtNode {
     result.setSource_id(sourceId_);
     result.setOrig_field_name(origFieldName_);
     result.setField_name(fieldName_);
-    result.setField_type(fieldType_);
+    result.setTransform(transform_.toThrift());
     return result;
   }
 }
diff --git a/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionTransform.java b/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionTransform.java
new file mode 100644
index 0000000..6db788c
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionTransform.java
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.analysis;
+
+import org.apache.impala.common.AnalysisException;
+import org.apache.impala.thrift.TIcebergPartitionTransform;
+import org.apache.impala.thrift.TIcebergPartitionTransformType;
+
+/**
+ * Represents the partition transform part of an Iceberg partition field.
+*/
+public class IcebergPartitionTransform extends StmtNode {
+
+    // Stores the transform type such as HOUR, YEAR, etc.
+    private TIcebergPartitionTransformType transformType_;
+
+    // Stores the parameter of BUCKET or TRUNCATE partition transforms (numBuckets,
+    // width respectively). This is null for partition transforms that don't have a
+    // parameter.
+    private Integer transformParam_;
+
+    // Constructor for parameterless partition transforms.
+    public IcebergPartitionTransform(TIcebergPartitionTransformType transformType) {
+        this(transformType, null);
+    }
+
+    public IcebergPartitionTransform(TIcebergPartitionTransformType transformType,
+        Integer transformParam) {
+      transformType_ = transformType;
+      transformParam_ = transformParam;
+    }
+
+    @Override
+    public void analyze(Analyzer analyzer) throws AnalysisException {
+      if (transformType_ == TIcebergPartitionTransformType.BUCKET ||
+          transformType_ == TIcebergPartitionTransformType.TRUNCATE) {
+        if (transformParam_ == null) {
+          throw new AnalysisException("BUCKET and TRUNCATE partition transforms should " +
+              "have a parameter.");
+        }
+        if (transformParam_ <= 0) {
+          throw new AnalysisException("The parameter of a partition transform should " +
+              "be greater than zero.");
+        }
+      } else {
+        if (transformParam_ != null) {
+          throw new AnalysisException("Only BUCKET and TRUNCATE partition transforms " +
+              "accept a parameter.");
+        }
+      }
+      return;
+    }
+
+  @Override
+  public final String toSql() {
+    return toSql(ToSqlOptions.DEFAULT);
+  }
+
+  @Override
+  public String toSql(ToSqlOptions options) {
+    StringBuilder builder = new StringBuilder();
+    builder.append(transformType_.toString());
+    if (transformParam_ != null) builder.append(" " + transformParam_.toString());
+    return builder.toString();
+  }
+
+  public TIcebergPartitionTransform toThrift() {
+    TIcebergPartitionTransform transform = new TIcebergPartitionTransform();
+    transform.setTransform_type(transformType_);
+    if (transformParam_ != null) transform.setTransform_param(transformParam_);
+    return transform;
+  }
+}
\ No newline at end of file
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
index 11dddd1..dd7c05e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
@@ -244,15 +244,17 @@ public interface FeIcebergTable extends FeFsTable {
 
       TableMetadata metadata = IcebergUtil.getIcebergTableMetadata(table);
       if (!metadata.specs().isEmpty()) {
-        // Just show the latest PartitionSpec from iceberg table metadata
-        PartitionSpec latestSpec = metadata.specs().get(metadata.specs().size() - 1);
+        // Just show the current PartitionSpec from Iceberg table metadata
+        PartitionSpec latestSpec = metadata.spec();
+        HashMap<String, Integer> transformParams =
+            IcebergUtil.getPartitionTransformParams(latestSpec);
         for(PartitionField field : latestSpec.fields()) {
           TResultRowBuilder builder = new TResultRowBuilder();
           builder.add(latestSpec.specId());
           builder.add(field.sourceId());
           builder.add(field.fieldId());
           builder.add(field.name());
-          builder.add(IcebergUtil.getPartitionTransform(field).toString());
+          builder.add(IcebergUtil.getPartitionTransform(field, transformParams).toSql());
           result.addToRows(builder.get());
         }
       }
@@ -369,10 +371,12 @@ public interface FeIcebergTable extends FeFsTable {
       List<IcebergPartitionSpec> ret = new ArrayList<>();
       for (PartitionSpec spec : metadata.specs()) {
         List<IcebergPartitionField> fields = new ArrayList<>();;
+        HashMap<String, Integer> transformParams =
+            IcebergUtil.getPartitionTransformParams(spec);
         for (PartitionField field : spec.fields()) {
           fields.add(new IcebergPartitionField(field.sourceId(), field.fieldId(),
               spec.schema().findColumnName(field.sourceId()), field.name(),
-              IcebergUtil.getPartitionTransform(field)));
+              IcebergUtil.getPartitionTransform(field, transformParams)));
         }
         ret.add(new IcebergPartitionSpec(spec.specId(), fields));
       }
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
index 0845c4f..b0f384a 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
@@ -32,6 +32,7 @@ import org.apache.iceberg.TableMetadata;
 import org.apache.iceberg.types.Types;
 import org.apache.impala.analysis.IcebergPartitionField;
 import org.apache.impala.analysis.IcebergPartitionSpec;
+import org.apache.impala.analysis.IcebergPartitionTransform;
 import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
 import org.apache.impala.thrift.TCatalogObjectType;
 import org.apache.impala.thrift.THdfsFileDesc;
@@ -312,8 +313,14 @@ public class IcebergTable extends Table implements FeIcebergTable {
       if (param.getPartition_fields() != null) {
         List<IcebergPartitionField> fields = new ArrayList<>();
         for (TIcebergPartitionField field : param.getPartition_fields()) {
+          Integer transformParam = null;
+          if (field.getTransform().isSetTransform_param()) {
+            transformParam = field.getTransform().getTransform_param();
+          }
           fields.add(new IcebergPartitionField(field.getSource_id(), field.getField_id(),
-              field.getOrig_field_name(), field.getField_name(), field.getField_type()));
+              field.getOrig_field_name(), field.getField_name(),
+              new IcebergPartitionTransform(field.getTransform().getTransform_type(),
+                  transformParam)));
         }
         ret.add(new IcebergPartitionSpec(param.getPartition_id(),
             fields));
diff --git a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
index af4ac31..79444f1 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
@@ -18,10 +18,14 @@
 package org.apache.impala.util;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 
+import com.google.common.collect.Maps;
 import com.google.common.hash.Hasher;
 import com.google.common.hash.Hashing;
+
+import org.apache.impala.common.Pair;
 import org.apache.iceberg.BaseTable;
 import org.apache.iceberg.catalog.TableIdentifier;
 import org.apache.iceberg.DataFile;
@@ -34,7 +38,10 @@ import org.apache.iceberg.PartitionField;
 import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.Schema;
 import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.transforms.PartitionSpecVisitor;
+import org.apache.iceberg.transforms.Transform;
 import org.apache.iceberg.types.Types;
+import org.apache.impala.analysis.IcebergPartitionTransform;
 import org.apache.impala.catalog.ArrayType;
 import org.apache.impala.catalog.Catalog;
 import org.apache.impala.catalog.FeIcebergTable;
@@ -54,6 +61,7 @@ import org.apache.impala.thrift.TIcebergCatalog;
 import org.apache.impala.thrift.TIcebergFileFormat;
 import org.apache.impala.thrift.TIcebergPartitionField;
 import org.apache.impala.thrift.TIcebergPartitionTransform;
+import org.apache.impala.thrift.TIcebergPartitionTransformType;
 
 public class IcebergUtil {
 
@@ -154,19 +162,27 @@ public class IcebergUtil {
     List<TIcebergPartitionField> partitionFields =
         params.getPartition_spec().getPartition_fields();
     for (TIcebergPartitionField partitionField : partitionFields) {
-      if (partitionField.getField_type() == TIcebergPartitionTransform.IDENTITY) {
+      TIcebergPartitionTransformType transformType =
+          partitionField.getTransform().getTransform_type();
+      if (transformType == TIcebergPartitionTransformType.IDENTITY) {
         builder.identity(partitionField.getField_name());
-      } else if (partitionField.getField_type() == TIcebergPartitionTransform.HOUR) {
+      } else if (transformType == TIcebergPartitionTransformType.HOUR) {
         builder.hour(partitionField.getField_name());
-      } else if (partitionField.getField_type() == TIcebergPartitionTransform.DAY) {
+      } else if (transformType == TIcebergPartitionTransformType.DAY) {
         builder.day(partitionField.getField_name());
-      } else if (partitionField.getField_type() == TIcebergPartitionTransform.MONTH) {
+      } else if (transformType == TIcebergPartitionTransformType.MONTH) {
         builder.month(partitionField.getField_name());
-      } else if (partitionField.getField_type() == TIcebergPartitionTransform.YEAR) {
+      } else if (transformType == TIcebergPartitionTransformType.YEAR) {
         builder.year(partitionField.getField_name());
+      } else if (transformType == TIcebergPartitionTransformType.BUCKET) {
+        builder.bucket(partitionField.getField_name(),
+            partitionField.getTransform().getTransform_param());
+      } else if (transformType == TIcebergPartitionTransformType.TRUNCATE) {
+        builder.truncate(partitionField.getField_name(),
+            partitionField.getTransform().getTransform_param());
       } else {
         throw new ImpalaRuntimeException(String.format("Skip partition: %s, %s",
-            partitionField.getField_name(), partitionField.getField_type()));
+            partitionField.getField_name(), transformType));
       }
     }
     return builder.build();
@@ -212,36 +228,126 @@ public class IcebergUtil {
     return null;
   }
 
-  /**
-   * Build TIcebergPartitionTransform by iceberg PartitionField
-   */
-  public static TIcebergPartitionTransform getPartitionTransform(PartitionField field)
+  public static IcebergPartitionTransform getPartitionTransform(
+      PartitionField field, HashMap<String, Integer> transformParams)
       throws TableLoadingException {
     String type = field.transform().toString();
-    return getPartitionTransform(type);
+    String transformMappingKey = getPartitonTransformMappingKey(field.sourceId(),
+        getPartitionTransformType(type));
+    return getPartitionTransform(type, transformParams.get(transformMappingKey));
   }
 
-  public static TIcebergPartitionTransform getPartitionTransform(String type)
+  public static IcebergPartitionTransform getPartitionTransform(String transformType,
+      Integer transformParam) throws TableLoadingException {
+    return new IcebergPartitionTransform(getPartitionTransformType(transformType),
+        transformParam);
+  }
+
+  public static IcebergPartitionTransform getPartitionTransform(String transformType)
       throws TableLoadingException {
-    type = type.toUpperCase();
-    if ("IDENTITY".equals(type)) {
-      return TIcebergPartitionTransform.IDENTITY;
-    } else if ("HOUR".equals(type)) {
-      return TIcebergPartitionTransform.HOUR;
-    } else if ("DAY".equals(type)) {
-      return TIcebergPartitionTransform.DAY;
-    } else if ("MONTH".equals(type)) {
-      return TIcebergPartitionTransform.MONTH;
-    } else if ("YEAR".equals(type)) {
-      return TIcebergPartitionTransform.YEAR;
-    } else if ("BUCKET".equals(type)) {
-      return TIcebergPartitionTransform.BUCKET;
-    } else if ("TRUNCATE".equals(type)) {
-      return TIcebergPartitionTransform.TRUNCATE;
+    return getPartitionTransform(transformType, null);
+  }
+
+  public static TIcebergPartitionTransformType getPartitionTransformType(
+      String transformType) throws TableLoadingException {
+    transformType = transformType.toUpperCase();
+    if ("IDENTITY".equals(transformType)) {
+      return TIcebergPartitionTransformType.IDENTITY;
+    } else if ("HOUR".equals(transformType)) {
+      return TIcebergPartitionTransformType.HOUR;
+    } else if ("DAY".equals(transformType)) {
+      return TIcebergPartitionTransformType.DAY;
+    } else if ("MONTH".equals(transformType)) {
+      return TIcebergPartitionTransformType.MONTH;
+    } else if ("YEAR".equals(transformType)) {
+      return TIcebergPartitionTransformType.YEAR;
+    } else if (transformType != null && transformType.startsWith("BUCKET")) {
+      return TIcebergPartitionTransformType.BUCKET;
+    } else if (transformType != null && transformType.startsWith("TRUNCATE")) {
+      return TIcebergPartitionTransformType.TRUNCATE;
     } else {
       throw new TableLoadingException("Unsupported iceberg partition type: " +
-          type);
+      transformType);
+    }
+  }
+
+  private static String getPartitonTransformMappingKey(int sourceId,
+      TIcebergPartitionTransformType transformType) {
+    return sourceId + "_" + transformType.toString();
+  }
+
+  /**
+   * Gets a PartitionSpec object and returns a mapping between a field in the
+   * PartitionSpec and its transform's parameter. Only Bucket and Truncate transforms
+   * have a parameter, for other transforms this mapping will have a null.
+   * source ID and the transform type are needed together to uniquely identify a specific
+   * field in the PartitionSpec. (Unfortunaltely, fieldId is not available in the Visitor
+   * class below.)
+   * The reason for implementing the PartitionSpecVisitor below was that Iceberg doesn't
+   * expose the interface of the transform types outside of their package and the only
+   * way to get the transform's parameter is implementing this visitor class.
+   */
+  public static HashMap<String, Integer> getPartitionTransformParams(PartitionSpec spec)
+      throws TableLoadingException {
+    List<Pair<String, Integer>> transformParams = PartitionSpecVisitor.visit(
+        spec.schema(), spec, new PartitionSpecVisitor<Pair<String, Integer>>() {
+          @Override
+          public Pair<String, Integer> identity(String sourceName, int sourceId) {
+            String mappingKey = getPartitonTransformMappingKey(sourceId,
+                TIcebergPartitionTransformType.IDENTITY);
+            return new Pair<String, Integer>(mappingKey, null);
+          }
+
+          @Override
+          public Pair<String, Integer> bucket(String sourceName, int sourceId,
+              int numBuckets) {
+            String mappingKey = getPartitonTransformMappingKey(sourceId,
+                TIcebergPartitionTransformType.BUCKET);
+            return new Pair<String, Integer>(mappingKey, numBuckets);
+          }
+
+          @Override
+          public Pair<String, Integer> truncate(String sourceName, int sourceId,
+              int width) {
+            String mappingKey = getPartitonTransformMappingKey(sourceId,
+                TIcebergPartitionTransformType.TRUNCATE);
+            return new Pair<String, Integer>(mappingKey, width);
+          }
+
+          @Override
+          public Pair<String, Integer> year(String sourceName, int sourceId) {
+            String mappingKey = getPartitonTransformMappingKey(sourceId,
+                TIcebergPartitionTransformType.YEAR);
+            return new Pair<String, Integer>(mappingKey, null);
+          }
+
+          @Override
+          public Pair<String, Integer> month(String sourceName, int sourceId) {
+            String mappingKey = getPartitonTransformMappingKey(sourceId,
+                TIcebergPartitionTransformType.MONTH);
+            return new Pair<String, Integer>(mappingKey, null);
+          }
+
+          @Override
+          public Pair<String, Integer> day(String sourceName, int sourceId) {
+            String mappingKey = getPartitonTransformMappingKey(sourceId,
+                TIcebergPartitionTransformType.DAY);
+            return new Pair<String, Integer>(mappingKey, null);
+          }
+
+          @Override
+          public Pair<String, Integer> hour(String sourceName, int sourceId) {
+            String mappingKey = getPartitonTransformMappingKey(sourceId,
+                TIcebergPartitionTransformType.HOUR);
+            return new Pair<String, Integer>(mappingKey, null);
+          }
+        });
+    // Move the content of the List into a HashMap for faster querying in the future.
+    HashMap<String, Integer> result = Maps.newHashMap();
+    for (Pair<String, Integer> transformParam : transformParams) {
+      result.put(transformParam.first, transformParam.second);
     }
+    return result;
   }
 
   /**
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index 5b4d4db..bbce69b 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -4836,4 +4836,26 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
             + "or they should both return 'STRING' or 'VARCHAR' or 'CHAR' types, "
             + "but they return types 'NULL_TYPE' and 'STRING'.");
   }
+
+  @Test
+  public void testCreatePartitionedIcebergTable() throws ImpalaException {
+    String tblProperties = " TBLPROPERTIES ('iceberg.catalog'='hadoop.tables')";
+    AnalyzesOk("CREATE TABLE tbl1 (i int, p1 int, p2 timestamp) " +
+        "PARTITION BY SPEC (p1 BUCKET 10, p1 TRUNCATE 5, p2 DAY) STORED AS ICEBERG" +
+            tblProperties);
+    AnalysisError("CREATE TABLE tbl1 (i int, p1 int, p2 timestamp) " +
+        "PARTITION BY SPEC (p1 BUCKET, p2 DAY) STORED AS ICEBERG" + tblProperties,
+        "BUCKET and TRUNCATE partition transforms should have a parameter.");
+    AnalysisError("CREATE TABLE tbl1 (i int, p1 int, p2 timestamp) " +
+        "PARTITION BY SPEC (p1 BUCKET 0, p2 DAY) STORED AS ICEBERG" + tblProperties,
+        "The parameter of a partition transform should be greater than zero.");
+    AnalysisError("CREATE TABLE tbl1 (i int, p1 int, p2 timestamp) " +
+        "PARTITION BY SPEC (p1 TRUNCATE 0, p2 DAY) STORED AS ICEBERG" + tblProperties,
+        "The parameter of a partition transform should be greater than zero.");
+
+    AnalysisError("CREATE TABLE tbl1 (i int, p1 int, p2 timestamp) " +
+        "PARTITION BY SPEC (p1 BUCKET 10, p2 DAY 10) STORED AS ICEBERG" + tblProperties,
+        "Only BUCKET and TRUNCATE partition transforms accept a parameter.");
+  }
+
 }
diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
index 582125b..73111e5 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -2984,6 +2984,10 @@ public class ParserTest extends FrontendTestBase {
     ParserError("CREATE TABLE Foo(a int PRIMARY KEY, b int BLOCK_SIZE 1+1) " +
         "STORED AS KUDU");
     ParserError("CREATE TABLE Foo(a int PRIMARY KEY BLOCK_SIZE -1) STORED AS KUDU");
+
+    // Iceberg TRUNCATE partition transform without parameter results a parse error.
+    ParserError("CREATE TABLE tbl1 (i int, p1 int) PARTITION BY SPEC (p1 TRUNCATE) " +
+        "STORED AS ICEBERG TBLPROPERTIES ('iceberg.catalog'='hadoop.tables')");
   }
 
   @Test
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
index 436b326..24f5239 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
@@ -12,8 +12,10 @@ CREATE TABLE iceberg_test1(
 PARTITION BY SPEC
 (
   level IDENTITY,
+  level TRUNCATE 10,
   event_time IDENTITY,
   event_time HOUR,
+  event_time BUCKET 1000,
   register_time DAY
 )
 STORED AS ICEBERG
@@ -38,9 +40,11 @@ STRING,STRING,STRING
 SHOW PARTITIONS iceberg_test1;
 ---- RESULTS
 0,1,1000,'level','IDENTITY'
-0,2,1001,'event_time','IDENTITY'
-0,2,1002,'event_time_hour','HOUR'
-0,3,1003,'register_time_day','DAY'
+0,1,1001,'level_trunc','TRUNCATE 10'
+0,2,1002,'event_time','IDENTITY'
+0,2,1003,'event_time_hour','HOUR'
+0,2,1004,'event_time_bucket','BUCKET 1000'
+0,3,1005,'register_time_day','DAY'
 ---- TYPES
 BIGINT,BIGINT,BIGINT,STRING,STRING
 ====
@@ -76,7 +80,9 @@ CREATE TABLE iceberg_test4(
 )
 PARTITION BY SPEC
 (
-  level IDENTITY
+  level IDENTITY,
+  level BUCKET 12345,
+  level TRUNCATE 15
 )
 STORED AS ICEBERG
 LOCATION '/$DATABASE.iceberg_test_with_location'
@@ -101,6 +107,8 @@ STRING,STRING,STRING
 SHOW PARTITIONS iceberg_test_external;
 ---- RESULTS
 0,1,1000,'level','IDENTITY'
+0,1,1001,'level_bucket','BUCKET 12345'
+0,1,1002,'level_trunc','TRUNCATE 15'
 ---- TYPES
 BIGINT,BIGINT,BIGINT,STRING,STRING
 ====
@@ -123,6 +131,8 @@ STRING,STRING,STRING
 SHOW PARTITIONS iceberg_test_external_empty_column;
 ---- RESULTS
 0,1,1000,'level','IDENTITY'
+0,1,1001,'level_bucket','BUCKET 12345'
+0,1,1002,'level_trunc','TRUNCATE 15'
 ---- TYPES
 BIGINT,BIGINT,BIGINT,STRING,STRING
 ====
@@ -146,8 +156,10 @@ CREATE TABLE iceberg_test5(
 PARTITION BY SPEC
 (
   level IDENTITY,
+  level TRUNCATE 10,
   event_time IDENTITY,
   event_time HOUR,
+  event_time BUCKET 1000,
   register_time DAY
 )
 STORED AS ICEBERG
@@ -173,9 +185,11 @@ STRING,STRING,STRING
 SHOW PARTITIONS iceberg_test5;
 ---- RESULTS
 0,1,1000,'level','IDENTITY'
-0,2,1001,'event_time','IDENTITY'
-0,2,1002,'event_time_hour','HOUR'
-0,3,1003,'register_time_day','DAY'
+0,1,1001,'level_trunc','TRUNCATE 10'
+0,2,1002,'event_time','IDENTITY'
+0,2,1003,'event_time_hour','HOUR'
+0,2,1004,'event_time_bucket','BUCKET 1000'
+0,3,1005,'register_time_day','DAY'
 ---- TYPES
 BIGINT,BIGINT,BIGINT,STRING,STRING
 ====
@@ -190,7 +204,9 @@ CREATE TABLE iceberg_test6(
 )
 PARTITION BY SPEC
 (
-  level IDENTITY
+  level IDENTITY,
+  level BUCKET 12345,
+  level TRUNCATE 10
 )
 STORED AS ICEBERG
 TBLPROPERTIES('iceberg.catalog'='hadoop.catalog',
@@ -215,6 +231,8 @@ STRING,STRING,STRING
 SHOW PARTITIONS iceberg_test6_external;
 ---- RESULTS
 0,1,1000,'level','IDENTITY'
+0,1,1001,'level_bucket','BUCKET 12345'
+0,1,1002,'level_trunc','TRUNCATE 10'
 ---- TYPES
 BIGINT,BIGINT,BIGINT,STRING,STRING
 ====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
index e46d5e8..5c649b6 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
@@ -630,12 +630,17 @@ CREATE TABLE iceberg_test1_partitioned (
   level STRING,
   p1 DATE,
   p2 STRING,
-  p3 TIMESTAMP
+  p3 TIMESTAMP,
+  p4 INT
 )
 PARTITION BY SPEC (
   p1 YEAR,
   p2 IDENTITY,
-  p3 HOUR
+  p2 BUCKET 500,
+  p2 TRUNCATE 15,
+  p3 HOUR,
+  p4 BUCKET 10,
+  p4 TRUNCATE 5
 )
 STORED AS ICEBERG
 TBLPROPERTIES('iceberg.file_format'='parquet',
@@ -645,13 +650,18 @@ CREATE TABLE show_create_table_test_db.iceberg_test1_partitioned (
   level STRING,
   p1 DATE,
   p2 STRING,
-  p3 TIMESTAMP
+  p3 TIMESTAMP,
+  p4 INT
 )
 PARTITION BY SPEC
 (
   p1 YEAR,
   p2 IDENTITY,
-  p3 HOUR
+  p2 BUCKET 500,
+  p2 TRUNCATE 15,
+  p3 HOUR,
+  p4 BUCKET 10,
+  p4 TRUNCATE 5
 )
 STORED AS ICEBERG
 TBLPROPERTIES('iceberg.file_format'='parquet',
@@ -661,13 +671,18 @@ CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_test1_partitioned (
   level STRING,
   p1 DATE,
   p2 STRING,
-  p3 TIMESTAMP
+  p3 TIMESTAMP,
+  p4 INT
 )
 PARTITION BY SPEC
 (
   p1 YEAR,
   p2 IDENTITY,
-  p3 HOUR
+  p2 BUCKET 500,
+  p2 TRUNCATE 15,
+  p3 HOUR,
+  p4 BUCKET 10,
+  p4 TRUNCATE 5,
 )
 STORED AS ICEBERG
 TBLPROPERTIES ('external.table.purge'='TRUE', 'iceberg.catalog'='hadoop.catalog',


[impala] 05/06: IMPALA-10159: Supporting ORC file format for Iceberg table

Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 0c0985a825fba8d9702639e3e679d2e1b9070fe1
Author: skyyws <sk...@163.com>
AuthorDate: Sun Sep 6 11:22:06 2020 +0800

    IMPALA-10159: Supporting ORC file format for Iceberg table
    
    This patch mainly realizes querying Iceberg table with ORC
    file format. We can using following SQL to create table with
    ORC file format:
      CREATE TABLE default.iceberg_test (
        level string,
        event_time timestamp,
        message string,
      )
      STORED AS ICEBERG
      LOCATION 'hdfs://xxx'
      TBLPROPERTIES ('iceberg.file_format'='orc', 'iceberg.catalog'='hadoop.tables');
    But pay attention, there still some problems when scan ORC files
    with Timestamp, more details please refer IMPALA-9967. We may add
    new tests with Timestmap type after this JIRA fixed.
    
    Testing:
    - Create table tests in functional_schema_template.sql
    - Iceberg table create test in test_iceberg.py
    - Iceberg table query test in test_scanners.py
    
    Change-Id: Ib579461aa57348c9893a6d26a003a0d812346c4d
    Reviewed-on: http://gerrit.cloudera.org:8080/16568
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 .../java/org/apache/impala/util/IcebergUtil.java   |   6 +-
 ...-0982a5d3-48c0-4dd0-ab87-d24190894251-00000.orc | Bin 0 -> 460 bytes
 ...-dc56d2c8-e285-428d-b81e-f3d07ec53c12-00000.orc | Bin 0 -> 460 bytes
 ...-d4a071f4-5d57-493e-8d21-8f82655087e4-00000.orc | Bin 0 -> 461 bytes
 ...-e9fa9638-8693-43dc-b1ba-2d8dc390b585-00000.orc | Bin 0 -> 460 bytes
 ...-4526ff61-8991-4645-b448-bf3535492111-00000.orc | Bin 0 -> 460 bytes
 ...-439376ff-3d17-4494-94ba-586fa00caadd-00000.orc | Bin 0 -> 461 bytes
 ...-fb7663c5-aade-4690-bdfc-2334d605a891-00000.orc | Bin 0 -> 474 bytes
 ...-fc53a761-1ee2-4ce4-9507-c9917917d2c9-00000.orc | Bin 0 -> 474 bytes
 ...-8887c762-b6a8-4e2c-b1e0-6069d6141917-00000.orc | Bin 0 -> 474 bytes
 ...-a85e02aa-ff28-4311-b996-a83f67a5e39b-00000.orc | Bin 0 -> 470 bytes
 ...-dea5a8ec-ea06-4688-8293-486a62563ef3-00000.orc | Bin 0 -> 474 bytes
 ...-d182fe7e-de19-4df8-ac19-27bbfe17b5bd-00000.orc | Bin 0 -> 474 bytes
 ...-d5df20b1-6ce6-486a-b084-ea55b7954431-00000.orc | Bin 0 -> 439 bytes
 ...-b30cc130-03a1-45e5-b927-d80545593f9a-00000.orc | Bin 0 -> 435 bytes
 ...-6476cc9e-9b5a-48d3-8cce-ea7ebb2d904f-00000.orc | Bin 0 -> 439 bytes
 ...-c2ab6dc5-3586-4b45-8a3c-860fdc71980d-00000.orc | Bin 0 -> 439 bytes
 ...-fd03c61f-8c73-4639-8974-1f6431a30bc5-00000.orc | Bin 0 -> 439 bytes
 ...-9baec6d2-8e31-4d19-8259-1a9e546aaea5-00000.orc | Bin 0 -> 439 bytes
 ...-433b1afa-9fee-4581-89fb-105483ebf1a2-00000.orc | Bin 0 -> 439 bytes
 ...-e0156a2d-4240-4c8e-9724-6ad3231cf178-00000.orc | Bin 0 -> 439 bytes
 .../02fb8198-e791-4d89-8afa-c75fb5443346-m0.avro   | Bin 0 -> 5018 bytes
 ...670-1-02fb8198-e791-4d89-8afa-c75fb5443346.avro | Bin 0 -> 2582 bytes
 .../metadata/v1.metadata.json                      |  47 +++++++++++
 .../metadata/v2.metadata.json                      |  66 +++++++++++++++
 .../metadata/version-hint.text                     |   1 +
 .../functional/functional_schema_template.sql      |  12 +++
 .../datasets/functional/schema_constraints.csv     |   2 +
 .../queries/QueryTest/iceberg-create.test          |  51 +++++++++++
 .../queries/QueryTest/iceberg-query.test           |  94 +++++++++++++++++++++
 .../queries/QueryTest/show-create-table.test       |  22 +++++
 31 files changed, 300 insertions(+), 1 deletion(-)

diff --git a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
index 79444f1..d5688dc 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
@@ -224,7 +224,11 @@ public class IcebergUtil {
    * Get TIcebergFileFormat from a string, usually from table properties
    */
   public static TIcebergFileFormat getIcebergFileFormat(String format){
-    if ("PARQUET".equalsIgnoreCase(format)) return TIcebergFileFormat.PARQUET;
+    if ("PARQUET".equalsIgnoreCase(format)) {
+      return TIcebergFileFormat.PARQUET;
+    } else if ("ORC".equalsIgnoreCase(format)) {
+      return TIcebergFileFormat.ORC;
+    }
     return null;
   }
 
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00004-4-0982a5d3-48c0-4dd0-ab87-d24190894251-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00004-4-0982a5d3-48c0-4dd0-ab87-d24190894251-00000.orc
new file mode 100644
index 0000000..435f1f5
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00004-4-0982a5d3-48c0-4dd0-ab87-d24190894251-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00014-14-dc56d2c8-e285-428d-b81e-f3d07ec53c12-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00014-14-dc56d2c8-e285-428d-b81e-f3d07ec53c12-00000.orc
new file mode 100644
index 0000000..4ca7c90
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00014-14-dc56d2c8-e285-428d-b81e-f3d07ec53c12-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00015-15-d4a071f4-5d57-493e-8d21-8f82655087e4-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00015-15-d4a071f4-5d57-493e-8d21-8f82655087e4-00000.orc
new file mode 100644
index 0000000..3b32dd2
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00015-15-d4a071f4-5d57-493e-8d21-8f82655087e4-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00019-19-e9fa9638-8693-43dc-b1ba-2d8dc390b585-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00019-19-e9fa9638-8693-43dc-b1ba-2d8dc390b585-00000.orc
new file mode 100644
index 0000000..e083888
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00019-19-e9fa9638-8693-43dc-b1ba-2d8dc390b585-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00020-20-4526ff61-8991-4645-b448-bf3535492111-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00020-20-4526ff61-8991-4645-b448-bf3535492111-00000.orc
new file mode 100644
index 0000000..711e00b
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00020-20-4526ff61-8991-4645-b448-bf3535492111-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00028-28-439376ff-3d17-4494-94ba-586fa00caadd-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00028-28-439376ff-3d17-4494-94ba-586fa00caadd-00000.orc
new file mode 100644
index 0000000..866f5df
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00028-28-439376ff-3d17-4494-94ba-586fa00caadd-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00003-3-fb7663c5-aade-4690-bdfc-2334d605a891-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00003-3-fb7663c5-aade-4690-bdfc-2334d605a891-00000.orc
new file mode 100644
index 0000000..2859593
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00003-3-fb7663c5-aade-4690-bdfc-2334d605a891-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00007-7-fc53a761-1ee2-4ce4-9507-c9917917d2c9-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00007-7-fc53a761-1ee2-4ce4-9507-c9917917d2c9-00000.orc
new file mode 100644
index 0000000..b44d885
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00007-7-fc53a761-1ee2-4ce4-9507-c9917917d2c9-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00011-11-8887c762-b6a8-4e2c-b1e0-6069d6141917-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00011-11-8887c762-b6a8-4e2c-b1e0-6069d6141917-00000.orc
new file mode 100644
index 0000000..2687eda
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00011-11-8887c762-b6a8-4e2c-b1e0-6069d6141917-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00012-12-a85e02aa-ff28-4311-b996-a83f67a5e39b-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00012-12-a85e02aa-ff28-4311-b996-a83f67a5e39b-00000.orc
new file mode 100644
index 0000000..c4cf486
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00012-12-a85e02aa-ff28-4311-b996-a83f67a5e39b-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00022-22-dea5a8ec-ea06-4688-8293-486a62563ef3-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00022-22-dea5a8ec-ea06-4688-8293-486a62563ef3-00000.orc
new file mode 100644
index 0000000..86ac4e0
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00022-22-dea5a8ec-ea06-4688-8293-486a62563ef3-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00025-25-d182fe7e-de19-4df8-ac19-27bbfe17b5bd-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00025-25-d182fe7e-de19-4df8-ac19-27bbfe17b5bd-00000.orc
new file mode 100644
index 0000000..38833fc
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00025-25-d182fe7e-de19-4df8-ac19-27bbfe17b5bd-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00001-1-d5df20b1-6ce6-486a-b084-ea55b7954431-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00001-1-d5df20b1-6ce6-486a-b084-ea55b7954431-00000.orc
new file mode 100644
index 0000000..da3d170
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00001-1-d5df20b1-6ce6-486a-b084-ea55b7954431-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00006-6-b30cc130-03a1-45e5-b927-d80545593f9a-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00006-6-b30cc130-03a1-45e5-b927-d80545593f9a-00000.orc
new file mode 100644
index 0000000..1ce1eba
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00006-6-b30cc130-03a1-45e5-b927-d80545593f9a-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00009-9-6476cc9e-9b5a-48d3-8cce-ea7ebb2d904f-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00009-9-6476cc9e-9b5a-48d3-8cce-ea7ebb2d904f-00000.orc
new file mode 100644
index 0000000..c531b59
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00009-9-6476cc9e-9b5a-48d3-8cce-ea7ebb2d904f-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00017-17-c2ab6dc5-3586-4b45-8a3c-860fdc71980d-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00017-17-c2ab6dc5-3586-4b45-8a3c-860fdc71980d-00000.orc
new file mode 100644
index 0000000..fe14583
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00017-17-c2ab6dc5-3586-4b45-8a3c-860fdc71980d-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00023-23-fd03c61f-8c73-4639-8974-1f6431a30bc5-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00023-23-fd03c61f-8c73-4639-8974-1f6431a30bc5-00000.orc
new file mode 100644
index 0000000..1d83368
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00023-23-fd03c61f-8c73-4639-8974-1f6431a30bc5-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00027-27-9baec6d2-8e31-4d19-8259-1a9e546aaea5-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00027-27-9baec6d2-8e31-4d19-8259-1a9e546aaea5-00000.orc
new file mode 100644
index 0000000..baed8f1
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00027-27-9baec6d2-8e31-4d19-8259-1a9e546aaea5-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00030-30-433b1afa-9fee-4581-89fb-105483ebf1a2-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00030-30-433b1afa-9fee-4581-89fb-105483ebf1a2-00000.orc
new file mode 100644
index 0000000..e423097
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00030-30-433b1afa-9fee-4581-89fb-105483ebf1a2-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00031-31-e0156a2d-4240-4c8e-9724-6ad3231cf178-00000.orc b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00031-31-e0156a2d-4240-4c8e-9724-6ad3231cf178-00000.orc
new file mode 100644
index 0000000..c343c76
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00031-31-e0156a2d-4240-4c8e-9724-6ad3231cf178-00000.orc differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/02fb8198-e791-4d89-8afa-c75fb5443346-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/02fb8198-e791-4d89-8afa-c75fb5443346-m0.avro
new file mode 100644
index 0000000..f6a6e41
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/02fb8198-e791-4d89-8afa-c75fb5443346-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/snap-3506237933060603670-1-02fb8198-e791-4d89-8afa-c75fb5443346.avro b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/snap-3506237933060603670-1-02fb8198-e791-4d89-8afa-c75fb5443346.avro
new file mode 100644
index 0000000..ae0c45a
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/snap-3506237933060603670-1-02fb8198-e791-4d89-8afa-c75fb5443346.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/v1.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/v1.metadata.json
new file mode 100644
index 0000000..6f69b2d
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/v1.metadata.json
@@ -0,0 +1,47 @@
+{
+  "format-version" : 1,
+  "table-uuid" : "ea1384cd-938f-4e85-86be-666434dedaaa",
+  "location" : "/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc",
+  "last-updated-ms" : 1600952811299,
+  "last-column-id" : 3,
+  "schema" : {
+    "type" : "struct",
+    "fields" : [ {
+      "id" : 1,
+      "name" : "id",
+      "required" : false,
+      "type" : "int"
+    }, {
+      "id" : 2,
+      "name" : "user",
+      "required" : true,
+      "type" : "string"
+    }, {
+      "id" : 3,
+      "name" : "action",
+      "required" : true,
+      "type" : "string"
+    } ]
+  },
+  "partition-spec" : [ {
+    "name" : "action",
+    "transform" : "identity",
+    "source-id" : 3,
+    "field-id" : 1000
+  } ],
+  "default-spec-id" : 0,
+  "partition-specs" : [ {
+    "spec-id" : 0,
+    "fields" : [ {
+      "name" : "action",
+      "transform" : "identity",
+      "source-id" : 3,
+      "field-id" : 1000
+    } ]
+  } ],
+  "properties" : { },
+  "current-snapshot-id" : -1,
+  "snapshots" : [ ],
+  "snapshot-log" : [ ],
+  "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/v2.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/v2.metadata.json
new file mode 100644
index 0000000..53fd9db
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/v2.metadata.json
@@ -0,0 +1,66 @@
+{
+  "format-version" : 1,
+  "table-uuid" : "ea1384cd-938f-4e85-86be-666434dedaaa",
+  "location" : "/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc",
+  "last-updated-ms" : 1600952815220,
+  "last-column-id" : 3,
+  "schema" : {
+    "type" : "struct",
+    "fields" : [ {
+      "id" : 1,
+      "name" : "id",
+      "required" : false,
+      "type" : "int"
+    }, {
+      "id" : 2,
+      "name" : "user",
+      "required" : true,
+      "type" : "string"
+    }, {
+      "id" : 3,
+      "name" : "action",
+      "required" : true,
+      "type" : "string"
+    } ]
+  },
+  "partition-spec" : [ {
+    "name" : "action",
+    "transform" : "identity",
+    "source-id" : 3,
+    "field-id" : 1000
+  } ],
+  "default-spec-id" : 0,
+  "partition-specs" : [ {
+    "spec-id" : 0,
+    "fields" : [ {
+      "name" : "action",
+      "transform" : "identity",
+      "source-id" : 3,
+      "field-id" : 1000
+    } ]
+  } ],
+  "properties" : { },
+  "current-snapshot-id" : 3506237933060603670,
+  "snapshots" : [ {
+    "snapshot-id" : 3506237933060603670,
+    "timestamp-ms" : 1600952815220,
+    "summary" : {
+      "operation" : "append",
+      "spark.app.id" : "local-1600952804995",
+      "added-data-files" : "20",
+      "added-records" : "20",
+      "changed-partition-count" : "3",
+      "total-records" : "20",
+      "total-data-files" : "20"
+    },
+    "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/snap-3506237933060603670-1-02fb8198-e791-4d89-8afa-c75fb5443346.avro"
+  } ],
+  "snapshot-log" : [ {
+    "timestamp-ms" : 1600952815220,
+    "snapshot-id" : 3506237933060603670
+  } ],
+  "metadata-log" : [ {
+    "timestamp-ms" : 1600952811299,
+    "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/v1.metadata.json"
+  } ]
+}
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/version-hint.text b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/version-hint.text
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/version-hint.text
@@ -0,0 +1 @@
+2
\ No newline at end of file
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index cd8a0d2..467798f 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -2973,3 +2973,15 @@ TBLPROPERTIES('iceberg.file_format'='parquet', 'iceberg.catalog'='hadoop.catalog
 `hadoop fs -mkdir -p /test-warehouse/iceberg_test/hadoop_catalog && \
 hadoop fs -put -f ${IMPALA_HOME}/testdata/data/iceberg_test/hadoop_catalog/hadoop_catalog_test /test-warehouse/iceberg_test/hadoop_catalog/
 ====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+iceberg_partitioned_orc_external
+---- CREATE
+CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name}
+STORED AS ICEBERG
+TBLPROPERTIES('iceberg.file_format'='orc', 'iceberg.catalog'='hadoop.catalog', 'iceberg.catalog_location'='/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc', 'iceberg.table_identifier'='functional_parquet.iceberg_partitioned_orc');
+---- DEPENDENT_LOAD
+`hadoop fs -mkdir -p /test-warehouse/iceberg_test/hadoop_catalog && \
+hadoop fs -put -f ${IMPALA_HOME}/testdata/data/iceberg_test/hadoop_catalog/iceberg_partitioned_orc /test-warehouse/iceberg_test/hadoop_catalog/
+====
diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv
index 39b724f..83136ea 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -63,6 +63,8 @@ table_name:hudi_non_partitioned, constraint:restrict_to, table_format:parquet/no
 table_name:hudi_as_parquet, constraint:restrict_to, table_format:parquet/none/none
 table_name:iceberg_partitioned, constraint:restrict_to, table_format:parquet/none/none
 table_name:iceberg_non_partitioned, constraint:restrict_to, table_format:parquet/none/none
+table_name:hadoop_catalog_test_external, constraint:restrict_to, table_format:parquet/none/none
+table_name:iceberg_partitioned_orc_external, constraint:restrict_to, table_format:parquet/none/none
 
 # TODO: Support Avro. Data loading currently fails for Avro because complex types
 # cannot be converted to the corresponding Avro types yet.
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
index 24f5239..fd06875 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
@@ -288,3 +288,54 @@ DESCRIBE FORMATTED iceberg_test9;
 ---- TYPES
 string, string, string
 ====
+---- QUERY
+CREATE TABLE iceberg_test10(
+  level STRING,
+  event_time TIMESTAMP,
+  register_time DATE,
+  message STRING,
+  price DECIMAL(8,1),
+  map_test MAP <STRING, array <STRING>>,
+  struct_test STRUCT <f1: BIGINT, f2: BIGINT>
+)
+PARTITION BY SPEC
+(
+  level IDENTITY,
+  event_time IDENTITY,
+  event_time HOUR,
+  register_time DAY
+)
+STORED AS ICEBERG
+TBLPROPERTIES('iceberg.file_format'='orc','iceberg_catalog'='hadoop.tables',
+'iceberg.catalog_location'='/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test');
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+DESCRIBE iceberg_test10;
+---- RESULTS
+'level','string',''
+'event_time','timestamp',''
+'register_time','date',''
+'message','string',''
+'price','decimal(8,1)',''
+'map_test','map<string,array<string>>',''
+'struct_test','struct<\n  f1:bigint,\n  f2:bigint\n>',''
+---- TYPES
+STRING,STRING,STRING
+====
+---- QUERY
+SHOW PARTITIONS iceberg_test10;
+---- RESULTS
+0,1,1000,'level','IDENTITY'
+0,2,1001,'event_time','IDENTITY'
+0,2,1002,'event_time_hour','HOUR'
+0,3,1003,'register_time_day','DAY'
+---- TYPES
+BIGINT,BIGINT,BIGINT,STRING,STRING
+====
+---- QUERY
+DROP TABLE iceberg_test10;
+---- RESULTS
+'Table has been dropped.'
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
index 9c94d7f..5081644 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
@@ -282,6 +282,89 @@ Path,Size,Partition
 STRING,STRING,STRING
 ====
 ---- QUERY
+SELECT count(*) from iceberg_partitioned_orc_external;
+---- TYPES
+bigint
+---- RESULTS
+20
+====
+---- QUERY
+SELECT count(*) from iceberg_partitioned_orc_external
+where id > 10
+---- TYPES
+bigint
+---- RESULTS
+10
+====
+---- QUERY
+SELECT count(*) from iceberg_partitioned_orc_external
+where action <> 'click'
+---- TYPES
+bigint
+---- RESULTS
+14
+====
+---- QUERY
+# Test distinct count
+SELECT count(distinct id),count(distinct action) from iceberg_partitioned_orc_external
+---- TYPES
+bigint,bigint
+---- RESULTS
+20,3
+====
+---- QUERY
+SELECT * from iceberg_partitioned_orc_external
+where id < 5
+ORDER BY id
+---- TYPES
+int,string,string
+---- RESULTS
+1,'Alex','view'
+2,'Lisa','download'
+3,'Alan','click'
+4,'Alex','view'
+====
+---- QUERY
+SHOW PARTITIONS iceberg_partitioned_orc_external
+---- TYPES
+bigint,bigint,bigint,string,string
+---- RESULTS
+0,3,1000,'action','IDENTITY'
+====
+---- QUERY
+SHOW FILES in iceberg_partitioned_orc_external
+---- LABELS
+Path,Size,Partition
+---- RESULTS
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00004-4-0982a5d3-48c0-4dd0-ab87-d24190894251-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00014-14-dc56d2c8-e285-428d-b81e-f3d07ec53c12-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00015-15-d4a071f4-5d57-493e-8d21-8f82655087e4-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00019-19-e9fa9638-8693-43dc-b1ba-2d8dc390b585-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00020-20-4526ff61-8991-4645-b448-bf3535492111-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=click/00028-28-439376ff-3d17-4494-94ba-586fa00caadd-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00003-3-fb7663c5-aade-4690-bdfc-2334d605a891-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00007-7-fc53a761-1ee2-4ce4-9507-c9917917d2c9-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00011-11-8887c762-b6a8-4e2c-b1e0-6069d6141917-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00012-12-a85e02aa-ff28-4311-b996-a83f67a5e39b-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00022-22-dea5a8ec-ea06-4688-8293-486a62563ef3-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=download/00025-25-d182fe7e-de19-4df8-ac19-27bbfe17b5bd-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00001-1-d5df20b1-6ce6-486a-b084-ea55b7954431-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00006-6-b30cc130-03a1-45e5-b927-d80545593f9a-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00009-9-6476cc9e-9b5a-48d3-8cce-ea7ebb2d904f-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00017-17-c2ab6dc5-3586-4b45-8a3c-860fdc71980d-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00023-23-fd03c61f-8c73-4639-8974-1f6431a30bc5-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00027-27-9baec6d2-8e31-4d19-8259-1a9e546aaea5-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00030-30-433b1afa-9fee-4581-89fb-105483ebf1a2-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/data/action=view/00031-31-e0156a2d-4240-4c8e-9724-6ad3231cf178-00000.orc',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/02fb8198-e791-4d89-8afa-c75fb5443346-m0.avro',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/snap-3506237933060603670-1-02fb8198-e791-4d89-8afa-c75fb5443346.avro',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/v1.metadata.json',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/v2.metadata.json',regex:.*,''
+'$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc/metadata/version-hint.text',regex:.*,''
+---- TYPES
+STRING,STRING,STRING
+====
+---- QUERY
 describe formatted iceberg_partitioned;
 ---- RESULTS: VERIFY_IS_SUBSET
 'Location:           ','$NAMENODE/test-warehouse/iceberg_test/iceberg_partitioned','NULL'
@@ -310,3 +393,14 @@ describe formatted hadoop_catalog_test_external;
 ---- TYPES
 string, string, string
 ====
+---- QUERY
+describe formatted iceberg_partitioned_orc_external;
+---- RESULTS: VERIFY_IS_SUBSET
+'Location:           ','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc','NULL'
+'','iceberg.catalog_location','/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc'
+'','iceberg.table_identifier','functional_parquet.iceberg_partitioned_orc'
+'','iceberg.file_format ','orc                 '
+'','iceberg.catalog     ','hadoop.catalog      '
+---- TYPES
+string, string, string
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
index 5c649b6..61bd879 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
@@ -689,3 +689,25 @@ TBLPROPERTIES ('external.table.purge'='TRUE', 'iceberg.catalog'='hadoop.catalog'
     'iceberg.file_format'='parquet',
     'iceberg.catalog_location'='/test-warehouse/hadoop_catalog_test')
 ====
+---- CREATE_TABLE
+CREATE TABLE iceberg_test_orc (
+  level STRING
+)
+STORED AS ICEBERG
+TBLPROPERTIES('iceberg.file_format'='orc', 'iceberg.catalog'='hadoop.tables')
+---- RESULTS-HIVE
+CREATE TABLE show_create_table_test_db.iceberg_test_orc (
+  level STRING
+)
+STORED AS ICEBERG
+LOCATION '$$location_uri$$'
+TBLPROPERTIES('iceberg.file_format'='orc', 'iceberg.catalog'='hadoop.tables')
+---- RESULTS-HIVE-3
+CREATE EXTERNAL TABLE show_create_table_test_db.iceberg_test_orc (
+  level STRING
+)
+STORED AS ICEBERG
+LOCATION '$$location_uri$$'
+TBLPROPERTIES ('external.table.purge'='TRUE', 'iceberg.file_format'='orc',
+'iceberg.catalog'='hadoop.tables')
+====


[impala] 02/06: IMPALA-10113: Add feature flag for incremental metadata updates

Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 308d692a1ba0c86c7ff718425495d787571c1484
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Wed Sep 9 10:20:46 2020 +0800

    IMPALA-10113: Add feature flag for incremental metadata updates
    
    This patch adds a feature flag, enable_incremental_metadata_updates, to
    turn off incremental metadata (i.e. partition level metadata)
    propagation from catalogd to coordinators. It defaults to true. When
    setting to false, catalogd will send metadata updates in table
    granularity (the legacy behavior).
    
    Also fixes a bug of logging an empty aggregated partition update log
    when no partitions are changed in a DDL.
    
    Tests:
     - Run CORE tests with this flag set to true and false.
     - Add tests with enable_incremental_metadata_updates=false.
    
    Change-Id: I98676fc8ca886f3d9f550f9b96fa6d6bff178ebb
    Reviewed-on: http://gerrit.cloudera.org:8080/16436
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/common/global-flags.cc                         |  8 ++++++++
 be/src/util/backend-gflag-util.cc                     |  3 +++
 common/thrift/BackendGflags.thrift                    |  2 ++
 .../apache/impala/catalog/CatalogServiceCatalog.java  | 19 +++++++++++++------
 .../java/org/apache/impala/catalog/HdfsTable.java     | 15 +++++++++++----
 .../org/apache/impala/catalog/ImpaladCatalog.java     |  2 +-
 .../apache/impala/catalog/PartitionMetaSummary.java   |  5 +++++
 .../java/org/apache/impala/service/BackendConfig.java |  4 ++++
 tests/custom_cluster/test_disable_features.py         | 14 ++++++++++++++
 tests/custom_cluster/test_local_catalog.py            |  3 ++-
 10 files changed, 63 insertions(+), 12 deletions(-)

diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index 55a62b9..3025be9 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -341,6 +341,14 @@ DEFINE_bool_hidden(use_customized_user_groups_mapper_for_ranger, false,
 DEFINE_bool(enable_column_masking, true,
     "If false, disable the column masking feature. Defaults to be true.");
 
+DEFINE_bool(enable_incremental_metadata_updates, true,
+    "If true, Catalog Server will send incremental table updates in partition level in "
+    "the statestore topic updates. Legacy coordinators will apply the partition updates "
+    "incrementally, i.e. reuse unchanged partition metadata. Disable this feature by "
+    "setting this to false in the Catalog Server. Then metadata of each table will be "
+    "propagated as a whole object in the statestore topic updates. Note that legacy "
+    "coordinators can apply incremental or full table updates so don't need this flag.");
+
 // ++========================++
 // || Startup flag graveyard ||
 // ++========================++
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
index 8401b3b..3a2c470 100644
--- a/be/src/util/backend-gflag-util.cc
+++ b/be/src/util/backend-gflag-util.cc
@@ -86,6 +86,7 @@ DECLARE_int32(num_check_authorization_threads);
 DECLARE_bool(use_customized_user_groups_mapper_for_ranger);
 DECLARE_bool(enable_column_masking);
 DECLARE_bool(compact_catalog_topic);
+DECLARE_bool(enable_incremental_metadata_updates);
 
 namespace impala {
 
@@ -174,6 +175,8 @@ Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
       FLAGS_use_customized_user_groups_mapper_for_ranger);
   cfg.__set_enable_column_masking(FLAGS_enable_column_masking);
   cfg.__set_compact_catalog_topic(FLAGS_compact_catalog_topic);
+  cfg.__set_enable_incremental_metadata_updates(
+      FLAGS_enable_incremental_metadata_updates);
   RETURN_IF_ERROR(SerializeThriftMsg(jni_env, &cfg, cfg_bytes));
   return Status::OK();
 }
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
index 48ac997..017ad18 100644
--- a/common/thrift/BackendGflags.thrift
+++ b/common/thrift/BackendGflags.thrift
@@ -159,4 +159,6 @@ struct TBackendGflags {
   67: required bool enable_insert_events
 
   68: required bool compact_catalog_topic
+
+  69: required bool enable_incremental_metadata_updates
 }
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index 0c22e6b..5c6bf1a 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -861,9 +861,15 @@ public class CatalogServiceCatalog extends Catalog {
           Catalog.toCatalogObjectKey(removedObject))) {
         ctx.addCatalogObject(removedObject, true);
       }
-      // If this is a HdfsTable, make sure we send deletes for removed partitions.
-      // So we won't leak partition topic entries in the statestored catalog topic.
-      if (removedObject.type == TCatalogObjectType.TABLE
+      // If this is a HdfsTable and incremental metadata updates are enabled, make sure we
+      // send deletes for removed partitions. So we won't leak partition topic entries in
+      // the statestored catalog topic. Partitions are only included as objects in topic
+      // updates if incremental metadata updates are enabled. Don't need this if
+      // incremental metadata updates are disabled, because in this case the table
+      // snapshot will be sent as a complete object. See more details in
+      // addTableToCatalogDeltaHelper().
+      if (BackendConfig.INSTANCE.isIncrementalMetadataUpdatesEnabled()
+          && removedObject.type == TCatalogObjectType.TABLE
           && removedObject.getTable().getTable_type() == TTableType.HDFS_TABLE) {
         THdfsTable hdfsTable = removedObject.getTable().getHdfs_table();
         Preconditions.checkState(
@@ -885,7 +891,7 @@ public class CatalogServiceCatalog extends Catalog {
             ctx.addCatalogObject(removedPart, true, deleteSummary);
           }
         }
-        LOG.info(deleteSummary.toString());
+        if (deleteSummary.hasUpdates()) LOG.info(deleteSummary.toString());
       }
     }
     // Each topic update should contain a single "TCatalog" object which is used to
@@ -1320,7 +1326,8 @@ public class CatalogServiceCatalog extends Catalog {
         return;
       }
       try {
-        if (tbl instanceof HdfsTable) {
+        if (BackendConfig.INSTANCE.isIncrementalMetadataUpdatesEnabled()
+            && tbl instanceof HdfsTable) {
           catalogTbl.setTable(((HdfsTable) tbl).toThriftWithMinimalPartitions());
           addHdfsPartitionsToCatalogDelta((HdfsTable) tbl, ctx);
         } else {
@@ -1364,7 +1371,7 @@ public class CatalogServiceCatalog extends Catalog {
     }
     hdfsTable.resetDroppedPartitions();
 
-    LOG.info(updateSummary.toString());
+    if (updateSummary.hasUpdates()) LOG.info(updateSummary.toString());
   }
 
   /**
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index c287af6..361de4e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -63,6 +63,7 @@ import org.apache.impala.common.Pair;
 import org.apache.impala.common.PrintUtils;
 import org.apache.impala.compat.MetastoreShim;
 import org.apache.impala.fb.FbFileBlock;
+import org.apache.impala.service.BackendConfig;
 import org.apache.impala.thrift.CatalogLookupStatus;
 import org.apache.impala.thrift.CatalogObjectsConstants;
 import org.apache.impala.thrift.TAccessLevel;
@@ -291,7 +292,7 @@ public class HdfsTable extends Table implements FeFsTable {
 
   // Dropped partitions since last catalog update. These partitions need to be removed
   // in coordinator's cache if there are no updates on them.
-  private final Set<HdfsPartition> droppedPartitions = new HashSet<>();
+  private final Set<HdfsPartition> droppedPartitions_ = new HashSet<>();
 
   // Represents a set of storage-related statistics aggregated at the table or partition
   // level.
@@ -987,7 +988,10 @@ public class HdfsTable extends Table implements FeFsTable {
     // nullPartitionIds_ and partitionValuesMap_ are only maintained in coordinators.
     if (!isStoredInImpaladCatalogCache()) {
       dirtyPartitions_.remove(partitionId);
-      droppedPartitions.add(partition.genMinimalPartition());
+      // Only tracks the dropped partition instances when we need partition-level updates.
+      if (BackendConfig.INSTANCE.isIncrementalMetadataUpdatesEnabled()) {
+        droppedPartitions_.add(partition.genMinimalPartition());
+      }
       return partition;
     }
     for (int i = 0; i < partition.getPartitionValues().size(); ++i) {
@@ -1910,6 +1914,9 @@ public class HdfsTable extends Table implements FeFsTable {
   @Override
   public TCatalogObject toMinimalTCatalogObject() {
     TCatalogObject catalogObject = super.toMinimalTCatalogObject();
+    if (!BackendConfig.INSTANCE.isIncrementalMetadataUpdatesEnabled()) {
+      return catalogObject;
+    }
     catalogObject.getTable().setTable_type(TTableType.HDFS_TABLE);
     THdfsTable hdfsTable = new THdfsTable(hdfsBaseDir_, getColumnNames(),
         nullPartitionKeyValue_, nullColumnValue_,
@@ -1949,13 +1956,13 @@ public class HdfsTable extends Table implements FeFsTable {
    * Gets the deleted/replaced partition instances since last catalog topic update.
    */
   public List<HdfsPartition> getDroppedPartitions() {
-    return ImmutableList.copyOf(droppedPartitions);
+    return ImmutableList.copyOf(droppedPartitions_);
   }
 
   /**
    * Clears the deleted/replaced partition instance set.
    */
-  public void resetDroppedPartitions() { droppedPartitions.clear(); }
+  public void resetDroppedPartitions() { droppedPartitions_.clear(); }
 
   /**
    * Gets catalog objects of new partitions since last catalog update. They are partitions
diff --git a/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java b/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
index 54ea1ca..85b447d 100644
--- a/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
@@ -246,7 +246,7 @@ public class ImpaladCatalog extends Catalog implements FeCatalog {
       }
     }
     for (PartitionMetaSummary summary : partUpdates.values()) {
-      LOG.info(summary.toString());
+      if (summary.hasUpdates()) LOG.info(summary.toString());
     }
 
     for (TCatalogObject catalogObject: sequencer.getUpdatedObjects()) {
diff --git a/fe/src/main/java/org/apache/impala/catalog/PartitionMetaSummary.java b/fe/src/main/java/org/apache/impala/catalog/PartitionMetaSummary.java
index aac98e0..a307c1d 100644
--- a/fe/src/main/java/org/apache/impala/catalog/PartitionMetaSummary.java
+++ b/fe/src/main/java/org/apache/impala/catalog/PartitionMetaSummary.java
@@ -188,6 +188,11 @@ public class PartitionMetaSummary {
     }
   }
 
+  /**
+   * Returns whether we have collected any partition updates/deletes.
+   */
+  public boolean hasUpdates() { return numUpdatedParts_ > 0 || numDeletedParts_ > 0; }
+
   @Override
   public String toString() {
     StringBuilder res = new StringBuilder();
diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index 675040c..c1a5264 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -212,6 +212,10 @@ public class BackendConfig {
 
   public boolean isCompactCatalogTopic() { return backendCfg_.compact_catalog_topic; }
 
+  public boolean isIncrementalMetadataUpdatesEnabled() {
+    return backendCfg_.enable_incremental_metadata_updates;
+  }
+
   // Inits the auth_to_local configuration in the static KerberosName class.
   private static void initAuthToLocal() {
     // If auth_to_local is enabled, we read the configuration hadoop.security.auth_to_local
diff --git a/tests/custom_cluster/test_disable_features.py b/tests/custom_cluster/test_disable_features.py
index 067051c..1434711 100644
--- a/tests/custom_cluster/test_disable_features.py
+++ b/tests/custom_cluster/test_disable_features.py
@@ -18,6 +18,7 @@
 import pytest
 
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.parametrize import UniqueDatabase
 
 
 class TestDisableFeatures(CustomClusterTestSuite):
@@ -31,3 +32,16 @@ class TestDisableFeatures(CustomClusterTestSuite):
   @CustomClusterTestSuite.with_args("--enable_orc_scanner=false")
   def test_disable_orc_scanner(self, vector):
     self.run_test_case('QueryTest/disable-orc-scanner', vector)
+
+  @pytest.mark.execute_serially
+  @UniqueDatabase.parametrize(sync_ddl=True)
+  @CustomClusterTestSuite.with_args(
+    catalogd_args="--enable_incremental_metadata_updates=false")
+  def test_disable_incremental_metadata_updates(self, vector, unique_database):
+    """Canary tests for disabling incremental metadata updates. Copy some partition
+    related tests in metadata/test_ddl.py here."""
+    vector.get_value('exec_option')['sync_ddl'] = True
+    self.run_test_case('QueryTest/alter-table-hdfs-caching', vector,
+        use_db=unique_database, multiple_impalad=True)
+    self.run_test_case('QueryTest/alter-table-set-column-stats', vector,
+        use_db=unique_database, multiple_impalad=True)
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index 109665f..98d6e35 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -160,7 +160,8 @@ class TestCompactCatalogUpdates(CustomClusterTestSuite):
   @pytest.mark.execute_serially
   @CustomClusterTestSuite.with_args(
     impalad_args="--use_local_catalog=true",
-    catalogd_args="--catalog_topic_mode=minimal")
+    catalogd_args="--catalog_topic_mode=minimal "
+                  "--enable_incremental_metadata_updates=true")
   def test_invalidate_stale_partitions(self, unique_database):
     """
     Test that partition level invalidations are sent from catalogd and processed


[impala] 06/06: IMPALA-10220: Fix negative value bug in RpcNetworkTime counter.

Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit c7581b5d8a9e00006a34cb82e88fa0ede26cfb3f
Author: Riza Suminto <ri...@cloudera.com>
AuthorDate: Mon Oct 5 18:32:07 2020 -0700

    IMPALA-10220: Fix negative value bug in RpcNetworkTime counter.
    
    Total RPC time was incorrectly computed using
    resp_.receiver_latency_ns() in function EndDataStreamCompleteCb(). This
    patch fix the bug by replacing it with eos_rsp_.receiver_latency_ns().
    This patch also fix logging mistakes in LogSlowRpc() to use its 'resp'
    parameter instead of 'resp_' field member.
    
    Testing:
    - Manually run data loading query that exhibit the bug for several times
      and verify that the Min value of RpcNetworkTime counter is always
      positive after the patch. The query used in testing is insert query to
      TPC-DS fact table store_sales of scale 10GB in single machine mini
      cluster.
    - Add DCHECK to verify that total rpc time is greater than or equal to
      receiver_latency_ns.
    - Run and pass core tests.
    
    Change-Id: I2a4d65a3e0f88349bd4ee1b01290bd2c386acc69
    Reviewed-on: http://gerrit.cloudera.org:8080/16552
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/runtime/krpc-data-stream-sender.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/be/src/runtime/krpc-data-stream-sender.cc b/be/src/runtime/krpc-data-stream-sender.cc
index 9a0f28e..42bcec9 100644
--- a/be/src/runtime/krpc-data-stream-sender.cc
+++ b/be/src/runtime/krpc-data-stream-sender.cc
@@ -390,12 +390,12 @@ void KrpcDataStreamSender::Channel::MarkDone(const Status& status) {
 template <typename ResponsePBType>
 void KrpcDataStreamSender::Channel::LogSlowRpc(
     const char* rpc_name, int64_t total_time_ns, const ResponsePBType& resp) {
-  int64_t network_time_ns = total_time_ns - resp_.receiver_latency_ns();
+  int64_t network_time_ns = total_time_ns - resp.receiver_latency_ns();
   LOG(INFO) << "Slow " << rpc_name << " RPC to " << address_
             << " (fragment_instance_id=" << PrintId(fragment_instance_id_) << "): "
             << "took " << PrettyPrinter::Print(total_time_ns, TUnit::TIME_NS) << ". "
             << "Receiver time: "
-            << PrettyPrinter::Print(resp_.receiver_latency_ns(), TUnit::TIME_NS)
+            << PrettyPrinter::Print(resp.receiver_latency_ns(), TUnit::TIME_NS)
             << " Network time: " << PrettyPrinter::Print(network_time_ns, TUnit::TIME_NS);
 }
 
@@ -489,15 +489,16 @@ void KrpcDataStreamSender::Channel::HandleFailedRPC(const DoRpcFn& rpc_fn,
 }
 
 void KrpcDataStreamSender::Channel::TransmitDataCompleteCb() {
-  DCHECK_NE(rpc_start_time_ns_, 0);
-  int64_t total_time = MonotonicNanos() - rpc_start_time_ns_;
   std::unique_lock<SpinLock> l(lock_);
   DCHECK(rpc_in_flight_);
+  DCHECK_NE(rpc_start_time_ns_, 0);
+  int64_t total_time = MonotonicNanos() - rpc_start_time_ns_;
   const kudu::Status controller_status = rpc_controller_.status();
   if (LIKELY(controller_status.ok())) {
     DCHECK(rpc_in_flight_batch_ != nullptr);
     // 'receiver_latency_ns' is calculated with MonoTime, so it must be non-negative.
     DCHECK_GE(resp_.receiver_latency_ns(), 0);
+    DCHECK_GE(total_time, resp_.receiver_latency_ns());
     int64_t row_batch_size = RowBatch::GetSerializedSize(*rpc_in_flight_batch_);
     int64_t network_time = total_time - resp_.receiver_latency_ns();
     COUNTER_ADD(parent_->bytes_sent_counter_, row_batch_size);
@@ -631,8 +632,9 @@ void KrpcDataStreamSender::Channel::EndDataStreamCompleteCb() {
   const kudu::Status controller_status = rpc_controller_.status();
   if (LIKELY(controller_status.ok())) {
     // 'receiver_latency_ns' is calculated with MonoTime, so it must be non-negative.
-    DCHECK_GE(resp_.receiver_latency_ns(), 0);
-    int64_t network_time_ns = total_time_ns - resp_.receiver_latency_ns();
+    DCHECK_GE(eos_resp_.receiver_latency_ns(), 0);
+    DCHECK_GE(total_time_ns, eos_resp_.receiver_latency_ns());
+    int64_t network_time_ns = total_time_ns - eos_resp_.receiver_latency_ns();
     parent_->network_time_stats_->UpdateCounter(network_time_ns);
     parent_->recvr_time_stats_->UpdateCounter(eos_resp_.receiver_latency_ns());
     if (IsSlowRpc(total_time_ns)) LogSlowRpc("EndDataStream", total_time_ns, eos_resp_);