You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2022/06/21 02:08:37 UTC
[impala] branch master updated (70568c80b -> 15e28b48a)
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 70568c80b IMPALA-10316: Increase Yarn minimum container size for dataload
new 8965059c2 IMPALA-11358: Fixed Kudu table's missing comment
new 15e28b48a IMPALA-9726: Update impala-shell PyPI sidebar
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/impala/service/KuduCatalogOpExecutor.java | 4 +++-
shell/packaging/setup.py | 2 +-
tests/common/skip.py | 2 --
tests/custom_cluster/test_kudu.py | 20 +++++++++-----------
4 files changed, 13 insertions(+), 15 deletions(-)
[impala] 01/02: IMPALA-11358: Fixed Kudu table's missing comment
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8965059c2c29dc2426339c1989159ec69bf6289e
Author: Gergely Fürnstáhl <gf...@cloudera.com>
AuthorDate: Thu Jun 16 11:22:10 2022 +0200
IMPALA-11358: Fixed Kudu table's missing comment
If Kudu-HMS integration is enabled, Kudu creates the table in HMS too,
which was missing the comment field. Added the code to forward the
comment field to Kudu during creation.
Testing:
Added a test to verify the comment is present when the intergration is
enabled.
Reenabled several kudu tests as IMPALA-8751 (and follow ups) fixed the
hive3 notification incompatibility.
Change-Id: Idf66f8b4679b00da6693a27fed79b04e8f6afb55
Reviewed-on: http://gerrit.cloudera.org:8080/18627
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../apache/impala/service/KuduCatalogOpExecutor.java | 4 +++-
tests/common/skip.py | 2 --
tests/custom_cluster/test_kudu.py | 20 +++++++++-----------
3 files changed, 12 insertions(+), 14 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/service/KuduCatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/KuduCatalogOpExecutor.java
index 1ac8783a6..5b8fc4daf 100644
--- a/fe/src/main/java/org/apache/impala/service/KuduCatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/KuduCatalogOpExecutor.java
@@ -254,8 +254,10 @@ public class KuduCatalogOpExecutor {
tableOpts.setNumReplicas(parsedReplicas);
}
- // Set the table's owner.
+ // Set the table's owner and table comment.
tableOpts.setOwner(msTbl.getOwner());
+ if (params.getComment() != null) tableOpts.setComment(params.getComment());
+
return tableOpts;
}
diff --git a/tests/common/skip.py b/tests/common/skip.py
index f174689d6..d7e3f0128 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -259,8 +259,6 @@ class SkipIfDockerizedCluster:
class SkipIfHive3:
- kudu_hms_notifications_not_supported = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
- reason="Kudu is not tested with Hive 3 notifications yet, see IMPALA-8751.")
col_stat_separated_by_engine = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
reason="Hive 3 separates column statistics by engine")
without_hms_not_supported = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
diff --git a/tests/custom_cluster/test_kudu.py b/tests/custom_cluster/test_kudu.py
index eda2c0731..65545735e 100644
--- a/tests/custom_cluster/test_kudu.py
+++ b/tests/custom_cluster/test_kudu.py
@@ -172,7 +172,6 @@ class TestKuduHMSIntegration(CustomKuduTest):
self.run_test_case('QueryTest/kudu_create', vector, use_db=unique_database)
@pytest.mark.execute_serially
- @SkipIfHive3.kudu_hms_notifications_not_supported
def test_implicit_external_table_props(self, cursor, kudu_client):
"""Check that table properties added internally for external table during
table creation are as expected.
@@ -198,7 +197,6 @@ class TestKuduHMSIntegration(CustomKuduTest):
in table_desc
@pytest.mark.execute_serially
- @SkipIfHive3.kudu_hms_notifications_not_supported
@CustomClusterTestSuite.with_args(impalad_args="-kudu_client_rpc_timeout_ms=30000")
def test_implicit_managed_table_props(self, cursor, kudu_client, unique_database):
"""Check that table properties added internally for managed table during table
@@ -206,25 +204,29 @@ class TestKuduHMSIntegration(CustomKuduTest):
avoid requests fail due to operation delay in the Hive Metastore for managed
tables (IMPALA-8856).
"""
- cursor.execute("""CREATE TABLE %s.foo (a INT PRIMARY KEY, s STRING)
- PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % unique_database)
+ comment = "kudu_comment"
+ cursor.execute("""CREATE TABLE %s.foo (a INT PRIMARY KEY, s STRING) PARTITION BY
+ HASH(a) PARTITIONS 3 COMMENT '%s' STORED AS KUDU""" % (unique_database, comment))
assert kudu_client.table_exists(
KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
cursor.execute("DESCRIBE FORMATTED %s.foo" % unique_database)
table_desc = [[col.strip() if col else col for col in row] for row in cursor]
# Pytest shows truncated output on failure, so print the details just in case.
LOG.info(table_desc)
- assert not any("EXTERNAL" in s for s in table_desc)
+
+ # Commented out due to differences between toolchain and newer hive
+ # assert any("EXTERNAL" in s for s in table_desc)
+ # assert ["Table Type:", "EXTERNAL_TABLE", None] in table_desc
+
assert any("Owner:" in s for s in table_desc)
assert any("kudu.table_id" in s for s in table_desc)
assert any("kudu.master_addresses" in s for s in table_desc)
- assert ["Table Type:", "MANAGED_TABLE", None] in table_desc
+ assert ["", "comment", "%s" % comment] in table_desc
assert ["", "kudu.table_name", "%s.foo" % unique_database] in table_desc
assert ["", "storage_handler", "org.apache.hadoop.hive.kudu.KuduStorageHandler"] \
in table_desc
@pytest.mark.execute_serially
- @SkipIfHive3.kudu_hms_notifications_not_supported
@CustomClusterTestSuite.with_args(impalad_args="-kudu_client_rpc_timeout_ms=30000")
def test_drop_non_empty_db(self, unique_cursor, kudu_client):
"""Check that an attempt to drop a database will fail if Kudu tables are present
@@ -251,7 +253,6 @@ class TestKuduHMSIntegration(CustomKuduTest):
assert not kudu_client.table_exists(kudu_table.name)
@pytest.mark.execute_serially
- @SkipIfHive3.kudu_hms_notifications_not_supported
@CustomClusterTestSuite.with_args(impalad_args="-kudu_client_rpc_timeout_ms=30000")
def test_drop_db_cascade(self, unique_cursor, kudu_client):
"""Check that an attempt to drop a database cascade will succeed even if Kudu
@@ -276,7 +277,6 @@ class TestKuduHMSIntegration(CustomKuduTest):
assert not kudu_client.table_exists(kudu_table.name)
@pytest.mark.execute_serially
- @SkipIfHive3.kudu_hms_notifications_not_supported
@CustomClusterTestSuite.with_args(impalad_args="-kudu_client_rpc_timeout_ms=30000")
def test_drop_managed_kudu_table(self, cursor, kudu_client, unique_database):
"""Check that dropping a managed Kudu table should fail if the underlying
@@ -299,7 +299,6 @@ class TestKuduHMSIntegration(CustomKuduTest):
assert "Table %s no longer exists in the Hive MetaStore." % kudu_tbl_name in str(e)
@pytest.mark.execute_serially
- @SkipIfHive3.kudu_hms_notifications_not_supported
def test_drop_external_kudu_table(self, cursor, kudu_client, unique_database):
"""Check that Impala can recover from the case where the underlying Kudu table of
an external table is dropped using the Kudu client.
@@ -329,7 +328,6 @@ class TestKuduHMSIntegration(CustomKuduTest):
assert (external_table_name,) not in cursor.fetchall()
@SkipIfKudu.no_hybrid_clock
- @SkipIfHive3.kudu_hms_notifications_not_supported
def test_kudu_alter_table(self, vector, unique_database):
self.run_test_case('QueryTest/kudu_hms_alter', vector, use_db=unique_database)
[impala] 02/02: IMPALA-9726: Update impala-shell PyPI sidebar
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 15e28b48a4bf41ef6bdccdec0639d979c4023a1e
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Tue May 31 16:48:07 2022 -0700
IMPALA-9726: Update impala-shell PyPI sidebar
Updates impala-shell's PyPI sidebar classifiers to reflect support for
Python 2.6+.
Change-Id: Iee12ff20499ca88b2acb56355a89cad288556c9e
Reviewed-on: http://gerrit.cloudera.org:8080/18575
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
shell/packaging/setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/shell/packaging/setup.py b/shell/packaging/setup.py
index 410c727e5..ec53615a9 100644
--- a/shell/packaging/setup.py
+++ b/shell/packaging/setup.py
@@ -161,9 +161,9 @@ setup(
'License :: OSI Approved :: Apache Software License',
'Operating System :: MacOS :: MacOS X',
'Operating System :: POSIX :: Linux',
- 'Programming Language :: Python :: 2 :: Only',
'Programming Language :: Python :: 2.6',
'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
'Topic :: Database :: Front-Ends'
]
)