You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2021/07/07 18:30:53 UTC
[impala] 03/04: Bump up the GBN to 14842939
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 46ae99a36b38f59eef6b0de6d27eb6d9be926e7f
Author: Vihang Karajgaonkar <vi...@apache.org>
AuthorDate: Tue Jun 22 16:14:23 2021 -0700
Bump up the GBN to 14842939
This patch bumps up the GBN to 14842939. This build
includes HIVE-23995 and HIVE-24175 and some of the tests
were modified to take into account of that.
Also, fixes a minor bug in environ.py
Testing done:
1. Core tests.
Change-Id: I78f167c1c0d8e90808e387aba0e86b697067ed8f
Reviewed-on: http://gerrit.cloudera.org:8080/17628
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
---
bin/impala-config.sh | 28 +++++-----
.../queries/QueryTest/create-database.test | 18 +++----
.../queries/QueryTest/describe-db.test | 63 +++++++++++++++++-----
.../queries/QueryTest/describe-hive-db.test | 55 -------------------
tests/common/environ.py | 32 ++++++++---
tests/custom_cluster/test_event_processing.py | 1 -
tests/metadata/test_metadata_query_statements.py | 5 +-
7 files changed, 97 insertions(+), 105 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index aafdfde..d461582 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -170,24 +170,20 @@ fi
: ${IMPALA_TOOLCHAIN_HOST:=native-toolchain.s3.amazonaws.com}
export IMPALA_TOOLCHAIN_HOST
-#TODO: On next CDP_BUILD_NUMBER bump check if the following issues are resolved
-# and adjust existing tests that mentions them:
-# * HIVE-23995
-# * HIVE-24175
-export CDP_BUILD_NUMBER=14156131
+export CDP_BUILD_NUMBER=14842939
export CDP_MAVEN_REPOSITORY=\
"https://${IMPALA_TOOLCHAIN_HOST}/build/cdp_components/${CDP_BUILD_NUMBER}/maven"
-export CDP_AVRO_JAVA_VERSION=1.8.2.7.2.11.0-94
-export CDP_HADOOP_VERSION=3.1.1.7.2.11.0-94
-export CDP_HBASE_VERSION=2.2.6.7.2.11.0-94
-export CDP_HIVE_VERSION=3.1.3000.7.2.11.0-94
-export CDP_ICEBERG_VERSION=0.9.1.7.2.11.0-94
-export CDP_KNOX_VERSION=1.3.0.7.2.11.0-94
-export CDP_OZONE_VERSION=1.1.0.7.2.11.0-94
-export CDP_PARQUET_VERSION=1.10.99.7.2.11.0-94
-export CDP_RANGER_VERSION=2.1.0.7.2.11.0-94
-export CDP_TEZ_VERSION=0.9.1.7.2.11.0-94
-export CDP_GCS_VERSION=2.1.2.7.2.11.0-94
+export CDP_AVRO_JAVA_VERSION=1.8.2.7.2.12.0-35
+export CDP_HADOOP_VERSION=3.1.1.7.2.12.0-35
+export CDP_HBASE_VERSION=2.2.6.7.2.12.0-35
+export CDP_HIVE_VERSION=3.1.3000.7.2.12.0-35
+export CDP_ICEBERG_VERSION=0.9.1.7.2.12.0-35
+export CDP_KNOX_VERSION=1.3.0.7.2.12.0-35
+export CDP_OZONE_VERSION=1.1.0.7.2.12.0-35
+export CDP_PARQUET_VERSION=1.10.99.7.2.12.0-35
+export CDP_RANGER_VERSION=2.1.0.7.2.12.0-35
+export CDP_TEZ_VERSION=0.9.1.7.2.12.0-35
+export CDP_GCS_VERSION=2.1.2.7.2.12.0-35
export ARCH_NAME=$(uname -p)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/create-database.test b/testdata/workloads/functional-query/queries/QueryTest/create-database.test
index 2ac56d4..650db4d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/create-database.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/create-database.test
@@ -14,10 +14,11 @@ STRING, STRING
---- QUERY
# Test that DESCRIBE shows the proper database location
# for a newly created database (regression test for IMPALA-7439)
+# After HIVE-24175 the managed location is only available when
+# explicitly set in the create database statement.
describe database $DATABASE_2
---- RESULTS
'$DATABASE_2','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/$DATABASE_2.db','For testing'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE_2.db',''
---- TYPES
string, string, string
====
@@ -26,7 +27,6 @@ string, string, string
describe database extended $DATABASE_2
---- RESULTS
'$DATABASE_2','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/$DATABASE_2.db','For testing'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE_2.db',''
'Owner: ','',''
'','$USER','USER'
---- TYPES
@@ -70,7 +70,6 @@ create database if not exists $DATABASE_loc comment "For testing"
describe database $DATABASE_loc
---- RESULTS
'$DATABASE_loc','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/specified_location','For testing'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE_loc.db',''
---- TYPES
string, string, string
====
@@ -79,7 +78,6 @@ string, string, string
describe database extended $DATABASE_loc
---- RESULTS
'$DATABASE_loc','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/specified_location','For testing'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE_loc.db',''
'Owner: ','',''
'','$USER','USER'
---- TYPES
@@ -168,10 +166,6 @@ show databases like '$DATABASE_restrict'
====
---- QUERY
# Test CREATE DATABASE ... MANAGEDLOCATION
-#
-# TODO: Currently Hive ignores the specified managedlocation due to HIVE-24175,
-# so we are using the default locations in the checks. Restore the specified
-# location once HIVE-24175 is resolved.
create database if not exists $DATABASE_loc comment "For testing"
managedlocation '$NAMENODE/$MANAGED_WAREHOUSE_DIR/specified_managedlocation'
---- RESULTS
@@ -183,7 +177,7 @@ create database if not exists $DATABASE_loc comment "For testing"
describe database $DATABASE_loc
---- RESULTS
'$DATABASE_loc','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/$DATABASE_loc.db','For testing'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE_loc.db',''
+'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/specified_managedlocation',''
---- TYPES
string, string, string
====
@@ -192,7 +186,7 @@ string, string, string
describe database extended $DATABASE_loc
---- RESULTS
'$DATABASE_loc','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/$DATABASE_loc.db','For testing'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE_loc.db',''
+'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/specified_managedlocation',''
'Owner: ','',''
'','$USER','USER'
---- TYPES
@@ -217,7 +211,7 @@ create database if not exists $DATABASE_loc comment "For testing"
describe database $DATABASE_loc
---- RESULTS
'$DATABASE_loc','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/specified_location','For testing'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE_loc.db',''
+'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/specified_managedlocation',''
---- TYPES
string, string, string
====
@@ -226,7 +220,7 @@ string, string, string
describe database extended $DATABASE_loc
---- RESULTS
'$DATABASE_loc','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/specified_location','For testing'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/$DATABASE_loc.db',''
+'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/specified_managedlocation',''
'Owner: ','',''
'','$USER','USER'
---- TYPES
diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
index 5219ea0..a43b3f8 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
@@ -4,7 +4,6 @@
describe database default
---- RESULTS
'default','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR','Default Hive database'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR',''
---- TYPES
string, string, string
====
@@ -14,7 +13,6 @@ describe database extended impala_test_desc_db1
'','$USER','USER'
'Owner: ','',''
'impala_test_desc_db1','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db1.db',''
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/impala_test_desc_db1.db',''
---- TYPES
string, string, string
====
@@ -24,7 +22,6 @@ describe database extended impala_test_desc_db2
'','$USER','USER'
'Owner: ','',''
'impala_test_desc_db2','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db2.db','test comment'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/impala_test_desc_db2.db',''
---- TYPES
string, string, string
====
@@ -34,7 +31,6 @@ describe database extended impala_test_desc_db3
'','$USER','USER'
'Owner: ','',''
'impala_test_desc_db3','$NAMENODE/testdb',''
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/impala_test_desc_db3.db',''
---- TYPES
string, string, string
====
@@ -44,18 +40,14 @@ describe database extended impala_test_desc_db4
'','$USER','USER'
'Owner: ','',''
'impala_test_desc_db4','$NAMENODE/test2.db','test comment'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/impala_test_desc_db4.db',''
---- TYPES
string, string, string
====
---- QUERY
-# TODO: Currently Hive ignores the specified managedlocation due to HIVE-24175,
-# so we are using the default locations in the checks. Restore the specified
-# location once HIVE-24175 is resolved.
describe database impala_test_desc_db5
---- RESULTS
'impala_test_desc_db5','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db5.db','test comment'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/impala_test_desc_db5.db',''
+'managedlocation:','$NAMENODE/test2.db',''
---- TYPES
string, string, string
====
@@ -65,7 +57,7 @@ describe database extended impala_test_desc_db5
'','$USER','USER'
'Owner: ','',''
'impala_test_desc_db5','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db5.db','test comment'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/impala_test_desc_db5.db',''
+'managedlocation:','$NAMENODE/test2.db',''
---- TYPES
string, string, string
====
@@ -75,7 +67,7 @@ describe database extended impala_test_desc_db5
'','$USER','USER'
'Owner: ','',''
'impala_test_desc_db5','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db5.db','test comment'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/impala_test_desc_db5.db',''
+'managedlocation:','$NAMENODE/test2.db',''
---- TYPES
string, string, string
====
@@ -85,7 +77,54 @@ describe database formatted impala_test_desc_db5
'','$USER','USER'
'Owner: ','',''
'impala_test_desc_db5','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/impala_test_desc_db5.db','test comment'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/impala_test_desc_db5.db',''
+'managedlocation:','$NAMENODE/test2.db',''
+---- TYPES
+string, string, string
+====
+---- QUERY
+# Test printing of default database.
+describe database default
+---- RESULTS
+'default','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR','Default Hive database'
+---- TYPES
+string, string, string
+====
+---- QUERY
+# Test printing of hive_test_desc_db database.
+describe database hive_test_desc_db
+---- RESULTS
+'hive_test_desc_db','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/hive_test_desc_db.db','test comment'
+---- TYPES
+string, string, string
+====
+---- QUERY
+# Test printing of hive_test_desc_db database with extended information.
+describe database extended hive_test_desc_db
+---- RESULTS
+'hive_test_desc_db','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/hive_test_desc_db.db','test comment'
+'Parameter: ','',''
+'','$USER','USER'
+'Owner: ','',''
+'','e','2.82'
+'','pi','3.14'
+---- TYPES
+string, string, string
+====
+---- QUERY
+describe database hive_test_desc_db2
+---- RESULTS
+'hive_test_desc_db2','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/hive_test_desc_db2.db','test comment'
+'managedlocation:','$NAMENODE/test2.db',''
+---- TYPES
+string, string, string
+====
+---- QUERY
+describe database extended hive_test_desc_db2
+---- RESULTS
+'hive_test_desc_db2','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/hive_test_desc_db2.db','test comment'
+'managedlocation:','$NAMENODE/test2.db',''
+'','$USER','USER'
+'Owner: ','',''
---- TYPES
string, string, string
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe-hive-db.test b/testdata/workloads/functional-query/queries/QueryTest/describe-hive-db.test
deleted file mode 100644
index d616c9d..0000000
--- a/testdata/workloads/functional-query/queries/QueryTest/describe-hive-db.test
+++ /dev/null
@@ -1,55 +0,0 @@
-====
----- QUERY
-# Test printing of default database.
-describe database default
----- RESULTS
-'default','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR','Default Hive database'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR',''
----- TYPES
-string, string, string
-====
----- QUERY
-# Test printing of hive_test_desc_db database.
-describe database hive_test_desc_db
----- RESULTS
-'hive_test_desc_db','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/hive_test_desc_db.db','test comment'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/hive_test_desc_db.db',''
----- TYPES
-string, string, string
-====
----- QUERY
-# Test printing of hive_test_desc_db database with extended information.
-describe database extended hive_test_desc_db
----- RESULTS
-'hive_test_desc_db','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/hive_test_desc_db.db','test comment'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/hive_test_desc_db.db',''
-'Parameter: ','',''
-'','$USER','USER'
-'Owner: ','',''
-'','e','2.82'
-'','pi','3.14'
----- TYPES
-string, string, string
-====
----- QUERY
-# Test printing of database with managedlocation.
-# TODO: Currently Hive ignores the specified managedlocation due to HIVE-24175,
-# so we are using the default locations in the checks. Restore the specified
-# location once HIVE-24175 is resolved.
-describe database hive_test_desc_db2
----- RESULTS
-'hive_test_desc_db2','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/hive_test_desc_db2.db','test comment'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/hive_test_desc_db2.db',''
----- TYPES
-string, string, string
-====
----- QUERY
-describe database extended hive_test_desc_db2
----- RESULTS
-'hive_test_desc_db2','$NAMENODE/$EXTERNAL_WAREHOUSE_DIR/hive_test_desc_db2.db','test comment'
-'managedlocation:','$NAMENODE/$MANAGED_WAREHOUSE_DIR/hive_test_desc_db2.db',''
-'','$USER','USER'
-'Owner: ','',''
----- TYPES
-string, string, string
-====
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 3c8d130..59df363 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -32,6 +32,7 @@ IMPALA_REMOTE_URL = os.environ.get("IMPALA_REMOTE_URL", "")
# Default web UI URL for local test cluster
DEFAULT_LOCAL_WEB_UI_URL = "http://localhost:25000"
+DEFAULT_LOCAL_CATALOGD_WEB_UI_URL = "http://localhost:25020"
# Find the local build version. May be None if Impala wasn't built locally.
IMPALA_LOCAL_BUILD_VERSION = None
@@ -223,12 +224,16 @@ class ImpalaTestClusterProperties(object):
"""
Acquires and provides characteristics about the way the Impala under test was compiled
and its likely effects on its responsiveness to automated test timings.
- """
- def __init__(self, build_flavor, library_link_type, web_ui_url):
+ TODO: Support remote urls for catalogd web UI."""
+
+ def __init__(self, build_flavor, library_link_type, web_ui_url,
+ catalogd_web_ui_url=DEFAULT_LOCAL_CATALOGD_WEB_UI_URL):
self._build_flavor = build_flavor
self._library_link_type = library_link_type
self._web_ui_url = web_ui_url
+ self._catalogd_web_ui_url = catalogd_web_ui_url
self._runtime_flags = None # Lazily populated to avoid unnecessary web UI calls.
+ self._catalogd_runtime_flags = None # Lazily populated
@classmethod
def get_instance(cls):
@@ -336,13 +341,27 @@ class ImpalaTestClusterProperties(object):
if self._runtime_flags is None:
response = requests.get(self._web_ui_url + "/varz?json")
assert response.status_code == requests.codes.ok,\
- "Offending url: " + impala_url
+ "Offending url: " + self._web_ui_url
assert "application/json" in response.headers['Content-Type']
self._runtime_flags = {}
for flag_dict in json.loads(response.text)["flags"]:
self._runtime_flags[flag_dict["name"]] = flag_dict
return self._runtime_flags
+ @property
+ def catalogd_runtime_flags(self):
+ """Return the command line flags from the catalogd web UI. Returns a Python map with
+ the flag name as the key and a dictionary of flag properties as the value."""
+ if self._catalogd_runtime_flags is None:
+ response = requests.get(self._catalogd_web_ui_url + "/varz?json")
+ assert response.status_code == requests.codes.ok,\
+ "Offending url: " + self._catalogd_web_ui_url
+ assert "application/json" in response.headers['Content-Type']
+ self._catalogd_runtime_flags = {}
+ for flag_dict in json.loads(response.text)["flags"]:
+ self._catalogd_runtime_flags[flag_dict["name"]] = flag_dict
+ return self._catalogd_runtime_flags
+
def is_catalog_v2_cluster(self):
"""Checks whether we use local catalog."""
try:
@@ -361,12 +380,13 @@ class ImpalaTestClusterProperties(object):
Checks if --hms_event_polling_interval_s is set to non-zero value"""
try:
key = "hms_event_polling_interval_s"
- # --use_local_catalog is hidden so does not appear in JSON if disabled.
- return key in self.runtime_flags and int(self.runtime_flags[key]["current"]) > 0
+ return key in self.catalogd_runtime_flags and int(
+ self._catalogd_runtime_flags[key]["current"]) > 0
except Exception:
if self.is_remote_cluster():
# IMPALA-8553: be more tolerant of failures on remote cluster builds.
- LOG.exception("Failed to get flags from web UI, assuming catalog V1")
+ LOG.exception(
+ "Failed to get flags from web UI, assuming event polling is disabled")
return False
def build_flavor_timeout(default_timeout, slow_build_timeout=None,
diff --git a/tests/custom_cluster/test_event_processing.py b/tests/custom_cluster/test_event_processing.py
index 3ef2795..e14985e 100644
--- a/tests/custom_cluster/test_event_processing.py
+++ b/tests/custom_cluster/test_event_processing.py
@@ -244,7 +244,6 @@ class TestEventProcessing(CustomClusterTestSuite):
self.__run_self_events_test(unique_database, True)
self.__run_self_events_test(unique_database, False)
- @pytest.mark.xfail(run=False, reason="This is failing due to HIVE-23995")
@CustomClusterTestSuite.with_args(catalogd_args="--hms_event_polling_interval_s=1")
def test_event_based_replication(self):
self.__run_event_based_replication_tests()
diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py
index 194c330..cdc242e 100644
--- a/tests/metadata/test_metadata_query_statements.py
+++ b/tests/metadata/test_metadata_query_statements.py
@@ -28,6 +28,7 @@ from tests.common.test_dimensions import ALL_NODES_ONLY
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.filesystem_utils import get_fs_path
+from tests.util.event_processor_utils import EventProcessorUtils
# TODO: For these tests to pass, all table metadata must be created exhaustively.
# the tests should be modified to remove that requirement.
@@ -192,13 +193,11 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
"managedlocation '" + get_fs_path("/test2.db") + "'")
if cluster_properties.is_event_polling_enabled():
# Using HMS event processor - wait until the database shows up.
- self.wait_for_db_to_appear("hive_test_desc_db", timeout_s=30)
+ EventProcessorUtils.wait_for_event_processing(self)
else:
# Invalidate metadata to pick up hive-created db.
self.client.execute("invalidate metadata")
self.run_test_case('QueryTest/describe-db', vector)
- if not cluster_properties.is_catalog_v2_cluster():
- self.run_test_case('QueryTest/describe-hive-db', vector)
finally:
self.__test_describe_db_cleanup()