You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/08/07 03:27:18 UTC
[impala] 01/02: IMPALA-8627: Enable catalog-v2 in tests
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 39613c8226aeb48f639bccc361f002c7085cf75a
Author: Vihang Karajgaonkar <vi...@cloudera.com>
AuthorDate: Fri Jul 26 16:55:33 2019 -0700
IMPALA-8627: Enable catalog-v2 in tests
This patch enables catalog-v2 by default in all the tests.
Test fixes:
1. Modified test_observability which fails on catalog-v2 since
the profile emits different metadata load events. The test now looks for
the right events on the profile depending on whether catalogv2 is
enabled or not.
2. TableName.java constructor allows non-lowercased
table and database names. This causes problems at the local catalog
cache which expects the tablenames to be always in lowercase. More
details on this failure are available in IMPALA-8627. The patch makes
sure that the loadTable requests in local catalog do a explicit
conversion of tablename to lowercase in order to get around the issue.
3. Fixes the JdbcTest which checks for existence of table comment in the
getTables metadata jdbc call. In catalog-v2 since the columns are not
requested, LocalTable is not loaded and hence the test needs to be
modified to check if catalog-v2 is enabled.
4. Skips test_sanity which creates a Hive db and issues a invalidate
metadata to make it visible in catalog. Unfortunately, in catalog-v2
currently there is no way to see a newly created database when event
polling is disabled.
5. Similar to above (4) test_metadata_query_statements.py creates a hive
db and issues a invalidate metadata. The test runs QueryTest/describe-db
which is split into two one for checking the hive-db and other contains
rest of the queries of the original describe-db. The split makes it
possible to only execute the test partially when catalog-v2 is enabled
Change-Id: Iddbde666de2b780c0e40df716a9dfe54524e092d
Reviewed-on: http://gerrit.cloudera.org:8080/13933
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
docker/catalogd/Dockerfile | 2 +-
docker/impalad_coord_exec/Dockerfile | 2 +-
docker/impalad_coordinator/Dockerfile | 2 +-
.../org/apache/impala/catalog/local/LocalDb.java | 5 +-
.../java/org/apache/impala/service/JdbcTest.java | 7 ++-
.../java/org/apache/impala/testutil/TestUtils.java | 36 ++++++++++++
.../queries/QueryTest/describe-db.test | 21 -------
.../queries/QueryTest/describe-hive-db.test | 30 ++++++++++
tests/common/environ.py | 15 ++++-
tests/common/skip.py | 6 ++
tests/hs2/hs2_test_suite.py | 67 ++++++++++++++++------
tests/hs2/test_hs2.py | 19 +++---
tests/metadata/test_hms_integration.py | 5 +-
tests/metadata/test_metadata_query_statements.py | 13 ++++-
tests/metadata/test_refresh_partition.py | 2 +-
tests/query_test/test_observability.py | 44 ++++++++++++--
16 files changed, 208 insertions(+), 68 deletions(-)
diff --git a/docker/catalogd/Dockerfile b/docker/catalogd/Dockerfile
index 5473c12..ce9ca5a 100644
--- a/docker/catalogd/Dockerfile
+++ b/docker/catalogd/Dockerfile
@@ -25,5 +25,5 @@ EXPOSE 25020
ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/catalogd",\
"-log_dir=/opt/impala/logs",\
"-abort_on_config_error=false", "-state_store_host=statestored",\
- "-catalog_topic_mode=full", "-hms_event_polling_interval_s=0",\
+ "-catalog_topic_mode=minimal", "-hms_event_polling_interval_s=0",\
"-invalidate_tables_on_memory_pressure=true"]
diff --git a/docker/impalad_coord_exec/Dockerfile b/docker/impalad_coord_exec/Dockerfile
index 542734b..24e024f 100644
--- a/docker/impalad_coord_exec/Dockerfile
+++ b/docker/impalad_coord_exec/Dockerfile
@@ -32,4 +32,4 @@ ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
"-log_dir=/opt/impala/logs",\
"-abort_on_config_error=false", "-state_store_host=statestored",\
"-catalog_service_host=catalogd", "-mem_limit_includes_jvm=true",\
- "-use_local_catalog=false", "--rpc_use_loopback=true"]
+ "-use_local_catalog=true", "--rpc_use_loopback=true"]
diff --git a/docker/impalad_coordinator/Dockerfile b/docker/impalad_coordinator/Dockerfile
index b12814d..8e07f9f 100644
--- a/docker/impalad_coordinator/Dockerfile
+++ b/docker/impalad_coordinator/Dockerfile
@@ -32,5 +32,5 @@ ENTRYPOINT ["/opt/impala/bin/daemon_entrypoint.sh", "/opt/impala/bin/impalad",\
"-log_dir=/opt/impala/logs",\
"-abort_on_config_error=false", "-state_store_host=statestored",\
"-catalog_service_host=catalogd", "-is_executor=false", \
- "-mem_limit_includes_jvm=true", "-use_local_catalog=false", \
+ "-mem_limit_includes_jvm=true", "-use_local_catalog=true", \
"--rpc_use_loopback=true"]
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java
index 2dcce7a..f14e255 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalDb.java
@@ -115,7 +115,10 @@ class LocalDb implements FeDb {
}
@Override
- public FeTable getTable(String tblName) {
+ public FeTable getTable(String tableName) {
+ // the underlying layers of the cache expect all the table name to be in lowercase
+ String tblName = Preconditions.checkNotNull(tableName,
+ "Received a null table name").toLowerCase();
FeTable tbl = getTableIfCached(tblName);
if (tbl instanceof LocalIncompleteTable) {
// The table exists but hasn't been loaded yet.
diff --git a/fe/src/test/java/org/apache/impala/service/JdbcTest.java b/fe/src/test/java/org/apache/impala/service/JdbcTest.java
index 0c91de6..e11feca 100644
--- a/fe/src/test/java/org/apache/impala/service/JdbcTest.java
+++ b/fe/src/test/java/org/apache/impala/service/JdbcTest.java
@@ -35,6 +35,7 @@ import java.util.List;
import java.util.Map;
import org.apache.impala.testutil.ImpalaJdbcClient;
+import org.apache.impala.testutil.TestUtils;
import org.apache.impala.util.Metrics;
import org.junit.AfterClass;
import org.junit.Before;
@@ -414,7 +415,11 @@ public class JdbcTest extends JdbcTestBase {
assertTrue(rs.next());
assertEquals("Incorrect table name", "jdbc_column_comments_test",
rs.getString("TABLE_NAME"));
- assertEquals("Incorrect table comment", "table comment", rs.getString("REMARKS"));
+ // if this is a catalog-v2 cluster the getTables call does not load the localTable
+ // since it does not request columns. See IMPALA-8606 for more details
+ if (!TestUtils.isCatalogV2Enabled("localhost", 25020)) {
+ assertEquals("Incorrect table comment", "table comment", rs.getString("REMARKS"));
+ }
}
@Test
diff --git a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
index 9a1c852..3849b2e 100644
--- a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
+++ b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
@@ -17,15 +17,19 @@
package org.apache.impala.testutil;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
+import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
+import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
@@ -393,4 +397,36 @@ public class TestUtils {
"IMPALA_HIVE_MAJOR_VERSION"));
return Integer.parseInt(hiveMajorVersion);
}
+
+ /**
+ * Gets checks if the catalog server running on the given host and port has
+ * catalog-v2 enabled
+ * @return
+ * @throws IOException
+ */
+ public static boolean isCatalogV2Enabled(String host, int port) throws IOException {
+ Preconditions.checkNotNull(host);
+ Preconditions.checkState(port >= 0);
+ String topicMode = getConfigValue(new URL(String.format("http://%s:%s"
+ + "/varz?json", host, port)), "catalog_topic_mode");
+ Preconditions.checkNotNull(topicMode);
+ return topicMode.equals("minimal");
+ }
+
+ /**
+ * Gets a flag value from the given URL. Useful to scrubbing the catalog/coordinator
+ * varz json output to look for interesting configurations
+ */
+ private static String getConfigValue(URL url, String key) throws IOException {
+ Map<Object, Object> map = new ObjectMapper().readValue(url, Map.class);
+ if (map.containsKey("flags")) {
+ Preconditions.checkState(map.containsKey("flags"));
+ ArrayList<LinkedHashMap<String, String>> flags =
+ (ArrayList<LinkedHashMap<String, String>>) map.get("flags");
+ for (LinkedHashMap<String, String> flag : flags) {
+ if (flag.getOrDefault("name", "").equals(key)) return flag.get("current");
+ }
+ }
+ return null;
+ }
}
diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
index 97159ab..c08ebdd 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/describe-db.test
@@ -8,27 +8,6 @@ describe database default
string, string, string
====
---- QUERY
-# Test printing of hive_test_desc_db database.
-describe database hive_test_desc_db
----- RESULTS
-'hive_test_desc_db','$NAMENODE/test-warehouse/hive_test_desc_db.db','test comment'
----- TYPES
-string, string, string
-====
----- QUERY
-# Test printing of hive_test_desc_db database with extended information.
-describe database extended hive_test_desc_db
----- RESULTS
-'hive_test_desc_db','$NAMENODE/test-warehouse/hive_test_desc_db.db','test comment'
-'Parameter: ','',''
-'','$USER','USER'
-'Owner: ','',''
-'','e','2.82'
-'','pi','3.14'
----- TYPES
-string, string, string
-====
----- QUERY
describe database extended impala_test_desc_db1
---- RESULTS
'','$USER','USER'
diff --git a/testdata/workloads/functional-query/queries/QueryTest/describe-hive-db.test b/testdata/workloads/functional-query/queries/QueryTest/describe-hive-db.test
new file mode 100644
index 0000000..10d9101
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/describe-hive-db.test
@@ -0,0 +1,30 @@
+====
+---- QUERY
+# Test printing of default database.
+describe database default
+---- RESULTS
+'default','$NAMENODE/test-warehouse','Default Hive database'
+---- TYPES
+string, string, string
+====
+---- QUERY
+# Test printing of hive_test_desc_db database.
+describe database hive_test_desc_db
+---- RESULTS
+'hive_test_desc_db','$NAMENODE/test-warehouse/hive_test_desc_db.db','test comment'
+---- TYPES
+string, string, string
+====
+---- QUERY
+# Test printing of hive_test_desc_db database with extended information.
+describe database extended hive_test_desc_db
+---- RESULTS
+'hive_test_desc_db','$NAMENODE/test-warehouse/hive_test_desc_db.db','test comment'
+'Parameter: ','',''
+'','$USER','USER'
+'Owner: ','',''
+'','e','2.82'
+'','pi','3.14'
+---- TYPES
+string, string, string
+====
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 64582a7..36b4a8a 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -333,8 +333,7 @@ class ImpalaTestClusterProperties(object):
return self._runtime_flags
def is_catalog_v2_cluster(self):
- """Whether we use CATALOG_V2 options, including local catalog and HMS notifications.
- For now, assume that --use_local_catalog=true implies that the others are enabled."""
+ """Checks whether we use local catalog."""
try:
key = "use_local_catalog"
# --use_local_catalog is hidden so does not appear in JSON if disabled.
@@ -346,6 +345,18 @@ class ImpalaTestClusterProperties(object):
return False
raise
+ def is_event_polling_enabled(self):
+ """Whether we use HMS notifications to automatically refresh catalog service.
+ Checks if --hms_event_polling_interval_s is set to non-zero value"""
+ try:
+ key = "hms_event_polling_interval_s"
+ # --use_local_catalog is hidden so does not appear in JSON if disabled.
+ return key in self.runtime_flags and int(self.runtime_flags[key]["current"]) > 0
+ except Exception:
+ if self.is_remote_cluster():
+ # IMPALA-8553: be more tolerant of failures on remote cluster builds.
+ LOG.exception("Failed to get flags from web UI, assuming catalog V1")
+ return False
def build_flavor_timeout(default_timeout, slow_build_timeout=None,
asan_build_timeout=None, code_coverage_build_timeout=None):
diff --git a/tests/common/skip.py b/tests/common/skip.py
index 0547049..afad729 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -264,6 +264,12 @@ class SkipIfCatalogV2:
reason="IMPALA-7539: support HDFS permission checks for LocalCatalog")
@classmethod
+ def impala_7506(self):
+ return pytest.mark.skipif(
+ IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
+ reason="IMPALA-7506: Support global INVALIDATE METADATA on fetch-on-demand impalad")
+
+ @classmethod
def hms_event_polling_enabled(self):
return pytest.mark.skipif(
IMPALA_TEST_CLUSTER_PROPERTIES.is_catalog_v2_cluster(),
diff --git a/tests/hs2/hs2_test_suite.py b/tests/hs2/hs2_test_suite.py
index d627316..f89aaac 100644
--- a/tests/hs2/hs2_test_suite.py
+++ b/tests/hs2/hs2_test_suite.py
@@ -26,33 +26,62 @@ from thrift.protocol import TBinaryProtocol
from tests.common.impala_test_suite import ImpalaTestSuite, IMPALAD_HS2_HOST_PORT
from time import sleep, time
+
+def add_session_helper(self, protocol_version, conf_overlay, close_session, fn):
+ """Helper function used in the various needs_session decorators before to set up
+ a session, call fn(), then optionally tear down the session."""
+ open_session_req = TCLIService.TOpenSessionReq()
+ open_session_req.username = getuser()
+ open_session_req.configuration = dict()
+ if conf_overlay is not None:
+ open_session_req.configuration = conf_overlay
+ open_session_req.client_protocol = protocol_version
+ resp = self.hs2_client.OpenSession(open_session_req)
+ HS2TestSuite.check_response(resp)
+ self.session_handle = resp.sessionHandle
+ assert protocol_version <= resp.serverProtocolVersion
+ try:
+ fn()
+ finally:
+ if close_session:
+ close_session_req = TCLIService.TCloseSessionReq()
+ close_session_req.sessionHandle = resp.sessionHandle
+ HS2TestSuite.check_response(self.hs2_client.CloseSession(close_session_req))
+ self.session_handle = None
+
def needs_session(protocol_version=
TCLIService.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6,
conf_overlay=None,
- close_session=True):
+ close_session=True,
+ cluster_properties=None):
def session_decorator(fn):
"""Decorator that establishes a session and sets self.session_handle. When the test is
finished, the session is closed.
"""
def add_session(self):
- open_session_req = TCLIService.TOpenSessionReq()
- open_session_req.username = getuser()
- open_session_req.configuration = dict()
- if conf_overlay is not None:
- open_session_req.configuration = conf_overlay
- open_session_req.client_protocol = protocol_version
- resp = self.hs2_client.OpenSession(open_session_req)
- HS2TestSuite.check_response(resp)
- self.session_handle = resp.sessionHandle
- assert protocol_version <= resp.serverProtocolVersion
- try:
- fn(self)
- finally:
- if close_session:
- close_session_req = TCLIService.TCloseSessionReq()
- close_session_req.sessionHandle = resp.sessionHandle
- HS2TestSuite.check_response(self.hs2_client.CloseSession(close_session_req))
- self.session_handle = None
+ add_session_helper(self, protocol_version, conf_overlay, close_session,
+ lambda: fn(self))
+ return add_session
+
+ return session_decorator
+
+
+# same as needs_session but takes in a cluster_properties as a argument
+# cluster_properties is defined as a fixture in conftest.py which allows us
+# to pass it as an argument to a test. However, it does not work well with
+# decorators without installing new modules.
+# Ref: https://stackoverflow.com/questions/19614658
+def needs_session_cluster_properties(protocol_version=
+ TCLIService.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6,
+ conf_overlay=None,
+ close_session=True):
+ def session_decorator(fn):
+ """Decorator that establishes a session and sets self.session_handle. When the test is
+ finished, the session is closed.
+ """
+ def add_session(self, cluster_properties, unique_database):
+ add_session_helper(self, protocol_version, conf_overlay, close_session,
+ lambda: fn(self, cluster_properties, unique_database))
return add_session
return session_decorator
diff --git a/tests/hs2/test_hs2.py b/tests/hs2/test_hs2.py
index 0294b13..223e29c 100644
--- a/tests/hs2/test_hs2.py
+++ b/tests/hs2/test_hs2.py
@@ -30,7 +30,7 @@ from tests.common.environ import ImpalaTestClusterProperties
from tests.common.skip import SkipIfDockerizedCluster
from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session,
operation_id_to_query_id, create_session_handle_without_secret,
- create_op_handle_without_secret)
+ create_op_handle_without_secret, needs_session_cluster_properties)
from TCLIService import TCLIService
LOG = logging.getLogger('test_hs2')
@@ -436,15 +436,13 @@ class TestHS2(HS2TestSuite):
self.session_handle)
TestHS2.check_invalid_session(self.hs2_client.GetSchemas(get_schemas_req))
- @pytest.mark.execute_serially
- @needs_session()
- def test_get_tables(self):
+ @needs_session_cluster_properties()
+ def test_get_tables(self, cluster_properties, unique_database):
"""Basic test for the GetTables() HS2 method. Needs to execute serially because
the test depends on controlling whether a table is loaded or not and other
concurrent tests loading or invalidating tables could interfere with it."""
- # TODO: unique_database would be better, but it doesn't work with @needs_session
- # at the moment.
table = "__hs2_column_comments_test"
+ self.execute_query("use {0}".format(unique_database))
self.execute_query("drop table if exists {0}".format(table))
self.execute_query("""
create table {0} (a int comment 'column comment')
@@ -452,7 +450,7 @@ class TestHS2(HS2TestSuite):
try:
req = TCLIService.TGetTablesReq()
req.sessionHandle = self.session_handle
- req.schemaName = "default"
+ req.schemaName = unique_database
req.tableName = table
# Execute the request twice, the first time with the table unloaded and the second
@@ -470,13 +468,14 @@ class TestHS2(HS2TestSuite):
table_type = results.columns[3].stringVal.values[0]
table_remarks = results.columns[4].stringVal.values[0]
assert table_cat == ''
- assert table_schema == "default"
+ assert table_schema == unique_database
assert table_name == table
assert table_type == "TABLE"
if i == 0:
assert table_remarks == ""
else:
- assert table_remarks == "table comment"
+ if not cluster_properties.is_catalog_v2_cluster():
+ assert table_remarks == "table comment"
# Ensure the table is loaded for the second iteration.
self.execute_query("describe {0}".format(table))
@@ -484,7 +483,7 @@ class TestHS2(HS2TestSuite):
invalid_req = TCLIService.TGetTablesReq()
invalid_req.sessionHandle = create_session_handle_without_secret(
self.session_handle)
- invalid_req.schemaName = "default"
+ invalid_req.schemaName = unique_database
invalid_req.tableName = table
TestHS2.check_invalid_session(self.hs2_client.GetTables(invalid_req))
finally:
diff --git a/tests/metadata/test_hms_integration.py b/tests/metadata/test_hms_integration.py
index f7d2595..9b1fd04 100644
--- a/tests/metadata/test_hms_integration.py
+++ b/tests/metadata/test_hms_integration.py
@@ -32,7 +32,7 @@ from subprocess import call
from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfHive2,
- SkipIfIsilon, SkipIfLocal)
+ SkipIfIsilon, SkipIfLocal, SkipIfCatalogV2)
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -57,7 +57,10 @@ class TestHmsIntegrationSanity(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_dimension(
create_uncompressed_text_dimension(cls.get_workload()))
+ # Skip this test if catalogv2 is enabled since global invalidate is not
+ # supported. #TODO This can be re-enabled when event polling is turned on
@pytest.mark.execute_serially
+ @SkipIfCatalogV2.impala_7506()
def test_sanity(self, vector, cluster_properties):
"""Verifies that creating a catalog entity (database, table) in Impala using
'IF NOT EXISTS' while the entity exists in HMS, does not throw an error."""
diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py
index 93fb111..313e2c3 100644
--- a/tests/metadata/test_metadata_query_statements.py
+++ b/tests/metadata/test_metadata_query_statements.py
@@ -169,13 +169,20 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
"location \"" + get_fs_path("/test2.db") + "\"")
self.run_stmt_in_hive("create database hive_test_desc_db comment 'test comment' "
"with dbproperties('pi' = '3.14', 'e' = '2.82')")
- if cluster_properties.is_catalog_v2_cluster():
- # Using local catalog + HMS event processor - wait until the database shows up.
+ if cluster_properties.is_event_polling_enabled():
+ # Using HMS event processor - wait until the database shows up.
self.wait_for_db_to_appear("hive_test_desc_db", timeout_s=30)
- else:
+ elif not cluster_properties.is_catalog_v2_cluster():
+ # Hive created database is visible
# Using traditional catalog - need to invalidate to pick up hive-created db.
self.client.execute("invalidate metadata")
+ else:
+ # In local catalog mode global invalidate metadata is not supported.
+ # TODO Once IMPALA-7506 is fixed, re-enable global invalidate for catalog-v2
+ pass
self.run_test_case('QueryTest/describe-db', vector)
+ if not cluster_properties.is_catalog_v2_cluster():
+ self.run_test_case('QueryTest/describe-hive-db', vector)
finally:
self.__test_describe_db_cleanup()
diff --git a/tests/metadata/test_refresh_partition.py b/tests/metadata/test_refresh_partition.py
index dc197ef..67901da 100644
--- a/tests/metadata/test_refresh_partition.py
+++ b/tests/metadata/test_refresh_partition.py
@@ -134,7 +134,7 @@ class TestRefreshPartition(ImpalaTestSuite):
# Make sure its still shows the same result before refreshing
result = self.client.execute("select count(*) from %s" % table_name)
valid_counts = [0]
- if cluster_properties.is_catalog_v2_cluster():
+ if cluster_properties.is_event_polling_enabled():
# HMS notifications may pick up added partition racily.
valid_counts.append(1)
assert int(result.data[0]) in valid_counts
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index 2012bab..fe2d92b 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -293,19 +293,51 @@ class TestObservability(ImpalaTestSuite):
runtime_profile = self.execute_query(query).runtime_profile
self.__verify_profile_event_sequence(event_regexes, runtime_profile)
- def test_query_profile_contains_query_compilation_metadata_load_events(self):
+ def test_query_profile_contains_query_compilation_metadata_load_events(self,
+ cluster_properties):
"""Test that the Metadata load started and finished events appear in the query
profile when Catalog cache is evicted."""
invalidate_query = "invalidate metadata functional.alltypes"
select_query = "select * from functional.alltypes"
self.execute_query(invalidate_query).runtime_profile
runtime_profile = self.execute_query(select_query).runtime_profile
- event_regexes = [r'Query Compilation:',
- r'Metadata load started:',
- r'Metadata load finished. loaded-tables=.*/.* load-requests=.* '
+ # Depending on whether this is a catalog-v2 cluster or not some of the metadata
+ # loading events are different
+ if not cluster_properties.is_catalog_v2_cluster():
+ load_event_regexes = [r'Query Compilation:', r'Metadata load started:',
+ r'Metadata load finished. loaded-tables=.*/.* load-requests=.* '
r'catalog-updates=.*:',
- r'Analysis finished:']
- self.__verify_profile_event_sequence(event_regexes, runtime_profile)
+ r'Analysis finished:']
+ else:
+ load_event_regexes = [
+ r'Frontend:',
+ r'CatalogFetch.ColumnStats.Misses',
+ r'CatalogFetch.ColumnStats.Requests',
+ r'CatalogFetch.ColumnStats.Time',
+ # The value of this counter may or not be present if it has a value of zero
+ r'CatalogFetch.Config.Misses|CatalogFetch.Config.Hits',
+ r'CatalogFetch.Config.Requests',
+ r'CatalogFetch.Config.Time',
+ r'CatalogFetch.DatabaseList.Hits',
+ r'CatalogFetch.DatabaseList.Requests',
+ r'CatalogFetch.DatabaseList.Time',
+ r'CatalogFetch.PartitionLists.Misses',
+ r'CatalogFetch.PartitionLists.Requests',
+ r'CatalogFetch.PartitionLists.Time',
+ r'CatalogFetch.Partitions.Hits',
+ r'CatalogFetch.Partitions.Misses',
+ r'CatalogFetch.Partitions.Requests',
+ r'CatalogFetch.Partitions.Time',
+ r'CatalogFetch.RPCs.Bytes',
+ r'CatalogFetch.RPCs.Requests',
+ r'CatalogFetch.RPCs.Time',
+ r'CatalogFetch.TableNames.Hits',
+ r'CatalogFetch.TableNames.Requests',
+ r'CatalogFetch.TableNames.Time',
+ r'CatalogFetch.Tables.Misses',
+ r'CatalogFetch.Tables.Requests',
+ r'CatalogFetch.Tables.Time']
+ self.__verify_profile_event_sequence(load_event_regexes, runtime_profile)
def test_query_profile_contains_query_compilation_metadata_cached_event(self):
"""Test that the Metadata cache available event appears in the query profile when