You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2024/03/29 22:41:21 UTC

(impala) branch master updated (9071030f7 -> 83734a122)

This is an automated email from the ASF dual-hosted git repository.

michaelsmith pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git


    from 9071030f7 IMPALA-12809: Iceberg metadata table scanner should always be scheduled to the coordinator
     new c529b855e IMPALA-12626: Add Tables Queried to profile/history
     new deee153c7 IMPALA-12426: Skip Inserting HS2 Operation Queries into the Completed Queries Table
     new 83734a122 IMPALA-12944: Fixes Workload Management Test Flakiness

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 be/src/exec/system-table-scanner.cc                |   5 +
 be/src/service/client-request-state.cc             |  36 ++---
 be/src/service/client-request-state.h              |   2 +
 be/src/service/impala-hs2-server.cc                |  10 +-
 be/src/service/query-state-record.cc               |   1 +
 be/src/service/query-state-record.h                |   3 +
 be/src/service/workload-management-fields.cc       |   6 +
 be/src/service/workload-management.cc              |  22 ++-
 be/src/util/debug-util.cc                          |   9 ++
 be/src/util/debug-util.h                           |   3 +
 be/src/util/error-util.cc                          |   8 +-
 common/thrift/Frontend.thrift                      |   2 +
 common/thrift/Query.thrift                         |   3 +
 common/thrift/SystemTables.thrift                  |   1 +
 .../java/org/apache/impala/service/Frontend.java   |   3 +
 tests/common/impala_test_suite.py                  |  16 +-
 tests/custom_cluster/test_query_live.py            |  12 ++
 tests/custom_cluster/test_query_log.py             | 171 ++++++++++++++++++++-
 tests/query_test/test_observability.py             |  14 ++
 tests/util/workload_management.py                  |  15 +-
 20 files changed, 293 insertions(+), 49 deletions(-)


(impala) 01/03: IMPALA-12626: Add Tables Queried to profile/history

Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit c529b855e95e5b4fe4fed504afa4e34b3e2481dc
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Wed Jan 10 16:57:17 2024 -0800

    IMPALA-12626: Add Tables Queried to profile/history
    
    Adds "Tables Queried" to the query profile, enumerating a
    comma-separated list of tables accessed during a query:
    
      Tables Queried: tpch.customer,tpch.lineitem
    
    Also adds "tables_queried" to impala_query_log and impala_query_live
    with the same content.
    
    Requires 'drop table sys.impala_query_log' to recreate it with the new
    column.
    
    Change-Id: I9c9c80b2adf7f3e44225a191fe8eb9df3c4bc5aa
    Reviewed-on: http://gerrit.cloudera.org:8080/20886
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/exec/system-table-scanner.cc                |  5 +++
 be/src/service/client-request-state.cc             | 36 +++++++---------------
 be/src/service/client-request-state.h              |  1 +
 be/src/service/query-state-record.cc               |  1 +
 be/src/service/query-state-record.h                |  3 ++
 be/src/service/workload-management-fields.cc       |  6 ++++
 be/src/util/debug-util.cc                          |  9 ++++++
 be/src/util/debug-util.h                           |  3 ++
 be/src/util/error-util.cc                          |  8 ++---
 common/thrift/Frontend.thrift                      |  2 ++
 common/thrift/SystemTables.thrift                  |  1 +
 .../java/org/apache/impala/service/Frontend.java   |  3 ++
 tests/custom_cluster/test_query_live.py            | 12 ++++++++
 tests/query_test/test_observability.py             | 14 +++++++++
 tests/util/workload_management.py                  | 15 ++++++++-
 15 files changed, 87 insertions(+), 32 deletions(-)

diff --git a/be/src/exec/system-table-scanner.cc b/be/src/exec/system-table-scanner.cc
index 3d0dbad6b..fa91fb165 100644
--- a/be/src/exec/system-table-scanner.cc
+++ b/be/src/exec/system-table-scanner.cc
@@ -335,6 +335,11 @@ Status QueryScanner::MaterializeNextTuple(
         RETURN_IF_ERROR(WriteStringSlot(
             trim_left_copy_if(record.plan, is_any_of("\n")), pool, slot));
         break;
+      case TQueryTableColumn::TABLES_QUERIED:
+        if (!query.tables.empty()) {
+          RETURN_IF_ERROR(WriteStringSlot(PrintTableList(query.tables), pool, slot));
+        }
+        break;
       default:
         DCHECK(false) << "Unknown column position " << slot_desc->col_pos();
     }
diff --git a/be/src/service/client-request-state.cc b/be/src/service/client-request-state.cc
index 538fde8c1..3704716a3 100644
--- a/be/src/service/client-request-state.cc
+++ b/be/src/service/client-request-state.cc
@@ -229,12 +229,16 @@ void ClientRequestState::SetBlacklistedExecutorAddresses(
 Status ClientRequestState::Exec() {
   MarkActive();
 
+  const TExecRequest& exec_req = exec_request();
   profile_->AddChild(server_profile_);
   summary_profile_->AddInfoString("Query Type", PrintValue(stmt_type()));
   summary_profile_->AddInfoString("Query Options (set by configuration)",
       DebugQueryOptions(query_ctx_.client_request.query_options));
   summary_profile_->AddInfoString("Query Options (set by configuration and planner)",
-      DebugQueryOptions(exec_request().query_options));
+      DebugQueryOptions(exec_req.query_options));
+  if (!exec_req.tables.empty()) {
+    summary_profile_->AddInfoString("Tables Queried", PrintTableList(exec_req.tables));
+  }
   if (query_ctx_.__isset.overridden_mt_dop_value) {
     DCHECK(query_ctx_.client_request.query_options.__isset.mt_dop);
     summary_profile_->AddInfoString("MT_DOP limited by admission control",
@@ -243,7 +247,6 @@ Status ClientRequestState::Exec() {
             query_ctx_.client_request.query_options.mt_dop));
   }
 
-  const TExecRequest& exec_req = exec_request();
   switch (exec_req.stmt_type) {
     case TStmtType::QUERY:
     case TStmtType::DML:
@@ -533,38 +536,21 @@ Status ClientRequestState::ExecQueryOrDmlRequest(
   if (!query_exec_request.query_ctx.__isset.parent_query_id &&
       query_exec_request.query_ctx.__isset.tables_missing_stats &&
       !query_exec_request.query_ctx.tables_missing_stats.empty()) {
-    stringstream ss;
-    const vector<TTableName>& tbls = query_exec_request.query_ctx.tables_missing_stats;
-    for (int i = 0; i < tbls.size(); ++i) {
-      if (i != 0) ss << ",";
-      ss << tbls[i].db_name << "." << tbls[i].table_name;
-    }
-    summary_profile_->AddInfoString(TABLES_MISSING_STATS_KEY, ss.str());
+    summary_profile_->AddInfoString(TABLES_MISSING_STATS_KEY,
+        PrintTableList(query_exec_request.query_ctx.tables_missing_stats));
   }
 
   if (!query_exec_request.query_ctx.__isset.parent_query_id &&
       query_exec_request.query_ctx.__isset.tables_with_corrupt_stats &&
       !query_exec_request.query_ctx.tables_with_corrupt_stats.empty()) {
-    stringstream ss;
-    const vector<TTableName>& tbls =
-        query_exec_request.query_ctx.tables_with_corrupt_stats;
-    for (int i = 0; i < tbls.size(); ++i) {
-      if (i != 0) ss << ",";
-      ss << tbls[i].db_name << "." << tbls[i].table_name;
-    }
-    summary_profile_->AddInfoString(TABLES_WITH_CORRUPT_STATS_KEY, ss.str());
+    summary_profile_->AddInfoString(TABLES_WITH_CORRUPT_STATS_KEY,
+        PrintTableList(query_exec_request.query_ctx.tables_with_corrupt_stats));
   }
 
   if (query_exec_request.query_ctx.__isset.tables_missing_diskids &&
       !query_exec_request.query_ctx.tables_missing_diskids.empty()) {
-    stringstream ss;
-    const vector<TTableName>& tbls =
-        query_exec_request.query_ctx.tables_missing_diskids;
-    for (int i = 0; i < tbls.size(); ++i) {
-      if (i != 0) ss << ",";
-      ss << tbls[i].db_name << "." << tbls[i].table_name;
-    }
-    summary_profile_->AddInfoString(TABLES_WITH_MISSING_DISK_IDS_KEY, ss.str());
+    summary_profile_->AddInfoString(TABLES_WITH_MISSING_DISK_IDS_KEY,
+        PrintTableList(query_exec_request.query_ctx.tables_missing_diskids));
   }
 
   {
diff --git a/be/src/service/client-request-state.h b/be/src/service/client-request-state.h
index d0d1b3d78..c96fcc73e 100644
--- a/be/src/service/client-request-state.h
+++ b/be/src/service/client-request-state.h
@@ -334,6 +334,7 @@ class ClientRequestState {
   }
   /// Returns 0:0 if this is a root query
   TUniqueId parent_query_id() const { return query_ctx_.parent_query_id; }
+  const vector<TTableName>& tables() const { return exec_request().tables; }
 
   const std::vector<std::string>& GetAnalysisWarnings() const {
     return exec_request().analysis_warnings;
diff --git a/be/src/service/query-state-record.cc b/be/src/service/query-state-record.cc
index 0b594df83..8c7458a03 100644
--- a/be/src/service/query-state-record.cc
+++ b/be/src/service/query-state-record.cc
@@ -203,6 +203,7 @@ QueryStateExpanded::QueryStateExpanded(const ClientRequestState& exec_state,
       .query_exec_request.dedicated_coord_mem_estimate;
   row_materialization_rate = exec_state.row_materialization_rate();
   row_materialization_time = exec_state.row_materialization_timer();
+  tables = exec_state.tables();
 
   // Update name_rows_fetched with the final count after query close.
   base_state->num_rows_fetched = exec_state.num_rows_fetched_counter();
diff --git a/be/src/service/query-state-record.h b/be/src/service/query-state-record.h
index c3fec5518..6ada7a0e9 100644
--- a/be/src/service/query-state-record.h
+++ b/be/src/service/query-state-record.h
@@ -340,6 +340,9 @@ struct QueryStateExpanded {
   /// Events Timeline Iterator
   EventsTimelineIterator EventsTimeline() const;
 
+  // Source tables accessed by this query.
+  std::vector<TTableName> tables;
+
   /// Required data will be copied from the provided ClientRequestState into members of
   /// the struct.
   QueryStateExpanded(const ClientRequestState& exec_state,
diff --git a/be/src/service/workload-management-fields.cc b/be/src/service/workload-management-fields.cc
index 19ef98fb7..a95bb671c 100644
--- a/be/src/service/workload-management-fields.cc
+++ b/be/src/service/workload-management-fields.cc
@@ -411,6 +411,12 @@ const std::list<FieldDefinition> FIELD_DEFINITIONS = {
               << "'";
         }),
 
+    // Tables Queried
+    FieldDefinition("tables_queried", TPrimitiveType::STRING,
+        [](FieldParserContext& ctx){
+          ctx.sql << "'" << PrintTableList(ctx.record->tables) << "'";
+        }),
+
     }; // FIELDS_PARSERS constant list
 
 } //namespace workload_management
diff --git a/be/src/util/debug-util.cc b/be/src/util/debug-util.cc
index 927319de8..005bf1c7e 100644
--- a/be/src/util/debug-util.cc
+++ b/be/src/util/debug-util.cc
@@ -277,6 +277,15 @@ string PrintNumericPath(const SchemaPath& path) {
   return ss.str();
 }
 
+string PrintTableList(const vector<TTableName>& tbls) {
+  stringstream ss;
+  for (int i = 0; i < tbls.size(); ++i) {
+    if (i != 0) ss << ",";
+    ss << tbls[i].db_name << "." << tbls[i].table_name;
+  }
+  return ss.str();
+}
+
 string GetBuildVersion(bool compact) {
   stringstream ss;
   ss << GetDaemonBuildVersion()
diff --git a/be/src/util/debug-util.h b/be/src/util/debug-util.h
index 4179eefcd..4d7255e45 100644
--- a/be/src/util/debug-util.h
+++ b/be/src/util/debug-util.h
@@ -91,6 +91,9 @@ template<typename ThriftStruct> std::string PrintThrift(const ThriftStruct& t) {
   return apache::thrift::ThriftDebugString(t);
 }
 
+/// Return a list of TTableName as a comma-separated string.
+std::string PrintTableList(const std::vector<TTableName>& tbls);
+
 /// Parse 's' into a TUniqueId object.  The format of s needs to be the output format
 /// from PrintId.  (<hi_part>:<low_part>)
 /// Returns true if parse succeeded.
diff --git a/be/src/util/error-util.cc b/be/src/util/error-util.cc
index 430f2c833..0d9443b20 100644
--- a/be/src/util/error-util.cc
+++ b/be/src/util/error-util.cc
@@ -16,6 +16,7 @@
 // under the License.
 
 #include "util/error-util-internal.h"
+#include "util/debug-util.h"
 #include "util/string-util.h"
 
 #include <errno.h>
@@ -47,12 +48,7 @@ string GetTablesMissingStatsWarning(const vector<TTableName>& tables_missing_sta
   stringstream ss;
   if (tables_missing_stats.empty()) return string("");
   ss << "WARNING: The following tables are missing relevant table and/or column "
-     << "statistics.\n";
-  for (int i = 0; i < tables_missing_stats.size(); ++i) {
-    const TTableName& table_name = tables_missing_stats[i];
-    if (i != 0) ss << ",";
-    ss << table_name.db_name << "." << table_name.table_name;
-  }
+     << "statistics.\n" << PrintTableList(tables_missing_stats);
   return ss.str();
 }
 
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index d564fd631..93a0d92bf 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -678,6 +678,8 @@ struct TExecRequest {
 
   // Request for "ALTER TABLE ... CONVERT TO" statements.
   19: optional TConvertTableRequest convert_table_request
+
+  20: optional list<CatalogObjects.TTableName> tables
 }
 
 // Parameters to FeSupport.cacheJar().
diff --git a/common/thrift/SystemTables.thrift b/common/thrift/SystemTables.thrift
index 224fcca41..3ca81db47 100644
--- a/common/thrift/SystemTables.thrift
+++ b/common/thrift/SystemTables.thrift
@@ -68,4 +68,5 @@ enum TQueryTableColumn {
     PERNODE_PEAK_MEM_MEAN
     SQL
     PLAN
+    TABLES_QUERIED
 }
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index f68ecb2eb..2c7649430 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -2412,6 +2412,9 @@ public class Frontend {
     }
     Preconditions.checkNotNull(analysisResult.getStmt());
     TExecRequest result = createBaseExecRequest(queryCtx, analysisResult);
+    for (TableName table : stmtTableCache.tables.keySet()) {
+      result.addToTables(table.toThrift());
+    }
 
     // Transfer the expected number of executors in executor group set to
     // analyzer's global state. The info is needed to compute the number of nodes to be
diff --git a/tests/custom_cluster/test_query_live.py b/tests/custom_cluster/test_query_live.py
index ee8c17e0c..ca44122e0 100644
--- a/tests/custom_cluster/test_query_live.py
+++ b/tests/custom_cluster/test_query_live.py
@@ -98,6 +98,18 @@ class TestQueryLive(CustomClusterTestSuite):
         'select * from sys.impala_query_live where cluster_id = "test_query_live_0"')
     assert len(result5.data) == 0
 
+    result = self.execute_query("""
+        select count(*) from functional.alltypestiny a
+          inner join functional.alltypes b on a.id = b.id
+          inner join functional.alltypessmall c on b.id = c.id
+    """)
+    result5 = self.execute_query(
+        'select tables_queried from sys.impala_query_live where query_id = "'
+        + result.query_id + '"')
+    assert len(result5.data) == 1
+    assert result5.data[0] == \
+        "functional.alltypes,functional.alltypestiny,functional.alltypessmall"
+
   @CustomClusterTestSuite.with_args(impalad_args="--enable_workload_mgmt "
                                                  "--cluster_id=test_query_live",
                                     catalogd_args="--enable_workload_mgmt",
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index b11453585..60761e57c 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -188,6 +188,20 @@ class TestObservability(ImpalaTestSuite):
     expected_str = expected_str.format(timezone=server_timezone)
     assert expected_str in profile, profile
 
+  def test_profile(self):
+    """Test that expected fields are populated in the profile."""
+    query = """select count(distinct a.int_col) from functional.alltypes a
+        inner join functional.alltypessmall b on (a.id = b.id + cast(sleep(15) as INT))"""
+    result = self.execute_query(query)
+
+    assert "Query Type: QUERY" in result.runtime_profile
+    assert "Query State: " in result.runtime_profile
+    assert "Default Db: default" in result.runtime_profile
+    tables = re.search(r'\n\s+Tables Queried:\s+(.*?)\n', result.runtime_profile)
+    assert tables is not None
+    assert sorted(tables.group(1).split(",")) \
+        == ["functional.alltypes", "functional.alltypessmall"]
+
   def test_exec_summary(self):
     """Test that the exec summary is populated correctly in every query state"""
     query = "select count(*) from functional.alltypes"
diff --git a/tests/util/workload_management.py b/tests/util/workload_management.py
index 900e523b3..2ee3eba3f 100644
--- a/tests/util/workload_management.py
+++ b/tests/util/workload_management.py
@@ -25,7 +25,7 @@ from tests.util.assert_time import assert_time_str, convert_to_nanos
 from tests.util.memory import assert_byte_str, convert_to_bytes
 
 DEDICATED_COORD_SAFETY_BUFFER_BYTES = 104857600
-EXPECTED_QUERY_COLS = 48
+EXPECTED_QUERY_COLS = 49
 
 
 CLUSTER_ID = "CLUSTER_ID"
@@ -76,6 +76,7 @@ PERNODE_PEAK_MEM_MAX = "PERNODE_PEAK_MEM_MAX"
 PERNODE_PEAK_MEM_MEAN = "PERNODE_PEAK_MEM_MEAN"
 SQL = "SQL"
 PLAN = "PLAN"
+TABLES_QUERIED = "TABLES_QUERIED"
 
 
 def assert_query(query_tbl, client, expected_cluster_id, raw_profile=None, impalad=None,
@@ -699,6 +700,18 @@ def assert_query(query_tbl, client, expected_cluster_id, raw_profile=None, impal
     assert plan is not None
     assert data[index] == plan.group(1)
 
+  # Tables Queried
+  index += 1
+  assert sql_results.column_labels[index] == TABLES_QUERIED
+  ret_data[TABLES_QUERIED] = data[index]
+  tables = re.search(r'\n\s+Tables Queried:\s+(.*?)\n', profile_text)
+  if query_state_value == "EXCEPTION" or query_type == "DDL":
+    assert tables is None
+    assert data[index] == ""
+  else:
+    assert tables is not None
+    assert data[index] == tables.group(1)
+
   return ret_data
 # function assert_query
 


(impala) 02/03: IMPALA-12426: Skip Inserting HS2 Operation Queries into the Completed Queries Table

Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit deee153c76d2856dd5405137f45fb47bc267f99b
Author: jasonmfehr <jf...@cloudera.com>
AuthorDate: Tue Mar 26 14:15:01 2024 -0700

    IMPALA-12426: Skip Inserting HS2 Operation Queries into the Completed Queries Table
    
    Prevents queries associated with HS2 metadata operations
    from being written to the completed queries table. These
    queries are represented by the TMetadataOpcode enum.
    
    A Custom cluster test that makes an HS2 connection to
    Impala and runs these operations has been added. This test
    asserts that none of the operations have their queries
    written to the completed queries table.
    
    Change-Id: Ie19cf5953522fa85941e6c0b9c15a9c9ba9dc362
    Reviewed-on: http://gerrit.cloudera.org:8080/21207
    Reviewed-by: Michael Smith <mi...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/service/client-request-state.h  |   1 +
 be/src/service/impala-hs2-server.cc    |  10 ++-
 be/src/service/workload-management.cc  |  22 ++++-
 common/thrift/Query.thrift             |   3 +
 tests/custom_cluster/test_query_log.py | 142 +++++++++++++++++++++++++++++++++
 5 files changed, 173 insertions(+), 5 deletions(-)

diff --git a/be/src/service/client-request-state.h b/be/src/service/client-request-state.h
index c96fcc73e..d5c582a57 100644
--- a/be/src/service/client-request-state.h
+++ b/be/src/service/client-request-state.h
@@ -332,6 +332,7 @@ class ClientRequestState {
   const TQueryOptions& query_options() const {
     return query_ctx_.client_request.query_options;
   }
+  bool hs2_metadata_op() const { return query_ctx_.client_request.hs2_metadata_op; }
   /// Returns 0:0 if this is a root query
   TUniqueId parent_query_id() const { return query_ctx_.parent_query_id; }
   const vector<TTableName>& tables() const { return exec_request().tables; }
diff --git a/be/src/service/impala-hs2-server.cc b/be/src/service/impala-hs2-server.cc
index 9392e1a02..e651a8439 100644
--- a/be/src/service/impala-hs2-server.cc
+++ b/be/src/service/impala-hs2-server.cc
@@ -145,9 +145,13 @@ void ImpalaServer::ExecuteMetadataOp(const THandleIdentifier& session_handle,
   // from an RPC. As a best effort, we use the type of the operation.
   map<int, const char*>::const_iterator query_text_it =
       _TMetadataOpcode_VALUES_TO_NAMES.find(request->opcode);
-  const string& query_text = query_text_it == _TMetadataOpcode_VALUES_TO_NAMES.end() ?
-      "N/A" : query_text_it->second;
-  query_ctx.client_request.stmt = query_text;
+  if (query_text_it == _TMetadataOpcode_VALUES_TO_NAMES.end()) {
+    query_ctx.client_request.stmt = "N/A";
+  } else {
+    query_ctx.client_request.stmt = query_text_it->second;
+    query_ctx.client_request.__set_hs2_metadata_op(true);
+  }
+
   QueryHandle query_handle;
   QueryDriver::CreateNewDriver(this, &query_handle, query_ctx, session);
   Status register_status = RegisterQuery(query_ctx.query_id, session, &query_handle);
diff --git a/be/src/service/workload-management.cc b/be/src/service/workload-management.cc
index 6394c508d..611395071 100644
--- a/be/src/service/workload-management.cc
+++ b/be/src/service/workload-management.cc
@@ -225,12 +225,27 @@ void ImpalaServer::EnqueueCompletedQuery(const QueryHandle& query_handle,
       case TCatalogOpType::DESCRIBE_TABLE:
       case TCatalogOpType::DESCRIBE_DB:
       case TCatalogOpType::DESCRIBE_HISTORY:
-        return;  // Note: early return
+        VLOG_QUERY << "skipping enqueue of completed query '" <<
+            PrintId(query_handle->query_id()) << "' with type '" <<
+          query_handle->stmt_type() << "' and catalog op type '" <<
+          query_handle->catalog_op_type() << "'";
+        return;  // Note: early return, query will not be added to the queue
       case TCatalogOpType::RESET_METADATA:
       case TCatalogOpType::DDL:
+        // No-op, continue execution of this function.
         break;
       default:
-        LOG(FATAL) << "unknown ddl type: " << to_string(query_handle->catalog_op_type());
+        LOG(WARNING) << "unknown ddl type: " <<
+            to_string(query_handle->catalog_op_type());
+        DCHECK(false);
+        return;  // Note: early return
+    }
+  } else if(query_handle->stmt_type() == TStmtType::UNKNOWN) {
+    if (query_handle->hs2_metadata_op()) {
+      VLOG_QUERY << "skipping enqueue of completed query '" <<
+          PrintId(query_handle->query_id()) << "' with type '" <<
+          query_handle->stmt_type() << "'";
+      return;  // Note: early return, query will not be added to the queue
     }
   }
 
@@ -246,6 +261,9 @@ void ImpalaServer::EnqueueCompletedQuery(const QueryHandle& query_handle,
       completed_queries_cv_.notify_all();
     }
   }
+
+  VLOG_QUERY << "enqueued completed '" << query_handle->stmt_type() << "' query '" <<
+      PrintId(query_handle->query_id()) << "'";
 } // ImpalaServer::EnqueueCompletedQuery
 
 void ImpalaServer::CompletedQueriesThread() {
diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift
index 75043a489..57f322e7d 100644
--- a/common/thrift/Query.thrift
+++ b/common/thrift/Query.thrift
@@ -723,6 +723,9 @@ struct TClientRequest {
 
   // Redacted SQL stmt
   3: optional string redacted_stmt
+
+  // Indicates if an HS2 metadata operation code was provided in the client request
+  4: optional bool hs2_metadata_op
 }
 
 // Per-client session state
diff --git a/tests/custom_cluster/test_query_log.py b/tests/custom_cluster/test_query_log.py
index 1d39b2da2..e2f9a08fb 100644
--- a/tests/custom_cluster/test_query_log.py
+++ b/tests/custom_cluster/test_query_log.py
@@ -22,9 +22,16 @@ import pytest
 import string
 import tempfile
 
+from getpass import getuser
+from ImpalaService import ImpalaHiveServer2Service
 from random import choice, randint
 from signal import SIGRTMIN
+from TCLIService import TCLIService
+from thrift.transport.TSocket import TSocket
+from thrift.transport.TTransport import TBufferedTransport
+from thrift.protocol import TBinaryProtocol
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.impala_test_suite import IMPALAD_HS2_HOST_PORT
 from tests.common.test_dimensions import create_single_exec_option_dimension
 from tests.common.test_vector import ImpalaTestDimension
 from tests.util.retry import retry
@@ -441,12 +448,147 @@ class TestQueryLogTableHS2(TestQueryLogTableBase):
   """Tests to assert the query log table is correctly populated when using the HS2
      client protocol."""
 
+  HS2_OPERATIONS_CLUSTER_ID = "hs2-operations-" + str(int(time()))
+
   @classmethod
   def add_test_dimensions(cls):
     super(TestQueryLogTableHS2, cls).add_test_dimensions()
     cls.ImpalaTestMatrix.add_constraint(lambda v:
         v.get_value('protocol') == 'hs2')
 
+  @CustomClusterTestSuite.with_args(impalad_args="--enable_workload_mgmt "
+                                                 "--query_log_write_interval_s=1 "
+                                                 "--cluster_id={} "
+                                                 "--shutdown_grace_period_s=10 "
+                                                 "--shutdown_deadline_s=60"
+                                                 .format(HS2_OPERATIONS_CLUSTER_ID),
+                                    catalogd_args="--enable_workload_mgmt",
+                                    cluster_size=2,
+                                    impalad_graceful_shutdown=True)
+  def test_query_log_table_hs2_operations(self, vector):
+    """Certain HS2 operations appear to Impala as a special kind of query. Specifically,
+       these operations have a type of unknown and a normally invalid sql syntax. This
+       test asserts those queries are not written to the completed queries table since
+       they are trivial."""
+    host, port = IMPALAD_HS2_HOST_PORT.split(":")
+    socket = TSocket(host, port)
+    transport = TBufferedTransport(socket)
+    transport.open()
+    protocol = TBinaryProtocol.TBinaryProtocol(transport)
+    hs2_client = ImpalaHiveServer2Service.Client(protocol)
+
+    # Asserts the response from an HS2 operation indicates success.
+    def assert_resp(resp):
+      assert resp.status.statusCode == TCLIService.TStatusCode.SUCCESS_STATUS
+
+    # Closes an HS2 operation.
+    def close_op(client, resp):
+      close_operation_req = TCLIService.TCloseOperationReq()
+      close_operation_req.operationHandle = resp.operationHandle
+      assert_resp(hs2_client.CloseOperation(close_operation_req))
+
+    try:
+      # Open a new HS2 session.
+      open_session_req = TCLIService.TOpenSessionReq()
+      open_session_req.username = getuser()
+      open_session_req.configuration = dict()
+      open_sess_resp = hs2_client.OpenSession(open_session_req)
+      assert_resp(open_sess_resp)
+
+      # Test the get_type_info query.
+      get_typeinfo_req = TCLIService.TGetTypeInfoReq()
+      get_typeinfo_req.sessionHandle = open_sess_resp.sessionHandle
+      get_typeinfo_resp = hs2_client.GetTypeInfo(get_typeinfo_req)
+      assert_resp(get_typeinfo_resp)
+      close_op(hs2_client, get_typeinfo_resp)
+
+      # Test the get_catalogs query.
+      get_cats_req = TCLIService.TGetCatalogsReq()
+      get_cats_req.sessionHandle = open_sess_resp.sessionHandle
+      get_cats_resp = hs2_client.GetCatalogs(get_cats_req)
+      assert_resp(get_cats_resp)
+      close_op(hs2_client, get_cats_resp)
+
+      # Test the get_schemas query.
+      get_schemas_req = TCLIService.TGetSchemasReq()
+      get_schemas_req.sessionHandle = open_sess_resp.sessionHandle
+      get_schemas_resp = hs2_client.GetSchemas(get_schemas_req)
+      assert_resp(get_schemas_resp)
+      close_op(hs2_client, get_schemas_resp)
+
+      # Test the get_tables query.
+      get_tables_req = TCLIService.TGetTablesReq()
+      get_tables_req.sessionHandle = open_sess_resp.sessionHandle
+      get_tables_req.schemaName = self.WM_DB
+      get_tables_resp = hs2_client.GetTables(get_tables_req)
+      assert_resp(get_tables_resp)
+      close_op(hs2_client, get_tables_resp)
+
+      # Test the get_table_types query.
+      get_tbl_typ_req = TCLIService.TGetTableTypesReq()
+      get_tbl_typ_req.sessionHandle = open_sess_resp.sessionHandle
+      get_tbl_typ_req.schemaName = self.WM_DB
+      get_tbl_typ_resp = hs2_client.GetTableTypes(get_tbl_typ_req)
+      assert_resp(get_tbl_typ_resp)
+      close_op(hs2_client, get_tbl_typ_resp)
+
+      # Test the get_columns query.
+      get_cols_req = TCLIService.TGetColumnsReq()
+      get_cols_req.sessionHandle = open_sess_resp.sessionHandle
+      get_cols_req.schemaName = 'functional'
+      get_cols_req.tableName = 'parent_table'
+      get_cols_resp = hs2_client.GetColumns(get_cols_req)
+      assert_resp(get_cols_resp)
+      close_op(hs2_client, get_cols_resp)
+
+      # Test the get_primary_keys query.
+      get_pk_req = TCLIService.TGetPrimaryKeysReq()
+      get_pk_req.sessionHandle = open_sess_resp.sessionHandle
+      get_pk_req.schemaName = 'functional'
+      get_pk_req.tableName = 'parent_table'
+      get_pk_resp = hs2_client.GetPrimaryKeys(get_pk_req)
+      assert_resp(get_pk_resp)
+      close_op(hs2_client, get_pk_resp)
+
+      # Test the get_cross_reference query.
+      get_cr_req = TCLIService.TGetCrossReferenceReq()
+      get_cr_req.sessionHandle = open_sess_resp.sessionHandle
+      get_cr_req.parentSchemaName = "functional"
+      get_cr_req.foreignSchemaName = "functional"
+      get_cr_req.parentTableName = "parent_table"
+      get_cr_req.foreignTableName = "child_table"
+      get_cr_resp = hs2_client.GetCrossReference(get_cr_req)
+      assert_resp(get_cr_resp)
+      close_op(hs2_client, get_cr_resp)
+
+      close_session_req = TCLIService.TCloseSessionReq()
+      close_session_req.sessionHandle = open_sess_resp.sessionHandle
+      resp = hs2_client.CloseSession(close_session_req)
+      assert resp.status.statusCode == TCLIService.TStatusCode.SUCCESS_STATUS
+    finally:
+      socket.close()
+
+    # Assert none of the queries were written to the completed queries table.
+    client = self.create_impala_client(protocol=vector.get_value('protocol'))
+
+    # Execute a general query and wait for it to appear in the completed queries table to
+    # ensure there are no false positives caused by the assertion query executing before
+    # Impala has a chance to write queued queries to the completed queries table.
+    assert client.execute("select 1").success
+    self.cluster.get_first_impalad().service.wait_for_metric_value(
+          "impala-server.completed-queries.written", 1, 30)
+
+    # Force Impala to process the inserts to the completed queries table.
+    client.execute("refresh {}".format(self.QUERY_TBL))
+
+    try:
+      assert_results = client.execute("select count(*) from {} where cluster_id='{}'"
+          .format(self.QUERY_TBL, self.HS2_OPERATIONS_CLUSTER_ID))
+      assert assert_results.success
+      assert assert_results.data[0] == "1"
+    finally:
+      client.close()
+
   @CustomClusterTestSuite.with_args(impalad_args="--enable_workload_mgmt "
                                                  "--query_log_write_interval_s=1 "
                                                  "--cluster_id=test_query_hist_mult "


(impala) 03/03: IMPALA-12944: Fixes Workload Management Test Flakiness

Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 83734a1220bf99261779bbc5037708a7e1d78df0
Author: jasonmfehr <jf...@cloudera.com>
AuthorDate: Thu Mar 28 10:43:03 2024 -0700

    IMPALA-12944: Fixes Workload Management Test Flakiness
    
    The custom cluster workload management tests are flaky
    because the tests can actually run before the completed
    queries table has been fully created by the Impala startup
    process. The table create sql runs asynchronously during
    startup and thus can take longer to finish than the custom
    cluster tests take to execute.
    
    This change adds checks at the beginning of each test to
    ensure the completed queries table sql has finished before
    any of the test code runs.
    
    Change-Id: I428702a210e024db95808dc2518da497426922f8
    Reviewed-on: http://gerrit.cloudera.org:8080/21221
    Reviewed-by: Michael Smith <mi...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 tests/common/impala_test_suite.py      | 16 ++++++++---
 tests/custom_cluster/test_query_log.py | 49 +++++++++++++++++++++-------------
 2 files changed, 43 insertions(+), 22 deletions(-)

diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 72b0e1fa3..40020e3d0 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -1267,14 +1267,16 @@ class ImpalaTestSuite(BaseTestSuite):
     """
     Convenience wrapper around assert_log_contains for impalad logs.
     """
-    self.assert_log_contains("impalad", level, line_regex, expected_count, timeout_s)
+    return self.assert_log_contains(
+        "impalad", level, line_regex, expected_count, timeout_s)
 
   def assert_catalogd_log_contains(self, level, line_regex, expected_count=1,
       timeout_s=6):
     """
     Convenience wrapper around assert_log_contains for catalogd logs.
     """
-    self.assert_log_contains("catalogd", level, line_regex, expected_count, timeout_s)
+    return self.assert_log_contains(
+        "catalogd", level, line_regex, expected_count, timeout_s)
 
   def assert_log_contains(self, daemon, level, line_regex, expected_count=1, timeout_s=6):
     """
@@ -1287,6 +1289,9 @@ class ImpalaTestSuite(BaseTestSuite):
     make sure that log buffering has been disabled, for example by adding
     '-logbuflevel=-1' to the daemon startup options or set timeout_s to a value higher
     than the log flush interval.
+
+    Returns the result of the very last call to line_regex.search or None if
+    expected_count is 0 or the line_regex did not match any lines.
     """
     pattern = re.compile(line_regex)
     start_time = time.time()
@@ -1300,10 +1305,13 @@ class ImpalaTestSuite(BaseTestSuite):
         log_file_path = os.path.join(log_dir, daemon + "." + level)
         # Resolve symlinks to make finding the file easier.
         log_file_path = os.path.realpath(log_file_path)
+        last_re_result = None
         with open(log_file_path) as log_file:
           for line in log_file:
-            if pattern.search(line):
+            re_result = pattern.search(line)
+            if re_result:
               found += 1
+              last_re_result = re_result
         if expected_count == -1:
           assert found > 0, "Expected at least one line in file %s matching regex '%s'"\
             ", but found none." % (log_file_path, line_regex)
@@ -1312,7 +1320,7 @@ class ImpalaTestSuite(BaseTestSuite):
             "Expected %d lines in file %s matching regex '%s', but found %d lines. "\
             "Last line was: \n%s" %\
             (expected_count, log_file_path, line_regex, found, line)
-        return
+        return last_re_result
       except AssertionError as e:
         # Re-throw the exception to the caller only when the timeout is expired. Otherwise
         # sleep before retrying.
diff --git a/tests/custom_cluster/test_query_log.py b/tests/custom_cluster/test_query_log.py
index e2f9a08fb..e02993217 100644
--- a/tests/custom_cluster/test_query_log.py
+++ b/tests/custom_cluster/test_query_log.py
@@ -56,9 +56,25 @@ class TestQueryLogTableBase(CustomClusterTestSuite):
     cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('protocol',
         cls.PROTOCOL_BEESWAX, cls.PROTOCOL_HS2))
 
-  def get_client(self, protocol):
+  def get_client(self, protocol, query_table_name=""):
     """Retrieves the default Impala client for the specified protocol. This client is
-       automatically closed after the test completes."""
+       automatically closed after the test completes. Also ensures the completed queries
+       table has been successfully created by checking the logs to verify the create
+       table sql has finished."""
+    if query_table_name == "":
+      query_table_name = self.QUERY_TBL
+
+    # These tests run very quickly and can actually complete before Impala has finished
+    # creating the completed queries table. Thus, to make these tests more robust, this
+    # code checks to make sure the table create has finished before returning.
+    create_re = r'\]\s+(\w+:\w+)\]\s+Analyzing query: CREATE TABLE IF NOT EXISTS {}' \
+        .format(query_table_name)
+    create_match = self.assert_impalad_log_contains("INFO", create_re)
+
+    finish_re = r'Query successfully unregistered: query_id={}' \
+        .format(create_match.group(1))
+    self.assert_impalad_log_contains("INFO", finish_re)
+
     if protocol == self.PROTOCOL_BEESWAX:
       return self.client
     elif protocol == self.PROTOCOL_HS2:
@@ -241,9 +257,8 @@ class TestQueryLogTableBeeswax(TestQueryLogTableBase):
   @CustomClusterTestSuite.with_args(impalad_args="--enable_workload_mgmt "
                                                  "--query_log_write_interval_s=5 "
                                                  "--shutdown_grace_period_s=10 "
-                                                 "--shutdown_deadline_s=60 "
-                                                 "--log_dir={0}"
-                                                 .format(LOG_DIR_MAX_WRITES),
+                                                 "--shutdown_deadline_s=60 ",
+                                    impala_log_dir=LOG_DIR_MAX_WRITES,
                                     catalogd_args="--enable_workload_mgmt",
                                     impalad_graceful_shutdown=True)
   def test_query_log_max_attempts_exceeded(self, vector):
@@ -365,13 +380,14 @@ class TestQueryLogTableBeeswax(TestQueryLogTableBase):
   def test_query_log_table_different_table(self, vector):
     """Asserts that the completed queries table can be renamed."""
 
-    client = self.get_client(vector.get_value('protocol'))
+    client = self.get_client(vector.get_value('protocol'),
+        "{}.{}".format(self.WM_DB, self.OTHER_TBL))
 
     try:
       res = client.execute("show tables in {0}".format(self.WM_DB))
       assert res.success
       assert len(res.data) > 0, "could not find any tables in database {0}" \
-          .format(self.DB)
+          .format(self.WM_DB)
 
       tbl_found = False
       for tbl in res.data:
@@ -379,7 +395,7 @@ class TestQueryLogTableBeeswax(TestQueryLogTableBase):
           tbl_found = True
           break
       assert tbl_found, "could not find table '{0}' in database '{1}'" \
-          .format(self.OTHER_TBL, self.DB)
+          .format(self.OTHER_TBL, self.WM_DB)
     finally:
       client.execute("drop table {0}.{1} purge".format(self.WM_DB, self.OTHER_TBL))
 
@@ -470,6 +486,8 @@ class TestQueryLogTableHS2(TestQueryLogTableBase):
        these operations have a type of unknown and a normally invalid sql syntax. This
        test asserts those queries are not written to the completed queries table since
        they are trivial."""
+    client = self.get_client(vector.get_value('protocol'))
+
     host, port = IMPALAD_HS2_HOST_PORT.split(":")
     socket = TSocket(host, port)
     transport = TBufferedTransport(socket)
@@ -568,9 +586,6 @@ class TestQueryLogTableHS2(TestQueryLogTableBase):
     finally:
       socket.close()
 
-    # Assert none of the queries were written to the completed queries table.
-    client = self.create_impala_client(protocol=vector.get_value('protocol'))
-
     # Execute a general query and wait for it to appear in the completed queries table to
     # ensure there are no false positives caused by the assertion query executing before
     # Impala has a chance to write queued queries to the completed queries table.
@@ -581,13 +596,11 @@ class TestQueryLogTableHS2(TestQueryLogTableBase):
     # Force Impala to process the inserts to the completed queries table.
     client.execute("refresh {}".format(self.QUERY_TBL))
 
-    try:
-      assert_results = client.execute("select count(*) from {} where cluster_id='{}'"
-          .format(self.QUERY_TBL, self.HS2_OPERATIONS_CLUSTER_ID))
-      assert assert_results.success
-      assert assert_results.data[0] == "1"
-    finally:
-      client.close()
+    # Assert only the one expected query was written to the completed queries table.
+    assert_results = client.execute("select count(*) from {} where cluster_id='{}'"
+        .format(self.QUERY_TBL, self.HS2_OPERATIONS_CLUSTER_ID))
+    assert assert_results.success
+    assert assert_results.data[0] == "1"
 
   @CustomClusterTestSuite.with_args(impalad_args="--enable_workload_mgmt "
                                                  "--query_log_write_interval_s=1 "