You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2025/02/28 02:56:46 UTC
(impala) branch master updated (768527c89 -> 7a1f4b671)
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 768527c89 IMPALA-13804: Use redacted statement in live table
new 3d57e4d05 IMPALA-13805: Fix TSAN build failures for the runtime-profile-test
new 7a1f4b671 IMPALA-13671: Add Additional Debug Output
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/util/runtime-profile.cc | 8 +++-----
be/src/util/runtime-profile.h | 2 +-
tests/util/workload_management.py | 14 ++++++++++----
3 files changed, 14 insertions(+), 10 deletions(-)
(impala) 02/02: IMPALA-13671: Add Additional Debug Output
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 7a1f4b671395a567c89e2cd3561d8560df22f7e6
Author: jasonmfehr <jf...@cloudera.com>
AuthorDate: Wed Feb 26 14:40:51 2025 -0800
IMPALA-13671: Add Additional Debug Output
The custom_cluster/test_workload_mgmt_init.py tests failed with a
mismatch between the expected and actual start time. The error that
is logged only says "start time incorrect".
This patch adds the expected and actual query start time values to
the assertion failed message.
Testing accomplished by the following test suites passing locally:
* tests/custom_cluster/test_query_live.py
* tests/custom_cluster/test_query_log.py
* tests/custom_cluster/test_workload_mgmt_init.py
* tests/custom_cluster/test_workload_mgmt_sql_details.py
Change-Id: Ieb1627ea719f0e20459ba6052685320a169d5176
Reviewed-on: http://gerrit.cloudera.org:8080/22555
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/util/workload_management.py | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/tests/util/workload_management.py b/tests/util/workload_management.py
index 3a8a7c5b7..5053a2a2c 100644
--- a/tests/util/workload_management.py
+++ b/tests/util/workload_management.py
@@ -34,10 +34,14 @@ QUERY_TBL_LOG_NAME = "impala_query_log"
QUERY_TBL_LOG = "{0}.{1}".format(WM_DB, QUERY_TBL_LOG_NAME)
QUERY_TBL_LIVE_NAME = "impala_query_live"
QUERY_TBL_LIVE = "{0}.{1}".format(WM_DB, QUERY_TBL_LIVE_NAME)
+
# Time in seconds the assert_query and assert_csv_col will wait for the query to become
# available in the relevant workload management table.
ASSERT_QUERY_TIMEOUT_S = 30
+# String parsing format for query start/end time fields in a text query profile.
+QUERY_PROFILE_DT_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
+
def round_to_3(val):
# The differences between round in Python 2 and Python 3 do not matter here.
@@ -216,15 +220,17 @@ def assert_query(query_tbl, client, expected_cluster_id="", raw_profile=None,
# Start Time
start_time = re.search(r'\n\s+Start Time:\s+(.*?)\n', profile_text)
assert start_time is not None
- start_time_obj = datetime.strptime(start_time.group(1)[:-3], "%Y-%m-%d %H:%M:%S.%f")
+ start_time_obj = datetime.strptime(start_time.group(1)[:-3], QUERY_PROFILE_DT_FORMAT)
start_time_obj_utc = start_time_obj + utc_offset
- assert column_val(TQueryTableColumn.START_TIME_UTC)[:-3] \
- == start_time_obj_utc.strftime("%Y-%m-%d %H:%M:%S.%f"), "start time incorrect"
+ expected = start_time_obj_utc.strftime(QUERY_PROFILE_DT_FORMAT)
+ actual = column_val(TQueryTableColumn.START_TIME_UTC)[:-3]
+ assert actual == expected, "start time incorrect, expected '{}' but was '{}'" \
+ .format(expected, actual)
# End Time (not in table, but needed for duration calculation)
end_time = re.search(r'\n\s+End Time:\s+(.*?)\n', profile_text)
assert end_time is not None
- end_time_obj = datetime.strptime(end_time.group(1)[:-3], "%Y-%m-%d %H:%M:%S.%f")
+ end_time_obj = datetime.strptime(end_time.group(1)[:-3], QUERY_PROFILE_DT_FORMAT)
# Query Duration (allow values that are within 1 second)
value = column_val(TQueryTableColumn.TOTAL_TIME_MS)
(impala) 01/02: IMPALA-13805: Fix TSAN build failures for the runtime-profile-test
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 3d57e4d05e4b9d7601eadfdd7eb442b13241a49f
Author: Surya Hebbar <sh...@cloudera.com>
AuthorDate: Thu Feb 27 19:38:36 2025 +0530
IMPALA-13805: Fix TSAN build failures for the runtime-profile-test
The shared memory of a rapidjson document's value containing
event sequences was being utilized through a reference pointer,
instead of referencing by value, while generating the aggregated
event sequences.
During thread sanitization builds, such values were cleared,
resulting in test failures.
This has been fixed by correctly passing values through reference,
instead of the shared memory address.
Tested locally by building with TSAN opts.
Change-Id: Ibdea503d032d456746e9cb4f19d0057197ddb117
Reviewed-on: http://gerrit.cloudera.org:8080/22560
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/util/runtime-profile.cc | 8 +++-----
be/src/util/runtime-profile.h | 2 +-
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/be/src/util/runtime-profile.cc b/be/src/util/runtime-profile.cc
index fc053b4d5..f36c7a42e 100644
--- a/be/src/util/runtime-profile.cc
+++ b/be/src/util/runtime-profile.cc
@@ -2972,10 +2972,10 @@ void AggregatedRuntimeProfile::ToJsonSubclass(
if (es_count > 0) {
Value event_sequences_json(kArrayType);
- Value* event_sequence_json;
for (AggEventSequence& agg_event_sequence : agg_event_sequences) {
+ Value event_sequence_json(kObjectType);
agg_event_sequence.ToJson(event_sequence_json, d);
- event_sequences_json.PushBack(*event_sequence_json, allocator);
+ event_sequences_json.PushBack(event_sequence_json, allocator);
}
parent->AddMember("event_sequences", event_sequences_json, allocator);
}
@@ -3001,7 +3001,7 @@ void AggregatedRuntimeProfile::ToJsonSubclass(
}
}
-void AggregatedRuntimeProfile::AggEventSequence::ToJson(Value*& val,
+void AggregatedRuntimeProfile::AggEventSequence::ToJson(Value& event_sequence_json,
rapidjson::Document* d) {
Document::AllocatorType& allocator = d->GetAllocator();
@@ -3068,7 +3068,6 @@ void AggregatedRuntimeProfile::AggEventSequence::ToJson(Value*& val,
static vector<size_t> inst_count_list(BUCKET_SIZE);
// Perform aggregation to limit the size and complexity of event sequences JSON
- Value event_sequence_json(kObjectType);
Value events_json(kArrayType);
if (num_instances <= BUCKET_SIZE || BUCKET_SIZE == 0) {
// Group event timestamps, when number of instances is less than bucket size
@@ -3165,7 +3164,6 @@ void AggregatedRuntimeProfile::AggEventSequence::ToJson(Value*& val,
event_sequence_json.AddMember("unreported_event_instance_idxs",
missing_event_instance_idxs, allocator);
}
- val = &event_sequence_json;
}
void AggregatedRuntimeProfile::CollectInfoStringIntoJson(const string& info_string_name,
diff --git a/be/src/util/runtime-profile.h b/be/src/util/runtime-profile.h
index 882ba5701..476357427 100644
--- a/be/src/util/runtime-profile.h
+++ b/be/src/util/runtime-profile.h
@@ -944,7 +944,7 @@ class AggregatedRuntimeProfile : public RuntimeProfileBase {
std::vector<std::vector<int64_t>> timestamps;
// Builds the JSON for the aggregate event sequence's metrics into the provided 'val'
- void ToJson(rapidjson::Value*& val, rapidjson::Document* d);
+ void ToJson(rapidjson::Value& event_sequence_json, rapidjson::Document* d);
};
// Map containing event sequences, with its type in string format as the key