You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/11/13 11:30:42 UTC
[incubator-kyuubi] branch branch-1.6 updated: [KYUUBI #3801] Correctly calculate active stages in SQLOperationListener
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch branch-1.6
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/branch-1.6 by this push:
new 3af7979d2 [KYUUBI #3801] Correctly calculate active stages in SQLOperationListener
3af7979d2 is described below
commit 3af7979d22c8020044075c0b1b29acf2aa5291c2
Author: lcy999 <15...@qq.com>
AuthorDate: Sun Nov 13 11:30:22 2022 +0000
[KYUUBI #3801] Correctly calculate active stages in SQLOperationListener
### _Code of Conduct_
- [x] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct)
### _Search before asking_
- [x] I have searched in the [issues](https://github.com/apache/incubator-kyuubi/issues?q=is%3Aissue) and found no similar issues.
### _Describe the bug_
activeStages key is StageAttempt, and its value is StageInfo. The contains function defaults to 'containsValue'
### _Affects Version(s)_
master/1.7/1.6
### _Are you willing to submit PR?_
- [x] Yes. I can submit a PR independently to fix.
- [ ] Yes. I would be willing to submit a PR with guidance from the Kyuubi community to fix.
- [ ] No. I cannot submit a PR at this time.
Closes #3801 from lcy999/bug_1113.
Closes #3801
d6d96d30 [lcy999] active stage can't calculate the number of tasks correctly in sql operation listener
Authored-by: lcy999 <15...@qq.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
(cherry picked from commit 979881d6877bef42faee3410f5796966258e235e)
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala
index 3cae5c5f8..1a57fcf29 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala
@@ -144,7 +144,7 @@ class SQLOperationListener(
override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = activeStages.synchronized {
val stageAttempt = StageAttempt(taskStart.stageId, taskStart.stageAttemptId)
- if (activeStages.contains(stageAttempt)) {
+ if (activeStages.containsKey(stageAttempt)) {
activeStages.get(stageAttempt).numActiveTasks += 1
super.onTaskStart(taskStart)
}
@@ -152,7 +152,7 @@ class SQLOperationListener(
override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = activeStages.synchronized {
val stageAttempt = StageAttempt(taskEnd.stageId, taskEnd.stageAttemptId)
- if (activeStages.contains(stageAttempt)) {
+ if (activeStages.containsKey(stageAttempt)) {
activeStages.get(stageAttempt).numActiveTasks -= 1
if (taskEnd.reason == org.apache.spark.Success) {
activeStages.get(stageAttempt).numCompleteTasks += 1