You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ul...@apache.org on 2021/10/13 05:16:43 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #1223] Remove
dead module kyuubi-spark-monitor
This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 556b5ad [KYUUBI #1223] Remove dead module kyuubi-spark-monitor
556b5ad is described below
commit 556b5ad57ee7913bebcea7d1ef40ab3a0ffa2409
Author: Kent Yao <ya...@apache.org>
AuthorDate: Wed Oct 13 13:15:33 2021 +0800
[KYUUBI #1223] Remove dead module kyuubi-spark-monitor
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
clean dead code
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #1223 from yaooqinn/mon.
Closes #1223
3b7a9a57 [Kent Yao] Remove dead module kyuubi-spark-monitor
b1e2239e [Kent Yao] Remove dead module kyuubi-spark-monitor
Authored-by: Kent Yao <ya...@apache.org>
Signed-off-by: ulysses-you <ul...@apache.org>
---
externals/kyuubi-spark-monitor/README.md | 6 -
externals/kyuubi-spark-monitor/pom.xml | 75 ------------
.../spark/monitor/KyuubiStatementMonitor.scala | 136 ---------------------
.../spark/monitor/entity/KyuubiJobInfo.scala | 45 -------
.../spark/monitor/entity/KyuubiStatementInfo.scala | 46 -------
.../src/test/resources/log4j.properties | 40 ------
externals/kyuubi-spark-sql-engine/pom.xml | 1 -
.../spark/kyuubi/SparkSQLEngineListener.scala | 6 -
.../engine/spark/KyuubiStatementMonitorSuite.scala | 76 ------------
pom.xml | 1 -
10 files changed, 432 deletions(-)
diff --git a/externals/kyuubi-spark-monitor/README.md b/externals/kyuubi-spark-monitor/README.md
deleted file mode 100644
index c20825e..0000000
--- a/externals/kyuubi-spark-monitor/README.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# for kyuubi-spark-monitor
-
-This module's feature:
-- get all SQL's metrics during runtime
-- monitor the sqlOperation
-- for auto-configuring
diff --git a/externals/kyuubi-spark-monitor/pom.xml b/externals/kyuubi-spark-monitor/pom.xml
deleted file mode 100644
index 01c8243..0000000
--- a/externals/kyuubi-spark-monitor/pom.xml
+++ /dev/null
@@ -1,75 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.kyuubi</groupId>
- <artifactId>kyuubi-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>kyuubi-spark-monitor_2.12</artifactId>
- <packaging>jar</packaging>
- <name>Kyuubi Project Spark Monitor</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.kyuubi</groupId>
- <artifactId>kyuubi-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.binary.version}</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
- <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
-
- <plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/externals/kyuubi-spark-monitor/src/main/scala/org/apache/kyuubi/engine/spark/monitor/KyuubiStatementMonitor.scala b/externals/kyuubi-spark-monitor/src/main/scala/org/apache/kyuubi/engine/spark/monitor/KyuubiStatementMonitor.scala
deleted file mode 100644
index 178cca4..0000000
--- a/externals/kyuubi-spark-monitor/src/main/scala/org/apache/kyuubi/engine/spark/monitor/KyuubiStatementMonitor.scala
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kyuubi.engine.spark.monitor
-
-import java.util.concurrent.{ArrayBlockingQueue, ConcurrentHashMap}
-
-import org.apache.spark.scheduler.SparkListenerJobEnd
-
-import org.apache.kyuubi.Logging
-import org.apache.kyuubi.engine.spark.monitor.entity.{KyuubiJobInfo, KyuubiStatementInfo}
-
-// TODO: Thread Safe need to consider
-object KyuubiStatementMonitor extends Logging{
-
- // TODO: Just for test. We will remove them in the future
- private val maxCapacity: Int = 10
- private val maxSize: Int = 7
-
- /**
- * This blockingQueue store kyuubiStatementInfo.
- *
- * Notice:
- * 1. When we remove items from this queue, we should ensure those statements have finished.
- * If not, we should put them into this queue again.
- * 2. There have two kinds of threshold to trigger when to remove items from this queue:
- * a. time
- * b. this queue's current size
- */
- // TODO: Capacity should make configurable
- private val kyuubiStatementQueue = new ArrayBlockingQueue[KyuubiStatementInfo](maxCapacity)
-
- /**
- * This map store the relationship between jobId and jobInfo.
- * When the job has finished, all we can get is jobId from the object-jobEnd.
- * So we need to maintain a mapping relationship to store endTime and jobResult
- * when this job has finished.
- *
- * Key is jobId, value is KyuubiJobInfo.
- *
- * Notice:
- * 1. There have two kinds of threshold to trigger when to remove and dump items from this map:
- * a. time
- * b. this map's current size
- */
- // TODO: Capacity should make configurable
- private val kyuubiJobIdToJobInfoMap = new ConcurrentHashMap[Int, KyuubiJobInfo](maxCapacity)
-
- /**
- * This method is used for putting kyuubiStatementInfo into blockingQueue(statementQueue).
- * Every time we put an item into this queue, we should judge this queue's current size at first.
- * If the size is less than threshold, we need to remove items from this queue.
- *
- * @param kyuubiStatementInfo
- */
- // TODO: Lack size type threshold and time type threshold
- def putStatementInfoIntoQueue(kyuubiStatementInfo: KyuubiStatementInfo): Unit = {
- if (kyuubiStatementQueue.size() >= maxSize) {
- removeAndDumpStatementInfoFromQueue()
- }
- val isSuccess = kyuubiStatementQueue.add(kyuubiStatementInfo)
- info(s"Add kyuubiStatementInfo into queue is [$isSuccess], " +
- s"statementId is [${kyuubiStatementInfo.statementId}]")
- }
-
- /**
- * This method is used for removing kyuubiStatementInfo from blockingQueue(statementQueue)
- * and dumpping them to a file by threshold.
- */
- // TODO: Need ensure those items have finished. If not, we should put them into this queue again.
- private def removeAndDumpStatementInfoFromQueue(): Unit = {
- // TODO: Just for test
- kyuubiStatementQueue.clear()
- }
-
- /**
- * This method is used for putting kyuubiJobInfo into hashMap(kyuubiJobIdToJobInfoMap)
- * and storing the mapping relationship between jobId and jobInfo.
- * The reason that we need to maintain a mapping relationship
- * is we need to store endTime and jobResult
- * when this job has finished but the object-jobEnd has nothing but jobId.
- *
- * @param kyuubiJobInfo
- */
- // TODO: Lack size type threshold and time type threshold
- def putJobInfoIntoMap(kyuubiJobInfo: KyuubiJobInfo): Unit = {
- if (kyuubiJobIdToJobInfoMap.size() >= maxSize) {
- removeAndDumpJobInfoFromMap()
- }
- // Put kyuubiJobInfo into kyuubiJobIdToJobInfoMap
- kyuubiJobIdToJobInfoMap.put(kyuubiJobInfo.jobId, kyuubiJobInfo)
- }
-
- /**
- * This method is used for removing kyuubiJobInfo from hashMap(kyuubiJobIdToJobInfoMap)
- * and dumpping them to a file by threshold.
- */
- private def removeAndDumpJobInfoFromMap(): Unit = {
- // TODO: Just for test
- kyuubiJobIdToJobInfoMap.clear()
- }
-
- /**
- * This method is used for inserting endTime and jobResult.
- * Those fields can only get when this job has finished.
- *
- * Notice:
- * 1. endTime and jobResult should consider the thread safe.
- *
- * @param jobEnd
- */
- def insertJobEndTimeAndResult(jobEnd: SparkListenerJobEnd): Unit = {
- val jobInfo = kyuubiJobIdToJobInfoMap.get(jobEnd.jobId)
- if (jobInfo != null) {
- jobInfo.endTime = jobEnd.time
- jobInfo.jobResult = jobEnd.jobResult
- debug(s"Job finished. Query [${jobInfo.statementId}]: JobId is [${jobInfo.jobId}]")
- } else {
- warn(s"JobStartEvent is lost. JobId is [${jobInfo.jobId}]")
- }
- }
-}
diff --git a/externals/kyuubi-spark-monitor/src/main/scala/org/apache/kyuubi/engine/spark/monitor/entity/KyuubiJobInfo.scala b/externals/kyuubi-spark-monitor/src/main/scala/org/apache/kyuubi/engine/spark/monitor/entity/KyuubiJobInfo.scala
deleted file mode 100644
index 4cb599d..0000000
--- a/externals/kyuubi-spark-monitor/src/main/scala/org/apache/kyuubi/engine/spark/monitor/entity/KyuubiJobInfo.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kyuubi.engine.spark.monitor.entity
-
-import org.apache.spark.scheduler.JobResult
-
-/**
- * This object is used for storing the basic data for job.
- * You can use statementId to get all jobs that belong to this statemnent.
- * And also you can use statementId and jobId to get all stages that belong to this job.
- *
- * Introduce:
- * 1. According to startTime and endTime, you can get how long did it run,
- * and get which stage took the longest time by stageIds.
- * 2. If this job failed, you can look up which stage cause this situation by stageIds.
- *
- * @param jobId
- * @param statementId
- * @param stageIds: is array
- * @param startTime
- */
-case class KyuubiJobInfo(
- jobId: Int,
- statementId: String,
- stageIds: Seq[Int],
- startTime: Long) {
-
- var endTime: Long = 0
- var jobResult: JobResult = null
-}
diff --git a/externals/kyuubi-spark-monitor/src/main/scala/org/apache/kyuubi/engine/spark/monitor/entity/KyuubiStatementInfo.scala b/externals/kyuubi-spark-monitor/src/main/scala/org/apache/kyuubi/engine/spark/monitor/entity/KyuubiStatementInfo.scala
deleted file mode 100644
index 7bf34f0..0000000
--- a/externals/kyuubi-spark-monitor/src/main/scala/org/apache/kyuubi/engine/spark/monitor/entity/KyuubiStatementInfo.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kyuubi.engine.spark.monitor.entity
-
-import scala.collection.mutable.Map
-
-import org.apache.spark.sql.execution.QueryExecution
-
-import org.apache.kyuubi.KyuubiSQLException
-import org.apache.kyuubi.cli.HandleIdentifier
-import org.apache.kyuubi.operation.OperationState.OperationState
-
-/**
- * This object store the summary infomation about statement.
- * You can use statementId to get all jobs' or stages' metric that this statement has.
- * @param statementId
- * @param statement
- * @param appId
- * @param sessionId
- * @param stateToTime: store this statement's every state and the time of occurrence
- */
-case class KyuubiStatementInfo(
- statementId: String,
- statement: String,
- appId: String,
- sessionId: HandleIdentifier,
- stateToTime: Map[OperationState, Long]) {
-
- var queryExecution: QueryExecution = null
- var exception: KyuubiSQLException = null
-}
diff --git a/externals/kyuubi-spark-monitor/src/test/resources/log4j.properties b/externals/kyuubi-spark-monitor/src/test/resources/log4j.properties
deleted file mode 100644
index fee75dc..0000000
--- a/externals/kyuubi-spark-monitor/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set everything to be logged to the file target/unit-tests.log
-log4j.rootLogger=DEBUG, CA, FA
-
-#Console Appender
-log4j.appender.CA=org.apache.log4j.ConsoleAppender
-log4j.appender.CA.layout=org.apache.log4j.PatternLayout
-log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
-log4j.appender.CA.Threshold = FATAL
-
-#File Appender
-log4j.appender.FA=org.apache.log4j.FileAppender
-log4j.appender.FA.append=false
-log4j.appender.FA.file=target/unit-tests.log
-log4j.appender.FA.layout=org.apache.log4j.PatternLayout
-log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n
-
-# Set the logger level of File Appender to WARN
-log4j.appender.FA.Threshold = DEBUG
-
-# SPARK-34128：Suppress undesirable TTransportException warnings involved in THRIFT-4805
-log4j.appender.console.filter.1=org.apache.log4j.varia.StringMatchFilter
-log4j.appender.console.filter.1.StringToMatch=Thrift error occurred during processing of message
-log4j.appender.console.filter.1.AcceptOnMatch=false
diff --git a/externals/kyuubi-spark-sql-engine/pom.xml b/externals/kyuubi-spark-sql-engine/pom.xml
index 64d9a83..7b7eb2a 100644
--- a/externals/kyuubi-spark-sql-engine/pom.xml
+++ b/externals/kyuubi-spark-sql-engine/pom.xml
@@ -196,7 +196,6 @@
<includes>
<include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
<include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
- <include>org.apache.kyuubi:kyuubi-spark-monitor_${scala.binary.version}</include>
<include>org.apache.curator:curator-client</include>
<include>org.apache.curator:curator-framework</include>
<include>org.apache.curator:curator-recipes</include>
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkSQLEngineListener.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkSQLEngineListener.scala
index f29bcdc..ef767aa 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkSQLEngineListener.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkSQLEngineListener.scala
@@ -31,8 +31,6 @@ import org.apache.kyuubi.Logging
import org.apache.kyuubi.Utils.stringifyException
import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.engine.spark.events.{EngineEventsStore, SessionEvent}
-import org.apache.kyuubi.engine.spark.monitor.KyuubiStatementMonitor
-import org.apache.kyuubi.engine.spark.monitor.entity.KyuubiJobInfo
import org.apache.kyuubi.service.{Serverable, ServiceState}
/**
@@ -69,15 +67,11 @@ class SparkSQLEngineListener(
override def onJobStart(jobStart: SparkListenerJobStart): Unit = {
val statementId = jobStart.properties.getProperty(KYUUBI_STATEMENT_ID_KEY)
- val kyuubiJobInfo = KyuubiJobInfo(
- jobStart.jobId, statementId, jobStart.stageIds, jobStart.time)
- KyuubiStatementMonitor.putJobInfoIntoMap(kyuubiJobInfo)
debug(s"Add jobStartInfo. Query [$statementId]: Job ${jobStart.jobId} started with " +
s"${jobStart.stageIds.length} stages")
}
override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = {
- KyuubiStatementMonitor.insertJobEndTimeAndResult(jobEnd)
info(s"Job end. Job ${jobEnd.jobId} state is ${jobEnd.jobResult.toString}")
jobEnd.jobResult match {
case JobFailed(e) if e != null =>
diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/KyuubiStatementMonitorSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/KyuubiStatementMonitorSuite.scala
deleted file mode 100644
index 4058557..0000000
--- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/KyuubiStatementMonitorSuite.scala
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kyuubi.engine.spark
-
-import java.util.concurrent.ConcurrentHashMap
-
-import org.apache.hive.service.rpc.thrift._
-import org.apache.spark.scheduler.JobSucceeded
-import org.scalatest.PrivateMethodTester
-import org.scalatest.time.SpanSugar._
-
-import org.apache.kyuubi.engine.spark.monitor.KyuubiStatementMonitor
-import org.apache.kyuubi.engine.spark.monitor.entity.KyuubiJobInfo
-import org.apache.kyuubi.operation.{HiveJDBCTests, OperationHandle}
-
-class KyuubiStatementMonitorSuite extends WithSparkSQLEngine with HiveJDBCTests
- with PrivateMethodTester {
-
- override protected def jdbcUrl: String = getJdbcUrl
- override def withKyuubiConf: Map[String, String] = Map.empty
-
- test("add kyuubiJobInfo into queue and remove them when threshold reached") {
- val sql = "select timestamp'2021-06-01'"
- val getJobMap = PrivateMethod[
- ConcurrentHashMap[Int, KyuubiJobInfo]](Symbol("kyuubiJobIdToJobInfoMap"))()
-
- val jobIdToJobInfoMap = KyuubiStatementMonitor.invokePrivate(getJobMap)
- jobIdToJobInfoMap.clear()
- withSessionHandle { (client, handle) =>
- val req = new TExecuteStatementReq()
- req.setSessionHandle(handle)
- req.setStatement(sql)
- val tExecuteStatementResp = client.ExecuteStatement(req)
- val opHandle = tExecuteStatementResp.getOperationHandle
-
- eventually(timeout(10.seconds), interval(100.milliseconds)) {
- val elements = jobIdToJobInfoMap.elements()
- while (elements.hasMoreElements) {
- val kyuubiJobInfo = elements.nextElement()
- assert(jobIdToJobInfoMap.size() === 1)
- assert(kyuubiJobInfo.statementId === OperationHandle(opHandle).identifier.toString)
- assert(kyuubiJobInfo.stageIds.length === 1)
- assert(kyuubiJobInfo.jobResult === JobSucceeded)
- assert(kyuubiJobInfo.endTime !== 0)
- }
- }
-
- // Test for clear kyuubiJobIdToJobInfoMap when threshold reached
- // This function is used for avoiding mem leak
- (1 to 7).foreach { _ =>
- val req = new TExecuteStatementReq()
- req.setSessionHandle(handle)
- req.setStatement(sql)
- val tExecuteStatementResp = client.ExecuteStatement(req)
- val operationHandle = tExecuteStatementResp.getOperationHandle
- waitForOperationToComplete(client, operationHandle)
- }
- assert(jobIdToJobInfoMap.size() === 1)
- }
- }
-}
diff --git a/pom.xml b/pom.xml
index 1c716c4..cfbcbeb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,7 +67,6 @@
<modules>
<module>dev/kyuubi-codecov</module>
<module>externals/kyuubi-download</module>
- <module>externals/kyuubi-spark-monitor</module>
<module>externals/kyuubi-spark-sql-engine</module>
<module>kyuubi-assembly</module>
<module>kyuubi-common</module>