You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2022/12/28 10:09:05 UTC
[GitHub] [spark] ulysses-you commented on a diff in pull request #39263: [SPARK-41726][SQL] Remove OptimizedCreateHiveTableAsSelectCommand
ulysses-you commented on code in PR #39263:
URL: https://github.com/apache/spark/pull/39263#discussion_r1058235116
##########
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala:
##########
@@ -2310,23 +2312,33 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
val targetTable = "targetTable"
withTable(targetTable) {
- val df = sql(s"CREATE TABLE $targetTable STORED AS $format AS SELECT id FROM p")
- checkAnswer(sql(s"SELECT id FROM $targetTable"),
- Row(1) :: Row(2) :: Row(3) :: Nil)
-
- val ctasDSCommand = df.queryExecution.analyzed.collect {
- case _: OptimizedCreateHiveTableAsSelectCommand => true
- }.headOption
- val ctasCommand = df.queryExecution.analyzed.collect {
- case _: CreateHiveTableAsSelectCommand => true
- }.headOption
-
- if (isConverted && isConvertedCtas) {
- assert(ctasDSCommand.nonEmpty)
- assert(ctasCommand.isEmpty)
- } else {
- assert(ctasDSCommand.isEmpty)
- assert(ctasCommand.nonEmpty)
+ var commands: Seq[SparkPlanInfo] = Seq.empty
+ val listener = new SparkListener {
+ override def onOtherEvent(event: SparkListenerEvent): Unit = {
+ event match {
+ case start: SparkListenerSQLExecutionStart =>
+ commands = commands ++ Seq(start.sparkPlanInfo)
+ case _ => // ignore other events
+ }
+ }
+ }
+ spark.sparkContext.addSparkListener(listener)
+ try {
+ val df = sql(s"CREATE TABLE $targetTable STORED AS $format AS SELECT id FROM p")
+ checkAnswer(sql(s"SELECT id FROM $targetTable"),
+ Row(1) :: Row(2) :: Row(3) :: Nil)
+ spark.sparkContext.listenerBus.waitUntilEmpty()
+ assert(commands.size == 3)
+ assert(commands.head.nodeName == "Execute CreateHiveTableAsSelectCommand")
+
+ val v1WriteCommand = commands(1)
+ if (isConverted && isConvertedCtas) {
Review Comment:
the removed test in [HiveExplainSuite.scala](https://github.com/apache/spark/pull/39263/files#diff-844c30ceddbb9b60013bef1e6b30dc8f65128779fa89e93733c537d392907f75) has been covered in this test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org