You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2016/07/04 05:45:18 UTC
spark git commit: [SPARK-16358][SQL] Remove InsertIntoHiveTable From
Logical Plan
Repository: spark
Updated Branches:
refs/heads/master 8cdb81fa8 -> 262833397
[SPARK-16358][SQL] Remove InsertIntoHiveTable From Logical Plan
#### What changes were proposed in this pull request?
LogicalPlan `InsertIntoHiveTable` is useless. Thus, we can remove it from the code base.
#### How was this patch tested?
The existing test cases
Author: gatorsmile <ga...@gmail.com>
Closes #14037 from gatorsmile/InsertIntoHiveTable.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/26283339
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/26283339
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/26283339
Branch: refs/heads/master
Commit: 26283339786f38c50722a7488d0bca8573b9c352
Parents: 8cdb81f
Author: gatorsmile <ga...@gmail.com>
Authored: Mon Jul 4 13:45:07 2016 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Mon Jul 4 13:45:07 2016 +0800
----------------------------------------------------------------------
.../spark/sql/hive/HiveMetastoreCatalog.scala | 39 --------------------
.../apache/spark/sql/hive/HiveStrategies.scala | 7 +---
.../sql/hive/execution/HiveComparisonTest.scala | 2 -
3 files changed, 1 insertion(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/26283339/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 2e0b5d5..151e456 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -372,12 +372,6 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log
if !r.hiveQlTable.isPartitioned && shouldConvertMetastoreParquet(r) =>
InsertIntoTable(convertToParquetRelation(r), partition, child, overwrite, ifNotExists)
- // Write path
- case InsertIntoHiveTable(r: MetastoreRelation, partition, child, overwrite, ifNotExists)
- // Inserting into partitioned table is not supported in Parquet data source (yet).
- if !r.hiveQlTable.isPartitioned && shouldConvertMetastoreParquet(r) =>
- InsertIntoTable(convertToParquetRelation(r), partition, child, overwrite, ifNotExists)
-
// Read path
case relation: MetastoreRelation if shouldConvertMetastoreParquet(relation) =>
val parquetRelation = convertToParquetRelation(relation)
@@ -416,12 +410,6 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log
if !r.hiveQlTable.isPartitioned && shouldConvertMetastoreOrc(r) =>
InsertIntoTable(convertToOrcRelation(r), partition, child, overwrite, ifNotExists)
- // Write path
- case InsertIntoHiveTable(r: MetastoreRelation, partition, child, overwrite, ifNotExists)
- // Inserting into partitioned table is not supported in Orc data source (yet).
- if !r.hiveQlTable.isPartitioned && shouldConvertMetastoreOrc(r) =>
- InsertIntoTable(convertToOrcRelation(r), partition, child, overwrite, ifNotExists)
-
// Read path
case relation: MetastoreRelation if shouldConvertMetastoreOrc(relation) =>
val orcRelation = convertToOrcRelation(relation)
@@ -489,30 +477,3 @@ private[hive] object MetaStorePartitionedTableFileCatalog {
}
}
}
-
-/**
- * A logical plan representing insertion into Hive table.
- * This plan ignores nullability of ArrayType, MapType, StructType unlike InsertIntoTable
- * because Hive table doesn't have nullability for ARRAY, MAP, STRUCT types.
- */
-private[hive] case class InsertIntoHiveTable(
- table: MetastoreRelation,
- partition: Map[String, Option[String]],
- child: LogicalPlan,
- overwrite: Boolean,
- ifNotExists: Boolean)
- extends LogicalPlan with Command {
-
- override def children: Seq[LogicalPlan] = child :: Nil
- override def output: Seq[Attribute] = Seq.empty
-
- val numDynamicPartitions = partition.values.count(_.isEmpty)
-
- // This is the expected schema of the table prepared to be inserted into,
- // including dynamic partition columns.
- val tableOutput = table.attributes ++ table.partitionKeys.takeRight(numDynamicPartitions)
-
- override lazy val resolved: Boolean = childrenResolved && child.output.zip(tableOutput).forall {
- case (childAttr, tableAttr) => childAttr.dataType.sameType(tableAttr.dataType)
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/26283339/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
index 71b180e..17956de 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
@@ -44,12 +44,7 @@ private[hive] trait HiveStrategies {
def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
case logical.InsertIntoTable(
table: MetastoreRelation, partition, child, overwrite, ifNotExists) =>
- execution.InsertIntoHiveTable(
- table, partition, planLater(child), overwrite, ifNotExists) :: Nil
- case hive.InsertIntoHiveTable(
- table: MetastoreRelation, partition, child, overwrite, ifNotExists) =>
- execution.InsertIntoHiveTable(
- table, partition, planLater(child), overwrite, ifNotExists) :: Nil
+ InsertIntoHiveTable(table, partition, planLater(child), overwrite, ifNotExists) :: Nil
case _ => Nil
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/26283339/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index f5d2f02..80e75aa 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -31,7 +31,6 @@ import org.apache.spark.sql.catalyst.planning.PhysicalOperation
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.hive.{InsertIntoHiveTable => LogicalInsertIntoHiveTable}
import org.apache.spark.sql.hive.test.{TestHive, TestHiveQueryExecution}
/**
@@ -349,7 +348,6 @@ abstract class HiveComparisonTest
val containsCommands = originalQuery.analyzed.collectFirst {
case _: Command => ()
case _: InsertIntoTable => ()
- case _: LogicalInsertIntoHiveTable => ()
}.nonEmpty
if (containsCommands) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org