You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2021/04/21 10:11:46 UTC
[GitHub] [spark] sandeep-katta0102 commented on a change in pull request #31993: [SPARK-34897][SQL] Support reconcile schemas based on index after nested column pruning
sandeep-katta0102 commented on a change in pull request #31993:
URL: https://github.com/apache/spark/pull/31993#discussion_r617395326
##########
File path: sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SchemaPruningSuite.scala
##########
@@ -60,44 +57,76 @@ class SchemaPruningSuite extends SparkFunSuite with SQLHelper {
arrayOfStruct :: StructField("b", structOfStruct) :: StructField("c", IntegerType) ::
mapOfStruct :: Nil)
- testPrunedSchema(complexStruct, StructField("a", ArrayType(StructType.fromDDL("b int"))),
- StructField("b", StructType.fromDDL("a int")))
testPrunedSchema(complexStruct,
- StructField("a", ArrayType(StructType.fromDDL("b int, c string"))),
- StructField("b", StructType.fromDDL("b int")))
+ Seq(StructField("a", ArrayType(StructType.fromDDL("b int"))),
+ StructField("b", StructType.fromDDL("a int"))),
+ StructType(
+ StructField("a", ArrayType(StructType.fromDDL("b int"))) ::
+ StructField("b", StructType.fromDDL("a int")) ::
+ StructField("c", IntegerType) ::
+ mapOfStruct :: Nil))
+ testPrunedSchema(complexStruct,
+ Seq(StructField("a", ArrayType(StructType.fromDDL("b int, c string"))),
+ StructField("b", StructType.fromDDL("b int"))),
+ StructType(
+ StructField("a", ArrayType(StructType.fromDDL("b int, c string"))) ::
+ StructField("b", StructType.fromDDL("b int")) ::
+ StructField("c", IntegerType) ::
+ mapOfStruct :: Nil))
val selectFieldInMap = StructField("d", MapType(StructType.fromDDL("a int, b int"),
StructType.fromDDL("e int, f string")))
- testPrunedSchema(complexStruct, StructField("c", IntegerType), selectFieldInMap)
+ testPrunedSchema(complexStruct,
+ Seq(StructField("c", IntegerType), selectFieldInMap),
+ StructType(
+ arrayOfStruct ::
+ StructField("b", structOfStruct) ::
+ StructField("c", IntegerType) ::
+ selectFieldInMap :: Nil))
}
test("SPARK-35096: test case insensitivity of pruned schema") {
- Seq(true, false).foreach(isCaseSensitive => {
+ val upperCaseSchema = StructType.fromDDL("A struct<A:int, B:int>, B int")
+ val lowerCaseSchema = StructType.fromDDL("a struct<a:int, b:int>, b int")
+ val upperCaseRequestedFields = Seq(StructField("A", StructType.fromDDL("A int")))
+ val lowerCaseRequestedFields = Seq(StructField("a", StructType.fromDDL("a int")))
+
+ Seq(true, false).foreach { isCaseSensitive =>
withSQLConf(CASE_SENSITIVE.key -> isCaseSensitive.toString) {
if (isCaseSensitive) {
- // Schema is case-sensitive
- val requestedFields = getRootFields(StructField("id", IntegerType))
- val prunedSchema = SchemaPruning.pruneDataSchema(
- StructType.fromDDL("ID int, name String"), requestedFields)
- assert(prunedSchema == StructType(Seq.empty))
- // Root fields are case-sensitive
- val rootFieldsSchema = SchemaPruning.pruneDataSchema(
- StructType.fromDDL("id int, name String"),
- getRootFields(StructField("ID", IntegerType)))
- assert(rootFieldsSchema == StructType(StructType(Seq.empty)))
+ testPrunedSchema(
+ upperCaseSchema,
+ upperCaseRequestedFields,
+ StructType.fromDDL("A struct<A:int>, B int"))
+ testPrunedSchema(
+ upperCaseSchema,
+ lowerCaseRequestedFields,
+ upperCaseSchema)
+
+ testPrunedSchema(
+ lowerCaseSchema,
+ upperCaseRequestedFields,
+ lowerCaseSchema)
+ testPrunedSchema(
+ lowerCaseSchema,
+ lowerCaseRequestedFields,
+ StructType.fromDDL("a struct<a:int>, b int"))
} else {
- // Schema is case-insensitive
- val prunedSchema = SchemaPruning.pruneDataSchema(
- StructType.fromDDL("ID int, name String"),
- getRootFields(StructField("id", IntegerType)))
- assert(prunedSchema == StructType(StructField("ID", IntegerType) :: Nil))
- // Root fields are case-insensitive
- val rootFieldsSchema = SchemaPruning.pruneDataSchema(
- StructType.fromDDL("id int, name String"),
- getRootFields(StructField("ID", IntegerType)))
- assert(rootFieldsSchema == StructType(StructField("id", IntegerType) :: Nil))
+ Seq(upperCaseRequestedFields, lowerCaseRequestedFields).foreach { requestedFields =>
+ testPrunedSchema(
+ upperCaseSchema,
+ requestedFields,
+ StructType.fromDDL("A struct<A:int>, B int"))
+ }
+
+ Seq(upperCaseRequestedFields, lowerCaseRequestedFields).foreach { requestedFields =>
+ testPrunedSchema(
+ lowerCaseSchema,
+ requestedFields,
+ StructType.fromDDL("a struct<a:int>, b int"))
+ }
}
}
- })
+ }
Review comment:
Tests LGTM, thanks for adding more scenarios
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org