[GitHub] carbondata pull request #1595: [CARBONDATA-1868][Spark-2.2]Carbon-Spark2.2 I...

[ravipesala <gi...@git.apache.org>]

Github user ravipesala commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1595#discussion_r155251303
  
    --- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala ---
    @@ -413,19 +414,35 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
           relation: BaseRelation,
           predicates: Seq[Expression]): (Seq[Expression], Seq[Filter]) = {
     
    +    def isComplexAttribute(attribute: Attribute) = attribute.dataType match {
    +      case ArrayType(dataType, _) => true
    +      case StructType(_) => true
    +      case _ => false
    +    }
    +
    +    // In case of ComplexType dataTypes no filters should be pushed down. IsNotNull is being
    +    // explicitly added by spark and pushed. That also has to be handled and pushed back to
    +    // Spark for handling.
    +    val predicatesWithoutComplex = predicates.filter(predicate => predicate.collect {
    +      case a: Attribute if (isComplexAttribute(a)) => a
    +    }.size == 0 )
    +
         // For conciseness, all Catalyst filter expressions of type `expressions.Expression` below are
         // called `predicate`s, while all data source filters of type `sources.Filter` are simply called
         // `filter`s.
     
         val translated: Seq[(Expression, Filter)] =
           for {
    -        predicate <- predicates
    +        predicate <- predicatesWithoutComplex
             filter <- translateFilter(predicate)
           } yield predicate -> filter
     
    +
    +    // val newTranslated = translated.toMap.filter()
    --- End diff --
    
    remove commented code


---