You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by li...@apache.org on 2021/03/05 08:01:51 UTC
[carbondata] branch master updated: [CARBONDATA-4137] Refactor
CarbonDataSourceScan without the soruces.Filter of Spark 3
This is an automated email from the ASF dual-hosted git repository.
litao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 8f2ee7f [CARBONDATA-4137] Refactor CarbonDataSourceScan without the soruces.Filter of Spark 3
8f2ee7f is described below
commit 8f2ee7f53b830b409ce3a510ff33b11e3e1c549d
Author: QiangCai <qi...@qq.com>
AuthorDate: Mon Feb 22 19:36:34 2021 +0800
[CARBONDATA-4137] Refactor CarbonDataSourceScan without the soruces.Filter of Spark 3
Why is this PR needed?
1. In spark version 3, org.apache.spark.sql.sources.Filter is sealed, carbon can't extend it in carbon code.
2. The name of CarbonLateDecodeStrategy class is incorrect, the code is complex and hard to read
3. CarbonDataSourceScan can be the same for 2.3 and 2.4, and should support both batch reading and row reading.
What changes were proposed in this PR?
1. translate spark Expression to carbon Expression directly, skip the spark Filter step. Remove all spark Filters in carbon code.
old follow: Spark Expression => Spark Filter => Carbon Expression
new follow: Spark Expression => Carbon Expression
2. Remove filter reorder, need to implement expression reorder (added CARBONDATA-4138).
3. separate CarbonLateDecodeStrategy to CarbonSourceStrategy and DMLStrategy, and simplify the code of CarbonSourceStrategy.
4. move CarbonDataSourceScan back to the source folder, use one CarbonDataSourceScan for all versions
CarbonDataSourceScan supports both VectorReader and RowReader, Carbon will not use RowDataSourceScanExec.
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
---
.../apache/spark/util/TestCarbonSegmentUtil.scala | 4 +-
.../org/apache/carbondata/geo/InPolygonUDF.scala | 21 -
.../org/apache/carbondata/index/TextMatchUDF.scala | 12 -
.../apache/spark/sql/CarbonBoundReference.scala | 17 -
.../spark/sql/CarbonDatasourceHadoopRelation.scala | 84 +-
.../org/apache/spark/sql/CarbonExtensions.scala | 10 +-
.../spark/sql/CustomDeterministicExpression.scala | 2 +-
.../sql/execution/CastExpressionOptimization.scala | 208 ++--
.../execution/strategy/CarbonDataSourceScan.scala | 180 ++++
.../strategy/CarbonLateDecodeStrategy.scala | 1091 --------------------
.../sql/execution/strategy/CarbonPlanHelper.scala | 101 +-
.../execution/strategy/CarbonSourceStrategy.scala | 432 ++++++++
.../spark/sql/execution/strategy/DDLStrategy.scala | 20 +-
.../spark/sql/execution/strategy/DMLStrategy.scala | 255 +++++
.../execution/strategy/MixedFormatHandler.scala | 30 +-
.../sql/execution/strategy/PushDownHelper.scala | 2 +-
.../strategy/StreamingTableStrategy.scala | 13 +-
.../apache/spark/sql/optimizer/CarbonFilters.scala | 799 +++++++-------
.../org/apache/spark/sql/optimizer/MVMatcher.scala | 11 +-
.../joins/BroadCastSIFilterPushJoin.scala | 22 +-
.../apache/spark/sql/CarbonToSparkAdapter.scala | 28 +-
.../execution/strategy/CarbonDataSourceScan.scala | 58 --
.../spark/sql/hive/CarbonSessionStateBuilder.scala | 8 +-
.../apache/spark/sql/CarbonToSparkAdapter.scala | 30 +-
.../execution/strategy/CarbonDataSourceScan.scala | 60 --
.../spark/sql/hive/CarbonSessionStateBuilder.scala | 10 +-
.../complexType/TestArrayContainsPushDown.scala | 36 +-
.../testsuite/cloud/AllDataSourceTestCase.scala | 2 +-
.../dataload/TestGlobalSortDataLoad.scala | 5 +-
.../filterexpr/AllDataTypesTestCaseFilter.scala | 8 +-
.../filterexpr/TestImplicitFilterExpression.scala | 5 +-
.../StandardPartitionTableLoadingTestCase.scala | 5 +-
.../StandardPartitionTableQueryTestCase.scala | 5 +-
.../carbondata/query/TestFilterReordering.scala | 75 --
.../vectorreader/VectorReaderTestCase.scala | 2 +-
.../apache/spark/sql/CarbonExtensionSuite.scala | 2 +-
36 files changed, 1650 insertions(+), 2003 deletions(-)
diff --git a/index/secondary-index/src/test/scala/org/apache/spark/util/TestCarbonSegmentUtil.scala b/index/secondary-index/src/test/scala/org/apache/spark/util/TestCarbonSegmentUtil.scala
index 5a030b2..55529b4 100644
--- a/index/secondary-index/src/test/scala/org/apache/spark/util/TestCarbonSegmentUtil.scala
+++ b/index/secondary-index/src/test/scala/org/apache/spark/util/TestCarbonSegmentUtil.scala
@@ -47,8 +47,8 @@ class TestCarbonSegmentUtil extends QueryTest {
createTable(tableName)
val dataFrame = sql(s"select * from $tableName")
val scanRdd = dataFrame.queryExecution.sparkPlan.collect {
- case b: CarbonDataSourceScan if b.rdd.isInstanceOf[CarbonScanRDD[InternalRow]] => b.rdd
- .asInstanceOf[CarbonScanRDD[InternalRow]]
+ case b: CarbonDataSourceScan if b.inputRDDs().head.isInstanceOf[CarbonScanRDD[InternalRow]] =>
+ b.inputRDDs().head.asInstanceOf[CarbonScanRDD[InternalRow]]
}.head
val expected = BroadCastSIFilterPushJoin.getFilteredSegments(scanRdd)
assert(expected.length == 4)
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/geo/InPolygonUDF.scala b/integration/spark/src/main/scala/org/apache/carbondata/geo/InPolygonUDF.scala
index b423978..ff618b5 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/geo/InPolygonUDF.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/geo/InPolygonUDF.scala
@@ -18,7 +18,6 @@
package org.apache.carbondata.geo
import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.sources.Filter
import org.apache.carbondata.common.annotations.InterfaceAudience
@@ -59,23 +58,3 @@ class InPolygonRangeListUDF extends ((String, String) => Boolean) with Serializa
true // Carbon applies the filter. So, Spark do not have to apply filter.
}
}
-
-@InterfaceAudience.Internal
-case class InPolygon(queryString: String) extends Filter {
- override def references: Array[String] = Array()
-}
-
-@InterfaceAudience.Internal
-case class InPolygonList(polygonListString: String, opType: String) extends Filter {
- override def references: Array[String] = Array()
-}
-
-@InterfaceAudience.Internal
-case class InPolylineList(polylineListString: String, buffer: String) extends Filter {
- override def references: Array[String] = Array()
-}
-
-@InterfaceAudience.Internal
-case class InPolygonRangeList(rangeListString: String, opType: String) extends Filter {
- override def references: Array[String] = Array()
-}
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/index/TextMatchUDF.scala b/integration/spark/src/main/scala/org/apache/carbondata/index/TextMatchUDF.scala
index 590fa7a..7c797dc 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/index/TextMatchUDF.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/index/TextMatchUDF.scala
@@ -17,8 +17,6 @@
package org.apache.carbondata.index
-import org.apache.spark.sql.sources.Filter
-
import org.apache.carbondata.common.annotations.InterfaceAudience
@InterfaceAudience.Internal
@@ -34,13 +32,3 @@ class TextMatchMaxDocUDF extends ((String, Int) => Boolean) with Serializable {
v1.length > 0
}
}
-
-@InterfaceAudience.Internal
-case class TextMatch(queryString: String) extends Filter {
- override def references: Array[String] = Array()
-}
-
-@InterfaceAudience.Internal
-case class TextMatchLimit(queryString: String, maxDoc: String) extends Filter {
- override def references: Array[String] = Array()
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
index e95ce34..589e49c 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
@@ -18,26 +18,9 @@
package org.apache.spark.sql
import org.apache.spark.sql.catalyst.expressions.Expression
-import org.apache.spark.sql.sources.Filter
object ExtractReferences {
def apply(expr: Expression): Array[String] = {
expr.references.map(_.name).toArray
}
}
-
-case class CastExpr(expr: Expression) extends Filter {
- override def references: Array[String] = ExtractReferences(expr)
-}
-
-case class FalseExpr() extends Filter {
- override def references: Array[String] = Array.empty
-}
-
-case class CarbonEndsWith(expr: Expression) extends Filter {
- override def references: Array[String] = ExtractReferences(expr)
-}
-
-case class CarbonContainsWith(expr: Expression) extends Filter {
- override def references: Array[String] = ExtractReferences(expr)
-}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index a1273af..0d60144 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -17,30 +17,13 @@
package org.apache.spark.sql
-import scala.collection.JavaConverters._
-
-import org.apache.spark.CarbonInputMetrics
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{GetStructField, NamedExpression}
-import org.apache.spark.sql.execution.command.management.CarbonInsertIntoCommand
-import org.apache.spark.sql.execution.strategy.PushDownHelper
import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.spark.sql.optimizer.CarbonFilters
-import org.apache.spark.sql.sources.{And, BaseRelation, Filter, InsertableRelation, Or}
+import org.apache.spark.sql.sources.{BaseRelation, Filter}
import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.util.CarbonException
-import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.index.IndexFilter
-import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.scan.expression.Expression
-import org.apache.carbondata.core.scan.expression.logical.AndExpression
-import org.apache.carbondata.hadoop.CarbonProjection
-import org.apache.carbondata.spark.rdd.CarbonScanRDD
case class CarbonDatasourceHadoopRelation(
sparkSession: SparkSession,
@@ -48,7 +31,7 @@ case class CarbonDatasourceHadoopRelation(
parameters: Map[String, String],
tableSchema: Option[StructType],
limit: Int = -1)
- extends BaseRelation with InsertableRelation {
+ extends BaseRelation {
val caseInsensitiveMap: Map[String, String] = parameters.map(f => (f._1.toLowerCase, f._2))
lazy val identifier: AbsoluteTableIdentifier = AbsoluteTableIdentifier.from(
@@ -68,48 +51,6 @@ case class CarbonDatasourceHadoopRelation(
override def schema: StructType = tableSchema.getOrElse(carbonRelation.schema)
- def buildScan(requiredColumns: Array[String],
- filterComplex: Seq[org.apache.spark.sql.catalyst.expressions.Expression],
- projects: Seq[NamedExpression],
- filters: Array[Filter],
- partitions: Seq[PartitionSpec]): RDD[InternalRow] = {
- val reorderedFilter = CarbonFilters.reorderFilter(filters, carbonTable)
- val filterExpression: Option[Expression] = reorderedFilter.flatMap { filter =>
- CarbonFilters.createCarbonFilter(schema, filter,
- carbonTable.getTableInfo.getFactTable.getTableProperties.asScala)
- }.reduceOption(new AndExpression(_, _))
-
- val projection = new CarbonProjection
-
- // As Filter push down for Complex datatype is not supported, if filter is applied on complex
- // column, then Projection push down on Complex Columns will not take effect. Hence, check if
- // filter contains Struct Complex Column.
- val complexFilterExists = filterComplex.map(col =>
- col.map(_.isInstanceOf[GetStructField]))
-
- if (!complexFilterExists.exists(f => f.contains(true))) {
- PushDownHelper.pushDownProjection(requiredColumns, projects, projection)
- } else {
- requiredColumns.foreach(projection.addColumn)
- }
-
- val inputMetricsStats: CarbonInputMetrics = new CarbonInputMetrics
- val filter = filterExpression.map(new IndexFilter(carbonTable, _, true)).orNull
- if (filter != null && filters.length == 1) {
- // push down the limit if only one filter
- filter.setLimit(limit)
- }
- new CarbonScanRDD(
- sparkSession,
- projection,
- filter,
- identifier,
- carbonTable.getTableInfo.serialize(),
- carbonTable.getTableInfo,
- inputMetricsStats,
- partitions)
- }
-
override def unhandledFilters(filters: Array[Filter]): Array[Filter] = new Array[Filter](0)
override def toString: String = {
@@ -117,25 +58,4 @@ case class CarbonDatasourceHadoopRelation(
}
override def sizeInBytes: Long = carbonRelation.sizeInBytes
-
- override def insert(data: DataFrame, overwrite: Boolean): Unit = {
- if (carbonRelation.output.size > CarbonCommonConstants.DEFAULT_MAX_NUMBER_OF_COLUMNS) {
- CarbonException.analysisException("Maximum supported column by carbon is: " +
- CarbonCommonConstants.DEFAULT_MAX_NUMBER_OF_COLUMNS)
- }
- if (data.logicalPlan.output.size >= carbonRelation.output.size) {
- CarbonInsertIntoCommand(
- databaseNameOp = Some(this.carbonRelation.databaseName),
- tableName = this.carbonRelation.tableName,
- options = scala.collection.immutable
- .Map("fileheader" -> this.tableSchema.get.fields.map(_.name).mkString(",")),
- isOverwriteTable = overwrite,
- logicalPlan = data.logicalPlan,
- tableInfo = this.carbonRelation.carbonTable.getTableInfo).run(sparkSession)
- } else {
- CarbonException.analysisException(
- "Cannot insert into target table because number of columns mismatch")
- }
- }
-
}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonExtensions.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonExtensions.scala
index de6bbcb..b371329 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonExtensions.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonExtensions.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql
import org.apache.spark.sql.catalyst.parser.ParserInterface
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
+import org.apache.spark.sql.execution.strategy.{CarbonSourceStrategy, DDLStrategy, DMLStrategy, StreamingTableStrategy}
import org.apache.spark.sql.hive.{CarbonIUDAnalysisRule, CarbonLoadDataAnalyzeRule, CarbonPreInsertionCasts}
import org.apache.spark.sql.parser.CarbonExtensionSqlParser
@@ -49,11 +49,13 @@ class CarbonExtensions extends (SparkSessionExtensions => Unit) {
// carbon planner strategies
extensions
- .injectPlannerStrategy((session: SparkSession) => new StreamingTableStrategy(session))
+ .injectPlannerStrategy(_ => StreamingTableStrategy)
extensions
- .injectPlannerStrategy((_: SparkSession) => new CarbonLateDecodeStrategy)
+ .injectPlannerStrategy(_ => DMLStrategy)
extensions
- .injectPlannerStrategy((session: SparkSession) => new DDLStrategy(session))
+ .injectPlannerStrategy(_ => DDLStrategy)
+ extensions
+ .injectPlannerStrategy(_ => CarbonSourceStrategy)
// init CarbonEnv
CarbonEnv.init()
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
index 8f8b196..c88828b 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CustomDeterministicExpression.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
-import org.apache.spark.sql.types.{DataType, StringType}
+import org.apache.spark.sql.types.DataType
/**
* Custom expression to override the deterministic property .
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/CastExpressionOptimization.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/CastExpressionOptimization.scala
index 6b125ee..0e0ece5 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/CastExpressionOptimization.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/CastExpressionOptimization.scala
@@ -24,21 +24,24 @@ import java.util.{Locale, TimeZone}
import scala.collection.JavaConverters._
import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast}
-import org.apache.spark.sql.CastExpr
-import org.apache.spark.sql.FalseExpr
-import org.apache.spark.sql.catalyst.expressions.{Attribute, EmptyRow, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, LessThan, LessThanOrEqual, Literal, Not}
+import org.apache.spark.sql.carbondata.execution.datasources.CarbonSparkDataSourceUtil.convertToJavaList
+import org.apache.spark.sql.catalyst.expressions.{Attribute, EmptyRow, EqualTo, GreaterThan, GreaterThanOrEqual, In, LessThan, LessThanOrEqual, Literal, Not}
+import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression}
import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.sources
-import org.apache.spark.sql.sources.Filter
-import org.apache.spark.sql.types._
+import org.apache.spark.sql.optimizer.CarbonFilters.{transformExpression, translateColumn, translateLiteral}
+import org.apache.spark.sql.types.{ArrayType, DateType, DoubleType, IntegerType, ShortType, StringType, TimestampType}
+import org.apache.spark.sql.types.{DataType => SparkDataType}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.{EqualToExpression, GreaterThanEqualToExpression, GreaterThanExpression, InExpression, LessThanEqualToExpression, LessThanExpression, ListExpression, NotEqualsExpression, NotInExpression}
+import org.apache.carbondata.core.scan.expression.logical.FalseExpression
import org.apache.carbondata.core.util.CarbonProperties
object CastExpressionOptimization {
- def typeCastStringToLong(v: Any, dataType: DataType): Any = {
+ def typeCastStringToLong(v: Any, dataType: SparkDataType): Any = {
if (dataType == TimestampType || dataType == DateType) {
val value = if (dataType == TimestampType) {
DateTimeUtils.stringToTimestamp(UTF8String.fromString(v.toString))
@@ -60,19 +63,19 @@ object CastExpressionOptimization {
parser.setTimeZone(TimeZone.getTimeZone("GMT"))
}
try {
- val value = parser.parse(v.toString).getTime() * 1000L
+ val value = parser.parse(v.toString).getTime * 1000L
value
} catch {
case e: ParseException =>
try {
val format: SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz")
- format.parse(v.toString).getTime() * 1000L
+ format.parse(v.toString).getTime * 1000L
} catch {
case e: ParseException =>
val gmtDay = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
gmtDay.setTimeZone(TimeZone.getTimeZone("GMT"))
try {
- gmtDay.parse(v.toString).getTime() * 1000L
+ gmtDay.parse(v.toString).getTime * 1000L
} catch {
case e: ParseException =>
v
@@ -92,13 +95,13 @@ object CastExpressionOptimization {
}
}
-
- def typeCastStringToLongList(list: Seq[Expression], dataType: DataType): Seq[Expression] = {
- val tempList = new util.ArrayList[Expression]()
+ def typeCastStringToLongList(list: Seq[SparkExpression],
+ dataType: SparkDataType): Seq[SparkExpression] = {
+ val tempList = new util.ArrayList[SparkExpression]()
list.foreach { value =>
val output = typeCastStringToLong(value, dataType)
if (!output.equals(value)) {
- tempList.add(output.asInstanceOf[Expression])
+ tempList.add(output.asInstanceOf[SparkExpression])
}
}
if (tempList.size() != list.size) {
@@ -108,12 +111,12 @@ object CastExpressionOptimization {
}
}
- def typeCastDoubleToIntList(list: Seq[Expression]): Seq[Expression] = {
- val tempList = new util.ArrayList[Expression]()
+ def typeCastDoubleToIntList(list: Seq[SparkExpression]): Seq[SparkExpression] = {
+ val tempList = new util.ArrayList[SparkExpression]()
list.foreach { value =>
val output = value.asInstanceOf[Double].toInt
if (value.asInstanceOf[Double].toInt.equals(output)) {
- tempList.add(output.asInstanceOf[Expression])
+ tempList.add(output.asInstanceOf[SparkExpression])
}
}
if (tempList.size() != list.size) {
@@ -123,12 +126,12 @@ object CastExpressionOptimization {
}
}
- def typeCastIntToShortList(list: Seq[Expression]): Seq[Expression] = {
- val tempList = new util.ArrayList[Expression]()
+ def typeCastIntToShortList(list: Seq[SparkExpression]): Seq[SparkExpression] = {
+ val tempList = new util.ArrayList[SparkExpression]()
list.foreach { value =>
val output = value.asInstanceOf[Integer].toShort
if (value.asInstanceOf[Integer].toShort.equals(output)) {
- tempList.add(output.asInstanceOf[Expression])
+ tempList.add(output.asInstanceOf[SparkExpression])
}
}
if (tempList.size() != list.size) {
@@ -153,57 +156,75 @@ object CastExpressionOptimization {
* @param expr
* @return
*/
- def checkIfCastCanBeRemove(expr: Expression): Option[sources.Filter] = {
+ def checkIfCastCanBeRemove(expr: SparkExpression): Option[Expression] = {
- def checkBinaryExpression(attributeType: DataType,
+ def checkBinaryExpression(attributeType: SparkDataType,
value: Any,
- valueType: DataType,
- nonEqual: Boolean = true): Option[Filter] = {
+ valueType: SparkDataType,
+ nonEqual: Boolean = true): Option[Expression] = {
attributeType match {
- case ts@(_: DateType | _: TimestampType) if valueType.sameType(StringType) =>
- val filter = updateFilterForTimeStamp(value, expr, ts)
+ case _: DateType | _: TimestampType if valueType.sameType(StringType) =>
+ val filter = updateFilterForTimeStamp(value, expr, attributeType)
if (nonEqual) {
updateFilterForNonEqualTimeStamp(value, expr, filter)
} else {
filter
}
case _: IntegerType if valueType.sameType(DoubleType) =>
- updateFilterForInt(value, expr)
+ updateFilterForInt(value, expr, attributeType)
case _: ShortType if valueType.sameType(IntegerType) =>
- updateFilterForShort(value, expr)
+ updateFilterForShort(value, expr, attributeType)
case arr: ArrayType if !nonEqual =>
checkBinaryExpression(arr.elementType, value, valueType, nonEqual)
- case _ => Some(CastExpr(expr))
+ case _ => Some(transformExpression(expr))
}
}
def checkInValueList(attributeName: String,
- list: Seq[Expression],
- newList: Seq[Expression],
- hasNot: Boolean = true): Option[Filter] = {
+ list: Seq[SparkExpression],
+ newList: Seq[SparkExpression],
+ dt: SparkDataType,
+ hasNot: Boolean = true): Option[Expression] = {
if (!newList.equals(list)) {
val hSet = list.map(e => e.eval(EmptyRow))
if (hasNot) {
- Some(sources.Not(sources.In(attributeName, hSet.toArray)))
+ if (hSet.contains(null)) {
+ Some(new FalseExpression(translateColumn(attributeName, dt)))
+ } else {
+ Some(new NotInExpression(translateColumn(attributeName, dt),
+ new ListExpression(convertToJavaList(
+ hSet.map(f => translateLiteral(f, dt)).toList))))
+ }
} else {
- Some(sources.In(attributeName, hSet.toArray))
+ if (hSet.length == 1 && hSet.head == null) {
+ Some(new FalseExpression(translateColumn(attributeName, dt)))
+ } else {
+ Some(new InExpression(translateColumn(attributeName, dt),
+ new ListExpression(convertToJavaList(hSet
+ .filterNot(_ == null)
+ .map(filterValues => translateLiteral(filterValues, dt))
+ .toList))))
+ }
}
} else {
- Some(CastExpr(expr))
+ Some(transformExpression(expr))
}
}
def checkInExpression(attribute: Attribute,
- list: Seq[Expression],
- hasNot: Boolean = true): Option[Filter] = {
+ list: Seq[SparkExpression],
+ hasNot: Boolean = true): Option[Expression] = {
attribute.dataType match {
- case ts@(_: DateType | _: TimestampType) if list.head.dataType.sameType(StringType) =>
- checkInValueList(attribute.name, list, typeCastStringToLongList(list, ts), hasNot)
+ case _: DateType | _: TimestampType if list.head.dataType.sameType(StringType) =>
+ checkInValueList(attribute.name, list, typeCastStringToLongList(list, attribute.dataType),
+ attribute.dataType, hasNot)
case _: IntegerType if list.head.dataType.sameType(DoubleType) =>
- checkInValueList(attribute.name, list, typeCastDoubleToIntList(list), hasNot)
+ checkInValueList(attribute.name, list, typeCastDoubleToIntList(list), attribute.dataType,
+ hasNot)
case _: ShortType if list.head.dataType.sameType(IntegerType) =>
- checkInValueList(attribute.name, list, typeCastIntToShortList(list), hasNot)
- case _ => Some(CastExpr(expr))
+ checkInValueList(attribute.name, list, typeCastIntToShortList(list), attribute.dataType,
+ hasNot)
+ case _ => Some(transformExpression(expr))
}
}
@@ -246,12 +267,14 @@ object CastExpressionOptimization {
* @param exp
* @return
*/
- def updateFilterForShort(actualValue: Any, exp: Expression): Option[sources.Filter] = {
+ def updateFilterForShort(actualValue: Any,
+ exp: SparkExpression,
+ dt: SparkDataType): Option[Expression] = {
val newValue = actualValue.asInstanceOf[Integer].toShort
if (newValue.toInt.equals(actualValue)) {
- updateFilterBasedOnFilterType(exp, newValue)
+ updateFilterBasedOnFilterType(exp, newValue, dt)
} else {
- Some(CastExpr(exp))
+ Some(transformExpression(exp))
}
}
@@ -262,12 +285,14 @@ object CastExpressionOptimization {
* @param exp
* @return
*/
- def updateFilterForInt(actualValue: Any, exp: Expression): Option[sources.Filter] = {
+ def updateFilterForInt(actualValue: Any,
+ exp: SparkExpression,
+ dt: SparkDataType): Option[Expression] = {
val newValue = actualValue.asInstanceOf[Double].toInt
if (newValue.toDouble.equals(actualValue)) {
- updateFilterBasedOnFilterType(exp, newValue)
+ updateFilterBasedOnFilterType(exp, newValue, dt)
} else {
- Some(CastExpr(exp))
+ Some(transformExpression(exp))
}
}
@@ -278,11 +303,12 @@ object CastExpressionOptimization {
* @param filter Filter Expression
* @return return CastExpression or same Filter
*/
- def updateFilterForNonEqualTimeStamp(actualValue: Any, exp: Expression, filter: Option[Filter]):
- Option[sources.Filter] = {
+ def updateFilterForNonEqualTimeStamp(actualValue: Any,
+ exp: SparkExpression,
+ filter: Option[Expression]): Option[Expression] = {
filter.get match {
- case FalseExpr() if (validTimeComparisionForSpark(actualValue)) =>
- Some(CastExpr(exp))
+ case _: FalseExpression if validTimeComparisionForSpark(actualValue) =>
+ Some(transformExpression(exp))
case _ =>
filter
}
@@ -313,18 +339,17 @@ object CastExpressionOptimization {
* @param exp
* @return
*/
- def updateFilterForTimeStamp(actualValue: Any, exp: Expression, dt: DataType):
- Option[sources.Filter] = {
+ def updateFilterForTimeStamp(actualValue: Any,
+ exp: SparkExpression,
+ dt: SparkDataType): Option[Expression] = {
val newValue = typeCastStringToLong(actualValue, dt)
if (!newValue.equals(actualValue)) {
- updateFilterBasedOnFilterType(exp, newValue)
+ updateFilterBasedOnFilterType(exp, newValue, dt)
} else {
- Some(FalseExpr())
+ Some(new FalseExpression(null))
}
-
}
-
/**
* the method removes the cast for the respective filter type
*
@@ -332,34 +357,47 @@ object CastExpressionOptimization {
* @param newValue
* @return
*/
- def updateFilterBasedOnFilterType(exp: Expression,
- newValue: Any): Some[Filter with Product with Serializable] = {
+ def updateFilterBasedOnFilterType(exp: SparkExpression,
+ newValue: Any,
+ dataType: SparkDataType): Some[Expression] = {
exp match {
- case c@EqualTo(Cast(a: Attribute, _), Literal(v, t)) =>
- Some(sources.EqualTo(a.name, newValue))
- case c@EqualTo(Literal(v, t), Cast(a: Attribute, _)) =>
- Some(sources.EqualTo(a.name, newValue))
- case c@Not(EqualTo(Cast(a: Attribute, _), Literal(v, t))) =>
- Some(sources.Not(sources.EqualTo(a.name, newValue)))
- case c@Not(EqualTo(Literal(v, t), Cast(a: Attribute, _))) =>
- Some(sources.Not(sources.EqualTo(a.name, newValue)))
- case GreaterThan(Cast(a: Attribute, _), Literal(v, t)) =>
- Some(sources.GreaterThan(a.name, newValue))
- case GreaterThan(Literal(v, t), Cast(a: Attribute, _)) =>
- Some(sources.LessThan(a.name, newValue))
- case c@LessThan(Cast(a: Attribute, _), Literal(v, t)) =>
- Some(sources.LessThan(a.name, newValue))
- case c@LessThan(Literal(v, t), Cast(a: Attribute, _)) =>
- Some(sources.GreaterThan(a.name, newValue))
- case c@GreaterThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) =>
- Some(sources.GreaterThanOrEqual(a.name, newValue))
- case c@GreaterThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
- Some(sources.LessThanOrEqual(a.name, newValue))
- case c@LessThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) =>
- Some(sources.LessThanOrEqual(a.name, newValue))
- case c@LessThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
- Some(sources.GreaterThanOrEqual(a.name, newValue))
- case _ => Some(CastExpr(exp))
+ case EqualTo(Cast(a: Attribute, _), _: Literal) =>
+ Some(new EqualToExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case EqualTo(_: Literal, Cast(a: Attribute, _)) =>
+ Some(new EqualToExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case Not(EqualTo(Cast(a: Attribute, _), _: Literal)) =>
+ Some(new NotEqualsExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case Not(EqualTo(_: Literal, Cast(a: Attribute, _))) =>
+ Some(new NotEqualsExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case GreaterThan(Cast(a: Attribute, _), _: Literal) =>
+ Some(new GreaterThanExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case GreaterThan(_: Literal, Cast(a: Attribute, _)) =>
+ Some(new LessThanExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case LessThan(Cast(a: Attribute, _), _: Literal) =>
+ Some(new LessThanExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case LessThan(_: Literal, Cast(a: Attribute, _)) =>
+ Some(new GreaterThanExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case GreaterThanOrEqual(Cast(a: Attribute, _), _: Literal) =>
+ Some(new GreaterThanEqualToExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case GreaterThanOrEqual(_: Literal, Cast(a: Attribute, _)) =>
+ Some(new LessThanEqualToExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case LessThanOrEqual(Cast(a: Attribute, _), _: Literal) =>
+ Some(new LessThanEqualToExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case LessThanOrEqual(_: Literal, Cast(a: Attribute, _)) =>
+ Some(new GreaterThanEqualToExpression(translateColumn(a.name, dataType),
+ translateLiteral(newValue, dataType)))
+ case _ => Some(transformExpression(exp))
}
}
}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala
new file mode 100644
index 0000000..b181659
--- /dev/null
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.strategy
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.CarbonInputMetrics
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.CarbonDatasourceHadoopRelation
+import org.apache.spark.sql.carbondata.execution.datasources.CarbonSparkDataSourceUtil
+import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, SortOrder, UnsafeProjection}
+import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression}
+import org.apache.spark.sql.catalyst.plans.QueryPlan
+import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, UnknownPartitioning}
+import org.apache.spark.sql.execution.{ColumnarBatchScan, DataSourceScanExec, WholeStageCodegenExec}
+import org.apache.spark.sql.optimizer.CarbonFilters
+import org.apache.spark.sql.types.AtomicType
+
+import org.apache.carbondata.core.index.IndexFilter
+import org.apache.carbondata.core.indexstore.PartitionSpec
+import org.apache.carbondata.core.metadata.schema.BucketingInfo
+import org.apache.carbondata.core.readcommitter.ReadCommittedScope
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.logical.AndExpression
+import org.apache.carbondata.hadoop.CarbonProjection
+import org.apache.carbondata.spark.rdd.CarbonScanRDD
+
+/**
+ * Physical plan node for scanning data. It is applied for both tables
+ * USING carbondata and STORED AS carbondata.
+ */
+case class CarbonDataSourceScan(
+ @transient relation: CarbonDatasourceHadoopRelation,
+ output: Seq[Attribute],
+ partitionFilters: Seq[SparkExpression],
+ dataFilters: Seq[SparkExpression],
+ @transient readComittedScope: ReadCommittedScope,
+ @transient pushedDownProjection: CarbonProjection,
+ @transient pushedDownFilters: Seq[Expression],
+ directScanSupport: Boolean,
+ @transient extraRDD: Option[(RDD[InternalRow], Boolean)] = None,
+ tableIdentifier: Option[TableIdentifier] = None)
+ extends DataSourceScanExec with ColumnarBatchScan {
+
+ override lazy val supportsBatch: Boolean = {
+ CarbonPlanHelper.supportBatchedDataSource(sqlContext, output, extraRDD)
+ }
+
+ override lazy val (outputPartitioning, outputOrdering): (Partitioning, Seq[SortOrder]) = {
+ val info: BucketingInfo = relation.carbonTable.getBucketingInfo
+ if (info != null) {
+ val cols = info.getListOfColumns.asScala
+ val numBuckets = info.getNumOfRanges
+ val bucketColumns = cols.flatMap { n =>
+ val attrRef = output.find(_.name.equalsIgnoreCase(n.getColumnName))
+ attrRef match {
+ case Some(attr: AttributeReference) =>
+ Some(AttributeReference(attr.name,
+ CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(n.getDataType),
+ attr.nullable,
+ attr.metadata)(attr.exprId, attr.qualifier))
+ case _ => None
+ }
+ }
+ if (bucketColumns.size == cols.size) {
+ // use HashPartitioning will not shuffle
+ (HashPartitioning(bucketColumns, numBuckets), Nil)
+ } else {
+ (UnknownPartitioning(0), Nil)
+ }
+ } else {
+ (UnknownPartitioning(0), Nil)
+ }
+ }
+
+ override lazy val metadata: Map[String, String] = {
+ def seqToString(seq: Seq[Any]) = seq.mkString("[", ", ", "]")
+ val metadata =
+ Map(
+ "ReadSchema" -> seqToString(pushedDownProjection.getAllColumns),
+ "Batched" -> supportsBatch.toString,
+ "DirectScan" -> (supportsBatch && directScanSupport).toString,
+ "PushedFilters" -> seqToString(pushedDownFilters.map(_.getStatement)))
+ if (relation.carbonTable.isHivePartitionTable) {
+ metadata + ("PartitionFilters" -> seqToString(partitionFilters)) +
+ ("PartitionCount" -> selectedPartitions.size.toString)
+ } else {
+ metadata
+ }
+ }
+
+ @transient private lazy val indexFilter: IndexFilter = {
+ val filter = pushedDownFilters.reduceOption(new AndExpression(_, _))
+ .map(new IndexFilter(relation.carbonTable, _, true)).orNull
+ if (filter != null && pushedDownFilters.length == 1) {
+ // push down the limit if only one filter
+ filter.setLimit(relation.limit)
+ }
+ filter
+ }
+
+ @transient private lazy val selectedPartitions: Seq[PartitionSpec] = {
+ CarbonFilters
+ .getPartitions(partitionFilters, relation.sparkSession, relation.carbonTable)
+ .orNull
+ }
+
+ private lazy val inputRDD: RDD[InternalRow] = {
+ val carbonRdd = new CarbonScanRDD[InternalRow](
+ relation.sparkSession,
+ pushedDownProjection,
+ indexFilter,
+ relation.identifier,
+ relation.carbonTable.getTableInfo.serialize(),
+ relation.carbonTable.getTableInfo,
+ new CarbonInputMetrics,
+ selectedPartitions)
+ carbonRdd.setVectorReaderSupport(supportsBatch)
+ carbonRdd.setDirectScanSupport(supportsBatch && directScanSupport)
+ extraRDD.map(_._1.union(carbonRdd)).getOrElse(carbonRdd)
+ }
+
+ override def inputRDDs(): Seq[RDD[InternalRow]] = inputRDD :: Nil
+
+ override protected def doExecute(): RDD[InternalRow] = {
+ if (supportsBatch) {
+ // in the case of fallback, this batched scan should never fail because of:
+ // 1) only primitive types are supported
+ // 2) the number of columns should be smaller than spark.sql.codegen.maxFields
+ WholeStageCodegenExec(this)(codegenStageId = 0).execute()
+ } else {
+ val unsafeRows = {
+ val scan = inputRDD
+ if (needsUnsafeRowConversion) {
+ scan.mapPartitionsWithIndexInternal { (index, iter) =>
+ val proj = UnsafeProjection.create(schema)
+ proj.initialize(index)
+ iter.map(proj)
+ }
+ } else {
+ scan
+ }
+ }
+ val numOutputRows = longMetric("numOutputRows")
+ unsafeRows.map { r =>
+ numOutputRows += 1
+ r
+ }
+ }
+ }
+
+ override protected def doCanonicalize(): CarbonDataSourceScan = {
+ CarbonDataSourceScan(
+ relation,
+ output.map(QueryPlan.normalizeExprId(_, output)),
+ QueryPlan.normalizePredicates(partitionFilters, output),
+ QueryPlan.normalizePredicates(dataFilters, output),
+ null,
+ null,
+ null,
+ directScanSupport,
+ extraRDD,
+ tableIdentifier)
+ }
+}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
deleted file mode 100644
index a3915bf..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ /dev/null
@@ -1,1091 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.strategy
-
-import java.util.Locale
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.log4j.Logger
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql._
-import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast}
-import org.apache.spark.sql.carbondata.execution.datasources.{CarbonFileIndex, CarbonSparkDataSourceUtil}
-import org.apache.spark.sql.catalyst.{expressions, InternalRow}
-import org.apache.spark.sql.catalyst.expressions.{Attribute, _}
-import org.apache.spark.sql.catalyst.planning.{ExtractEquiJoinKeys, PhysicalOperation}
-import org.apache.spark.sql.catalyst.plans.{Inner, LeftSemi}
-import org.apache.spark.sql.catalyst.plans.logical.{Filter => LogicalFilter, _}
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
-import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, UnknownPartitioning}
-import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, InMemoryFileIndex, LogicalRelation, SparkCarbonTableFormat}
-import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight}
-import org.apache.spark.sql.hive.MatchLogicalRelation
-import org.apache.spark.sql.index.CarbonIndexUtil
-import org.apache.spark.sql.optimizer.CarbonFilters
-import org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin
-import org.apache.spark.sql.sources.{BaseRelation, Filter}
-import org.apache.spark.sql.types._
-import org.apache.spark.util.{CarbonReflectionUtils, SparkUtil}
-
-import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.indexstore.PartitionSpec
-import org.apache.carbondata.core.metadata.schema.BucketingInfo
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.readcommitter.TableStatusReadCommittedScope
-import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.geo.{InPolygon, InPolygonList, InPolygonListUDF, InPolygonRangeList, InPolygonRangeListUDF, InPolygonUDF, InPolylineList, InPolylineListUDF}
-import org.apache.carbondata.index.{TextMatch, TextMatchLimit, TextMatchMaxDocUDF, TextMatchUDF}
-import org.apache.carbondata.spark.rdd.CarbonScanRDD
-
-/**
- * Carbon specific optimization
- * 1. filter push down
- * 2. count star
- */
-private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
- val PUSHED_FILTERS = "PushedFilters"
- val READ_SCHEMA = "ReadSchema"
-
- val LOGGER: Logger = LogServiceFactory.getLogService(this.getClass.getName)
-
- /*
- Spark 2.3.1 plan there can be case of multiple projections like below
- Project [substring(name, 1, 2)#124, name#123, tupleId#117, cast(rand(-6778822102499951904)#125
- as string) AS rand(-6778822102499951904)#137]
- +- Project [substring(name#123, 1, 2) AS substring(name, 1, 2)#124, name#123, UDF:getTupleId()
- AS tupleId#117, (rand(-6778822102499951904) AS rand(-6778822102499951904)#125]
- +- Relation[imei#118,age#119,task#120L,num#121,level#122,name#123]
- CarbonDatasourceHadoopRelation []
- */
- def apply(plan: LogicalPlan): Seq[SparkPlan] = {
- val transformedPlan = makeDeterministic(plan)
- transformedPlan match {
- case GlobalLimit(IntegerLiteral(limit),
- LocalLimit(IntegerLiteral(limitValue),
- _@PhysicalOperation(projects, filters, l: LogicalRelation))) if l.relation
- .isInstanceOf[CarbonDatasourceHadoopRelation] =>
- val relation = l.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- GlobalLimitExec(limit, LocalLimitExec(limitValue, pruneFilterProject(
- l,
- projects.filterNot(_.name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID)),
- filters,
- (a, f, p) =>
- setVectorReadSupport(
- l,
- a,
- relation.buildScan(a.map(_.name).toArray, filters, projects, f, p))
- ))) :: Nil
- case PhysicalOperation(projects, filters, l: LogicalRelation)
- if l.relation.isInstanceOf[CarbonDatasourceHadoopRelation] =>
- val relation = l.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- // In Spark 2.3.1 there is case of multiple projections like below
- // if 1 projection is failed then need to continue to other
- try {
- pruneFilterProject(
- l,
- projects,
- filters,
- (a, f, p) =>
- setVectorReadSupport(
- l,
- a,
- relation.buildScan(a.map(_.name).toArray, filters, projects, f, p))
- ) :: Nil
- } catch {
- case e: CarbonPhysicalPlanException => Nil
- }
- case CountStarPlan(colAttr, PhysicalOperation(projectList, predicates, l: LogicalRelation))
- if l.relation.isInstanceOf[CarbonDatasourceHadoopRelation] && driverSideCountStar(l) =>
- val relation = l.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- CarbonCountStar(colAttr, relation.carbonTable, SparkSession.getActiveSession.get) :: Nil
- case ExtractEquiJoinKeys(Inner, leftKeys, rightKeys, condition,
- left, right)
- if isCarbonPlan(left) && CarbonIndexUtil.checkIsIndexTable(right) =>
- LOGGER.info(s"pushing down for ExtractEquiJoinKeys:right")
- val carbon = apply(left).head
- // in case of SI Filter push join remove projection list from the physical plan
- // no need to have the project list in the main table physical plan execution
- // only join uses the projection list
- var carbonChild = carbon match {
- case projectExec: ProjectExec =>
- projectExec.child
- case _ =>
- carbon
- }
- // check if the outer and the inner project are matching, only then remove project
- if (left.isInstanceOf[Project]) {
- val leftOutput = left.output
- .filterNot(_.name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID))
- .map(c => (c.name.toLowerCase, c.dataType))
- val childOutput = carbonChild.output
- .filterNot(_.name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID))
- .map(c => (c.name.toLowerCase, c.dataType))
- if (!leftOutput.equals(childOutput)) {
- // if the projection list and the scan list are different(in case of alias)
- // we should not skip the project, so we are taking the original plan with project
- carbonChild = carbon
- }
- }
- val pushedDownJoin = BroadCastSIFilterPushJoin(
- leftKeys: Seq[Expression],
- rightKeys: Seq[Expression],
- Inner,
- BuildRight,
- carbonChild,
- planLater(right),
- condition)
- condition.map(FilterExec(_, pushedDownJoin)).getOrElse(pushedDownJoin) :: Nil
- case ExtractEquiJoinKeys(Inner, leftKeys, rightKeys, condition, left,
- right)
- if isCarbonPlan(right) && CarbonIndexUtil.checkIsIndexTable(left) =>
- LOGGER.info(s"pushing down for ExtractEquiJoinKeys:left")
- val carbon = planLater(right)
-
- val pushedDownJoin =
- BroadCastSIFilterPushJoin(
- leftKeys: Seq[Expression],
- rightKeys: Seq[Expression],
- Inner,
- BuildLeft,
- planLater(left),
- carbon,
- condition)
- condition.map(FilterExec(_, pushedDownJoin)).getOrElse(pushedDownJoin) :: Nil
- case ExtractEquiJoinKeys(LeftSemi, leftKeys, rightKeys, condition,
- left, right)
- if isLeftSemiExistPushDownEnabled &&
- isAllCarbonPlan(left) && isAllCarbonPlan(right) =>
- LOGGER.info(s"pushing down for ExtractEquiJoinKeysLeftSemiExist:right")
- val carbon = planLater(left)
- val pushedDownJoin = BroadCastSIFilterPushJoin(
- leftKeys: Seq[Expression],
- rightKeys: Seq[Expression],
- LeftSemi,
- BuildRight,
- carbon,
- planLater(right),
- condition)
- condition.map(FilterExec(_, pushedDownJoin)).getOrElse(pushedDownJoin) :: Nil
- case ExtractTakeOrderedAndProjectExec(carbonTakeOrderedAndProjectExec) =>
- carbonTakeOrderedAndProjectExec :: Nil
- case _ => Nil
- }
- }
-
- private def isAllCarbonPlan(plan: LogicalPlan): Boolean = {
- val allRelations = plan.collect { case logicalRelation: LogicalRelation => logicalRelation }
- allRelations.forall(x => x.relation.isInstanceOf[CarbonDatasourceHadoopRelation])
- }
-
- private def isCarbonPlan(plan: LogicalPlan): Boolean = {
- plan match {
- case PhysicalOperation(_, _,
- MatchLogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)) =>
- true
- case LogicalFilter(_, MatchLogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)) =>
- true
- case _ => false
- }
- }
-
- private def isLeftSemiExistPushDownEnabled: Boolean = {
- CarbonProperties.getInstance.getProperty(
- CarbonCommonConstants.CARBON_PUSH_LEFTSEMIEXIST_JOIN_AS_IN_FILTER,
- CarbonCommonConstants.CARBON_PUSH_LEFTSEMIEXIST_JOIN_AS_IN_FILTER_DEFAULT).toBoolean
- }
-
- /**
- * get deterministic expression for NamedExpression
- */
- private def makeDeterministicExp(exp: NamedExpression): Expression = {
- exp match {
- case alias: Alias if alias.child.isInstanceOf[CustomDeterministicExpression] =>
- alias
- case _ =>
- CustomDeterministicExpression(exp)
- }
- }
-
- /**
- * Convert all Expression to deterministic Expression
- */
- private def makeDeterministic(plan: LogicalPlan): LogicalPlan = {
- val transformedPlan = plan transform {
- case p@Project(projectList: Seq[NamedExpression], cd) =>
- if (cd.isInstanceOf[org.apache.spark.sql.catalyst.plans.logical.Filter] ||
- cd.isInstanceOf[LogicalRelation]) {
- p.transformAllExpressions {
- case a@Alias(exp, _)
- if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
- CarbonToSparkAdapter.createAliasRef(
- CustomDeterministicExpression(exp),
- a.name,
- a.exprId,
- a.qualifier,
- a.explicitMetadata,
- Some(a))
- case exp: NamedExpression
- if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
- makeDeterministicExp(exp)
- }
- } else {
- p
- }
- case f@org.apache.spark.sql.catalyst.plans.logical.Filter(condition: Expression, cd) =>
- if (cd.isInstanceOf[Project] || cd.isInstanceOf[LogicalRelation]) {
- f.transformAllExpressions {
- case a@Alias(exp, _)
- if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
- CarbonToSparkAdapter.createAliasRef(
- CustomDeterministicExpression(exp),
- a.name,
- a.exprId,
- a.qualifier,
- a.explicitMetadata,
- Some(a))
- case exp: NamedExpression
- if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
- makeDeterministicExp(exp)
- }
- } else {
- f
- }
- }
- transformedPlan
- }
-
- /**
- * Return true if driver-side count star optimization can be used.
- * Following case can't use driver-side count star:
- * 1. There is data update and delete
- * 2. It is streaming table
- */
- private def driverSideCountStar(logicalRelation: LogicalRelation): Boolean = {
- val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- val segmentUpdateStatusManager = new SegmentUpdateStatusManager(
- relation.carbonRelation.carbonTable)
- val updateDeltaMetadata = segmentUpdateStatusManager.readLoadMetadata()
- val hasNonCarbonSegment =
- segmentUpdateStatusManager.getLoadMetadataDetails.exists(!_.isCarbonFormat)
- if (hasNonCarbonSegment || updateDeltaMetadata != null && updateDeltaMetadata.nonEmpty) {
- false
- } else if (relation.carbonTable.isStreamingSink) {
- false
- } else {
- true
- }
- }
-
- /**
- * Converts to physical RDD of carbon after pushing down applicable filters.
- * @return
- */
- def pruneFilterProject(
- relation: LogicalRelation,
- projects: Seq[NamedExpression],
- filterPredicates: Seq[Expression],
- scanBuilder: (Seq[Attribute], Array[Filter], Seq[PartitionSpec]) => RDD[InternalRow])
- : CodegenSupport = {
- val names = relation.catalogTable match {
- case Some(table) => table.partitionColumnNames
- case _ => Seq.empty
- }
- // Get the current partitions from table.
- var partitions: Seq[PartitionSpec] = null
- if (names.nonEmpty) {
- val partitionSet = AttributeSet(names
- .map(p => relation.output.find(_.name.equalsIgnoreCase(p)).get))
- val partitionKeyFilters = CarbonToSparkAdapter
- .getPartitionKeyFilter(partitionSet, filterPredicates)
- // Update the name with lower case as it is case sensitive while getting partition info.
- val updatedPartitionFilters = partitionKeyFilters.map { exp =>
- exp.transform {
- case attr: AttributeReference =>
- CarbonToSparkAdapter.createAttributeReference(
- attr.name.toLowerCase,
- attr.dataType,
- attr.nullable,
- attr.metadata,
- attr.exprId,
- attr.qualifier)
- }
- }
- partitions =
- CarbonFilters.getPartitions(
- updatedPartitionFilters.toSeq,
- SparkSession.getActiveSession.get,
- relation.catalogTable.get.identifier).orNull
- }
- pruneFilterProjectRaw(
- relation,
- projects,
- filterPredicates,
- partitions,
- (requestedColumns, _, pushedFilters, p) => {
- scanBuilder(requestedColumns, pushedFilters.toArray, p)
- })
- }
-
- def setVectorReadSupport(
- relation: LogicalRelation,
- output: Seq[Attribute],
- rdd: RDD[InternalRow]): RDD[InternalRow] = {
- rdd.asInstanceOf[CarbonScanRDD[InternalRow]]
- .setVectorReaderSupport(supportBatchedDataSource(relation.relation.sqlContext, output))
- rdd
- }
-
- protected def pruneFilterProjectRaw(
- relation: LogicalRelation,
- rawProjects: Seq[NamedExpression],
- filterPredicates: Seq[Expression],
- partitions: Seq[PartitionSpec],
- scanBuilder: (Seq[Attribute], Seq[Expression], Seq[Filter], Seq[PartitionSpec])
- => RDD[InternalRow]): CodegenSupport = {
- val table = relation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- val readCommittedScope = new TableStatusReadCommittedScope(
- table.identifier,
- FileFactory.getConfiguration)
- val extraRdd = MixedFormatHandler.extraRDD(
- relation,
- rawProjects,
- filterPredicates,
- readCommittedScope,
- table.identifier)
- val projects = rawProjects.map {p =>
- p.transform {
- case CustomDeterministicExpression(exp) => exp
- }
- }.asInstanceOf[Seq[NamedExpression]]
-
- // contains the original order of the projection requested
- val projectsAttr = projects.flatMap(_.references)
- val projectSet = AttributeSet(projectsAttr)
- val filterSet = AttributeSet(filterPredicates.flatMap(_.references))
-
- val candidatePredicates = filterPredicates.map {
- _ transform {
- case a: AttributeReference => relation.attributeMap(a) // Match original case of attributes.
- }
- }
-
- val (unhandledPredicates, pushedFilters, handledFilters ) =
- selectFilters(relation.relation, candidatePredicates)
-
- // A set of column attributes that are only referenced by pushed down filters. We can eliminate
- // them from requested columns.
- val handledSet = {
- val handledPredicates = filterPredicates.filterNot(unhandledPredicates.contains)
- val unhandledSet = AttributeSet(unhandledPredicates.flatMap(_.references))
- try {
- AttributeSet(handledPredicates.flatMap(_.references)) --
- (projectSet ++ unhandledSet).map(relation.attributeMap)
- } catch {
- case e: Throwable => throw new CarbonPhysicalPlanException
- }
- }
- // Combines all Catalyst filter `Expression`s that are either not convertible to data source
- // `Filter`s or cannot be handled by `relation`.
- val filterCondition = unhandledPredicates.reduceLeftOption(expressions.And)
-
- val metadata: Map[String, String] = {
- val pairs = ArrayBuffer.empty[(String, String)]
-
- if (pushedFilters.nonEmpty) {
- pairs += (PUSHED_FILTERS -> pushedFilters.mkString("[", ", ", "]"))
- }
- pairs += (READ_SCHEMA -> projectSet.++(filterSet).toSeq.toStructType.catalogString)
- pairs.toMap
- }
-
- var vectorPushRowFilters = CarbonProperties.getInstance().isPushRowFiltersForVector
-
- // Spark cannot filter the rows from the pages in case of polygon query. So, we do the row
- // level filter at carbon and return the rows directly.
- if (candidatePredicates
- .exists(exp => exp.isInstanceOf[ScalaUDF] &&
- (exp.asInstanceOf[ScalaUDF].function.isInstanceOf[InPolygonUDF] ||
- exp.asInstanceOf[ScalaUDF].function.isInstanceOf[InPolygonListUDF] ||
- exp.asInstanceOf[ScalaUDF].function.isInstanceOf[InPolylineListUDF] ||
- exp.asInstanceOf[ScalaUDF].function.isInstanceOf[InPolygonRangeListUDF]))) {
- vectorPushRowFilters = true
- }
-
- // In case of mixed format, make the vectorPushRowFilters always false as other formats
- // filtering happens in spark layer.
- if (vectorPushRowFilters && extraRdd.isDefined) {
- vectorPushRowFilters = false
- }
- if (projects.map(_.toAttribute) == projects &&
- projectSet.size == projects.size &&
- filterSet.subsetOf(projectSet)) {
- // When it is possible to just use column pruning to get the right projection and
- // when the columns of this projection are enough to evaluate all filter conditions,
- // just do a scan followed by a filter, with no extra project.
- val requestedColumns = projects
- // Safe due to if above.
- .asInstanceOf[Seq[Attribute]]
- // Match original case of attributes.
- .map(relation.attributeMap)
- // Don't request columns that are only referenced by pushed filters.
- .filterNot(handledSet.contains)
- val updateRequestedColumns = requestedColumns
-
- val updateProject = projects.map { expr =>
- expr.toAttribute.asInstanceOf[AttributeReference]
- }
- val scan = getDataSourceScan(
- relation,
- (updateProject, partitions),
- scanBuilder,
- candidatePredicates,
- pushedFilters,
- handledFilters,
- metadata,
- updateRequestedColumns.asInstanceOf[Seq[Attribute]],
- extraRdd)
- if (scan.isInstanceOf[CarbonDataSourceScan] && table.carbonTable.isTransactionalTable) {
- // set the read committed scope and reuse in the scanRDD
- scan.inputRDDs().head match {
- case rdd: CarbonScanRDD[InternalRow] =>
- rdd.setReadCommittedScope(readCommittedScope)
- case _ =>
- }
- }
- // Check whether spark should handle row filters in case of vector flow.
- if (!vectorPushRowFilters && scan.isInstanceOf[CarbonDataSourceScan]) {
- // Here carbon only do page pruning and row level pruning will be done by spark.
- scan.inputRDDs().head match {
- case rdd: CarbonScanRDD[InternalRow] =>
- rdd.setDirectScanSupport(true)
- case _ =>
- }
- filterPredicates.reduceLeftOption(expressions.And).map(execution.FilterExec(_, scan))
- .getOrElse(scan)
- } else if (extraRdd.isDefined) {
- filterPredicates.reduceLeftOption(expressions.And).map(execution.FilterExec(_, scan))
- .getOrElse(scan)
- } else {
- filterCondition.map(execution.FilterExec(_, scan)).getOrElse(scan)
- }
- } else {
-
- var newProjectList: Seq[Attribute] = Seq.empty
- // In case of implicit exist we should disable vectorPushRowFilters as it goes in IUD flow
- // to get the positionId or tupleID
- var implicitExisted = false
- var updatedProjects = projects.map {
- case a@Alias(s: ScalaUDF, name)
- if name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID) ||
- name.equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID) =>
- val reference = AttributeReference(name, StringType, true)().withExprId(a.exprId)
- newProjectList :+= reference
- implicitExisted = true
- reference
- case a@Alias(s: ScalaUDF, name)
- if name.equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_SEGMENTID) =>
- implicitExisted = true
- val reference =
- AttributeReference(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID,
- StringType, true)().withExprId(a.exprId)
- newProjectList :+= reference
- a.transform {
- case s: ScalaUDF =>
- CarbonToSparkAdapter.createScalaUDF(s, reference)
- }
- case other => other
- }
- val updatedColumns: (Seq[Attribute], Seq[Expression]) = getRequestedColumns(relation,
- projectsAttr,
- filterSet,
- handledSet,
- newProjectList,
- updatedProjects)
- // Don't request columns that are only referenced by pushed filters.
- val requestedColumns = updatedColumns._1
- updatedProjects = updatedColumns._2
-
- var updateRequestedColumns =
- if (!vectorPushRowFilters && !implicitExisted) {
- (projectsAttr.to[mutable.LinkedHashSet] ++ filterSet).map(relation.attributeMap).toSeq
- } else {
- requestedColumns
- }
- val supportBatch =
- supportBatchedDataSource(relation.relation.sqlContext,
- updateRequestedColumns) && extraRdd.getOrElse((null, true))._2
- if (!vectorPushRowFilters && !supportBatch && !implicitExisted && filterSet.nonEmpty &&
- !filterSet.baseSet.exists(_.a.dataType.isInstanceOf[ArrayType])) {
- // revert for row scan
- updateRequestedColumns = requestedColumns
- }
- val newRequestedColumns = if (!vectorPushRowFilters && extraRdd.isDefined) {
- extractUniqueAttributes(projectsAttr, filterSet.toSeq)
- } else {
- updateRequestedColumns
- }
- val scan = getDataSourceScan(
- relation,
- (newRequestedColumns, partitions),
- scanBuilder,
- candidatePredicates,
- pushedFilters,
- handledFilters,
- metadata,
- newRequestedColumns,
- extraRdd)
- if (scan.isInstanceOf[CarbonDataSourceScan] && table.carbonTable.isTransactionalTable) {
- // set the read committed scope and reuse in the scanRDD
- scan.inputRDDs().head match {
- case rdd: CarbonScanRDD[InternalRow] =>
- rdd.setReadCommittedScope(readCommittedScope)
- case _ =>
- }
- }
- // Check whether spark should handle row filters in case of vector flow.
- if (!vectorPushRowFilters &&
- scan.isInstanceOf[CarbonDataSourceScan] &&
- !implicitExisted) {
- // Here carbon only do page pruning and row level pruning will be done by spark.
- scan.inputRDDs().head match {
- case rdd: CarbonScanRDD[InternalRow] =>
- rdd.setDirectScanSupport(true)
- case _ =>
- }
- execution.ProjectExec(
- updatedProjects.asInstanceOf[Seq[NamedExpression]],
- filterPredicates.reduceLeftOption(expressions.And).map(
- execution.FilterExec(_, scan)).getOrElse(scan))
- } else if (extraRdd.isDefined) {
- execution.ProjectExec(
- updatedProjects.asInstanceOf[Seq[NamedExpression]],
- filterPredicates.reduceLeftOption(expressions.And).map(
- execution.FilterExec(_, scan)).getOrElse(scan))
- } else {
- execution.ProjectExec(
- updatedProjects.asInstanceOf[Seq[NamedExpression]],
- filterCondition.map(execution.FilterExec(_, scan)).getOrElse(scan))
- }
-
- }
- }
-
- /*
- This function is used to get the Unique attributes from filter set
- and projection set based on their semantics.
- */
- private def extractUniqueAttributes(projections: Seq[Attribute],
- filter: Seq[Attribute]): Seq[Attribute] = {
- def checkSemanticEquals(filter: Attribute): Option[Attribute] = {
- projections.find(_.semanticEquals(filter))
- }
-
- filter.toList match {
- case head :: tail =>
- checkSemanticEquals(head) match {
- case Some(_) => extractUniqueAttributes(projections, tail)
- case None => extractUniqueAttributes(projections :+ head, tail)
- }
- case Nil => projections
- }
- }
-
- protected def getRequestedColumns(relation: LogicalRelation,
- projectsAttr: Seq[Attribute],
- filterSet: AttributeSet,
- handledSet: AttributeSet,
- newProjectList: Seq[Attribute],
- updatedProjects: Seq[Expression]): (Seq[Attribute], Seq[Expression]) = {
- val sparkSession = SparkSession.getActiveSession.get
- val pushDownJoinEnabled = sparkSession.sparkContext.getConf
- .getBoolean("spark.carbon.pushdown.join.as.filter", defaultValue = true)
-
- // positionId column can be added in two cases
- // case 1: SI pushdown case, SI rewritten plan adds positionId column
- // case 2: if the user requested positionId column thru getPositionId() UDF
- // positionId column should be removed only in case 1, as it is manually added
- // Below code is added to handle case 2. But getPositionId() UDF is almost used only for testing
- val isPositionIDRequested = relation.catalogTable match {
- case Some(table) =>
- val tblProperties = CarbonEnv.getCarbonTable(table.identifier)(sparkSession).getTableInfo
- .getFactTable
- .getTableProperties
- val isPosIDRequested = if (tblProperties.containsKey("isPositionIDRequested")) {
- java.lang.Boolean.parseBoolean(tblProperties.get("isPositionIDRequested"))
- } else {
- false
- }
- isPosIDRequested
- case _ => false
- }
- // remove positionId col only if pushdown is enabled and
- // positionId col is not requested in the query
- if (pushDownJoinEnabled && !isPositionIDRequested) {
- ((projectsAttr.to[scala.collection.mutable.LinkedHashSet] ++ filterSet -- handledSet)
- .map(relation.attributeMap).toSeq ++ newProjectList
- .filterNot(attr => attr.name
- .equalsIgnoreCase(CarbonCommonConstants.POSITION_ID)), updatedProjects
- .filterNot(attr => attr.isInstanceOf[AttributeReference] &&
- attr.asInstanceOf[AttributeReference].name
- .equalsIgnoreCase(CarbonCommonConstants.POSITION_ID)))
- } else {
- ((projectsAttr.to[scala.collection.mutable.LinkedHashSet] ++ filterSet -- handledSet)
- .map(relation.attributeMap).toSeq ++ newProjectList, updatedProjects)
- }
- }
-
- private def getDataSourceScan(relation: LogicalRelation,
- outputAndPartitions: (Seq[Attribute], Seq[PartitionSpec]),
- scanBuilder: (Seq[Attribute], Seq[Expression], Seq[Filter],
- Seq[PartitionSpec]) => RDD[InternalRow],
- candidatePredicates: Seq[Expression],
- pushedFilters: Seq[Filter], handledFilters: Seq[Filter],
- metadata: Map[String, String],
- updateRequestedColumns: Seq[Attribute],
- extraRDD: Option[(RDD[InternalRow], Boolean)]): DataSourceScanExec = {
- val table = relation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- var rdd = scanBuilder(updateRequestedColumns, candidatePredicates,
- pushedFilters, outputAndPartitions._2)
- if (supportBatchedDataSource(relation.relation.sqlContext, updateRequestedColumns) &&
- extraRDD.getOrElse((rdd, true))._2) {
- rdd = extraRDD.map(_._1.union(rdd)).getOrElse(rdd)
- new CarbonDataSourceScan(
- outputAndPartitions._1,
- rdd,
- createHadoopFSRelation(relation),
- getPartitioning(table.carbonTable, updateRequestedColumns),
- metadata,
- relation.catalogTable.map(_.identifier), relation)
- } else {
- rdd match {
- case cs: CarbonScanRDD[InternalRow] => cs.setVectorReaderSupport(false)
- case _ =>
- }
- rdd = extraRDD.map(_._1.union(rdd)).getOrElse(rdd)
- val partition = getPartitioning(table.carbonTable, updateRequestedColumns)
- CarbonReflectionUtils.getRowDataSourceScanExecObj(relation, outputAndPartitions._1,
- pushedFilters, handledFilters,
- rdd, partition, metadata)
- }
- }
-
- private def getPartitioning(carbonTable: CarbonTable,
- output: Seq[Attribute]): Partitioning = {
- val info: BucketingInfo = carbonTable.getBucketingInfo()
- if (info != null) {
- val cols = info.getListOfColumns.asScala
- val numBuckets = info.getNumOfRanges
- val bucketColumns = cols.flatMap { n =>
- val attrRef = output.find(_.name.equalsIgnoreCase(n.getColumnName))
- attrRef match {
- case Some(attr: AttributeReference) =>
- Some(AttributeReference(attr.name,
- CarbonSparkDataSourceUtil.convertCarbonToSparkDataType(n.getDataType),
- attr.nullable,
- attr.metadata)(attr.exprId, attr.qualifier))
- case _ => None
- }
- }
- if (bucketColumns.size == cols.size) {
- // use HashPartitioning will not shuffle
- HashPartitioning(bucketColumns, numBuckets)
- } else {
- UnknownPartitioning(0)
- }
- } else {
- UnknownPartitioning(0)
- }
- }
-
- private def isComplexAttribute(attribute: Attribute) = attribute.dataType match {
- case ArrayType(dataType, _) => true
- case StructType(_) => true
- case MapType(_, _, _) => true
- case _ => false
- }
-
- protected[sql] def selectFilters(
- relation: BaseRelation,
- predicates: Seq[Expression]): (Seq[Expression], Seq[Filter], Seq[Filter]) = {
-
- // In case of ComplexType dataTypes no filters should be pushed down. IsNotNull is being
- // explicitly added by spark and pushed. That also has to be handled and pushed back to
- // Spark for handling.
- // allow array_contains() push down
- val filteredPredicates = predicates.filter { predicate =>
- predicate.isInstanceOf[ArrayContains] ||
- predicate.collect {
- case a: Attribute if isComplexAttribute(a) => a
- }.isEmpty
- }
-
- // For conciseness, all Catalyst filter expressions of type `expressions.Expression` below are
- // called `predicate`s, while all data source filters of type `sources.Filter` are simply called
- // `filter`s. And block filters for lucene with more than one text_match udf
- // Todo: handle when lucene and normal query filter is supported
-
- var count = 0
- val translated: Seq[(Expression, Filter)] = filteredPredicates.flatMap {
- predicate =>
- if (predicate.isInstanceOf[ScalaUDF]) {
- predicate match {
- case u: ScalaUDF if u.function.isInstanceOf[TextMatchUDF] ||
- u.function.isInstanceOf[TextMatchMaxDocUDF] => count = count + 1
- case _ =>
- }
- }
- if (count > 1) {
- throw new MalformedCarbonCommandException(
- "Specify all search filters for Lucene within a single text_match UDF")
- }
- val filter = translateFilter(predicate)
- if (filter.isDefined) {
- Some(predicate, filter.get)
- } else {
- None
- }
- }
-
- // A map from original Catalyst expressions to corresponding translated data source filters.
- val translatedMap: Map[Expression, Filter] = translated.toMap
-
-
- // Catalyst predicate expressions that cannot be translated to data source filters.
- val unrecognizedPredicates = predicates.filterNot(translatedMap.contains)
-
- // Data source filters that cannot be handled by `relation`. The semantic of a unhandled filter
- // at here is that a data source may not be able to apply this filter to every row
- // of the underlying dataset.
- val unhandledFilters = relation.unhandledFilters(translatedMap.values.toArray).toSet
-
- val (unhandled, handled) = translated.partition {
- case (predicate, filter) =>
- unhandledFilters.contains(filter)
- }
-
- // Catalyst predicate expressions that can be translated to data source filters, but cannot be
- // handled by `relation`.
- val (unhandledPredicates, _) = unhandled.unzip
-
- // Translated data source filters that can be handled by `relation`
- val (_, handledFilters) = handled.unzip
-
- // translated contains all filters that have been converted to the public Filter interface.
- // We should always push them to the data source no matter whether the data source can apply
- // a filter to every row or not.
- val (_, translatedFilters) = translated.unzip
-
- (unrecognizedPredicates ++ unhandledPredicates, translatedFilters, handledFilters)
- }
-
- /**
- * Tries to translate a Catalyst [[Expression]] into data source [[Filter]].
- *
- * @return a `Some[Filter]` if the input [[Expression]] is convertible, otherwise a `None`.
- */
- protected[sql] def translateFilter(predicate: Expression, or: Boolean = false): Option[Filter] = {
- predicate match {
- case u: ScalaUDF if u.function.isInstanceOf[TextMatchUDF] =>
- if (u.children.size > 1) {
- throw new MalformedCarbonCommandException(
- "TEXT_MATCH UDF syntax: TEXT_MATCH('luceneQuerySyntax')")
- }
- Some(TextMatch(u.children.head.toString()))
-
- case u: ScalaUDF if u.function.isInstanceOf[TextMatchMaxDocUDF] =>
- if (u.children.size > 2) {
- throw new MalformedCarbonCommandException(
- "TEXT_MATCH UDF syntax: TEXT_MATCH_LIMIT('luceneQuerySyntax')")
- }
- Some(TextMatchLimit(u.children.head.toString(), u.children.last.toString()))
-
- case u: ScalaUDF if u.function.isInstanceOf[InPolygonUDF] =>
- if (u.children.size > 1) {
- throw new MalformedCarbonCommandException("Expect one string in polygon")
- }
- Some(InPolygon(u.children.head.toString()))
-
- case u: ScalaUDF if u.function.isInstanceOf[InPolygonListUDF] =>
- if (u.children.size != 2) {
- throw new MalformedCarbonCommandException("Expect two string in polygon list")
- }
- Some(InPolygonList(u.children.head.toString(), u.children.last.toString()))
-
- case u: ScalaUDF if u.function.isInstanceOf[InPolylineListUDF] =>
- if (u.children.size != 2) {
- throw new MalformedCarbonCommandException("Expect two string in polyline list")
- }
- Some(InPolylineList(u.children.head.toString(), u.children.last.toString()))
-
- case u: ScalaUDF if u.function.isInstanceOf[InPolygonRangeListUDF] =>
- if (u.children.size != 2) {
- throw new MalformedCarbonCommandException("Expect two string in polygon range list")
- }
- Some(InPolygonRangeList(u.children.head.toString(), u.children.last.toString()))
-
- case or@Or(left, right) =>
-
- val leftFilter = translateFilter(left, true)
- val rightFilter = translateFilter(right, true)
- if (leftFilter.isDefined && rightFilter.isDefined) {
- Some(sources.Or(leftFilter.get, rightFilter.get))
- } else {
- None
- }
-
- case And(left, right) =>
- val leftFilter = translateFilter(left, or)
- val rightFilter = translateFilter(right, or)
- if (or) {
- if (leftFilter.isDefined && rightFilter.isDefined) {
- (translateFilter(left) ++ translateFilter(right)).reduceOption(sources.And)
- } else {
- None
- }
- } else {
- (translateFilter(left) ++ translateFilter(right)).reduceOption(sources.And)
- }
- case EqualTo(a: Attribute, Literal(v, t)) =>
- Some(sources.EqualTo(a.name, v))
- case EqualTo(l@Literal(v, t), a: Attribute) =>
- Some(sources.EqualTo(a.name, v))
- case c@EqualTo(Cast(a: Attribute, _), Literal(v, t)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case c@EqualTo(Literal(v, t), Cast(a: Attribute, _)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case Not(EqualTo(a: Attribute, Literal(v, t))) =>
- Some(sources.Not(sources.EqualTo(a.name, v)))
- case Not(EqualTo(Literal(v, t), a: Attribute)) =>
- Some(sources.Not(sources.EqualTo(a.name, v)))
- case c@Not(EqualTo(Cast(a: Attribute, _), Literal(v, t))) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case c@Not(EqualTo(Literal(v, t), Cast(a: Attribute, _))) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case IsNotNull(a: Attribute) => Some(sources.IsNotNull(a.name))
- case IsNull(a: Attribute) => Some(sources.IsNull(a.name))
- case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) =>
- val hSet = list.map(e => e.eval(EmptyRow))
- Some(sources.Not(sources.In(a.name, hSet.toArray)))
- case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
- val hSet = list.map(e => e.eval(EmptyRow))
- Some(sources.In(a.name, hSet.toArray))
- case c@Not(In(Cast(a: Attribute, _), list))
- if !list.exists(!_.isInstanceOf[Literal]) =>
- Some(CastExpr(c))
- case c@In(Cast(a: Attribute, _), list) if !list.exists(!_.isInstanceOf[Literal]) =>
- Some(CastExpr(c))
- case InSet(a: Attribute, set) =>
- Some(sources.In(a.name, set.toArray))
- case Not(InSet(a: Attribute, set)) =>
- Some(sources.Not(sources.In(a.name, set.toArray)))
- case GreaterThan(a: Attribute, Literal(v, t)) =>
- Some(sources.GreaterThan(a.name, v))
- case GreaterThan(Literal(v, t), a: Attribute) =>
- Some(sources.LessThan(a.name, v))
- case c@GreaterThan(Cast(a: Attribute, _), Literal(v, t)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case c@GreaterThan(Literal(v, t), Cast(a: Attribute, _)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case LessThan(a: Attribute, Literal(v, t)) =>
- Some(sources.LessThan(a.name, v))
- case LessThan(Literal(v, t), a: Attribute) =>
- Some(sources.GreaterThan(a.name, v))
- case c@LessThan(Cast(a: Attribute, _), Literal(v, t)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case c@LessThan(Literal(v, t), Cast(a: Attribute, _)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
- Some(sources.GreaterThanOrEqual(a.name, v))
- case GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
- Some(sources.LessThanOrEqual(a.name, v))
- case c@GreaterThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case c@GreaterThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case LessThanOrEqual(a: Attribute, Literal(v, t)) =>
- Some(sources.LessThanOrEqual(a.name, v))
- case LessThanOrEqual(Literal(v, t), a: Attribute) =>
- Some(sources.GreaterThanOrEqual(a.name, v))
- case c@LessThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case c@LessThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case s@StartsWith(a: Attribute, Literal(v, t)) =>
- Some(sources.StringStartsWith(a.name, v.toString))
- case c@EndsWith(a: Attribute, Literal(v, t)) =>
- Some(CarbonEndsWith(c))
- case c@Contains(a: Attribute, Literal(v, t)) =>
- Some(CarbonContainsWith(c))
- case c@Literal(v, t) if (v == null) =>
- Some(FalseExpr())
- case c@ArrayContains(a: Attribute, Literal(v, t)) =>
- a.dataType match {
- case arrayType: ArrayType =>
- if (SparkUtil.isPrimitiveType(arrayType.elementType)) {
- Some(sources.EqualTo(a.name, v))
- } else {
- None
- }
- case _ =>
- None
- }
- case c@ArrayContains(Cast(a: Attribute, _), Literal(v, t)) =>
- CastExpressionOptimization.checkIfCastCanBeRemove(
- EqualTo(
- predicate.asInstanceOf[ArrayContains].left,
- predicate.asInstanceOf[ArrayContains].right))
- case _ => None
- }
- }
-
- def supportBatchedDataSource(sqlContext: SQLContext, cols: Seq[Attribute]): Boolean = {
- val vectorizedReader = {
- if (sqlContext.sparkSession.conf.contains(CarbonCommonConstants.ENABLE_VECTOR_READER)) {
- sqlContext.sparkSession.conf.get(CarbonCommonConstants.ENABLE_VECTOR_READER)
- } else if (System.getProperty(CarbonCommonConstants.ENABLE_VECTOR_READER) != null) {
- System.getProperty(CarbonCommonConstants.ENABLE_VECTOR_READER)
- } else {
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ENABLE_VECTOR_READER,
- CarbonCommonConstants.ENABLE_VECTOR_READER_DEFAULT)
- }
- }
- val supportCodegen =
- sqlContext.conf.wholeStageEnabled && sqlContext.conf.wholeStageMaxNumFields >= cols.size
- supportCodegen && vectorizedReader.toBoolean &&
- cols.forall(_.dataType.isInstanceOf[AtomicType])
- }
-
- private def createHadoopFSRelation(relation: LogicalRelation): HadoopFsRelation = {
- val sparkSession = relation.relation.sqlContext.sparkSession
- relation.catalogTable match {
- case Some(catalogTable) =>
- val fileIndex = CarbonFileIndex(sparkSession,
- catalogTable.schema,
- catalogTable.storage.properties,
- new CatalogFileIndex(
- sparkSession,
- catalogTable,
- sizeInBytes = relation.relation.sizeInBytes))
- fileIndex.setDummy(true)
- HadoopFsRelation(
- fileIndex,
- catalogTable.partitionSchema,
- catalogTable.schema,
- catalogTable.bucketSpec,
- new SparkCarbonTableFormat,
- catalogTable.storage.properties)(sparkSession)
- case _ =>
- HadoopFsRelation(
- new InMemoryFileIndex(sparkSession, Seq.empty, Map.empty, None),
- new StructType(),
- relation.relation.schema,
- None,
- new SparkCarbonTableFormat,
- null)(sparkSession)
- }
- }
-
- object ExtractTakeOrderedAndProjectExec {
-
- def unapply(plan: LogicalPlan): Option[CarbonTakeOrderedAndProjectExec] = {
- val allRelations = plan.collect { case logicalRelation: LogicalRelation => logicalRelation }
- // push down order by limit to carbon map task,
- // only when there are only one CarbonDatasourceHadoopRelation
- if (allRelations.size != 1 ||
- allRelations.exists(x => !x.relation.isInstanceOf[CarbonDatasourceHadoopRelation])) {
- return None
- }
- // check and Replace TakeOrderedAndProject (physical plan node for order by + limit)
- // with CarbonTakeOrderedAndProjectExec.
- val relation = allRelations.head.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
- plan match {
- case ReturnAnswer(rootPlan) => rootPlan match {
- case Limit(IntegerLiteral(limit), Sort(order, true, child)) =>
- carbonTakeOrder(relation, limit,
- order,
- child.output,
- planLater(pushLimit(limit, child)))
- case Limit(IntegerLiteral(limit), Project(projectList, Sort(order, true, child))) =>
- carbonTakeOrder(relation, limit, order, projectList, planLater(pushLimit(limit, child)))
- case _ => None
- }
- case Limit(IntegerLiteral(limit), Sort(order, true, child)) =>
- carbonTakeOrder(relation, limit, order, child.output, planLater(pushLimit(limit, child)))
- case Limit(IntegerLiteral(limit), Project(projectList, Sort(order, true, child))) =>
- carbonTakeOrder(relation, limit, order, projectList, planLater(pushLimit(limit, child)))
- case _ => None
- }
- }
-
- private def carbonTakeOrder(relation: CarbonDatasourceHadoopRelation,
- limit: Int,
- orders: Seq[SortOrder],
- projectList: Seq[NamedExpression],
- child: SparkPlan): Option[CarbonTakeOrderedAndProjectExec] = {
- val latestOrder = orders.last
- val fromHead: Boolean = latestOrder.direction match {
- case Ascending => true
- case Descending => false
- }
- val (columnName, canPushDown) = latestOrder.child match {
- case attr: AttributeReference => (attr.name, true)
- case Alias(AttributeReference(name, _, _, _), _) => (name, true)
- case _ => (null, false)
- }
- val mapOrderPushDown = CarbonProperties.getInstance.getProperty(
- CarbonCommonConstants.CARBON_MAP_ORDER_PUSHDOWN + "." +
- s"${ relation.carbonTable.getTableUniqueName.toLowerCase(Locale.ROOT) }.column")
- // when this property is enabled and order by column is in sort column,
- // enable limit push down to map task, row scanner can use this limit.
- val sortColumns = relation.carbonTable.getSortColumns
- if (mapOrderPushDown != null && canPushDown && sortColumns.size() > 0
- && sortColumns.get(0).equalsIgnoreCase(columnName)
- && mapOrderPushDown.equalsIgnoreCase(columnName)) {
- // Replace TakeOrderedAndProject (which comes after physical plan with limit and order by)
- // with CarbonTakeOrderedAndProjectExec.
- // which will skip the order at map task as column data is already sorted
- Some(CarbonTakeOrderedAndProjectExec(limit, orders, projectList, child, skipMapOrder =
- true, readFromHead = fromHead))
- } else {
- None
- }
- }
-
- def pushLimit(limit: Int, plan: LogicalPlan): LogicalPlan = {
- val newPlan = plan transform {
- case lr: LogicalRelation =>
- val newRelation = lr.copy(relation = lr.relation
- .asInstanceOf[CarbonDatasourceHadoopRelation]
- .copy(limit = limit))
- newRelation
- case other => other
- }
- newPlan
- }
- }
-}
-
-class CarbonPhysicalPlanException extends Exception
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonPlanHelper.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonPlanHelper.scala
index dd335fc..c351ca9 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonPlanHelper.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonPlanHelper.scala
@@ -18,19 +18,25 @@
package org.apache.spark.sql.execution.strategy
import org.apache.commons.lang3.StringUtils
-import org.apache.spark.sql.{CarbonEnv, InsertIntoCarbonTable, SparkSession}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{CarbonEnv, CarbonToSparkAdapter, CustomDeterministicExpression, InsertIntoCarbonTable, SparkSession, SQLContext}
import org.apache.spark.sql.carbondata.execution.datasources.CarbonSparkDataSourceUtil
-import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Expression, NamedExpression}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
import org.apache.spark.sql.execution.SparkPlan
import org.apache.spark.sql.execution.command.{ExecutedCommandExec, RunnableCommand}
import org.apache.spark.sql.execution.command.management.{CarbonAlterTableCompactionCommand, CarbonInsertIntoCommand}
import org.apache.spark.sql.execution.command.schema.{CarbonAlterTableAddColumnCommand, CarbonAlterTableColRenameDataTypeChangeCommand, CarbonAlterTableDropColumnCommand}
-import org.apache.spark.sql.types.StructField
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.types.{AtomicType, StructField}
+import org.apache.spark.sql.util.SparkSQLUtil
import org.apache.spark.util.{CarbonReflectionUtils, SparkUtil}
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
-import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.{CarbonProperties, DataTypeUtil}
object CarbonPlanHelper {
@@ -123,6 +129,10 @@ object CarbonPlanHelper {
}
}
+ def isCarbonTable(tableIdent: TableIdentifier): Boolean = {
+ isCarbonTable(tableIdent, SparkSQLUtil.getSparkSession)
+ }
+
def isCarbonTable(tableIdent: TableIdentifier, sparkSession: SparkSession): Boolean = {
val dbOption = tableIdent.database.map(_.toLowerCase)
val tableIdentifier = TableIdentifier(tableIdent.table.toLowerCase(), dbOption)
@@ -168,4 +178,87 @@ object CarbonPlanHelper {
throw new UnsupportedOperationException(message)
}
}
+
+ /**
+ * get deterministic expression for NamedExpression
+ */
+ private def makeDeterministicExp(exp: NamedExpression): Expression = {
+ exp match {
+ case alias: Alias if alias.child.isInstanceOf[CustomDeterministicExpression] =>
+ alias
+ case _ =>
+ CustomDeterministicExpression(exp)
+ }
+ }
+
+ /**
+ * Convert all Expression to deterministic Expression
+ */
+ def makeDeterministic(plan: LogicalPlan): LogicalPlan = {
+ val transformedPlan = plan transform {
+ case p@Project(projectList: Seq[NamedExpression], cd) =>
+ if (cd.isInstanceOf[org.apache.spark.sql.catalyst.plans.logical.Filter] ||
+ cd.isInstanceOf[LogicalRelation]) {
+ p.transformAllExpressions {
+ case a@Alias(exp, _)
+ if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
+ CarbonToSparkAdapter.createAliasRef(
+ CustomDeterministicExpression(exp),
+ a.name,
+ a.exprId,
+ a.qualifier,
+ a.explicitMetadata,
+ Some(a))
+ case exp: NamedExpression
+ if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
+ makeDeterministicExp(exp)
+ }
+ } else {
+ p
+ }
+ case f@org.apache.spark.sql.catalyst.plans.logical.Filter(condition: Expression, cd) =>
+ if (cd.isInstanceOf[Project] || cd.isInstanceOf[LogicalRelation]) {
+ f.transformAllExpressions {
+ case a@Alias(exp, _)
+ if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
+ CarbonToSparkAdapter.createAliasRef(
+ CustomDeterministicExpression(exp),
+ a.name,
+ a.exprId,
+ a.qualifier,
+ a.explicitMetadata,
+ Some(a))
+ case exp: NamedExpression
+ if !exp.deterministic && !exp.isInstanceOf[CustomDeterministicExpression] =>
+ makeDeterministicExp(exp)
+ }
+ } else {
+ f
+ }
+ }
+ transformedPlan
+ }
+
+ def vectorReaderEnabled(): Boolean = {
+ val vectorizedReader =
+ if (SparkSQLUtil.getSparkSession.conf.contains(CarbonCommonConstants.ENABLE_VECTOR_READER)) {
+ SparkSQLUtil.getSparkSession.conf.get(CarbonCommonConstants.ENABLE_VECTOR_READER)
+ } else if (System.getProperty(CarbonCommonConstants.ENABLE_VECTOR_READER) != null) {
+ System.getProperty(CarbonCommonConstants.ENABLE_VECTOR_READER)
+ } else {
+ CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ENABLE_VECTOR_READER,
+ CarbonCommonConstants.ENABLE_VECTOR_READER_DEFAULT)
+ }
+ vectorizedReader.toBoolean
+ }
+
+ def supportBatchedDataSource(sqlContext: SQLContext,
+ cols: Seq[Attribute],
+ extraRDD: Option[(RDD[InternalRow], Boolean)]): Boolean = {
+ vectorReaderEnabled() &&
+ extraRDD.getOrElse((null, true))._2 &&
+ sqlContext.conf.wholeStageEnabled &&
+ sqlContext.conf.wholeStageMaxNumFields >= cols.size &&
+ cols.forall(_.dataType.isInstanceOf[AtomicType])
+ }
}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala
new file mode 100644
index 0000000..1e7f0bf
--- /dev/null
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonSourceStrategy.scala
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.strategy
+
+import scala.collection.mutable
+
+import org.apache.log4j.Logger
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.{expressions, InternalRow, TableIdentifier}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, _}
+import org.apache.spark.sql.catalyst.planning.PhysicalOperation
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, _}
+import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.optimizer.CarbonFilters
+import org.apache.spark.sql.types._
+
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.readcommitter.TableStatusReadCommittedScope
+import org.apache.carbondata.core.scan.expression.{Expression => CarbonFilter}
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.geo.{InPolygonListUDF, InPolygonRangeListUDF, InPolygonUDF, InPolylineListUDF}
+import org.apache.carbondata.hadoop.CarbonProjection
+import org.apache.carbondata.index.{TextMatchMaxDocUDF, TextMatchUDF}
+
+private[sql] object CarbonSourceStrategy extends SparkStrategy {
+ val PUSHED_FILTERS = "PushedFilters"
+ val READ_SCHEMA = "ReadSchema"
+
+ val LOGGER: Logger = LogServiceFactory.getLogService(this.getClass.getName)
+
+ def apply(plan: LogicalPlan): Seq[SparkPlan] = {
+ val transformedPlan = CarbonPlanHelper.makeDeterministic(plan)
+ transformedPlan match {
+ case GlobalLimit(IntegerLiteral(limit), LocalLimit(IntegerLiteral(limitValue),
+ _@PhysicalOperation(projects, filters, l: LogicalRelation))) if isCarbonRelation(l) =>
+ GlobalLimitExec(limit, LocalLimitExec(limitValue, pruneFilterProject(l,
+ projects.filterNot(_.name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID)),
+ filters))) :: Nil
+ case PhysicalOperation(projects, filters, l: LogicalRelation) if isCarbonRelation(l) =>
+ try {
+ pruneFilterProject(l, projects, filters) :: Nil
+ } catch {
+ case _: CarbonPhysicalPlanException => Nil
+ }
+ case _ => Nil
+ }
+ }
+
+ private def isCarbonRelation(logicalRelation: LogicalRelation): Boolean = {
+ logicalRelation.relation.isInstanceOf[CarbonDatasourceHadoopRelation]
+ }
+
+ private def getPartitionFilter(relation: LogicalRelation,
+ filterPredicates: Seq[Expression]): Seq[Expression] = {
+ val names = relation.catalogTable.map(_.partitionColumnNames).getOrElse(Seq.empty)
+ // Get the current partitions from table.
+ if (names.nonEmpty) {
+ val partitionSet = AttributeSet(names
+ .map(p => relation.output.find(_.name.equalsIgnoreCase(p)).get))
+ // Update the name with lower case as it is case sensitive while getting partition info.
+ CarbonToSparkAdapter
+ .getPartitionFilter(partitionSet, filterPredicates)
+ .map(CarbonToSparkAdapter.lowerCaseAttribute)
+ } else {
+ Seq.empty
+ }
+ }
+
+ /**
+ * Converts to physical RDD of carbon after pushing down applicable filters.
+ * @return
+ */
+ private def pruneFilterProject(
+ relation: LogicalRelation,
+ rawProjects: Seq[NamedExpression],
+ allPredicates: Seq[Expression]): CodegenSupport = {
+ val partitionsFilter = getPartitionFilter(relation, allPredicates)
+ val table = relation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
+ val projects = rawProjects.map {p =>
+ p.transform {
+ case CustomDeterministicExpression(exp) => exp
+ }
+ }.asInstanceOf[Seq[NamedExpression]]
+
+ // contains the original order of the projection requested
+ val projectsAttr = projects.flatMap(_.references)
+ val projectSet = AttributeSet(projectsAttr)
+ val filterSet = AttributeSet(allPredicates.flatMap(_.references))
+
+ val relationPredicates = allPredicates.map {
+ _ transform {
+ case a: AttributeReference => relation.attributeMap(a) // Match original case of attributes.
+ }
+ }
+
+ val (unhandledPredicates, handledPredicates, pushedFilters) =
+ selectFilters(table, relationPredicates)
+
+ // A set of column attributes that are only referenced by pushed down filters. We can eliminate
+ // them from requested columns.
+ val handledSet = {
+ val handledPredicates = allPredicates.filterNot(unhandledPredicates.contains)
+ val unhandledSet = AttributeSet(unhandledPredicates.flatMap(_.references))
+ try {
+ AttributeSet(handledPredicates.flatMap(_.references)) --
+ (projectSet ++ unhandledSet).map(relation.attributeMap)
+ } catch {
+ case e: Throwable => throw new CarbonPhysicalPlanException
+ }
+ }
+ // Combines all Catalyst filter `Expression`s that are either not convertible to data source
+ // `Filter`s or cannot be handled by `relation`.
+ val filterCondition = unhandledPredicates.reduceLeftOption(expressions.And)
+
+ val readCommittedScope =
+ new TableStatusReadCommittedScope(table.identifier, FileFactory.getConfiguration)
+ val extraSegments = MixedFormatHandler.extraSegments(table.identifier, readCommittedScope)
+ val extraRDD = MixedFormatHandler.extraRDD(relation, rawProjects, allPredicates,
+ readCommittedScope, table.identifier, extraSegments)
+ val vectorPushRowFilters =
+ vectorPushRowFiltersEnabled(relationPredicates, extraSegments.nonEmpty)
+ var directScanSupport = !vectorPushRowFilters
+ val (updatedProjects, output, requiredColumns) = if (projects.map(_.toAttribute) == projects &&
+ projectSet.size == projects.size && filterSet.subsetOf(projectSet)) {
+ getRequiredColumnsWithoutProject(relation, projects, handledSet)
+ } else {
+ val tuple4 = getUpdatedProjects(relation, projects, projectsAttr, filterSet, handledSet,
+ directScanSupport, extraRDD)
+ if (directScanSupport) {
+ directScanSupport = tuple4._4
+ }
+ (tuple4._1, tuple4._2, tuple4._3)
+ }
+ // scan
+ val scan = CarbonDataSourceScan(
+ table,
+ output,
+ partitionsFilter,
+ handledPredicates,
+ readCommittedScope,
+ getCarbonProjection(relationPredicates, requiredColumns, projects),
+ pushedFilters,
+ directScanSupport,
+ extraRDD,
+ Some(TableIdentifier(table.identifier.getTableName, Option(table.identifier.getDatabaseName)))
+ )
+ // filter
+ val filterOption = if (directScanSupport && scan.supportsBatch) {
+ allPredicates.reduceLeftOption(expressions.And)
+ } else if (extraSegments.nonEmpty) {
+ allPredicates.reduceLeftOption(expressions.And)
+ } else {
+ filterCondition
+ }
+ val scanWithFilter = filterOption.map(execution.FilterExec(_, scan)).getOrElse(scan)
+ // project
+ updatedProjects.map(execution.ProjectExec(_, scanWithFilter)).getOrElse(scanWithFilter)
+ }
+
+ private def getRequiredColumnsWithoutProject(relation: LogicalRelation,
+ projects: Seq[NamedExpression],
+ handledSet: AttributeSet): (Option[Seq[NamedExpression]], Seq[Attribute], Seq[String]) = {
+ val newRequiredColumns = projects
+ // Safe due to if above.
+ .asInstanceOf[Seq[Attribute]]
+ // Match original case of attributes.
+ .map(relation.attributeMap)
+ // Don't request columns that are only referenced by pushed filters.
+ .filterNot(handledSet.contains)
+
+ (None, projects.map(_.toAttribute.asInstanceOf[AttributeReference]),
+ newRequiredColumns.map(_.name))
+ }
+
+ private def getUpdatedProjects(
+ relation: LogicalRelation,
+ projects: Seq[NamedExpression],
+ projectsAttr: Seq[Attribute],
+ filterSet: AttributeSet,
+ handledSet: AttributeSet,
+ oldDirectScanSupport: Boolean,
+ extraRDD: Option[(RDD[InternalRow], Boolean)]
+ ): (Option[Seq[NamedExpression]], Seq[Attribute], Seq[String], Boolean) = {
+ var newProjectList: Seq[Attribute] = Seq.empty
+ // In case of implicit exist we should disable vectorPushRowFilters as it goes in IUD flow
+ // to get the positionId or tupleID
+ var implicitExisted = false
+ var updatedProjects = projects.map {
+ case a@Alias(_: ScalaUDF, name)
+ if name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID) ||
+ name.equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID) =>
+ val reference = AttributeReference(name, StringType, true)().withExprId(a.exprId)
+ newProjectList :+= reference
+ implicitExisted = true
+ reference
+ case a@Alias(s: ScalaUDF, name)
+ if name.equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_SEGMENTID) =>
+ implicitExisted = true
+ val reference =
+ AttributeReference(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID,
+ StringType, true)().withExprId(a.exprId)
+ newProjectList :+= reference
+ a.transform {
+ case s: ScalaUDF =>
+ CarbonToSparkAdapter.createScalaUDF(s, reference)
+ }
+ case other => other
+ }.asInstanceOf[Seq[NamedExpression]]
+ val directScanSupport = if (oldDirectScanSupport && implicitExisted) {
+ false
+ } else {
+ oldDirectScanSupport
+ }
+ val updatedColumns: (Seq[Attribute], Seq[NamedExpression]) = getRequiredColumns(relation,
+ projectsAttr, filterSet, handledSet, newProjectList, updatedProjects)
+ // Don't request columns that are only referenced by pushed filters.
+ val requiredColumns = updatedColumns._1
+ updatedProjects = updatedColumns._2
+ var updateRequestedColumns = if (directScanSupport) {
+ (projectsAttr.to[mutable.LinkedHashSet] ++ filterSet).map(relation.attributeMap).toSeq
+ } else {
+ requiredColumns
+ }
+ val supportBatch = CarbonPlanHelper.supportBatchedDataSource(relation.relation.sqlContext,
+ updateRequestedColumns, extraRDD)
+ if (directScanSupport && !supportBatch && filterSet.nonEmpty &&
+ !filterSet.baseSet.exists(_.a.dataType.isInstanceOf[ArrayType])) {
+ // revert for row scan
+ updateRequestedColumns = requiredColumns
+ }
+ val newRequiredColumns = if (oldDirectScanSupport && extraRDD.isDefined) {
+ extractUniqueAttributes(projectsAttr, filterSet.toSeq)
+ } else {
+ updateRequestedColumns
+ }
+ (Some(updatedProjects), newRequiredColumns, newRequiredColumns.map(_.name), directScanSupport)
+ }
+
+ private def getCarbonProjection(scanFilters: Seq[Expression],
+ requiredColumns: Seq[String],
+ projects: Seq[NamedExpression]): CarbonProjection = {
+ val projection = new CarbonProjection
+
+ // As Filter push down for Complex datatype is not supported, if filter is applied on complex
+ // column, then Projection push down on Complex Columns will not take effect. Hence, check if
+ // filter contains Struct Complex Column.
+ val complexFilterExists = scanFilters.map(col =>
+ col.map(_.isInstanceOf[GetStructField]))
+
+ if (!complexFilterExists.exists(f => f.contains(true))) {
+ PushDownHelper.pushDownProjection(requiredColumns, projects, projection)
+ } else {
+ requiredColumns.foreach(projection.addColumn)
+ }
+ projection
+ }
+
+ private def vectorPushRowFiltersEnabled(candidatePredicates: Seq[Expression],
+ hasExtraSegment: Boolean): Boolean = {
+ var vectorPushRowFilters = CarbonProperties.getInstance().isPushRowFiltersForVector
+
+ // Spark cannot filter the rows from the pages in case of polygon query. So, we do the row
+ // level filter at carbon and return the rows directly.
+ if (candidatePredicates
+ .exists(exp => exp.isInstanceOf[ScalaUDF] &&
+ (exp.asInstanceOf[ScalaUDF].function.isInstanceOf[InPolygonUDF] ||
+ exp.asInstanceOf[ScalaUDF].function.isInstanceOf[InPolygonListUDF] ||
+ exp.asInstanceOf[ScalaUDF].function.isInstanceOf[InPolylineListUDF] ||
+ exp.asInstanceOf[ScalaUDF].function.isInstanceOf[InPolygonRangeListUDF]))) {
+ vectorPushRowFilters = true
+ }
+
+ // In case of mixed format, make the vectorPushRowFilters always false as other formats
+ // filtering happens in spark layer.
+ if (vectorPushRowFilters && hasExtraSegment) {
+ vectorPushRowFilters = false
+ }
+ vectorPushRowFilters
+ }
+
+ /*
+ This function is used to get the Unique attributes from filter set
+ and projection set based on their semantics.
+ */
+ private def extractUniqueAttributes(projections: Seq[Attribute],
+ filter: Seq[Attribute]): Seq[Attribute] = {
+ def checkSemanticEquals(filter: Attribute): Option[Attribute] = {
+ projections.find(_.semanticEquals(filter))
+ }
+
+ filter.toList match {
+ case head :: tail =>
+ checkSemanticEquals(head) match {
+ case Some(_) => extractUniqueAttributes(projections, tail)
+ case None => extractUniqueAttributes(projections :+ head, tail)
+ }
+ case Nil => projections
+ }
+ }
+
+ protected def getRequiredColumns(relation: LogicalRelation,
+ projectsAttr: Seq[Attribute],
+ filterSet: AttributeSet,
+ handledSet: AttributeSet,
+ newProjectList: Seq[Attribute],
+ updatedProjects: Seq[NamedExpression]): (Seq[Attribute], Seq[NamedExpression]) = {
+ val sparkSession = SparkSession.getActiveSession.get
+ val pushDownJoinEnabled = sparkSession.sparkContext.getConf
+ .getBoolean("spark.carbon.pushdown.join.as.filter", defaultValue = true)
+
+ // positionId column can be added in two cases
+ // case 1: SI pushdown case, SI rewritten plan adds positionId column
+ // case 2: if the user requested positionId column thru getPositionId() UDF
+ // positionId column should be removed only in case 1, as it is manually added
+ // Below code is added to handle case 2. But getPositionId() UDF is almost used only for testing
+ val isPositionIDRequested = relation.catalogTable match {
+ case Some(table) =>
+ val tblProperties = CarbonEnv.getCarbonTable(table.identifier)(sparkSession).getTableInfo
+ .getFactTable
+ .getTableProperties
+ val isPosIDRequested = if (tblProperties.containsKey("isPositionIDRequested")) {
+ java.lang.Boolean.parseBoolean(tblProperties.get("isPositionIDRequested"))
+ } else {
+ false
+ }
+ isPosIDRequested
+ case _ => false
+ }
+ // remove positionId col only if pushdown is enabled and
+ // positionId col is not requested in the query
+ if (pushDownJoinEnabled && !isPositionIDRequested) {
+ ((projectsAttr.to[scala.collection.mutable.LinkedHashSet] ++ filterSet -- handledSet)
+ .map(relation.attributeMap).toSeq ++ newProjectList
+ .filterNot(attr => attr.name
+ .equalsIgnoreCase(CarbonCommonConstants.POSITION_ID)), updatedProjects
+ .filterNot(attr => attr.isInstanceOf[AttributeReference] &&
+ attr.asInstanceOf[AttributeReference].name
+ .equalsIgnoreCase(CarbonCommonConstants.POSITION_ID)))
+ } else {
+ ((projectsAttr.to[scala.collection.mutable.LinkedHashSet] ++ filterSet -- handledSet)
+ .map(relation.attributeMap).toSeq ++ newProjectList, updatedProjects)
+ }
+ }
+
+ private def isComplexAttribute(attribute: Attribute) = {
+ attribute.dataType match {
+ case _: ArrayType => true
+ case _: StructType => true
+ case _: MapType => true
+ case _ => false
+ }
+ }
+
+ protected def selectFilters(relation: CarbonDatasourceHadoopRelation,
+ predicates: Seq[Expression]): (Seq[Expression], Seq[Expression], Seq[CarbonFilter]) = {
+ // In case of ComplexType dataTypes no filters should be pushed down. IsNotNull is being
+ // explicitly added by spark and pushed. That also has to be handled and pushed back to
+ // Spark for handling.
+ // allow array_contains() push down
+ val filteredPredicates = predicates.filter { predicate =>
+ predicate.isInstanceOf[ArrayContains] ||
+ predicate.collect {
+ case a: Attribute if isComplexAttribute(a) => a
+ }.isEmpty
+ }
+
+ val dataTypeOf = relation.schema.map { f =>
+ f.dataType match {
+ case arrayType: ArrayType => f.name -> arrayType.elementType
+ case _ => f.name -> f.dataType
+ }
+ }.toMap
+
+ var count = 0
+ val translated: Seq[(Expression, CarbonFilter)] = filteredPredicates.flatMap {
+ predicate =>
+ if (predicate.isInstanceOf[ScalaUDF]) {
+ predicate match {
+ case u: ScalaUDF if u.function.isInstanceOf[TextMatchUDF] ||
+ u.function.isInstanceOf[TextMatchMaxDocUDF] => count = count + 1
+ case _ =>
+ }
+ }
+ if (count > 1) {
+ throw new MalformedCarbonCommandException(
+ "Specify all search filters for Lucene within a single text_match UDF")
+ }
+
+ val filter = CarbonFilters.translateExpression(relation, predicate, dataTypeOf)
+ if (filter.isDefined) {
+ Some(predicate, filter.get)
+ } else {
+ None
+ }
+ }
+
+ // A map from original Catalyst expressions to corresponding translated data source filters.
+ val translatedMap: Map[Expression, CarbonFilter] = translated.toMap
+
+ // Catalyst predicate expressions that cannot be translated to data source filters.
+ val unrecognizedPredicates = predicates.filterNot(translatedMap.contains)
+
+ // translated contains all filters that have been converted to the public Filter interface.
+ // We should always push them to the data source no matter whether the data source can apply
+ // a filter to every row or not.
+ val (recognizedPredicates, translatedFilters) = translated.unzip
+
+ (unrecognizedPredicates, recognizedPredicates, translatedFilters)
+ }
+}
+
+class CarbonPhysicalPlanException extends Exception
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index e1032e9..5327976 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -28,10 +28,12 @@ import org.apache.spark.sql.execution.command.management.{CarbonAlterTableCompac
import org.apache.spark.sql.execution.command.mutation.CarbonTruncateCommand
import org.apache.spark.sql.execution.command.schema._
import org.apache.spark.sql.execution.command.table.{CarbonCreateTableLikeCommand, CarbonDropTableCommand, CarbonShowCreateTableCommand, CarbonShowTablesCommand}
-import org.apache.spark.sql.execution.datasources.{InsertIntoHadoopFsRelationCommand, RefreshResource, RefreshTable}
+import org.apache.spark.sql.execution.datasources.{RefreshResource, RefreshTable}
+import org.apache.spark.sql.execution.strategy.CarbonPlanHelper.isCarbonTable
import org.apache.spark.sql.hive.execution.CreateHiveTableAsSelectCommand
import org.apache.spark.sql.hive.execution.command.{CarbonDropDatabaseCommand, CarbonResetCommand, CarbonSetCommand, MatchResetCommand}
import org.apache.spark.sql.secondaryindex.command.CarbonCreateSecondaryIndexCommand
+import org.apache.spark.sql.util.SparkSQLUtil
import org.apache.carbondata.common.logging.LogServiceFactory
@@ -49,21 +51,13 @@ object MatchCreateDataSourceTable {
}
}
-class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
+object DDLStrategy extends SparkStrategy {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
def apply(plan: LogicalPlan): Seq[SparkPlan] = {
+ val sparkSession = SparkSQLUtil.getSparkSession
plan match {
case _: ReturnAnswer => Nil
- // load data / insert into
- case loadData: LoadDataCommand
- if isCarbonTable(loadData.table) =>
- ExecutedCommandExec(DMLHelper.loadData(loadData)) :: Nil
- case insert: InsertIntoCarbonTable =>
- ExecutedCommandExec(CarbonPlanHelper.insertInto(insert)) :: Nil
- case insert: InsertIntoHadoopFsRelationCommand
- if insert.catalogTable.isDefined && isCarbonTable(insert.catalogTable.get.identifier) =>
- DataWritingCommandExec(DMLHelper.insertInto(insert), planLater(insert.query)) :: Nil
// alter table
case renameTable: AlterTableRenameCommand
if isCarbonTable(renameTable.oldName) =>
@@ -230,10 +224,6 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
}
}
- private def isCarbonTable(tableIdent: TableIdentifier): Boolean = {
- CarbonPlanHelper.isCarbonTable(tableIdent, sparkSession)
- }
-
private def isCarbonHiveTable(table: CatalogTable): Boolean = {
table.provider.isDefined &&
DDLUtils.HIVE_PROVIDER == table.provider.get &&
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DMLStrategy.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DMLStrategy.scala
new file mode 100644
index 0000000..e2e41dc
--- /dev/null
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/DMLStrategy.scala
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.strategy
+
+import java.util.Locale
+
+import org.apache.log4j.Logger
+import org.apache.spark.sql.{CarbonCountStar, CarbonDatasourceHadoopRelation, CountStarPlan, InsertIntoCarbonTable, SparkSession}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, AttributeReference, Descending, Expression, IntegerLiteral, NamedExpression, SortOrder}
+import org.apache.spark.sql.catalyst.planning.{ExtractEquiJoinKeys, PhysicalOperation}
+import org.apache.spark.sql.catalyst.plans.{Inner, LeftSemi}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, Limit, LogicalPlan, Project, ReturnAnswer, Sort}
+import org.apache.spark.sql.execution.{CarbonTakeOrderedAndProjectExec, FilterExec, ProjectExec, SparkPlan, SparkStrategy}
+import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec, LoadDataCommand}
+import org.apache.spark.sql.execution.datasources.{InsertIntoHadoopFsRelationCommand, LogicalRelation}
+import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight}
+import org.apache.spark.sql.execution.strategy.CarbonPlanHelper.isCarbonTable
+import org.apache.spark.sql.hive.MatchLogicalRelation
+import org.apache.spark.sql.index.CarbonIndexUtil
+import org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager
+import org.apache.carbondata.core.util.CarbonProperties
+
+object DMLStrategy extends SparkStrategy {
+ val LOGGER: Logger = LogServiceFactory.getLogService(this.getClass.getName)
+
+ override def apply(plan: LogicalPlan): Seq[SparkPlan] = {
+ plan match {
+ // load data / insert into
+ case loadData: LoadDataCommand if isCarbonTable(loadData.table) =>
+ ExecutedCommandExec(DMLHelper.loadData(loadData)) :: Nil
+ case insert: InsertIntoCarbonTable =>
+ ExecutedCommandExec(CarbonPlanHelper.insertInto(insert)) :: Nil
+ case insert: InsertIntoHadoopFsRelationCommand
+ if insert.catalogTable.isDefined && isCarbonTable(insert.catalogTable.get.identifier) =>
+ DataWritingCommandExec(DMLHelper.insertInto(insert), planLater(insert.query)) :: Nil
+ case CountStarPlan(colAttr, PhysicalOperation(_, _, l: LogicalRelation))
+ if l.relation.isInstanceOf[CarbonDatasourceHadoopRelation] && driverSideCountStar(l) =>
+ val relation = l.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
+ CarbonCountStar(colAttr, relation.carbonTable, SparkSession.getActiveSession.get) :: Nil
+ case ExtractEquiJoinKeys(Inner, leftKeys, rightKeys, condition,
+ left, right)
+ if isCarbonPlan(left) && CarbonIndexUtil.checkIsIndexTable(right) =>
+ LOGGER.info(s"pushing down for ExtractEquiJoinKeys:right")
+ val carbon = CarbonSourceStrategy.apply(left).head
+ // in case of SI Filter push join remove projection list from the physical plan
+ // no need to have the project list in the main table physical plan execution
+ // only join uses the projection list
+ var carbonChild = carbon match {
+ case projectExec: ProjectExec =>
+ projectExec.child
+ case _ =>
+ carbon
+ }
+ // check if the outer and the inner project are matching, only then remove project
+ if (left.isInstanceOf[Project]) {
+ val leftOutput = left.output
+ .filterNot(_.name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID))
+ .map(c => (c.name.toLowerCase, c.dataType))
+ val childOutput = carbonChild.output
+ .filterNot(_.name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID))
+ .map(c => (c.name.toLowerCase, c.dataType))
+ if (!leftOutput.equals(childOutput)) {
+ // if the projection list and the scan list are different(in case of alias)
+ // we should not skip the project, so we are taking the original plan with project
+ carbonChild = carbon
+ }
+ }
+ val pushedDownJoin = BroadCastSIFilterPushJoin(
+ leftKeys: Seq[Expression],
+ rightKeys: Seq[Expression],
+ Inner,
+ BuildRight,
+ carbonChild,
+ planLater(right),
+ condition)
+ condition.map(FilterExec(_, pushedDownJoin)).getOrElse(pushedDownJoin) :: Nil
+ case ExtractEquiJoinKeys(Inner, leftKeys, rightKeys, condition, left,
+ right)
+ if isCarbonPlan(right) && CarbonIndexUtil.checkIsIndexTable(left) =>
+ LOGGER.info(s"pushing down for ExtractEquiJoinKeys:left")
+ val carbon = CarbonSourceStrategy.apply(right).head
+ val pushedDownJoin =
+ BroadCastSIFilterPushJoin(
+ leftKeys: Seq[Expression],
+ rightKeys: Seq[Expression],
+ Inner,
+ BuildLeft,
+ planLater(left),
+ carbon,
+ condition)
+ condition.map(FilterExec(_, pushedDownJoin)).getOrElse(pushedDownJoin) :: Nil
+ case ExtractEquiJoinKeys(LeftSemi, leftKeys, rightKeys, condition,
+ left, right)
+ if isLeftSemiExistPushDownEnabled &&
+ isAllCarbonPlan(left) && isAllCarbonPlan(right) =>
+ LOGGER.info(s"pushing down for ExtractEquiJoinKeysLeftSemiExist:right")
+ val pushedDownJoin = BroadCastSIFilterPushJoin(
+ leftKeys: Seq[Expression],
+ rightKeys: Seq[Expression],
+ LeftSemi,
+ BuildRight,
+ planLater(left),
+ planLater(right),
+ condition)
+ condition.map(FilterExec(_, pushedDownJoin)).getOrElse(pushedDownJoin) :: Nil
+ case ExtractTakeOrderedAndProjectExec(carbonTakeOrderedAndProjectExec) =>
+ carbonTakeOrderedAndProjectExec :: Nil
+ case _ => Nil
+ }
+ }
+
+ /**
+ * Return true if driver-side count star optimization can be used.
+ * Following case can't use driver-side count star:
+ * 1. There is data update and delete
+ * 2. It is streaming table
+ */
+ private def driverSideCountStar(logicalRelation: LogicalRelation): Boolean = {
+ val relation = logicalRelation.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
+ val segmentUpdateStatusManager = new SegmentUpdateStatusManager(
+ relation.carbonRelation.carbonTable)
+ val updateDeltaMetadata = segmentUpdateStatusManager.readLoadMetadata()
+ val hasNonCarbonSegment =
+ segmentUpdateStatusManager.getLoadMetadataDetails.exists(!_.isCarbonFormat)
+ if (hasNonCarbonSegment || updateDeltaMetadata != null && updateDeltaMetadata.nonEmpty) {
+ false
+ } else if (relation.carbonTable.isStreamingSink) {
+ false
+ } else {
+ true
+ }
+ }
+
+ private def isCarbonPlan(plan: LogicalPlan): Boolean = {
+ plan match {
+ case PhysicalOperation(_, _,
+ MatchLogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)) =>
+ true
+ case Filter(_, MatchLogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)) =>
+ true
+ case _ => false
+ }
+ }
+
+ private def isLeftSemiExistPushDownEnabled: Boolean = {
+ CarbonProperties.getInstance.getProperty(
+ CarbonCommonConstants.CARBON_PUSH_LEFTSEMIEXIST_JOIN_AS_IN_FILTER,
+ CarbonCommonConstants.CARBON_PUSH_LEFTSEMIEXIST_JOIN_AS_IN_FILTER_DEFAULT).toBoolean
+ }
+
+ private def isAllCarbonPlan(plan: LogicalPlan): Boolean = {
+ val allRelations = plan.collect { case logicalRelation: LogicalRelation => logicalRelation }
+ allRelations.forall(x => x.relation.isInstanceOf[CarbonDatasourceHadoopRelation])
+ }
+
+
+ object ExtractTakeOrderedAndProjectExec {
+
+ def unapply(plan: LogicalPlan): Option[CarbonTakeOrderedAndProjectExec] = {
+ val allRelations = plan.collect { case logicalRelation: LogicalRelation => logicalRelation }
+ // push down order by limit to carbon map task,
+ // only when there are only one CarbonDatasourceHadoopRelation
+ if (allRelations.size != 1 ||
+ allRelations.exists(x => !x.relation.isInstanceOf[CarbonDatasourceHadoopRelation])) {
+ return None
+ }
+ // check and Replace TakeOrderedAndProject (physical plan node for order by + limit)
+ // with CarbonTakeOrderedAndProjectExec.
+ val relation = allRelations.head.relation.asInstanceOf[CarbonDatasourceHadoopRelation]
+ plan match {
+ case ReturnAnswer(rootPlan) => rootPlan match {
+ case Limit(IntegerLiteral(limit), Sort(order, true, child)) =>
+ carbonTakeOrder(relation, limit,
+ order,
+ child.output,
+ planLater(pushLimit(limit, child)))
+ case Limit(IntegerLiteral(limit), Project(projectList, Sort(order, true, child))) =>
+ carbonTakeOrder(relation, limit, order, projectList, planLater(pushLimit(limit, child)))
+ case _ => None
+ }
+ case Limit(IntegerLiteral(limit), Sort(order, true, child)) =>
+ carbonTakeOrder(relation, limit, order, child.output, planLater(pushLimit(limit, child)))
+ case Limit(IntegerLiteral(limit), Project(projectList, Sort(order, true, child))) =>
+ carbonTakeOrder(relation, limit, order, projectList, planLater(pushLimit(limit, child)))
+ case _ => None
+ }
+ }
+
+ private def carbonTakeOrder(relation: CarbonDatasourceHadoopRelation,
+ limit: Int,
+ orders: Seq[SortOrder],
+ projectList: Seq[NamedExpression],
+ child: SparkPlan): Option[CarbonTakeOrderedAndProjectExec] = {
+ val latestOrder = orders.last
+ val fromHead: Boolean = latestOrder.direction match {
+ case Ascending => true
+ case Descending => false
+ }
+ val (columnName, canPushDown) = latestOrder.child match {
+ case attr: AttributeReference => (attr.name, true)
+ case Alias(AttributeReference(name, _, _, _), _) => (name, true)
+ case _ => (null, false)
+ }
+ val mapOrderPushDown = CarbonProperties.getInstance.getProperty(
+ CarbonCommonConstants.CARBON_MAP_ORDER_PUSHDOWN + "." +
+ s"${ relation.carbonTable.getTableUniqueName.toLowerCase(Locale.ROOT) }.column")
+ // when this property is enabled and order by column is in sort column,
+ // enable limit push down to map task, row scanner can use this limit.
+ val sortColumns = relation.carbonTable.getSortColumns
+ if (mapOrderPushDown != null && canPushDown && sortColumns.size() > 0
+ && sortColumns.get(0).equalsIgnoreCase(columnName)
+ && mapOrderPushDown.equalsIgnoreCase(columnName)) {
+ // Replace TakeOrderedAndProject (which comes after physical plan with limit and order by)
+ // with CarbonTakeOrderedAndProjectExec.
+ // which will skip the order at map task as column data is already sorted
+ Some(CarbonTakeOrderedAndProjectExec(limit, orders, projectList, child, skipMapOrder =
+ true, readFromHead = fromHead))
+ } else {
+ None
+ }
+ }
+
+ def pushLimit(limit: Int, plan: LogicalPlan): LogicalPlan = {
+ val newPlan = plan transform {
+ case lr: LogicalRelation =>
+ val newRelation = lr.copy(relation = lr.relation
+ .asInstanceOf[CarbonDatasourceHadoopRelation]
+ .copy(limit = limit))
+ newRelation
+ case other => other
+ }
+ newPlan
+ }
+ }
+}
+
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/MixedFormatHandler.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/MixedFormatHandler.scala
index 84e59f1..cc5a64c 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/MixedFormatHandler.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/MixedFormatHandler.scala
@@ -44,7 +44,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, SegmentFileStore}
import org.apache.carbondata.core.readcommitter.ReadCommittedScope
-import org.apache.carbondata.core.statusmanager.{FileFormat => FileFormatName, SegmentStatus, SegmentStatusManager}
+import org.apache.carbondata.core.statusmanager.{FileFormat => FileFormatName, LoadMetadataDetails, SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, SessionParams, ThreadLocalSessionInfo}
import org.apache.carbondata.core.util.path.CarbonTablePath
@@ -132,6 +132,21 @@ object MixedFormatHandler {
}
}
+ def extraSegments(identifier: AbsoluteTableIdentifier,
+ readCommittedScope: ReadCommittedScope): Array[LoadMetadataDetails] = {
+ val loadMetadataDetails = readCommittedScope.getSegmentList
+ val segsToAccess = getSegmentsToAccess(identifier)
+ loadMetadataDetails.filter { metaDetail =>
+ metaDetail.getSegmentStatus.equals(SegmentStatus.SUCCESS) ||
+ metaDetail.getSegmentStatus.equals(SegmentStatus.LOAD_PARTIAL_SUCCESS)
+ }.filterNot { currLoad =>
+ currLoad.getFileFormat.equals(FileFormatName.COLUMNAR_V3) ||
+ currLoad.getFileFormat.equals(FileFormatName.ROW_V1)
+ }.filter {
+ l => segsToAccess.isEmpty || segsToAccess.contains(l.getLoadName)
+ }
+ }
+
/**
* Generates the RDD for non carbon segments. It uses the spark underlying file formats and
* generates the RDD in its native format without changing any of its flow to keep the original
@@ -146,16 +161,9 @@ object MixedFormatHandler {
filters: Seq[Expression],
readCommittedScope: ReadCommittedScope,
identifier: AbsoluteTableIdentifier,
+ extraSegments: Array[LoadMetadataDetails],
supportBatch: Boolean = true): Option[(RDD[InternalRow], Boolean)] = {
- val loadMetadataDetails = readCommittedScope.getSegmentList
- val segsToAccess = getSegmentsToAccess(identifier)
- val rdds = loadMetadataDetails.filter(metaDetail =>
- (metaDetail.getSegmentStatus.equals(SegmentStatus.SUCCESS) ||
- metaDetail.getSegmentStatus.equals(SegmentStatus.LOAD_PARTIAL_SUCCESS)))
- .filterNot(currLoad =>
- currLoad.getFileFormat.equals(FileFormatName.COLUMNAR_V3) ||
- currLoad.getFileFormat.equals(FileFormatName.ROW_V1))
- .filter(l => segsToAccess.isEmpty || segsToAccess.contains(l.getLoadName))
+ val rdds = extraSegments
.groupBy(_.getFileFormat)
.map { case (format, details) =>
// collect paths as input to scan RDD
@@ -189,7 +197,7 @@ object MixedFormatHandler {
Some(rdds.head)
} else {
if (supportBatch && rdds.exists(!_._2)) {
- extraRDD(l, projects, filters, readCommittedScope, identifier, false)
+ extraRDD(l, projects, filters, readCommittedScope, identifier, extraSegments, false)
} else {
var rdd: RDD[InternalRow] = null
rdds.foreach { r =>
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/PushDownHelper.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/PushDownHelper.scala
index 8033d39..e532cd4 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/PushDownHelper.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/PushDownHelper.scala
@@ -27,7 +27,7 @@ import org.apache.carbondata.hadoop.CarbonProjection
object PushDownHelper {
def pushDownProjection(
- requiredColumns: Array[String],
+ requiredColumns: Seq[String],
projects: Seq[NamedExpression],
projection: CarbonProjection
): Unit = {
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
index 2fbe1cf..39cfcbb 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
@@ -17,29 +17,30 @@
package org.apache.spark.sql.execution.strategy
-import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.apache.spark.sql.CarbonEnv
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy}
import org.apache.spark.sql.execution.command.{AlterTableAddColumnsCommand, AlterTableChangeColumnCommand, AlterTableRenameCommand}
import org.apache.spark.sql.execution.command.mutation.{CarbonProjectForDeleteCommand, CarbonProjectForUpdateCommand}
import org.apache.spark.sql.execution.command.schema.{CarbonAlterTableAddColumnCommand, CarbonAlterTableColRenameDataTypeChangeCommand, CarbonAlterTableDropColumnCommand}
+import org.apache.spark.sql.util.SparkSQLUtil
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
/**
* Strategy for streaming table, like blocking unsupported operation
*/
-private[sql] class StreamingTableStrategy(sparkSession: SparkSession) extends SparkStrategy {
+private[sql] object StreamingTableStrategy extends SparkStrategy {
override def apply(plan: LogicalPlan): Seq[SparkPlan] = {
plan match {
- case CarbonProjectForUpdateCommand(_, databaseNameOp, tableName, columns) =>
+ case CarbonProjectForUpdateCommand(_, databaseNameOp, tableName, _) =>
rejectIfStreamingTable(
TableIdentifier(tableName, databaseNameOp),
"Data update")
Nil
- case CarbonProjectForDeleteCommand(_, databaseNameOp, tableName, timestamp) =>
+ case CarbonProjectForDeleteCommand(_, databaseNameOp, tableName, _) =>
rejectIfStreamingTable(
TableIdentifier(tableName, databaseNameOp),
"Data delete")
@@ -93,7 +94,7 @@ private[sql] class StreamingTableStrategy(sparkSession: SparkSession) extends Sp
var streaming = false
try {
streaming = CarbonEnv.getCarbonTable(
- tableIdentifier.database, tableIdentifier.table)(sparkSession)
+ tableIdentifier.database, tableIdentifier.table)(SparkSQLUtil.getSparkSession)
.isStreamingSink
} catch {
case e: Exception =>
@@ -106,6 +107,6 @@ private[sql] class StreamingTableStrategy(sparkSession: SparkSession) extends Sp
}
def isCarbonTable(tableIdent: TableIdentifier): Boolean = {
- CarbonPlanHelper.isCarbonTable(tableIdent, sparkSession)
+ CarbonPlanHelper.isCarbonTable(tableIdent, SparkSQLUtil.getSparkSession)
}
}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index c51d3d7..8bd680e 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -20,178 +20,362 @@ package org.apache.spark.sql.optimizer
import java.util.ArrayList
import scala.collection.JavaConverters._
-import scala.collection.mutable
import scala.util.Try
-import org.apache.spark.sql.{CarbonContainsWith, CarbonEndsWith, _}
+import org.apache.spark.sql.{CarbonBoundReference, CarbonDatasourceHadoopRelation, CarbonEnv, SparkSession, SparkUnknownExpression}
import org.apache.spark.sql.carbondata.execution.datasources.CarbonSparkDataSourceUtil
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.catalog.CatalogTablePartition
+import org.apache.spark.sql.catalyst.expressions.{And, ArrayContains, Attribute, AttributeReference, Cast, Contains, EmptyRow, EndsWith, EqualTo, GreaterThan, GreaterThanOrEqual, In, InSet, IsNotNull, IsNull, LessThan, LessThanOrEqual, Literal, Not, Or, ScalaUDF, StartsWith, StringTrim}
+import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression}
+import org.apache.spark.sql.execution.CastExpressionOptimization
import org.apache.spark.sql.hive.{CarbonHiveIndexMetadataUtil, CarbonSessionCatalogUtil}
-import org.apache.spark.sql.sources.Filter
-import org.apache.spark.sql.types._
-import org.apache.spark.util.CarbonReflectionUtils
+import org.apache.spark.sql.types.{ArrayType, BooleanType, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, StringType, StructType, TimestampType}
+import org.apache.spark.sql.types.{DataType => SparkDataType}
+import org.apache.spark.util.{CarbonReflectionUtils, SparkUtil}
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.indexstore.PartitionSpec
-import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
+import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression,
- Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression, MatchExpression}
-import org.apache.carbondata.core.scan.expression.conditional._
+import org.apache.carbondata.core.scan.expression.{ColumnExpression, Expression, LiteralExpression, MatchExpression}
+import org.apache.carbondata.core.scan.expression.conditional.{EqualToExpression, GreaterThanEqualToExpression, GreaterThanExpression, ImplicitExpression, InExpression, LessThanEqualToExpression, LessThanExpression, ListExpression, NotEqualsExpression, NotInExpression, StartsWithExpression}
import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
import org.apache.carbondata.core.scan.filter.intf.ExpressionType
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.geo.{GeoUtils, InPolygon, InPolygonList, InPolygonRangeList, InPolylineList}
-import org.apache.carbondata.geo.scan.expression.{PolygonExpression => CarbonPolygonExpression}
-import org.apache.carbondata.geo.scan.expression.{PolygonListExpression => CarbonPolygonListExpression}
-import org.apache.carbondata.geo.scan.expression.{PolylineListExpression => CarbonPolylineListExpression}
-import org.apache.carbondata.geo.scan.expression.{PolygonRangeListExpression => CarbonPolygonRangeListExpression}
-import org.apache.carbondata.index.{TextMatch, TextMatchLimit}
+import org.apache.carbondata.geo.{GeoUtils, InPolygonListUDF, InPolygonRangeListUDF, InPolygonUDF, InPolylineListUDF}
+import org.apache.carbondata.geo.scan.expression.{PolygonExpression, PolygonListExpression, PolygonRangeListExpression, PolylineListExpression}
+import org.apache.carbondata.index.{TextMatchMaxDocUDF, TextMatchUDF}
/**
* All filter conversions are done here.
*/
object CarbonFilters {
- val carbonProperties = CarbonProperties.getInstance()
/**
- * Converts data sources filters to carbon filter predicates.
+ * translate spark [[Expression]] into carbon [[Expression]].
+ *
+ * @return a `Some[Expression]` if the input [[Expression]] is convertible, otherwise a `None`.
*/
- def createCarbonFilter(schema: StructType,
- predicate: sources.Filter,
- tableProperties: mutable.Map[String, String]): Option[CarbonExpression] = {
- val dataTypeOf = schema.map { f =>
- f.dataType match {
- case arrayType: ArrayType => f.name -> arrayType.elementType
- case _ => f.name -> f.dataType
- }
- }.toMap
-
- def createFilter(predicate: sources.Filter): Option[CarbonExpression] = {
- predicate match {
- case sources.EqualTo(name, value) =>
- Some(new EqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.Not(sources.EqualTo(name, value)) =>
- Some(new NotEqualsExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.EqualNullSafe(name, value) =>
- Some(new EqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.Not(sources.EqualNullSafe(name, value)) =>
- Some(new NotEqualsExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.GreaterThan(name, value) =>
- Some(new GreaterThanExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.LessThan(name, value) =>
- Some(new LessThanExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.GreaterThanOrEqual(name, value) =>
- Some(new GreaterThanEqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.LessThanOrEqual(name, value) =>
- Some(new LessThanEqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.In(name, values) =>
- if (values.length == 1 && values(0) == null) {
- Some(new FalseExpression(getCarbonExpression(name)))
- } else {
- Some(new InExpression(getCarbonExpression(name),
- new ListExpression(
- convertToJavaList(values.filterNot(_ == null)
- .map(filterValues => getCarbonLiteralExpression(name, filterValues)).toList))))
- }
- case sources.Not(sources.In(name, values)) =>
- if (values.contains(null)) {
- Some(new FalseExpression(getCarbonExpression(name)))
- } else {
- Some(new NotInExpression(getCarbonExpression(name),
- new ListExpression(
- convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
- }
- case sources.IsNull(name) =>
- Some(new EqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, null), true))
- case sources.IsNotNull(name) =>
- Some(new NotEqualsExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, null), true))
- case sources.And(lhs, rhs) =>
- (createFilter(lhs) ++ createFilter(rhs)).reduceOption(new AndExpression(_, _))
- case sources.Or(lhs, rhs) =>
- for {
- lhsFilter <- createFilter(lhs)
- rhsFilter <- createFilter(rhs)
- } yield {
- new OrExpression(lhsFilter, rhsFilter)
- }
- case sources.StringStartsWith(name, value) if value.length > 0 =>
- Some(new StartsWithExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case CarbonEndsWith(expr: Expression) =>
- Some(getSparkUnknownExpression(expr, ExpressionType.ENDSWITH))
- case CarbonContainsWith(expr: Expression) =>
- Some(getSparkUnknownExpression(expr, ExpressionType.CONTAINSWITH))
- case CastExpr(expr: Expression) =>
- Some(transformExpression(expr))
- case FalseExpr() =>
- Some(new FalseExpression(null))
- case TextMatch(queryString) =>
- Some(new MatchExpression(queryString))
- case TextMatchLimit(queryString, maxDoc) =>
- Some(new MatchExpression(queryString, Try(maxDoc.toInt).getOrElse(Integer.MAX_VALUE)))
- case InPolygon(queryString) =>
- val (columnName, instance) = GeoUtils.getGeoHashHandler(tableProperties)
- Some(new CarbonPolygonExpression(queryString, columnName, instance))
- case InPolygonList(polygonListString, opType) =>
- val (columnName, instance) = GeoUtils.getGeoHashHandler(tableProperties)
- Some(new CarbonPolygonListExpression(polygonListString, opType, columnName, instance))
- case InPolylineList(polylineListString, buffer) =>
- val (columnName, instance) = GeoUtils.getGeoHashHandler(tableProperties)
- Some(new CarbonPolylineListExpression(polylineListString, buffer.toFloat,
- columnName, instance))
- case InPolygonRangeList(rangeListString, opType) =>
- val (columnName, instance) = GeoUtils.getGeoHashHandler(tableProperties)
- Some(new CarbonPolygonRangeListExpression(rangeListString, opType, columnName, instance))
- case _ => None
- }
+ protected[sql] def translateExpression(
+ relation: CarbonDatasourceHadoopRelation,
+ predicate: SparkExpression,
+ columnTypes: Map[String, SparkDataType],
+ isOr: Boolean = false): Option[Expression] = {
+
+ predicate match {
+ case u: ScalaUDF =>
+ translateUDF(u.children, u.function, relation)
+ case Or(left, right) =>
+ translateOr(left, right, relation, columnTypes)
+ case And(left, right) =>
+ translateAnd(left, right, relation, columnTypes, isOr)
+ case EqualTo(a: Attribute, Literal(v, _)) =>
+ translateEqualTo(a.name, v, columnTypes)
+ case EqualTo(Literal(v, _), a: Attribute) =>
+ translateEqualTo(a.name, v, columnTypes)
+ case c@EqualTo(Cast(_: Attribute, _, _), _: Literal) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@EqualTo(_: Literal, Cast(_: Attribute, _, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case Not(EqualTo(a: Attribute, Literal(v, _))) =>
+ translateNotEqualTo(a.name, v, columnTypes)
+ case Not(EqualTo(Literal(v, _), a: Attribute)) =>
+ translateNotEqualTo(a.name, v, columnTypes)
+ case c@Not(EqualTo(Cast(_: Attribute, _, _), _: Literal)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@Not(EqualTo(_: Literal, Cast(_: Attribute, _, _))) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case IsNotNull(a: Attribute) =>
+ translateNotEqualTo(a.name, null, columnTypes, true)
+ case IsNull(a: Attribute) =>
+ translateEqualTo(a.name, null, columnTypes, true)
+ case Not(In(a: Attribute, list)) if list.forall(_.isInstanceOf[Literal]) =>
+ translateNotIn(a.name, list.map(e => e.eval(EmptyRow)), columnTypes)
+ case In(a: Attribute, list) if list.forall(_.isInstanceOf[Literal]) =>
+ translateIn(a.name, list.map(e => e.eval(EmptyRow)), columnTypes)
+ case c@Not(In(Cast(_: Attribute, _, _), list)) if list.forall(_.isInstanceOf[Literal]) =>
+ Some(transformExpression(c))
+ case c@In(Cast(_: Attribute, _, _), list) if list.forall(_.isInstanceOf[Literal]) =>
+ Some(transformExpression(c))
+ case InSet(a: Attribute, set) =>
+ translateIn(a.name, set.toSeq, columnTypes)
+ case Not(InSet(a: Attribute, set)) =>
+ translateNotIn(a.name, set.toSeq, columnTypes)
+ case GreaterThan(a: Attribute, Literal(v, _)) =>
+ translateGreaterThan(a.name, v, columnTypes)
+ case GreaterThan(Literal(v, _), a: Attribute) =>
+ translateLessThan(a.name, v, columnTypes)
+ case c@GreaterThan(Cast(_: Attribute, _, _), _: Literal) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@GreaterThan(_: Literal, Cast(_: Attribute, _, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case LessThan(a: Attribute, Literal(v, _)) =>
+ translateLessThan(a.name, v, columnTypes)
+ case LessThan(Literal(v, _), a: Attribute) =>
+ translateGreaterThan(a.name, v, columnTypes)
+ case c@LessThan(Cast(_: Attribute, _, _), _: Literal) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@LessThan(_: Literal, Cast(_: Attribute, _, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case GreaterThanOrEqual(a: Attribute, Literal(v, _)) =>
+ translateGreaterThanEqual(a.name, v, columnTypes)
+ case GreaterThanOrEqual(Literal(v, _), a: Attribute) =>
+ translateLessThanEqual(a.name, v, columnTypes)
+ case c@GreaterThanOrEqual(Cast(_: Attribute, _, _), _: Literal) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@GreaterThanOrEqual(_: Literal, Cast(_: Attribute, _, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case LessThanOrEqual(a: Attribute, Literal(v, _)) =>
+ translateLessThanEqual(a.name, v, columnTypes)
+ case LessThanOrEqual(Literal(v, _), a: Attribute) =>
+ translateGreaterThanEqual(a.name, v, columnTypes)
+ case c@LessThanOrEqual(Cast(_: Attribute, _, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@LessThanOrEqual(_: Literal, Cast(_: Attribute, _, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case StartsWith(a: Attribute, Literal(v, _)) if v.toString.nonEmpty =>
+ translateStartsWith(a.name, v, columnTypes)
+ case c@EndsWith(_: Attribute, _: Literal) =>
+ Some(getSparkUnknownExpression(c, ExpressionType.ENDSWITH))
+ case c@Contains(_: Attribute, _: Literal) =>
+ Some(getSparkUnknownExpression(c, ExpressionType.CONTAINSWITH))
+ case Literal(v, _) if v == null =>
+ Some(new FalseExpression(null))
+ case ArrayContains(a: Attribute, Literal(v, _)) =>
+ translateArrayContains(a, v, columnTypes)
+ case ac@ArrayContains(Cast(_: Attribute, _, _), _: Literal) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(EqualTo(ac.left, ac.right))
+ case _ => None
}
+ }
- def getCarbonExpression(name: String) = {
- var sparkDatatype = dataTypeOf(name)
- new CarbonColumnExpression(name,
- CarbonSparkDataSourceUtil.convertSparkToCarbonDataType(sparkDatatype))
+ private def translateArrayContains(a: Attribute,
+ v: Any,
+ columnTypes: Map[String, SparkDataType]) = {
+ a.dataType match {
+ case arrayType: ArrayType =>
+ if (SparkUtil.isPrimitiveType(arrayType.elementType)) {
+ val dt = translateDataType(a.name, columnTypes)
+ Some(new EqualToExpression(translateColumn(a.name, dt), translateLiteral(v, dt)))
+ } else {
+ None
+ }
+ case _ =>
+ None
}
+ }
- def getCarbonLiteralExpression(name: String, value: Any): CarbonExpression = {
- var sparkDatatype = dataTypeOf(name)
- val dataTypeOfAttribute =
- CarbonSparkDataSourceUtil.convertSparkToCarbonDataType(sparkDatatype)
- val dataType = if (Option(value).isDefined
- && dataTypeOfAttribute == CarbonDataTypes.STRING
- && value.isInstanceOf[Double]) {
- CarbonDataTypes.DOUBLE
- } else {
- dataTypeOfAttribute
- }
- val dataValue = if (dataTypeOfAttribute.equals(CarbonDataTypes.BINARY)
- && Option(value).isDefined) {
- new String(value.asInstanceOf[Array[Byte]])
+ private def translateStartsWith(
+ name: String,
+ v: Any,
+ columnTypes: Map[String, SparkDataType]): Option[StartsWithExpression] = {
+ val dt = translateDataType(name, columnTypes)
+ Some(new StartsWithExpression(translateColumn(name, dt), translateLiteral(v.toString, dt)))
+ }
+
+ private def translateLessThanEqual(
+ name: String,
+ v: Any,
+ columnTypes: Map[String, SparkDataType]): Option[LessThanEqualToExpression] = {
+ val dt = translateDataType(name, columnTypes)
+ Some(new LessThanEqualToExpression(translateColumn(name, dt), translateLiteral(v, dt)))
+ }
+
+ private def translateGreaterThanEqual(
+ name: String,
+ v: Any,
+ columnTypes: Map[String, SparkDataType]): Option[GreaterThanEqualToExpression] = {
+ val dt = translateDataType(name, columnTypes)
+ Some(new GreaterThanEqualToExpression(translateColumn(name, dt), translateLiteral(v, dt)))
+ }
+
+ private def translateLessThan(
+ name: String,
+ v: Any,
+ columnTypes: Map[String, SparkDataType]): Option[LessThanExpression] = {
+ val dt = translateDataType(name, columnTypes)
+ Some(new LessThanExpression(translateColumn(name, dt), translateLiteral(v, dt)))
+ }
+
+ private def translateGreaterThan(
+ name: String,
+ v: Any,
+ columnTypes: Map[String, SparkDataType]): Option[GreaterThanExpression] = {
+ val dt = translateDataType(name, columnTypes)
+ Some(new GreaterThanExpression(translateColumn(name, dt), translateLiteral(v, dt)))
+ }
+
+ def translateUDF(
+ children: Seq[SparkExpression],
+ function: AnyRef,
+ relation: CarbonDatasourceHadoopRelation): Option[Expression] = {
+ function match {
+ case _: TextMatchUDF =>
+ if (children.size > 1) {
+ throw new MalformedCarbonCommandException(
+ "TEXT_MATCH UDF syntax: TEXT_MATCH('luceneQuerySyntax')")
+ }
+ Some(new MatchExpression(children.head.toString()))
+ case _: TextMatchMaxDocUDF =>
+ if (children.size > 2) {
+ throw new MalformedCarbonCommandException(
+ "TEXT_MATCH UDF syntax: TEXT_MATCH_LIMIT('luceneQuerySyntax')")
+ }
+ Some(new MatchExpression(children.head.toString(),
+ Try(children.last.toString().toInt).getOrElse(Integer.MAX_VALUE)))
+ case _: InPolygonUDF =>
+ if (children.size > 1) {
+ throw new MalformedCarbonCommandException("Expect one string in polygon")
+ }
+ val (columnName, instance) = getGeoHashHandler(relation.carbonTable)
+ Some(new PolygonExpression(children.head.toString(), columnName, instance))
+ case _: InPolygonListUDF =>
+ if (children.size != 2) {
+ throw new MalformedCarbonCommandException("Expect two string in polygon list")
+ }
+ val (columnName, instance) = getGeoHashHandler(relation.carbonTable)
+ Some(new PolygonListExpression(children.head.toString(), children.last.toString(),
+ columnName, instance))
+ case _: InPolylineListUDF =>
+ if (children.size != 2) {
+ throw new MalformedCarbonCommandException("Expect two string in polyline list")
+ }
+ val (columnName, instance) = getGeoHashHandler(relation.carbonTable)
+ Some(new PolylineListExpression(children.head.toString(),
+ children.last.toString().toFloat, columnName, instance))
+ case _: InPolygonRangeListUDF =>
+ if (children.size != 2) {
+ throw new MalformedCarbonCommandException("Expect two string in polygon range list")
+ }
+ val (columnName, instance) = getGeoHashHandler(relation.carbonTable)
+ Some(new PolygonRangeListExpression(children.head.toString(), children.last.toString(),
+ columnName, instance))
+ case _ => None
+ }
+ }
+
+ private def getGeoHashHandler(carbonTable: CarbonTable) = {
+ val tableProperties = carbonTable.getTableInfo.getFactTable.getTableProperties.asScala
+ GeoUtils.getGeoHashHandler(tableProperties)
+ }
+
+ def translateOr(
+ left : SparkExpression,
+ right: SparkExpression,
+ relation: CarbonDatasourceHadoopRelation,
+ columnTypes: Map[String, SparkDataType]) : Option[OrExpression] = {
+ val leftExpression = translateExpression(relation, left, columnTypes, true)
+ val rightExpression = translateExpression(relation, right, columnTypes, true)
+ if (leftExpression.isDefined && rightExpression.isDefined) {
+ Some(new OrExpression(leftExpression.get, rightExpression.get))
+ } else {
+ None
+ }
+ }
+
+ def translateAnd(
+ left : SparkExpression,
+ right: SparkExpression,
+ relation: CarbonDatasourceHadoopRelation,
+ columnTypes: Map[String, SparkDataType],
+ isOr: Boolean): Option[Expression] = {
+ val leftExpression = translateExpression(relation, left, columnTypes, isOr)
+ val rightExpression = translateExpression(relation, right, columnTypes, isOr)
+ if (isOr) {
+ if (leftExpression.isDefined && rightExpression.isDefined) {
+ (leftExpression ++ rightExpression).reduceOption(new AndExpression(_, _))
} else {
- value
+ None
}
- new CarbonLiteralExpression(dataValue, dataType)
+ } else {
+ (leftExpression ++ rightExpression).reduceOption(new AndExpression(_, _))
+ }
+ }
+
+ def translateDataType(name: String, columnTypes: Map[String, SparkDataType]): DataType = {
+ CarbonSparkDataSourceUtil.convertSparkToCarbonDataType(columnTypes(name))
+ }
+
+ def translateColumn(name: String, dataType: SparkDataType): ColumnExpression = {
+ translateColumn(name, CarbonSparkDataSourceUtil.convertSparkToCarbonDataType(dataType))
+ }
+
+ def translateColumn(name: String, dataType: DataType): ColumnExpression = {
+ new ColumnExpression(name, dataType)
+ }
+
+ def translateLiteral(value: Any, dataType: SparkDataType): Expression = {
+ translateLiteral(value, CarbonSparkDataSourceUtil.convertSparkToCarbonDataType(dataType))
+ }
+
+ def translateLiteral(value: Any, dataTypeOfAttribute: DataType): Expression = {
+ val dataType = if (Option(value).isDefined
+ && dataTypeOfAttribute == DataTypes.STRING
+ && value.isInstanceOf[Double]) {
+ DataTypes.DOUBLE
+ } else {
+ dataTypeOfAttribute
+ }
+ val dataValue = if (dataTypeOfAttribute.equals(DataTypes.BINARY)
+ && Option(value).isDefined) {
+ new String(value.asInstanceOf[Array[Byte]])
+ } else {
+ value
}
+ new LiteralExpression(dataValue, dataType)
+ }
+
+ def translateEqualTo(
+ name: String,
+ value: Any,
+ columnTypes: Map[String, SparkDataType],
+ isNull: Boolean = false): Option[EqualToExpression] = {
+ val dt = translateDataType(name, columnTypes)
+ Some(new EqualToExpression(translateColumn(name, dt), translateLiteral(value, dt), isNull))
+ }
+
+ def translateNotEqualTo(
+ name: String,
+ value: Any,
+ columnTypes: Map[String, SparkDataType],
+ isNotNull: Boolean = false): Option[NotEqualsExpression] = {
+ val dt = translateDataType(name, columnTypes)
+ Some(new NotEqualsExpression(translateColumn(name, dt), translateLiteral(value, dt), isNotNull))
+ }
+
+ private def translateIn(name: String,
+ list: Seq[Any],
+ columnTypes: Map[String, SparkDataType]): Option[Expression] = {
+ val dt = translateDataType(name, columnTypes)
+ if (list.length == 1 && list.head == null) {
+ Some(new FalseExpression(translateColumn(name, dt)))
+ } else {
+ Some(new InExpression(translateColumn(name, dt),
+ new ListExpression(convertToJavaList(list.filterNot(_ == null)
+ .map(filterValues => translateLiteral(filterValues, dt))))))
+ }
+ }
- createFilter(predicate)
+ private def translateNotIn(
+ name: String,
+ list: Seq[Any],
+ columnTypes: Map[String, SparkDataType]): Option[Expression] = {
+ val dt = translateDataType(name, columnTypes)
+ if (list.contains(null)) {
+ Some(new FalseExpression(translateColumn(name, dt)))
+ } else {
+ Some(new NotInExpression(translateColumn(name, dt),
+ new ListExpression(list.map(f => translateLiteral(f, dt)).asJava)))
+ }
}
- private def getSparkUnknownExpression(expr: Expression,
+ private def getSparkUnknownExpression(expr: SparkExpression,
exprType: ExpressionType = ExpressionType.UNKNOWN) = {
new SparkUnknownExpression(expr.transform {
case AttributeReference(name, dataType, _, _) =>
- CarbonBoundReference(new CarbonColumnExpression(name,
+ CarbonBoundReference(new ColumnExpression(name,
CarbonSparkDataSourceUtil.convertSparkToCarbonDataType(dataType)),
dataType, expr.nullable)
}, exprType)
@@ -216,103 +400,85 @@ object CarbonFilters {
}
}
- def transformExpression(expr: Expression): CarbonExpression = {
+ def transformExpression(expr: SparkExpression): Expression = {
expr match {
- case plan if (CarbonHiveIndexMetadataUtil.checkNIUDF(plan)) =>
+ case plan if CarbonHiveIndexMetadataUtil.checkNIUDF(plan) =>
transformExpression(CarbonHiveIndexMetadataUtil.getNIChildren(plan))
- case Or(left, right)
- if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) => new
- OrExpression(transformExpression(left), transformExpression(right))
- case And(left, right)
- if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) => new
- AndExpression(transformExpression(left), transformExpression(right))
- case EqualTo(left, right)
- if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) => new
- EqualToExpression(transformExpression(left), transformExpression(right))
- case Not(EqualTo(left, right))
- if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) => new
- NotEqualsExpression(transformExpression(left), transformExpression(right))
- case IsNotNull(child)
- if (isCarbonSupportedDataTypes(child)) => new
- NotEqualsExpression(transformExpression(child), transformExpression(Literal(null)), true)
- case IsNull(child)
- if (isCarbonSupportedDataTypes(child)) => new
- EqualToExpression(transformExpression(child), transformExpression(Literal(null)), true)
- case Not(In(left, right)) if (isCarbonSupportedDataTypes(left)) =>
+ case Or(left, right) if canPushDown(left, right) =>
+ new OrExpression(transformExpression(left), transformExpression(right))
+ case And(left, right) if canPushDown(left, right) =>
+ new AndExpression(transformExpression(left), transformExpression(right))
+ case EqualTo(left, right) if canPushDown(left, right) =>
+ new EqualToExpression(transformExpression(left), transformExpression(right))
+ case Not(EqualTo(left, right)) if canPushDown(left, right) =>
+ new NotEqualsExpression(transformExpression(left), transformExpression(right))
+ case IsNotNull(child) if canPushDown(child) =>
+ new NotEqualsExpression(transformExpression(child), transformExpression(Literal(null)),
+ true)
+ case IsNull(child) if canPushDown(child) =>
+ new EqualToExpression(transformExpression(child), transformExpression(Literal(null)), true)
+ case Not(In(left, right)) if canPushDown(left) =>
if (right.contains(null)) {
new FalseExpression(transformExpression(left))
- }
- else {
+ } else {
new NotInExpression(transformExpression(left),
new ListExpression(convertToJavaList(right.map(transformExpression)))
)
}
- case In(left, right) if (isCarbonSupportedDataTypes(left)) =>
+ case In(left, right) if canPushDown(left) =>
left match {
- case left: AttributeReference if (left.name
- .equalsIgnoreCase(CarbonCommonConstants.POSITION_ID)) =>
+ case left: AttributeReference
+ if left.name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID) =>
new InExpression(transformExpression(left),
new ImplicitExpression(convertToJavaList(right.filter(_ != null)
.filter(!isNullLiteral(_))
.map(transformExpression))))
case _ =>
- new InExpression(transformExpression(left),
- new ListExpression(convertToJavaList(right.filter(_ != null).filter(!isNullLiteral(_))
- .map(transformExpression))))
+ new InExpression(transformExpression(left), new ListExpression(convertToJavaList(
+ right.filter(_ != null).filter(!isNullLiteral(_)).map(transformExpression))))
}
- case InSet(left, right) if (isCarbonSupportedDataTypes(left)) =>
+ case InSet(left, right) if canPushDown(left) =>
val validData = right.filter(_ != null).map { x =>
- val e = Literal(x.toString)
- transformExpression(e)
+ transformExpression(Literal(x.toString))
}.toList
new InExpression(transformExpression(left),
new ListExpression(convertToJavaList(validData)))
- case Not(InSet(left, right)) if (isCarbonSupportedDataTypes(left)) =>
+ case Not(InSet(left, right)) if canPushDown(left) =>
if (right.contains(null)) {
new FalseExpression(transformExpression(left))
- }
- else {
+ } else {
val r = right.map { x =>
val strVal = if (null == x) {
x
} else {
x.toString
}
- val e = Literal(strVal)
- transformExpression(e)
+ transformExpression(Literal(strVal))
}.toList
new NotInExpression(transformExpression(left), new ListExpression(convertToJavaList(r)))
}
- case GreaterThan(left, right)
- if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) =>
+ case GreaterThan(left, right) if canPushDown(left, right) =>
new GreaterThanExpression(transformExpression(left), transformExpression(right))
- case GreaterThanOrEqual(left, right)
- if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) =>
+ case GreaterThanOrEqual(left, right) if canPushDown(left, right) =>
new GreaterThanEqualToExpression(transformExpression(left), transformExpression(right))
- case LessThan(left, right)
- if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) =>
+ case LessThan(left, right) if canPushDown(left, right) =>
new LessThanExpression(transformExpression(left), transformExpression(right))
- case LessThanOrEqual(left, right)
- if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) =>
+ case LessThanOrEqual(left, right) if canPushDown(left, right) =>
new LessThanEqualToExpression(transformExpression(left), transformExpression(right))
case AttributeReference(name, dataType, _, _) =>
- new CarbonColumnExpression(name.toString,
- CarbonSparkDataSourceUtil.convertSparkToCarbonDataType(dataType))
+ new ColumnExpression(name, CarbonSparkDataSourceUtil.convertSparkToCarbonDataType(dataType))
case Literal(name, dataType) => new
- CarbonLiteralExpression(name,
+ LiteralExpression(name,
CarbonSparkDataSourceUtil.convertSparkToCarbonDataType(dataType))
- case StartsWith(left, right@Literal(pattern, dataType)) if pattern.toString.size > 0 &&
- isCarbonSupportedDataTypes(left) &&
- isCarbonSupportedDataTypes
- (right) =>
+ case StartsWith(left, right@Literal(pattern, dataType))
+ if pattern.toString.nonEmpty && canPushDown(left, right) =>
val l = new GreaterThanEqualToExpression(transformExpression(left),
transformExpression(right))
- val maxValueLimit = pattern.toString.substring(0, pattern.toString.length - 1) +
- (pattern.toString.charAt(pattern.toString.length - 1).toInt + 1)
- .toChar
- val r = new LessThanExpression(
- transformExpression(left),
- new CarbonLiteralExpression(maxValueLimit,
+ val patternText = pattern.toString
+ val lastChar = (patternText.charAt(patternText.length - 1).toInt + 1).toChar
+ val maxValueLimit = patternText.substring(0, patternText.length - 1) + lastChar
+ val r = new LessThanExpression(transformExpression(left),
+ new LiteralExpression(maxValueLimit,
CarbonSparkDataSourceUtil.convertSparkToCarbonDataType(dataType)))
new AndExpression(l, r)
case strTrim: StringTrim if isStringTrimCompatibleWithCarbon(strTrim) =>
@@ -321,8 +487,7 @@ object CarbonFilters {
}
}
-
- private def isNullLiteral(exp: Expression): Boolean = {
+ private def isNullLiteral(exp: SparkExpression): Boolean = {
if (null != exp
&& exp.isInstanceOf[Literal]
&& (exp.asInstanceOf[Literal].dataType == org.apache.spark.sql.types.DataTypes.NullType)
@@ -333,7 +498,7 @@ object CarbonFilters {
}
}
- def isCarbonSupportedDataTypes(expr: Expression): Boolean = {
+ def canPushDown(expr: SparkExpression): Boolean = {
expr.dataType match {
case StringType => true
case IntegerType => true
@@ -350,14 +515,17 @@ object CarbonFilters {
}
}
+ def canPushDown(left: SparkExpression, right: SparkExpression): Boolean = {
+ canPushDown(left) && canPushDown(right)
+ }
+
// Convert scala list to java list, Cannot use scalaList.asJava as while deserializing it is
// not able find the classes inside scala list and gives ClassNotFoundException.
- private def convertToJavaList(
- scalaList: Seq[CarbonExpression]): java.util.List[CarbonExpression] = {
+ private def convertToJavaList(scalaList: Seq[Expression]): java.util.List[Expression] = {
CarbonSparkDataSourceUtil.convertToJavaList(scalaList)
}
- def preProcessExpressions(expressions: Seq[Expression]): Seq[Expression] = {
+ def preProcessExpressions(expressions: Seq[SparkExpression]): Seq[SparkExpression] = {
expressions match {
case left :: right :: rest => preProcessExpressions(List(And(left, right)) ::: rest)
case List(left, right) => List(And(left, right))
@@ -368,7 +536,7 @@ object CarbonFilters {
def getCurrentPartitions(sparkSession: SparkSession,
tableIdentifier: TableIdentifier): Option[Seq[PartitionSpec]] = {
- CarbonFilters.getPartitions(
+ getPartitions(
Seq.empty,
sparkSession,
tableIdentifier)
@@ -376,202 +544,83 @@ object CarbonFilters {
def getCurrentPartitions(sparkSession: SparkSession,
carbonTable: CarbonTable): Option[Seq[PartitionSpec]] = {
- CarbonFilters.getPartitions(
+ getPartitions(
Seq.empty,
sparkSession,
carbonTable)
}
- def getPartitions(partitionFilters: Seq[Expression],
+ def getPartitions(partitionFilters: Seq[SparkExpression],
sparkSession: SparkSession,
identifier: TableIdentifier): Option[Seq[PartitionSpec]] = {
val carbonTable = CarbonEnv.getCarbonTable(identifier)(sparkSession)
getPartitions(partitionFilters, sparkSession, carbonTable)
}
- def getStorageOrdinal(filter: Filter, carbonTable: CarbonTable): Int = {
- val column = filter.references.map(carbonTable.getColumnByName)
- if (column.isEmpty) {
- -1
- } else {
- if (column.head.isDimension) {
- column.head.getOrdinal
- } else {
- column.head.getOrdinal + carbonTable.getAllDimensions.size()
- }
- }
- }
-
- def collectSimilarExpressions(filter: Filter, table: CarbonTable): Seq[(Filter, Int)] = {
- filter match {
- case sources.And(left, right) =>
- collectSimilarExpressions(left, table) ++ collectSimilarExpressions(right, table)
- case sources.Or(left, right) => collectSimilarExpressions(left, table) ++
- collectSimilarExpressions(right, table)
- case others => Seq((others, getStorageOrdinal(others, table)))
- }
- }
-
/**
- * This method will reorder the filter based on the Storage Ordinal of the column references.
- *
- * Example1:
- * And And
- * Or And => Or And
- * col3 col1 col2 col1 col1 col3 col1 col2
- *
- * **Mixed expression filter reordered locally, but wont be reordered globally.**
- *
- * Example2:
- * And And
- * And And => And And
- * col3 col1 col2 col1 col1 col1 col2 col3
- *
- * Or Or
- * Or Or => Or Or
- * col3 col1 col2 col1 col1 col1 col2 col3
- *
- * **Similar expression filters are reordered globally**
- *
- * @param filters the filter expressions to be reordered
- * @return The reordered filter with the current ordinal
+ * Fetches partition information from hive
*/
- def reorderFilter(filters: Array[Filter], table: CarbonTable): Array[Filter] = {
- val filterMap = mutable.HashMap[String, List[(Filter, Int)]]()
- if (!CarbonProperties.isFilterReorderingEnabled) {
- filters
- } else {
- filters.collect {
- // If the filter size is one or the user has disabled reordering then no need to reorder.
- case filter if filter.references.toSet.size == 1 =>
- (filter, getStorageOrdinal(filter, table))
- case filter =>
- val sortedFilter = sortFilter(filter, filterMap, table)
- // If filter has only AND/OR expression then sort the nodes globally using the filterMap.
- // Else sort the subnodes individually
- if (!filterMap.contains("OR") && filterMap.contains("AND") && filterMap("AND").nonEmpty) {
- val sortedFilterAndOrdinal = filterMap("AND").sortBy(_._2)
- (sortedFilterAndOrdinal.map(_._1).reduce(sources.And), sortedFilterAndOrdinal.head._2)
- } else if (!filterMap.contains("AND") && filterMap.contains("OR") &&
- filterMap("OR").nonEmpty) {
- val sortedFilterAndOrdinal = filterMap("OR").sortBy(_._2)
- (sortedFilterAndOrdinal.map(_._1).reduce(sources.Or), sortedFilterAndOrdinal.head._2)
- } else {
- sortedFilter
- }
- }.sortBy(_._2).map(_._1)
+ def getPartitions(partitionFilters: Seq[SparkExpression],
+ sparkSession: SparkSession,
+ carbonTable: CarbonTable): Option[Seq[PartitionSpec]] = {
+ if (!carbonTable.isHivePartitionTable) {
+ return None
}
+ val partitions = getCatalogTablePartitions(partitionFilters, sparkSession, carbonTable)
+ Some(convertToPartitionSpec(partitions))
}
- def generateNewFilter(filterType: String, left: Filter, right: Filter,
- filterMap: mutable.HashMap[String, List[(Filter, Int)]],
- table: CarbonTable): (Filter, Int) = {
- filterMap.getOrElseUpdate(filterType, List())
- // Generate a function which can handle both AND/OR.
- val newFilter: (Filter, Filter) => Filter = filterType match {
- case "OR" => sources.Or
- case _ => sources.And
- }
- if (checkIfRightIsASubsetOfLeft(left, right)) {
- val (sorted, ordinal) = sortFilter(left, filterMap, table)
- val rightOrdinal = getStorageOrdinal(right, table)
- val orderedFilter = if (ordinal >= rightOrdinal) {
- (newFilter(right, sorted), rightOrdinal)
- } else {
- (newFilter(sorted, right), ordinal)
- }
- if (isLeafNode(left)) {
- filterMap.put(filterType, filterMap(filterType) ++ List((sorted, ordinal)))
- }
- if (isLeafNode(right)) {
- filterMap.put(filterType, filterMap(filterType) ++ List((right, rightOrdinal)))
- }
- orderedFilter
- } else {
- val (leftSorted, leftOrdinal) = sortFilter(left, filterMap, table)
- val (rightSorted, rightOrdinal) = sortFilter(right, filterMap, table)
- val orderedFilter = if (leftOrdinal > rightOrdinal) {
- (newFilter(rightSorted, leftSorted), rightOrdinal)
- } else {
- (newFilter(leftSorted, rightSorted), leftOrdinal)
- }
- if (isLeafNode(left)) {
- filterMap.put(filterType, filterMap(filterType) ++ List((leftSorted, leftOrdinal)))
- }
- if (isLeafNode(right)) {
- filterMap.put(filterType, filterMap(filterType) ++ List((rightSorted, rightOrdinal)))
- }
- orderedFilter
+ def convertToPartitionSpec(partitions: Seq[CatalogTablePartition]): Seq[PartitionSpec] = {
+ partitions.map { partition =>
+ new PartitionSpec(
+ new ArrayList[String](partition.spec.seq.map { case (column, value) =>
+ column + "=" + value
+ }.toList.asJava), partition.location)
}
}
- def sortFilter(filter: Filter, filterMap: mutable.HashMap[String, List[(Filter, Int)]],
- table: CarbonTable): (Filter, Int) = {
- filter match {
- case sources.And(left, right) =>
- generateNewFilter("AND", left, right, filterMap, table)
- case sources.Or(left, right) =>
- generateNewFilter("OR", left, right, filterMap, table)
- case others => (others, getStorageOrdinal(others, table))
+ def getCatalogTablePartitions(
+ partitionFilters: Seq[SparkExpression],
+ sparkSession: SparkSession,
+ carbonTable: CarbonTable): Seq[CatalogTablePartition] = {
+ val identifier = TableIdentifier(carbonTable.getTableName, Some(carbonTable.getDatabaseName))
+ try {
+ if (CarbonProperties.getInstance().
+ getProperty(CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT,
+ CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT_DEFAULT).toBoolean) {
+ // read partitions directly from hive metastore using filters
+ sparkSession.sessionState.catalog.listPartitionsByFilter(identifier, partitionFilters)
+ } else {
+ // Read partitions alternatively by first get all partitions then filter them
+ CarbonSessionCatalogUtil.getPartitionsAlternate(
+ partitionFilters,
+ sparkSession,
+ carbonTable)
+ }
+ } catch {
+ case e: Exception =>
+ // Get partition information alternatively.
+ CarbonSessionCatalogUtil.getPartitionsAlternate(
+ partitionFilters,
+ sparkSession,
+ carbonTable)
}
}
- /**
- * Checks if the filter node is a leaf node, here leaf node means node should not be AND/OR.
- *
- * @return true if leaf node, false otherwise
- */
- private def isLeafNode(filter: Filter): Boolean = {
- !filter.isInstanceOf[sources.Or] && !filter.isInstanceOf[sources.And]
- }
-
- /**
- * Checks if the references in right subtree of a filter are a subset of the left references.
- * @return true if right is a subset, otherwise false
- */
- private def checkIfRightIsASubsetOfLeft(left: Filter, right: Filter): Boolean = {
- left.references.toSeq == right.references.toSeq ||
- right.references.diff(left.references).length == 0
- }
-
- /**
- * Fetches partition information from hive
- */
- def getPartitions(partitionFilters: Seq[Expression],
+ def getCatalogTablePartitions(
+ partitionFilters: Seq[SparkExpression],
sparkSession: SparkSession,
- carbonTable: CarbonTable): Option[Seq[PartitionSpec]] = {
- val identifier = TableIdentifier(carbonTable.getTableName, Some(carbonTable.getDatabaseName))
+ identifier: TableIdentifier
+ ): (Seq[CatalogTablePartition], Seq[PartitionSpec], Seq[SparkExpression]) = {
+ val carbonTable = CarbonEnv.getCarbonTable(identifier)(sparkSession)
if (!carbonTable.isHivePartitionTable) {
- return None
+ return (null, null, partitionFilters)
}
- val partitions = {
- try {
- if (CarbonProperties.getInstance().
- getProperty(CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT,
- CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT_DEFAULT).toBoolean) {
- // read partitions directly from hive metastore using filters
- sparkSession.sessionState.catalog.listPartitionsByFilter(identifier, partitionFilters)
- } else {
- // Read partitions alternatively by first get all partitions then filter them
- CarbonSessionCatalogUtil.getPartitionsAlternate(
- partitionFilters,
- sparkSession,
- carbonTable)
- }
- } catch {
- case e: Exception =>
- // Get partition information alternatively.
- CarbonSessionCatalogUtil.getPartitionsAlternate(
- partitionFilters,
- sparkSession,
- carbonTable)
- }
+ val partititions = getCatalogTablePartitions(partitionFilters, sparkSession, carbonTable)
+ if (partititions.isEmpty) {
+ (Seq.empty, Seq.empty, partitionFilters)
+ } else {
+ (partititions, convertToPartitionSpec(partititions), partitionFilters)
}
- Some(partitions.map { partition =>
- new PartitionSpec(
- new ArrayList[String]( partition.spec.seq.map{case (column, value) =>
- column + "=" + value}.toList.asJava), partition.location)
- })
}
-
}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala
index 83fb83e..75af9bf 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVMatcher.scala
@@ -287,16 +287,7 @@ private abstract class MVMatchPattern extends Logging {
*/
protected def getTransformedCastOrImplicitCastExpression(
castOrImplicitCastExp: Expression): Expression = {
- castOrImplicitCastExp.transform {
- case reference: AttributeReference =>
- CarbonToSparkAdapter.createAttributeReference(
- reference.name.toLowerCase,
- reference.dataType,
- reference.nullable,
- reference.metadata,
- reference.exprId,
- reference.qualifier)
- }
+ CarbonToSparkAdapter.lowerCaseAttribute(castOrImplicitCastExp)
}
def tryMatch(a: modular.Matchable,
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/joins/BroadCastSIFilterPushJoin.scala b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/joins/BroadCastSIFilterPushJoin.scala
index 1402fa2..d925081 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/joins/BroadCastSIFilterPushJoin.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/secondaryindex/joins/BroadCastSIFilterPushJoin.scala
@@ -83,10 +83,7 @@ case class BroadCastSIFilterPushJoin(
}
lazy val secondaryIndexRDD: Seq[RDD[InternalRow]] = buildPlan.collect {
- case batchData: CarbonDataSourceScan =>
- batchData.rdd
- case rowData: RowDataSourceScanExec =>
- rowData.rdd
+ case scan: CarbonDataSourceScan => scan.inputRDDs().head
}
private lazy val inputCopy: Array[InternalRow] = {
@@ -119,10 +116,7 @@ case class BroadCastSIFilterPushJoin(
}
val mainTableRDD: Option[RDD[InternalRow]] = carbonScan.collectFirst {
- case batchData: CarbonDataSourceScan =>
- batchData.rdd
- case rowData: RowDataSourceScanExec =>
- rowData.rdd
+ case scan: CarbonDataSourceScan => scan.inputRDDs().head
}
override def doExecute(): RDD[InternalRow] = {
@@ -234,7 +228,7 @@ object BroadCastSIFilterPushJoin {
logger.info("Pushing down filter for broadcast join. Filter size:" + filters(0).length)
tableScan.get match {
case scan: CarbonDataSourceScan =>
- addPushDownToCarbonRDD(scan.rdd,
+ addPushDownToCarbonRDD(scan.inputRDDs().head,
addPushDownFilters(filterKeys, filters))
case _ =>
addPushDownToCarbonRDD(tableScan.get.asInstanceOf[RowDataSourceScanExec].rdd,
@@ -460,13 +454,9 @@ object BroadCastSIFilterPushJoin {
val collectRDD: Seq[CarbonScanRDD[InternalRow]] = dataFrame.queryExecution.sparkPlan
.collect {
case scan: CarbonDataSourceScan
- if scan.rdd.isInstanceOf[CarbonScanRDD[InternalRow]] &&
- isMainTableRDD(scan.rdd.asInstanceOf[CarbonScanRDD[InternalRow]]) =>
- scan.rdd.asInstanceOf[CarbonScanRDD[InternalRow]]
- case scan: RowDataSourceScanExec
- if scan.rdd.isInstanceOf[CarbonScanRDD[InternalRow]] &&
- isMainTableRDD(scan.rdd.asInstanceOf[CarbonScanRDD[InternalRow]]) =>
- scan.rdd.asInstanceOf[CarbonScanRDD[InternalRow]]
+ if scan.inputRDDs().head.isInstanceOf[CarbonScanRDD[InternalRow]] &&
+ isMainTableRDD(scan.inputRDDs().head.asInstanceOf[CarbonScanRDD[InternalRow]]) =>
+ scan.inputRDDs().head.asInstanceOf[CarbonScanRDD[InternalRow]]
}
// If collectRDD length is 0 or greater than 1 then throw exception
if (1 != collectRDD.length) {
diff --git a/integration/spark/src/main/spark2.3/org/apache/spark/sql/CarbonToSparkAdapter.scala b/integration/spark/src/main/spark2.3/org/apache/spark/sql/CarbonToSparkAdapter.scala
index 1215525..2eced66 100644
--- a/integration/spark/src/main/spark2.3/org/apache/spark/sql/CarbonToSparkAdapter.scala
+++ b/integration/spark/src/main/spark2.3/org/apache/spark/sql/CarbonToSparkAdapter.scala
@@ -51,6 +51,17 @@ object CarbonToSparkAdapter {
})
}
+ def lowerCaseAttribute(expression: Expression): Expression = expression.transform {
+ case attr: AttributeReference =>
+ CarbonToSparkAdapter.createAttributeReference(
+ attr.name.toLowerCase,
+ attr.dataType,
+ attr.nullable,
+ attr.metadata,
+ attr.exprId,
+ attr.qualifier)
+ }
+
def createAttributeReference(name: String, dataType: DataType, nullable: Boolean,
metadata: Metadata, exprId: ExprId, qualifier: Option[String],
attrRef : NamedExpression = null): AttributeReference = {
@@ -123,13 +134,18 @@ object CarbonToSparkAdapter {
* @param filterPredicates
* @return
*/
- def getPartitionKeyFilter(
+ def getPartitionFilter(
partitionSet: AttributeSet,
- filterPredicates: Seq[Expression]): ExpressionSet = {
- ExpressionSet(
- ExpressionSet(filterPredicates)
- .filterNot(SubqueryExpression.hasSubquery)
- .filter(_.references.subsetOf(partitionSet)))
+ filterPredicates: Seq[Expression]): Seq[Expression] = {
+ filterPredicates
+ .filterNot(SubqueryExpression.hasSubquery)
+ .filter { filter =>
+ filter.references.nonEmpty && filter.references.subsetOf(partitionSet)
+ }
+ }
+
+ def getDataFilter(partitionSet: AttributeSet, filter: Seq[Expression]): Seq[Expression] = {
+ filter
}
// As per SPARK-22520 OptimizeCodegen is removed in 2.3.1
diff --git a/integration/spark/src/main/spark2.3/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala b/integration/spark/src/main/spark2.3/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala
deleted file mode 100644
index 25d5543..0000000
--- a/integration/spark/src/main/spark2.3/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.execution.strategy
-
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder}
-import org.apache.spark.sql.catalyst.plans.physical.Partitioning
-import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
-import org.apache.spark.sql.execution.FileSourceScanExec
-import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
-
-/**
- * Physical plan node for scanning data. It is applied for both tables
- * USING carbondata and STORED AS CARBONDATA.
- */
-class CarbonDataSourceScan(
- override val output: Seq[Attribute],
- val rdd: RDD[InternalRow],
- @transient override val relation: HadoopFsRelation,
- val partitioning: Partitioning,
- val md: Map[String, String],
- identifier: Option[TableIdentifier],
- @transient private val logicalRelation: LogicalRelation)
- extends FileSourceScanExec(
- relation,
- output,
- relation.dataSchema,
- Seq.empty,
- Seq.empty,
- identifier) {
-
- // added lazy since spark 2.3.2 version (SPARK-PR#21815)
- override lazy val supportsBatch: Boolean = true
-
- // added lazy since spark 2.3.2 version (SPARK-PR#21815)
- override lazy val (outputPartitioning, outputOrdering): (Partitioning, Seq[SortOrder]) =
- (partitioning, Nil)
-
- // added lazy since spark 2.3.2 version (SPARK-PR#21815)
- override lazy val metadata: Map[String, String] = md
-
- override def inputRDDs(): Seq[RDD[InternalRow]] = rdd :: Nil
-
-}
diff --git a/integration/spark/src/main/spark2.3/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala b/integration/spark/src/main/spark2.3/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala
index 44e556a..aae7c26 100644
--- a/integration/spark/src/main/spark2.3/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala
+++ b/integration/spark/src/main/spark2.3/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala
@@ -27,9 +27,9 @@ import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.parser.ParserInterface
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier}
-import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
+import org.apache.spark.sql.execution.strategy.{CarbonSourceStrategy, DDLStrategy, DMLStrategy, StreamingTableStrategy}
import org.apache.spark.sql.hive.client.HiveClient
-import org.apache.spark.sql.internal.{SQLConf, SessionState}
+import org.apache.spark.sql.internal.{SessionState, SQLConf}
import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonUDFTransformRule}
import org.apache.spark.sql.parser.CarbonSparkSqlParser
import org.apache.spark.sql.{CarbonEnv, SparkSession}
@@ -149,8 +149,8 @@ class CarbonSessionStateBuilder(sparkSession: SparkSession,
override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession)
- experimentalMethods.extraStrategies =Seq(new StreamingTableStrategy(sparkSession),
- new CarbonLateDecodeStrategy, new DDLStrategy(sparkSession))
+ experimentalMethods.extraStrategies =
+ Seq(StreamingTableStrategy, DMLStrategy, DDLStrategy, CarbonSourceStrategy)
experimentalMethods.extraOptimizations = Seq(new CarbonIUDRule, new CarbonUDFTransformRule)
/**
diff --git a/integration/spark/src/main/spark2.4/org/apache/spark/sql/CarbonToSparkAdapter.scala b/integration/spark/src/main/spark2.4/org/apache/spark/sql/CarbonToSparkAdapter.scala
index a7a4bad..163d833 100644
--- a/integration/spark/src/main/spark2.4/org/apache/spark/sql/CarbonToSparkAdapter.scala
+++ b/integration/spark/src/main/spark2.4/org/apache/spark/sql/CarbonToSparkAdapter.scala
@@ -86,6 +86,17 @@ object CarbonToSparkAdapter {
metadata)(exprId, qualifier)
}
+ def lowerCaseAttribute(expression: Expression): Expression = expression.transform {
+ case attr: AttributeReference =>
+ CarbonToSparkAdapter.createAttributeReference(
+ attr.name.toLowerCase,
+ attr.dataType,
+ attr.nullable,
+ attr.metadata,
+ attr.exprId,
+ attr.qualifier)
+ }
+
def createAttributeReference(attr: AttributeReference,
attrName: String,
newSubsume: String): AttributeReference = {
@@ -151,18 +162,23 @@ object CarbonToSparkAdapter {
/**
* As a part of SPARK-24085 Hive tables supports scala subquery for
- * parition tables, so Carbon also needs to supports
+ * the partitioned tables,so Carbon also needs to supports
* @param partitionSet
* @param filterPredicates
* @return
*/
- def getPartitionKeyFilter(
+ def getPartitionFilter(
partitionSet: AttributeSet,
- filterPredicates: Seq[Expression]): ExpressionSet = {
- ExpressionSet(
- ExpressionSet(filterPredicates)
- .filterNot(SubqueryExpression.hasSubquery)
- .filter(_.references.subsetOf(partitionSet)))
+ filterPredicates: Seq[Expression]): Seq[Expression] = {
+ filterPredicates
+ .filterNot(SubqueryExpression.hasSubquery)
+ .filter { filter =>
+ filter.references.nonEmpty && filter.references.subsetOf(partitionSet)
+ }
+ }
+
+ def getDataFilter(partitionSet: AttributeSet, filter: Seq[Expression]): Seq[Expression] = {
+ filter
}
// As per SPARK-22520 OptimizeCodegen is removed in 2.3.1
diff --git a/integration/spark/src/main/spark2.4/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala b/integration/spark/src/main/spark2.4/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala
deleted file mode 100644
index 7627b0b..0000000
--- a/integration/spark/src/main/spark2.4/org/apache/spark/sql/execution/strategy/CarbonDataSourceScan.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.strategy
-
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
-import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder}
-import org.apache.spark.sql.catalyst.plans.physical.Partitioning
-import org.apache.spark.sql.execution.FileSourceScanExec
-import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
-
-/**
- * Physical plan node for scanning data. It is applied for both tables
- * USING carbondata and STORED AS CARBONDATA.
- */
-class CarbonDataSourceScan(
- override val output: Seq[Attribute],
- val rdd: RDD[InternalRow],
- @transient override val relation: HadoopFsRelation,
- val partitioning: Partitioning,
- val md: Map[String, String],
- identifier: Option[TableIdentifier],
- @transient private val logicalRelation: LogicalRelation)
- extends FileSourceScanExec(
- relation,
- output,
- relation.dataSchema,
- Seq.empty,
- None,
- Seq.empty,
- identifier) {
-
- // added lazy since spark 2.3.2 version (SPARK-PR#21815)
- override lazy val supportsBatch: Boolean = true
-
- // added lazy since spark 2.3.2 version (SPARK-PR#21815)
- override lazy val (outputPartitioning, outputOrdering): (Partitioning, Seq[SortOrder]) =
- (partitioning, Nil)
-
- // added lazy since spark 2.3.2 version (SPARK-PR#21815)
- override lazy val metadata: Map[String, String] = md
-
- override def inputRDDs(): Seq[RDD[InternalRow]] = rdd :: Nil
-
-}
diff --git a/integration/spark/src/main/spark2.4/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala b/integration/spark/src/main/spark2.4/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala
index 28b1e72..bace849 100644
--- a/integration/spark/src/main/spark2.4/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala
+++ b/integration/spark/src/main/spark2.4/org/apache/spark/sql/hive/CarbonSessionStateBuilder.scala
@@ -28,14 +28,13 @@ import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.parser.ParserInterface
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
+import org.apache.spark.sql.execution.strategy.{CarbonSourceStrategy, DDLStrategy, DMLStrategy, StreamingTableStrategy}
import org.apache.spark.sql.hive.client.HiveClient
-import org.apache.spark.sql.internal.{SQLConf, SessionState}
+import org.apache.spark.sql.internal.{SessionState, SQLConf}
import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonUDFTransformRule}
import org.apache.spark.sql.parser.CarbonSparkSqlParser
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
/**
* This class will have carbon catalog and refresh the relation from cache if the carbontable in
@@ -150,10 +149,7 @@ class CarbonSessionStateBuilder(sparkSession: SparkSession,
override lazy val sqlParser: ParserInterface = new CarbonSparkSqlParser(conf, sparkSession)
experimentalMethods.extraStrategies =
- Seq(new StreamingTableStrategy(sparkSession),
- new CarbonLateDecodeStrategy,
- new DDLStrategy(sparkSession)
- )
+ Seq(StreamingTableStrategy, DMLStrategy, DDLStrategy, CarbonSourceStrategy)
experimentalMethods.extraOptimizations = Seq(new CarbonIUDRule,
new CarbonUDFTransformRule)
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestArrayContainsPushDown.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestArrayContainsPushDown.scala
index 8ff94da..4bd8a86 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestArrayContainsPushDown.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestArrayContainsPushDown.scala
@@ -48,11 +48,11 @@ class TestArrayContainsPushDown extends QueryTest with BeforeAndAfterAll {
checkExistence(sql(" explain select * from complex1 where array_contains(arr,'sd')"),
true,
- "PushedFilters: [*EqualTo(arr,sd)]")
+ "PushedFilters: [arr = sd]")
checkExistence(sql(" explain select count(*) from complex1 where array_contains(arr,'sd')"),
true,
- "PushedFilters: [*EqualTo(arr,sd)]")
+ "PushedFilters: [arr = sd]")
checkAnswer(sql(" select * from complex1 where array_contains(arr,'sd')"),
Seq(Row(mutable.WrappedArray.make(Array("sd", "df", "gh"))),
@@ -101,11 +101,11 @@ class TestArrayContainsPushDown extends QueryTest with BeforeAndAfterAll {
checkExistence(sql(" explain select * from complex1 where array_contains(arr,true)"),
true,
- "PushedFilters: [*EqualTo(arr,true)]")
+ "PushedFilters: [arr = true]")
checkExistence(sql(" explain select count(*) from complex1 where array_contains(arr,true)"),
true,
- "PushedFilters: [*EqualTo(arr,true)]")
+ "PushedFilters: [arr = true]")
checkAnswer(sql(" select * from complex1 where array_contains(arr,true)"),
Seq(Row(mutable.WrappedArray.make(Array(true))),
@@ -127,11 +127,11 @@ class TestArrayContainsPushDown extends QueryTest with BeforeAndAfterAll {
checkExistence(sql(" explain select * from complex1 where array_contains(arr,31000)"),
true,
- "PushedFilters: [*EqualTo(arr,31000)]")
+ "PushedFilters: [arr = 31000]")
checkExistence(sql(" explain select count(*) from complex1 where array_contains(arr,31000)"),
true,
- "PushedFilters: [*EqualTo(arr,31000)]")
+ "PushedFilters: [arr = 31000]")
checkAnswer(sql(" select * from complex1 where array_contains(arr,31000)"),
Seq(Row(mutable.WrappedArray.make(Array(20, 30, 31000))),
@@ -153,11 +153,11 @@ class TestArrayContainsPushDown extends QueryTest with BeforeAndAfterAll {
checkExistence(sql(" explain select * from complex1 where array_contains(arr,33000)"),
true,
- "PushedFilters: [*EqualTo(arr,33000)]")
+ "PushedFilters: [arr = 33000]")
checkExistence(sql(" explain select count(*) from complex1 where array_contains(arr,33000)"),
true,
- "PushedFilters: [*EqualTo(arr,33000)]")
+ "PushedFilters: [arr = 33000]")
checkAnswer(sql(" select * from complex1 where array_contains(arr,33000)"),
Seq(Row(mutable.WrappedArray.make(Array(20, 30, 33000))),
@@ -179,11 +179,11 @@ class TestArrayContainsPushDown extends QueryTest with BeforeAndAfterAll {
checkExistence(sql(" explain select * from complex1 where array_contains(arr,33000)"),
true,
- "PushedFilters: [*EqualTo(arr,33000)]")
+ "PushedFilters: [arr = 33000]")
checkExistence(sql(" explain select count(*) from complex1 where array_contains(arr,33000)"),
true,
- "PushedFilters: [*EqualTo(arr,33000)]")
+ "PushedFilters: [arr = 33000]")
checkAnswer(sql(" select * from complex1 where array_contains(arr,33000)"),
Seq(Row(mutable.WrappedArray.make(Array(20, 30, 33000))),
@@ -204,12 +204,12 @@ class TestArrayContainsPushDown extends QueryTest with BeforeAndAfterAll {
checkExistence(
sql(" explain select * from complex1 where array_contains(arr,cast(2.2 as double))"),
true,
- "PushedFilters: [*EqualTo(arr,2.2)]")
+ "PushedFilters: [arr = 2.2]")
checkExistence(
sql(" explain select count(*) from complex1 where array_contains(arr,cast(2.2 as double))"),
true,
- "PushedFilters: [*EqualTo(arr,2.2)]")
+ "PushedFilters: [arr = 2.2]")
checkAnswer(sql(" select * from complex1 where array_contains(arr,cast(2.2 as double))"),
Seq(Row(mutable.WrappedArray.make(Array(2.2))),
@@ -231,13 +231,13 @@ class TestArrayContainsPushDown extends QueryTest with BeforeAndAfterAll {
checkExistence(
sql(" explain select * from complex1 where array_contains(arr,cast(2.2 as decimal(5,2)))"),
true,
- "PushedFilters: [*EqualTo(arr,2.20)]")
+ "PushedFilters: [arr = 2.20]")
checkExistence(
sql("explain select count(*) from complex1 " +
"where array_contains(arr,cast(2.2 as decimal(5,2)))"),
true,
- "PushedFilters: [*EqualTo(arr,2.20)]")
+ "PushedFilters: [arr = 2.20]")
checkAnswer(sql(" select * from complex1 where array_contains(arr,cast(2.2 as decimal(5,2)))"),
Seq(Row(mutable.WrappedArray.make(Array(java.math.BigDecimal.valueOf(2.20).setScale(2)))),
@@ -263,13 +263,13 @@ class TestArrayContainsPushDown extends QueryTest with BeforeAndAfterAll {
sql("explain select * from complex1 " +
"where array_contains(arr,cast('2018-01-01 00:00:00' as timestamp))"),
true,
- "PushedFilters: [*EqualTo(arr,1514793600000000)]")
+ "PushedFilters: [arr = 1514793600000000]")
checkExistence(
sql("explain select count(*) from complex1 " +
"where array_contains(arr,cast('2018-01-01 00:00:00' as timestamp))"),
true,
- "PushedFilters: [*EqualTo(arr,1514793600000000)]")
+ "PushedFilters: [arr = 1514793600000000]")
checkAnswer(
sql("select * from complex1 " +
@@ -295,13 +295,13 @@ class TestArrayContainsPushDown extends QueryTest with BeforeAndAfterAll {
checkExistence(
sql("explain select * from complex1 where array_contains(arr,cast('2018-01-01' as date))"),
true,
- "PushedFilters: [*EqualTo(arr,17532)]")
+ "PushedFilters: [arr = 17532]")
checkExistence(
sql("explain select count(*) from complex1 " +
"where array_contains(arr,cast('2018-01-01' as date))"),
true,
- "PushedFilters: [*EqualTo(arr,17532)]")
+ "PushedFilters: [arr = 17532]")
checkAnswer(sql(" select * from complex1 where array_contains(arr,cast('2018-01-01' as date))"),
Seq(Row(mutable.WrappedArray.make(
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cloud/AllDataSourceTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cloud/AllDataSourceTestCase.scala
index 2207e1c..bb71812 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cloud/AllDataSourceTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/cloud/AllDataSourceTestCase.scala
@@ -407,7 +407,7 @@ class AllDataSourceTestCase extends QueryTest with BeforeAndAfterAll {
checkExistence(
sql(s"explain select * from ${tableName} where col1 = 1"),
true,
- "FileScan")
+ "Scan")
checkExistence(
sql(s"explain update ${tableName} set (col2) = ('4aa') where col1 = 1"),
true,
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index 2a91833..c4a1647 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -402,8 +402,9 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
}
val df = sql("select * from carbon_globalsort")
val scanRdd = df.queryExecution.sparkPlan.collect {
- case b: CarbonDataSourceScan if b.rdd.isInstanceOf[CarbonScanRDD[InternalRow]] =>
- b.rdd.asInstanceOf[CarbonScanRDD[InternalRow]]
+ case b: CarbonDataSourceScan
+ if b.inputRDDs().head.isInstanceOf[CarbonScanRDD[InternalRow]] =>
+ b.inputRDDs().head.asInstanceOf[CarbonScanRDD[InternalRow]]
}.head
assertResult(defaultParallelism)(scanRdd.getPartitions.length)
assertResult(10)(df.count)
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
index e53e0ba..d4d2cfd 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
@@ -57,10 +57,10 @@ class AllDataTypesTestCaseFilter extends QueryTest with BeforeAndAfterAll {
.sparkPlan
if (df.isInstanceOf[CarbonDataSourceScan]) {
assert(df.asInstanceOf[CarbonDataSourceScan].metadata
- .get("PushedFilters").get.contains("CarbonEndsWith"))
+ .get("PushedFilters").get.contains("EndsWith(input[0], nandh)"))
} else {
assert(df.children.head.asInstanceOf[CarbonDataSourceScan].metadata
- .get("PushedFilters").get.contains("CarbonEndsWith"))
+ .get("PushedFilters").get.contains("EndsWith(input[0], nandh)"))
}
}
@@ -69,10 +69,10 @@ class AllDataTypesTestCaseFilter extends QueryTest with BeforeAndAfterAll {
.sparkPlan
if (df.isInstanceOf[CarbonDataSourceScan]) {
assert(df.asInstanceOf[CarbonDataSourceScan].metadata
- .get("PushedFilters").get.contains("CarbonContainsWith"))
+ .get("PushedFilters").get.contains("Contains(input[0], nand)"))
} else {
assert(df.children.head.asInstanceOf[CarbonDataSourceScan].metadata
- .get("PushedFilters").get.contains("CarbonContainsWith"))
+ .get("PushedFilters").get.contains("Contains(input[0], nand)"))
}
}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestImplicitFilterExpression.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestImplicitFilterExpression.scala
index dfe6e53..89fc9e0 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestImplicitFilterExpression.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestImplicitFilterExpression.scala
@@ -63,8 +63,9 @@ class TestImplicitFilterExpression extends QueryTest with BeforeAndAfterAll {
blockletId: Int): Unit = {
// from the plan extract the CarbonScanRDD
val scanRDD = query.queryExecution.sparkPlan.collect {
- case scan: CarbonDataSourceScan if (scan.rdd.isInstanceOf[CarbonScanRDD[InternalRow]]) =>
- scan.rdd.asInstanceOf[CarbonScanRDD[InternalRow]]
+ case scan: CarbonDataSourceScan
+ if scan.inputRDDs().head.isInstanceOf[CarbonScanRDD[InternalRow]] =>
+ scan.inputRDDs().head.asInstanceOf[CarbonScanRDD[InternalRow]]
}.head
// get carbon relation
val relation: CarbonRelation = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetaStore
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 09609e0..3a143b6 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -487,8 +487,9 @@ class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfte
FileUtils.deleteDirectory(folder)
val dataFrame = sql("select * from smallpartitionfilesread")
val scanRdd = dataFrame.queryExecution.sparkPlan.collect {
- case b: CarbonDataSourceScan if b.rdd.isInstanceOf[CarbonScanRDD[InternalRow]] => b.rdd
- .asInstanceOf[CarbonScanRDD[InternalRow]]
+ case b: CarbonDataSourceScan
+ if b.inputRDDs().head.isInstanceOf[CarbonScanRDD[InternalRow]] =>
+ b.inputRDDs().head.asInstanceOf[CarbonScanRDD[InternalRow]]
}.head
assert(scanRdd.getPartitions.length < 10)
assertResult(100)(dataFrame.count)
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
index 20d14a6..c7bb9cf 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableQueryTestCase.scala
@@ -535,8 +535,9 @@ test("Creation of partition table should fail if the colname in table schema and
private def verifyPartitionInfo(frame: DataFrame, partitionNames: Seq[String]) = {
val plan = frame.queryExecution.sparkPlan
val scanRDD = plan collect {
- case b: CarbonDataSourceScan if b.rdd.isInstanceOf[CarbonScanRDD[InternalRow]] => b.rdd
- .asInstanceOf[CarbonScanRDD[InternalRow]]
+ case b: CarbonDataSourceScan
+ if b.inputRDDs().head.isInstanceOf[CarbonScanRDD[InternalRow]] =>
+ b.inputRDDs().head.asInstanceOf[CarbonScanRDD[InternalRow]]
}
assert(scanRDD.nonEmpty)
assert(!partitionNames.map(f => scanRDD.head.partitionNames.exists(_.getPartitions.contains(f))).exists(!_))
diff --git a/integration/spark/src/test/scala/org/apache/spark/carbondata/query/TestFilterReordering.scala b/integration/spark/src/test/scala/org/apache/spark/carbondata/query/TestFilterReordering.scala
deleted file mode 100644
index 078bffb..0000000
--- a/integration/spark/src/test/scala/org/apache/spark/carbondata/query/TestFilterReordering.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.carbondata.query
-
-import org.apache.spark.sql.{CarbonEnv, CarbonThreadUtil}
-import org.apache.spark.sql.optimizer.CarbonFilters
-import org.apache.spark.sql.sources.{And, EqualTo, Filter, Or}
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-
-class TestFilterReordering extends QueryTest with BeforeAndAfterAll{
-
- override protected def beforeAll(): Unit = {
- sql("drop table if exists filter_reorder")
- sql("create table filter_reorder(one string, two string, three string, four int, " +
- "five int) stored as carbondata")
- }
-
- test("Test filter reorder with various conditions") {
- val filter1 = Or(And(EqualTo("four", 11), EqualTo("two", 11)), EqualTo("one", 11))
- val table = CarbonEnv.getCarbonTable(None, "filter_reorder")(sqlContext.sparkSession)
- var d: Array[Filter] = CarbonFilters.reorderFilter(Array(filter1), table)
- assert(d.head.references.sameElements(Array("one", "two", "four")))
-
- val filter2 = Or(Or(EqualTo("four", 11), EqualTo("two", 11)),
- Or(EqualTo("one", 11), Or(EqualTo("five", 11), EqualTo("three", 11))))
- d = CarbonFilters.reorderFilter(Array(filter2), table)
- assert(d.head.references.sameElements(Array("one", "two", "three", "four", "five")))
-
- val filter3 = Or(Or(EqualTo("four", 11), EqualTo("two", 11)),
- Or(EqualTo("one", 11), Or(EqualTo("five", 11),
- And(EqualTo("three", 11), EqualTo("three", 11)))))
- d = CarbonFilters.reorderFilter(Array(filter3), table)
- assert(d.head.references.sameElements(Array("one", "three", "three", "five", "two", "four")))
-
- val filter4: Array[Filter] = Array(EqualTo("four", 11), EqualTo("two", 11), EqualTo("one", 11),
- EqualTo("five", 11), EqualTo("three", 11))
- d = CarbonFilters.reorderFilter(filter4, table)
- assert(d.map(_.references.head) sameElements
- Array("one", "two", "three", "four", "five"))
- }
-
- test("test disabling filter reordering") {
- sqlContext.sparkSession.sql(s"set ${CarbonCommonConstants.CARBON_REORDER_FILTER}=false")
- CarbonThreadUtil.updateSessionInfoToCurrentThread(sqlContext.sparkSession)
- val filter1 = Or(And(EqualTo("four", 11), EqualTo("two", 11)), EqualTo("one", 11))
- val table = CarbonEnv.getCarbonTable(None, "filter_reorder")(sqlContext.sparkSession)
- val d: Array[Filter] = CarbonFilters.reorderFilter(Array(filter1), table)
- assert(d.head.references.sameElements(Array("four", "two", "one")))
- sqlContext.sparkSession.sql(s"set ${CarbonCommonConstants.CARBON_REORDER_FILTER}=true")
- }
-
- override protected def afterAll(): Unit = {
- sqlContext.sparkSession.sql(s"set ${CarbonCommonConstants.CARBON_REORDER_FILTER}=true")
- CarbonThreadUtil.updateSessionInfoToCurrentThread(sqlContext.sparkSession)
- sql("drop table if exists filter_reorder")
- }
-}
diff --git a/integration/spark/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala b/integration/spark/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala
index 5a4c369..009803c 100644
--- a/integration/spark/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala
@@ -57,7 +57,7 @@ class VectorReaderTestCase extends QueryTest with BeforeAndAfterAll {
"""select * from vectorreader""".stripMargin).queryExecution.executedPlan
var rowReader = false
plan.collect {
- case _: RowDataSourceScanExec => rowReader = true
+ case scan: CarbonDataSourceScan => rowReader = !scan.supportsBatch
}
assert(rowReader, "row reader should exist by default")
}
diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/CarbonExtensionSuite.scala b/integration/spark/src/test/scala/org/apache/spark/sql/CarbonExtensionSuite.scala
index 54a5088..ebaeac6 100644
--- a/integration/spark/src/test/scala/org/apache/spark/sql/CarbonExtensionSuite.scala
+++ b/integration/spark/src/test/scala/org/apache/spark/sql/CarbonExtensionSuite.scala
@@ -42,7 +42,7 @@ class CarbonExtensionSuite extends QueryTest with BeforeAndAfterAll {
}
test("test strategy injection") {
- assert(session.sessionState.planner.strategies.filter(_.isInstanceOf[DDLStrategy]).length == 1)
+ assert(session.sessionState.planner.strategies.count(_ == DDLStrategy) == 1)
session.sql("create table if not exists table1 (column1 String) using carbondata ").collect()
}
}