You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2019/07/22 10:56:30 UTC
[carbondata] branch master updated: [HOTFIX] Reset the hive catalog
table stats to none even after refresh lookup relation.
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new a77e4fd [HOTFIX] Reset the hive catalog table stats to none even after refresh lookup relation.
a77e4fd is described below
commit a77e4fd2a6d7fb4080ce2df5ee89d57a2bca8b8d
Author: ravipesala <ra...@gmail.com>
AuthorDate: Tue Jul 9 19:29:19 2019 +0530
[HOTFIX] Reset the hive catalog table stats to none even after refresh lookup relation.
Problem:
Spark plan picks up the catalog table stats in case of relookup of relation in case of modification time is updated.
Solution:
Even in case of relookup of relation set the stats to none.
This closes #3321
---
.../sql/hive/CarbonInMemorySessionState.scala | 28 ++++++---
.../apache/spark/sql/hive/CarbonSessionState.scala | 25 +++++---
.../apache/spark/sql/hive/CarbonSessionUtil.scala | 69 +++++++++++++++-------
3 files changed, 84 insertions(+), 38 deletions(-)
diff --git a/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonInMemorySessionState.scala b/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonInMemorySessionState.scala
index 5dfb16d..36f166d 100644
--- a/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonInMemorySessionState.scala
+++ b/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonInMemorySessionState.scala
@@ -16,16 +16,18 @@
*/
package org.apache.spark.sql.hive
+import java.util.concurrent.Callable
+
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.sql.carbondata.execution.datasources.CarbonSparkDataSourceUtil
-import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.optimizer.Optimizer
import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.execution.datasources._
import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
@@ -33,9 +35,9 @@ import org.apache.spark.sql.internal.{SQLConf, SessionResourceLoader, SessionSta
import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
import org.apache.spark.sql.parser.CarbonSparkSqlParser
import org.apache.spark.sql.types.{StructField, StructType}
-import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, CarbonEnv, SparkSession}
-import org.apache.carbondata.core.metadata.schema.table.column.{ColumnSchema => ColumnSchema}
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.format.TableInfo
@@ -154,14 +156,22 @@ class InMemorySessionCatalog(
}
override def lookupRelation(name: TableIdentifier): LogicalPlan = {
- val rtnRelation = super.lookupRelation(name)
+ var rtnRelation = super.lookupRelation(name)
val isRelationRefreshed =
- CarbonSessionUtil.refreshRelation(rtnRelation, name)(sparkSession)
+ CarbonSessionUtil.refreshRelationAndSetStats(rtnRelation, name)(sparkSession)
if (isRelationRefreshed) {
- super.lookupRelation(name)
- } else {
- rtnRelation
+ rtnRelation = super.lookupRelation(name)
+ // Reset the stats after lookup.
+ CarbonSessionUtil.refreshRelationAndSetStats(rtnRelation, name)(sparkSession)
}
+ rtnRelation
+ }
+
+
+ override def getCachedPlan(t: QualifiedTableName,
+ c: Callable[LogicalPlan]): LogicalPlan = {
+ val plan = super.getCachedPlan(t, c)
+ CarbonSessionUtil.updateCachedPlan(plan)
}
/**
diff --git a/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonSessionState.scala
index f991a78..f78c785 100644
--- a/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonSessionState.scala
+++ b/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -16,16 +16,18 @@
*/
package org.apache.spark.sql.hive
+import java.util.concurrent.Callable
+
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{Analyzer, FunctionRegistry}
import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.optimizer.Optimizer
import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.execution.datasources.{FindDataSourceTable, PreWriteCheck, ResolveSQLOnFile, _}
import org.apache.spark.sql.execution.strategy.{CarbonLateDecodeStrategy, DDLStrategy, StreamingTableStrategy}
@@ -34,7 +36,7 @@ import org.apache.spark.sql.internal.{SQLConf, SessionState}
import org.apache.spark.sql.optimizer.{CarbonIUDRule, CarbonLateDecodeRule, CarbonUDFTransformRule}
import org.apache.spark.sql.parser.CarbonSparkSqlParser
-import org.apache.carbondata.core.metadata.schema.table.column.{ColumnSchema => ColumnSchema}
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.spark.util.CarbonScalaUtil
/**
@@ -86,14 +88,21 @@ class CarbonHiveSessionCatalog(
CarbonEnv.init
override def lookupRelation(name: TableIdentifier): LogicalPlan = {
- val rtnRelation = super.lookupRelation(name)
+ var rtnRelation = super.lookupRelation(name)
val isRelationRefreshed =
- CarbonSessionUtil.refreshRelation(rtnRelation, name)(sparkSession)
+ CarbonSessionUtil.refreshRelationAndSetStats(rtnRelation, name)(sparkSession)
if (isRelationRefreshed) {
- super.lookupRelation(name)
- } else {
- rtnRelation
+ rtnRelation = super.lookupRelation(name)
+ // Reset the stats after lookup.
+ CarbonSessionUtil.refreshRelationAndSetStats(rtnRelation, name)(sparkSession)
}
+ rtnRelation
+ }
+
+ override def getCachedPlan(t: QualifiedTableName,
+ c: Callable[LogicalPlan]): LogicalPlan = {
+ val plan = super.getCachedPlan(t, c)
+ CarbonSessionUtil.updateCachedPlan(plan)
}
/**
diff --git a/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonSessionUtil.scala b/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonSessionUtil.scala
index 88a2565..febba35 100644
--- a/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonSessionUtil.scala
+++ b/integration/spark2/src/main/commonTo2.2And2.3/org/apache/spark/sql/hive/CarbonSessionUtil.scala
@@ -17,7 +17,9 @@
package org.apache.spark.sql.hive
-import org.apache.spark.sql.catalyst.TableIdentifier
+import java.util.concurrent.Callable
+
+import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier}
import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTablePartition, ExternalCatalogUtils}
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
@@ -29,9 +31,10 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.spark.sql.types.{StructField, StructType}
import org.apache.spark.sql.util.SparkTypeConverter
-
import scala.collection.mutable.ArrayBuffer
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+
/**
* This class refresh the relation from cache if the carbontable in
* carbon catalog is not same as cached carbon relation's carbon table.
@@ -48,16 +51,43 @@ object CarbonSessionUtil {
* @param sparkSession
* @return
*/
- def refreshRelation(rtnRelation: LogicalPlan, name: TableIdentifier)
+ def refreshRelationAndSetStats(rtnRelation: LogicalPlan, name: TableIdentifier)
(sparkSession: SparkSession): Boolean = {
var isRelationRefreshed = false
+
+ /**
+ * Set the stats to none in case of carbontable
+ */
+ def setStatsNone(catalogTable: CatalogTable): Unit = {
+ catalogTable.provider match {
+ case Some(provider)
+ if provider.equals("org.apache.spark.sql.CarbonSource") ||
+ provider.equalsIgnoreCase("carbondata") =>
+ // Update stats to none in case of carbon table as we are not expecting any stats from
+ // Hive. Hive gives wrong stats for carbon table.
+ catalogTable.stats match {
+ case Some(stats) =>
+ CarbonReflectionUtils.setFieldToCaseClass(catalogTable, "stats", None)
+ case _ =>
+ }
+ isRelationRefreshed =
+ CarbonEnv.refreshRelationFromCache(catalogTable.identifier)(sparkSession)
+ case _ =>
+ }
+ }
+
rtnRelation match {
case SubqueryAlias(_,
- MatchLogicalRelation(_: CarbonDatasourceHadoopRelation, _, _)
- ) =>
+ MatchLogicalRelation(_: CarbonDatasourceHadoopRelation, _, catalogTable)) =>
isRelationRefreshed = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
- case MatchLogicalRelation(_: CarbonDatasourceHadoopRelation, _, _) =>
+ if (catalogTable.isInstanceOf[Option[CatalogTable]]) {
+ catalogTable.asInstanceOf[Option[CatalogTable]].foreach(setStatsNone)
+ }
+ case MatchLogicalRelation(_: CarbonDatasourceHadoopRelation, _, catalogTable) =>
isRelationRefreshed = CarbonEnv.refreshRelationFromCache(name)(sparkSession)
+ if (catalogTable.isInstanceOf[Option[CatalogTable]]) {
+ catalogTable.asInstanceOf[Option[CatalogTable]].foreach(setStatsNone)
+ }
case SubqueryAlias(_, relation) if
relation.getClass.getName.equals("org.apache.spark.sql.catalyst.catalog.CatalogRelation") ||
relation.getClass.getName
@@ -70,21 +100,7 @@ object CarbonSessionUtil {
"tableMeta",
relation
).asInstanceOf[CatalogTable]
- catalogTable.provider match {
- case Some(provider)
- if provider.equals("org.apache.spark.sql.CarbonSource") ||
- provider.equalsIgnoreCase("carbondata") =>
- // Update stats to none in case of carbon table as we are not expecting any stats from
- // Hive. Hive gives wrong stats for carbon table.
- catalogTable.stats match {
- case Some(stats) =>
- CarbonReflectionUtils.setFieldToCaseClass(catalogTable, "stats", None)
- case _ =>
- }
- isRelationRefreshed =
- CarbonEnv.refreshRelationFromCache(catalogTable.identifier)(sparkSession)
- case _ =>
- }
+ setStatsNone(catalogTable)
case _ =>
}
isRelationRefreshed
@@ -159,4 +175,15 @@ object CarbonSessionUtil {
StructType(colArray))
}
+ def updateCachedPlan(plan: LogicalPlan): LogicalPlan = {
+ plan match {
+ case sa@SubqueryAlias(_,
+ MatchLogicalRelation(_: CarbonDatasourceHadoopRelation, _, catalogTable)) =>
+ sa.copy(child = sa.child.asInstanceOf[LogicalRelation].copy())
+ case MatchLogicalRelation(_: CarbonDatasourceHadoopRelation, _, _) =>
+ plan.asInstanceOf[LogicalRelation].copy()
+ case _ => plan
+ }
+ }
+
}