You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ma...@apache.org on 2017/12/13 04:23:10 UTC
carbondata git commit: [CARBONDATA-1826] Spark 2.2: Describe table &
Describe Formatted shows the same result
Repository: carbondata
Updated Branches:
refs/heads/master 34cb55194 -> b6f29dec4
[CARBONDATA-1826] Spark 2.2: Describe table & Describe Formatted shows the same result
Carbon 1.3.0 - Spark 2.2: Describe table & Describe Formatted shows the same result.
This closes #1630
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b6f29dec
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b6f29dec
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b6f29dec
Branch: refs/heads/master
Commit: b6f29dec4335d536fe371ba985fe2c9b98cb2772
Parents: 34cb551
Author: dhatchayani <dh...@gmail.com>
Authored: Thu Dec 7 12:50:05 2017 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Wed Dec 13 09:55:49 2017 +0530
----------------------------------------------------------------------
.../describeTable/TestDescribeTable.scala | 8 +++--
.../spark/util/CarbonReflectionUtils.scala | 8 +++++
.../sql/execution/strategy/DDLStrategy.scala | 33 +++++++++++++-------
3 files changed, 35 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6f29dec/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
index b2a1120..a87292f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
@@ -16,6 +16,7 @@
*/
package org.apache.carbondata.spark.testsuite.describeTable
+import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
@@ -34,8 +35,11 @@ class TestDescribeTable extends QueryTest with BeforeAndAfterAll {
sql("CREATE TABLE Desc2(Dec2Col1 BigInt, Dec2Col2 String, Dec2Col3 Bigint, Dec2Col4 Decimal) stored by 'carbondata'")
}
- ignore("test describe table") {
- checkAnswer(sql("DESC Desc1"), sql("DESC Desc2"))
+ test("test describe table") {
+ checkAnswer(sql("DESC Desc1"), Seq(Row("dec2col1","bigint",null),
+ Row("dec2col2","string",null),
+ Row("dec2col3","bigint",null),
+ Row("dec2col4","decimal(10,0)",null)))
}
test("test describe formatted table") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6f29dec/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
index d88f190..6cc7ae1 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/CarbonReflectionUtils.scala
@@ -175,6 +175,14 @@ object CarbonReflectionUtils {
}
}
+ def getDescribeTableFormattedField[T: TypeTag : reflect.ClassTag](obj: T): Boolean = {
+ val im = rm.reflect(obj)
+ val isFormatted = im.symbol.typeSignature.members
+ .find(_.name.toString.equalsIgnoreCase("isFormatted"))
+ .map(l => im.reflectField(l.asTerm).get).getOrElse("false").asInstanceOf[Boolean]
+ isFormatted
+ }
+
def createObject(className: String, conArgs: Object*): (Any, Class[_]) = {
val clazz = Utils.classForName(className)
val ctor = clazz.getConstructors.head
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6f29dec/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 9ae1979..8745900 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.execution.command.table.{CarbonDescribeFormattedComm
import org.apache.spark.sql.hive.execution.command.{CarbonDropDatabaseCommand, CarbonResetCommand, CarbonSetCommand}
import org.apache.spark.sql.CarbonExpressions.{CarbonDescribeTable => DescribeTableCommand}
import org.apache.spark.sql.execution.datasources.RefreshTable
-import org.apache.spark.util.FileUtils
+import org.apache.spark.util.{CarbonReflectionUtils, FileUtils}
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.util.CarbonProperties
@@ -134,17 +134,26 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
} else {
throw new MalformedCarbonCommandException("Unsupported alter operation on hive table")
}
- case desc@DescribeTableCommand(identifier, partitionSpec, isExtended)
- if CarbonEnv.getInstance(sparkSession).carbonMetastore
- .tableExists(identifier)(sparkSession) =>
- val resolvedTable =
- sparkSession.sessionState.executePlan(UnresolvedRelation(identifier)).analyzed
- val resultPlan = sparkSession.sessionState.executePlan(resolvedTable).executedPlan
- ExecutedCommandExec(
- CarbonDescribeFormattedCommand(
- resultPlan,
- plan.output,
- identifier)) :: Nil
+ case desc@DescribeTableCommand(identifier, partitionSpec, isExtended) =>
+ val isFormatted: Boolean = if (sparkSession.version.startsWith("2.1")) {
+ CarbonReflectionUtils
+ .getDescribeTableFormattedField(desc.asInstanceOf[DescribeTableCommand])
+ } else {
+ false
+ }
+ if (CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .tableExists(identifier)(sparkSession) && (isExtended || isFormatted)) {
+ val resolvedTable =
+ sparkSession.sessionState.executePlan(UnresolvedRelation(identifier)).analyzed
+ val resultPlan = sparkSession.sessionState.executePlan(resolvedTable).executedPlan
+ ExecutedCommandExec(
+ CarbonDescribeFormattedCommand(
+ resultPlan,
+ plan.output,
+ identifier)) :: Nil
+ } else {
+ Nil
+ }
case ShowPartitionsCommand(t, cols) =>
val isCarbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
.tableExists(t)(sparkSession)