You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/06/29 03:11:14 UTC
carbondata git commit: [CARBONDATA-2634][BloomDataMap] Add datamap
properties in show datamap outputs
Repository: carbondata
Updated Branches:
refs/heads/master f7552a97d -> 64ae5ae0b
[CARBONDATA-2634][BloomDataMap] Add datamap properties in show datamap outputs
add datamap properties in show datamap outputs
This closes #2404
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/64ae5ae0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/64ae5ae0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/64ae5ae0
Branch: refs/heads/master
Commit: 64ae5ae0bbd968750c1db97fc2ff5b2bf0d60976
Parents: f7552a9
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Sun Jun 24 11:49:27 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Jun 29 11:10:56 2018 +0800
----------------------------------------------------------------------
.../schema/datamap/DataMapProperty.java | 8 +++
.../core/metadata/schema/table/TableSchema.java | 6 +-
.../testsuite/datamap/TestDataMapCommand.scala | 63 ++++++++++++++++++++
.../datamap/CarbonDataMapShowCommand.scala | 28 ++++++---
.../preaaggregate/PreAggregateUtil.scala | 3 +-
5 files changed, 97 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/64ae5ae0/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProperty.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProperty.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProperty.java
index 9bd78da..37c051a 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProperty.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProperty.java
@@ -36,4 +36,12 @@ public class DataMapProperty {
* property internally
*/
public static final String DEFERRED_REBUILD = "_internal.deferred.rebuild";
+ /**
+ * for internal property 'CHILD_SELECT_QUERY'
+ */
+ public static final String CHILD_SELECT_QUERY = "CHILD_SELECT QUERY";
+ /**
+ * for internal property 'QUERYTYPE'
+ */
+ public static final String QUERY_TYPE = "QUERYTYPE";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/64ae5ae0/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index 3d9e068..b53a9d8 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -30,6 +30,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProperty;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -279,13 +280,12 @@ public class TableSchema implements Serializable, Writable {
new RelationIdentifier(databaseName, tableName, tableId);
Map<String, String> properties = new HashMap<>();
if (queryString != null) {
- properties.put(
- "CHILD_SELECT QUERY",
+ properties.put(DataMapProperty.CHILD_SELECT_QUERY,
CarbonUtil.encodeToString(queryString.trim().getBytes(
// replace = to with & as hive metastore does not allow = inside. For base 64
// only = is allowed as special character , so replace with &
CarbonCommonConstants.DEFAULT_CHARSET)).replace("=", "&"));
- properties.put("QUERYTYPE", queryType);
+ properties.put(DataMapProperty.QUERY_TYPE, queryType);
}
DataMapSchema dataMapSchema = new DataMapSchema(dataMapName, className);
dataMapSchema.setProperties(properties);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/64ae5ae0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index 642607c..afca3b2 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -204,6 +204,69 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
}
}
+ test("test show datamap: show datamap property related information") {
+ val tableName = "datamapshowtest"
+ val datamapName = "bloomdatamap"
+ val datamapName2 = "bloomdatamap2"
+ val datamapName3 = "bloomdatamap3"
+ sql(s"drop table if exists $tableName")
+ // for index datamap
+ sql(s"create table $tableName (a string, b string, c string) stored by 'carbondata'")
+ sql(
+ s"""
+ | create datamap $datamapName on table $tableName using 'bloomfilter'
+ | DMPROPERTIES ('index_columns'='a', 'bloom_size'='32000', 'bloom_fpp'='0.001')
+ """.stripMargin)
+ sql(
+ s"""
+ | create datamap $datamapName2 on table $tableName using 'bloomfilter'
+ | DMPROPERTIES ('index_columns'='b')
+ """.stripMargin)
+ sql(
+ s"""
+ | create datamap $datamapName3 on table $tableName using 'bloomfilter'
+ | with deferred rebuild
+ | DMPROPERTIES ('index_columns'='c')
+ """.stripMargin)
+ var result = sql(s"show datamap on table $tableName").cache()
+ checkAnswer(sql(s"show datamap on table $tableName"),
+ Seq(Row(datamapName, "bloomfilter", s"default.$tableName", "'bloom_fpp'='0.001', 'bloom_size'='32000', 'index_columns'='a'"),
+ Row(datamapName2, "bloomfilter", s"default.$tableName", "'index_columns'='b'"),
+ Row(datamapName3, "bloomfilter", s"default.$tableName", "'index_columns'='c'")))
+ result.unpersist()
+ sql(s"drop table if exists $tableName")
+
+ // for timeseries datamap
+ sql(s"CREATE TABLE $tableName(mytime timestamp, name string, age int) STORED BY 'org.apache.carbondata.format'")
+ sql(
+ s"""
+ | CREATE DATAMAP agg0_hour ON TABLE $tableName
+ | USING 'timeSeries'
+ | DMPROPERTIES (
+ | 'EVENT_TIME'='mytime',
+ | 'HOUR_GRANULARITY'='1')
+ | AS SELECT mytime, SUM(age) FROM $tableName
+ | GROUP BY mytime
+ """.stripMargin)
+ checkAnswer(sql(s"show datamap on table $tableName"),
+ Seq(Row("agg0_hour", "timeSeries", s"default.${tableName}_agg0_hour", "'event_time'='mytime', 'hour_granularity'='1'")))
+ sql(s"drop table if exists $tableName")
+
+ // for preaggreate datamap, the property is empty
+ sql(s"CREATE TABLE $tableName(id int, name string, city string, age string)" +
+ s" STORED BY 'org.apache.carbondata.format'")
+ sql (
+ s"""
+ | CREATE DATAMAP agg0 ON TABLE $tableName USING 'preaggregate' AS
+ | SELECT name,
+ | count(age)
+ | FROM $tableName GROUP BY name
+ | """.stripMargin)
+ checkAnswer(sql(s"show datamap on table $tableName"),
+ Seq(Row("agg0", "preaggregate", s"default.${tableName}_agg0", "")))
+ sql(s"drop table if exists $tableName")
+ }
+
test("test if preaggregate load is successfull for hivemetastore") {
try {
CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/64ae5ae0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
index 844d4c0..3ee8e67 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.execution.command.{Checker, DataCommand}
import org.apache.spark.sql.types.StringType
import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapClassProvider
+import org.apache.carbondata.core.metadata.schema.datamap.{DataMapClassProvider, DataMapProperty}
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
/**
@@ -42,7 +42,8 @@ case class CarbonDataMapShowCommand(tableIdentifier: Option[TableIdentifier])
override def output: Seq[Attribute] = {
Seq(AttributeReference("DataMapName", StringType, nullable = false)(),
AttributeReference("ClassName", StringType, nullable = false)(),
- AttributeReference("Associated Table", StringType, nullable = false)())
+ AttributeReference("Associated Table", StringType, nullable = false)(),
+ AttributeReference("DataMap Properties", StringType, nullable = false)())
}
override def processData(sparkSession: SparkSession): Seq[Row] = {
@@ -66,11 +67,24 @@ case class CarbonDataMapShowCommand(tableIdentifier: Option[TableIdentifier])
private def convertToRow(schemaList: util.List[DataMapSchema]) = {
if (schemaList != null && schemaList.size() > 0) {
- schemaList.asScala.map { s =>
- var table = "(NA)"
- val relationIdentifier = s.getRelationIdentifier
- table = relationIdentifier.getDatabaseName + "." + relationIdentifier.getTableName
- Row(s.getDataMapName, s.getProviderName, table)
+ schemaList.asScala
+ .map { s =>
+ val relationIdentifier = s.getRelationIdentifier
+ val table = relationIdentifier.getDatabaseName + "." + relationIdentifier.getTableName
+ // preaggregate datamap does not support user specified property, therefor we return empty
+ val dmPropertieStr = if (s.getProviderName.equalsIgnoreCase(
+ DataMapClassProvider.PREAGGREGATE.getShortName)) {
+ ""
+ } else {
+ s.getProperties.asScala
+ // ignore internal used property
+ .filter(p => !p._1.equalsIgnoreCase(DataMapProperty.DEFERRED_REBUILD) &&
+ !p._1.equalsIgnoreCase(DataMapProperty.CHILD_SELECT_QUERY) &&
+ !p._1.equalsIgnoreCase(DataMapProperty.QUERY_TYPE))
+ .map(p => s"'${ p._1 }'='${ p._2 }'").toSeq
+ .sorted.mkString(", ")
+ }
+ Row(s.getDataMapName, s.getProviderName, table, dmPropertieStr)
}
} else {
Seq.empty
http://git-wip-us.apache.org/repos/asf/carbondata/blob/64ae5ae0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index a7bcddb..06ebc43 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -40,6 +40,7 @@ import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProperty
import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable, DataMapSchema, TableSchema}
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.util.CarbonUtil
@@ -869,7 +870,7 @@ object PreAggregateUtil {
def getChildQuery(aggDataMapSchema: AggregationDataMapSchema): String = {
new String(
CarbonUtil.decodeStringToBytes(
- aggDataMapSchema.getProperties.get("CHILD_SELECT QUERY").replace("&", "=")),
+ aggDataMapSchema.getProperties.get(DataMapProperty.CHILD_SELECT_QUERY).replace("&", "=")),
CarbonCommonConstants.DEFAULT_CHARSET)
}