You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/02/03 19:43:49 UTC
[42/50] [abbrv] carbondata git commit: [CARBONDATA-2123] Refactor
datamap schema thrift and datamap provider to use short name and classname
[CARBONDATA-2123] Refactor datamap schema thrift and datamap provider to use short name and classname
Update schema thrift file for datamap schema to correct the typo errors and updated the names.
Added class name to schema file and updated short name for each enum.
This closes #1919
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/46d9bf96
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/46d9bf96
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/46d9bf96
Branch: refs/heads/branch-1.3
Commit: 46d9bf966910afb98a4e4e9cf879f2a9beef5b72
Parents: 4677fc6
Author: ravipesala <ra...@gmail.com>
Authored: Sat Feb 3 00:18:10 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Feb 3 22:05:37 2018 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 2 -
.../ThriftWrapperSchemaConverterImpl.java | 10 ++---
.../schema/datamap/DataMapProvider.java | 39 +++++++++++++++++++-
.../schema/table/DataMapSchemaFactory.java | 12 +++---
format/src/main/thrift/schema.thrift | 14 ++++---
.../preaggregate/TestPreAggCreateCommand.scala | 3 +-
.../timeseries/TestTimeSeriesCreateTable.scala | 4 +-
.../datamap/CarbonCreateDataMapCommand.scala | 28 ++++++++------
.../CreatePreAggregateTableCommand.scala | 5 ++-
9 files changed, 80 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/46d9bf96/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 8480758..a799e51 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1455,8 +1455,6 @@ public final class CarbonCommonConstants {
public static final String BITSET_PIPE_LINE_DEFAULT = "true";
-
- public static final String AGGREGATIONDATAMAPSCHEMA = "AggregateDataMapHandler";
/*
* The total size of carbon data
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/46d9bf96/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index e9c5505..21ab797 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -343,7 +343,7 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
.setDatabaseName(wrapperChildSchema.getRelationIdentifier().getDatabaseName());
relationIdentifier.setTableName(wrapperChildSchema.getRelationIdentifier().getTableName());
relationIdentifier.setTableId(wrapperChildSchema.getRelationIdentifier().getTableId());
- thriftChildSchema.setRelationIdentifire(relationIdentifier);
+ thriftChildSchema.setChildTableIdentifier(relationIdentifier);
}
thriftChildSchema.setProperties(wrapperChildSchema.getProperties());
thriftChildSchema.setClassName(wrapperChildSchema.getClassName());
@@ -648,11 +648,11 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
DataMapSchema childSchema = new DataMapSchema(thriftDataMapSchema.getDataMapName(),
thriftDataMapSchema.getClassName());
childSchema.setProperties(thriftDataMapSchema.getProperties());
- if (null != thriftDataMapSchema.getRelationIdentifire()) {
+ if (null != thriftDataMapSchema.getChildTableIdentifier()) {
RelationIdentifier relationIdentifier =
- new RelationIdentifier(thriftDataMapSchema.getRelationIdentifire().getDatabaseName(),
- thriftDataMapSchema.getRelationIdentifire().getTableName(),
- thriftDataMapSchema.getRelationIdentifire().getTableId());
+ new RelationIdentifier(thriftDataMapSchema.getChildTableIdentifier().getDatabaseName(),
+ thriftDataMapSchema.getChildTableIdentifier().getTableName(),
+ thriftDataMapSchema.getChildTableIdentifier().getTableId());
childSchema.setRelationIdentifier(relationIdentifier);
childSchema.setChildSchema(
fromExternalToWrapperTableSchema(thriftDataMapSchema.getChildTableSchema(),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/46d9bf96/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
index 65578b1..0052428 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
@@ -27,6 +27,41 @@ package org.apache.carbondata.core.metadata.schema.datamap;
*/
public enum DataMapProvider {
- PREAGGREGATE,
- TIMESERIES;
+ PREAGGREGATE("org.apache.carbondata.core.datamap.AggregateDataMap", "preaggregate"),
+ TIMESERIES("org.apache.carbondata.core.datamap.TimeSeriesDataMap", "timeseries");
+
+ /**
+ * Fully qualified class name of datamap
+ */
+ private String className;
+
+ /**
+ * Short name representation of datamap
+ */
+ private String shortName;
+
+ DataMapProvider(String className, String shortName) {
+ this.className = className;
+ this.shortName = shortName;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ private boolean isEqual(String dataMapClass) {
+ return (dataMapClass != null &&
+ (dataMapClass.equals(className) ||
+ dataMapClass.equalsIgnoreCase(shortName)));
+ }
+
+ public static DataMapProvider getDataMapProvider(String dataMapClass) {
+ if (TIMESERIES.isEqual(dataMapClass)) {
+ return TIMESERIES;
+ } else if (PREAGGREGATE.isEqual(dataMapClass)) {
+ return PREAGGREGATE;
+ } else {
+ throw new UnsupportedOperationException("Unknown datamap provider/class " + dataMapClass);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/46d9bf96/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
index 5729959..d0c7386 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
@@ -16,7 +16,7 @@
*/
package org.apache.carbondata.core.metadata.schema.table;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.AGGREGATIONDATAMAPSCHEMA;
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider;
public class DataMapSchemaFactory {
public static final DataMapSchemaFactory INSTANCE = new DataMapSchemaFactory();
@@ -28,11 +28,11 @@ public class DataMapSchemaFactory {
* @return data map schema
*/
public DataMapSchema getDataMapSchema(String dataMapName, String className) {
- switch (className) {
- case AGGREGATIONDATAMAPSCHEMA:
- return new AggregationDataMapSchema(dataMapName, className);
- default:
- return new DataMapSchema(dataMapName, className);
+ if (DataMapProvider.PREAGGREGATE.getClassName().equals(className) ||
+ DataMapProvider.TIMESERIES.getClassName().equals(className)) {
+ return new AggregationDataMapSchema(dataMapName, className);
+ } else {
+ return new DataMapSchema(dataMapName, className);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/46d9bf96/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index a924009..b44fe19 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -192,13 +192,15 @@ struct DataMapSchema {
1: required string dataMapName;
// class name
2: required string className;
- // relation indentifier
- 3: optional RelationIdentifier relationIdentifire;
- // in case of preaggregate it will be used to maintain the child schema
+ // to maintain properties which are mentioned in DMPROPERTIES of DDL and also it
+ // stores properties of select query, query type like groupby, join in
+ // case of preaggregate/timeseries
+ 3: optional map<string, string> properties;
+ // relation identifier of a table which stores data of datamaps like preaggregate/timeseries.
+ 4: optional RelationIdentifier childTableIdentifier;
+ // in case of preaggregate/timeseries datamap it will be used to maintain the child schema
// which will be usefull in case of query and data load
- 4: optional TableSchema childTableSchema;
- // to maintain properties like select query, query type like groupby, join
- 5: optional map<string, string> properties;
+ 5: optional TableSchema childTableSchema;
}
struct TableInfo{
http://git-wip-us.apache.org/repos/asf/carbondata/blob/46d9bf96/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index 5d0f61b..6988adc 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -296,7 +296,8 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
| GROUP BY column3,column5,column2
""".stripMargin)
}
- assert(e.getMessage.contains("Unknown data map type abc"))
+ assert(e.getMessage.contains(
+ s"Unknown datamap provider/class abc"))
sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/46d9bf96/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
index f3bbcaf..3d991a9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
@@ -201,7 +201,7 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
| GROUP BY dataTime
""".stripMargin)
}
- assert(e.getMessage.equals("Unknown data map type abc"))
+ assert(e.getMessage.equals("Unknown datamap provider/class abc"))
}
test("test timeseries create table 12: USING and catch MalformedCarbonCommandException") {
@@ -216,7 +216,7 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
| GROUP BY dataTime
""".stripMargin)
}
- assert(e.getMessage.equals("Unknown data map type abc"))
+ assert(e.getMessage.equals("Unknown datamap provider/class abc"))
}
test("test timeseries create table 13: Only one granularity level can be defined 1") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/46d9bf96/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 242087e..f2f001e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.execution.command.preaaggregate.CreatePreAggregateTa
import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider._
import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, MalformedDataMapCommandException}
@@ -60,7 +61,14 @@ case class CarbonCreateDataMapCommand(
} else {
dmProperties
}
-
+ val dataMapProvider = {
+ try {
+ DataMapProvider.getDataMapProvider(dmClassName)
+ } catch {
+ case e: UnsupportedOperationException =>
+ throw new MalformedDataMapCommandException(e.getMessage)
+ }
+ }
if (sparkSession.sessionState.catalog.listTables(dbName)
.exists(_.table.equalsIgnoreCase(tableName))) {
LOGGER.audit(
@@ -70,16 +78,16 @@ case class CarbonCreateDataMapCommand(
if (!ifNotExistsSet) {
throw new TableAlreadyExistsException(dbName, tableName)
}
- } else if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
- dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ } else {
TimeSeriesUtil.validateTimeSeriesGranularity(newDmProperties, dmClassName)
- createPreAggregateTableCommands = if (dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ createPreAggregateTableCommands = if (dataMapProvider == TIMESERIES) {
val details = TimeSeriesUtil
.getTimeSeriesGranularityDetails(newDmProperties, dmClassName)
val updatedDmProperties = newDmProperties - details._1
- CreatePreAggregateTableCommand(dataMapName,
+ CreatePreAggregateTableCommand(
+ dataMapName,
tableIdentifier,
- dmClassName,
+ dataMapProvider,
updatedDmProperties,
queryString.get,
Some(details._1),
@@ -88,14 +96,12 @@ case class CarbonCreateDataMapCommand(
CreatePreAggregateTableCommand(
dataMapName,
tableIdentifier,
- dmClassName,
+ dataMapProvider,
newDmProperties,
queryString.get,
ifNotExistsSet = ifNotExistsSet)
}
createPreAggregateTableCommands.processMetadata(sparkSession)
- } else {
- throw new MalformedDataMapCommandException("Unknown data map type " + dmClassName)
}
LOGGER.audit(s"DataMap $dataMapName successfully added to Table ${tableIdentifier.table}")
Seq.empty
@@ -110,7 +116,7 @@ case class CarbonCreateDataMapCommand(
Seq.empty
}
} else {
- throw new MalformedDataMapCommandException("Unknown data map type " + dmClassName)
+ throw new MalformedDataMapCommandException("Unknown datamap provider/class " + dmClassName)
}
}
@@ -123,7 +129,7 @@ case class CarbonCreateDataMapCommand(
Seq.empty
}
} else {
- throw new MalformedDataMapCommandException("Unknown data map type " + dmClassName)
+ throw new MalformedDataMapCommandException("Unknown datamap provider/class " + dmClassName)
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/46d9bf96/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index 31a3403..231a001 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
@@ -46,7 +47,7 @@ import org.apache.carbondata.spark.util.DataLoadingUtil
case class CreatePreAggregateTableCommand(
dataMapName: String,
parentTableIdentifier: TableIdentifier,
- dmClassName: String,
+ dataMapProvider: DataMapProvider,
dmProperties: Map[String, String],
queryString: String,
timeSeriesFunction: Option[String] = None,
@@ -125,7 +126,7 @@ case class CreatePreAggregateTableCommand(
// child schema object which will be updated on parent table about the
val childSchema = tableInfo.getFactTable.buildChildSchema(
dataMapName,
- CarbonCommonConstants.AGGREGATIONDATAMAPSCHEMA,
+ dataMapProvider.getClassName,
tableInfo.getDatabaseName,
queryString,
"AGGREGATION")