You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by qi...@apache.org on 2020/02/12 14:18:22 UTC
[carbondata] branch master updated: [HOTFIX] Remove unused
parameter in TabelModel
This is an automated email from the ASF dual-hosted git repository.
qiangcai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 4540441 [HOTFIX] Remove unused parameter in TabelModel
4540441 is described below
commit 45404413f7c8f9b012454874f4328a6cf5efaec5
Author: Jacky Li <ja...@qq.com>
AuthorDate: Wed Feb 12 16:28:01 2020 +0800
[HOTFIX] Remove unused parameter in TabelModel
Why is this PR needed?
parentTable and parentTableRelation are not used in the TableModel, they can be removed
What changes were proposed in this PR?
parentTable and parentTableRelation are removed in TableModel
This closes #3613
---
.../command/carbonTableSchemaCommon.scala | 140 +++++----------------
1 file changed, 32 insertions(+), 108 deletions(-)
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 4e5454d..4636179 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -36,8 +36,8 @@ import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, DatabaseLoc
import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes, DecimalType}
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema._
-import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, RelationIdentifier, TableInfo, TableSchema}
-import org.apache.carbondata.core.metadata.schema.table.column.{ColumnSchema, ParentColumnTableRelation}
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema}
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.service.impl.ColumnUniqueIdGenerator
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentUpdateStatusManager}
import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
@@ -62,9 +62,7 @@ case class TableModel(
colProps: Option[util.Map[String, util.List[ColumnProperty]]] = None,
bucketFields: Option[BucketFields],
partitionInfo: Option[PartitionInfo],
- tableComment: Option[String] = None,
- var parentTable: Option[CarbonTable] = None,
- var dataMapRelation: Option[scala.collection.mutable.LinkedHashMap[Field, DataMapField]] = None)
+ tableComment: Option[String] = None)
case class Field(column: String, var dataType: Option[String], name: Option[String],
children: Option[List[Field]], parent: String = null,
@@ -610,7 +608,6 @@ class TableNewProcessor(cm: TableModel) {
encoders,
true,
field,
- cm.dataMapRelation,
useDictionaryEncoding = useDictionaryEncoding)
allColumns ++= Seq(columnSchema)
if (field.children.get != null) {
@@ -633,24 +630,20 @@ class TableNewProcessor(cm: TableModel) {
encoders: java.util.List[Encoding],
isDimensionCol: Boolean,
field: Field,
- map: Option[scala.collection.mutable.LinkedHashMap[Field, DataMapField]],
useDictionaryEncoding: Boolean = true) : ColumnSchema = {
val columnSchema = new ColumnSchema()
columnSchema.setDataType(dataType)
columnSchema.setColumnName(colName)
- val isParentColumnRelation = map.isDefined && map.get.get(field).isDefined
- if (!isParentColumnRelation) {
- val highCardinalityDims = cm.highcardinalitydims.getOrElse(Seq())
- if (highCardinalityDims.contains(colName)) {
- encoders.remove(Encoding.DICTIONARY)
- }
+ val highCardinalityDims = cm.highcardinalitydims.getOrElse(Seq())
+ if (highCardinalityDims.contains(colName)) {
+ encoders.remove(Encoding.DICTIONARY)
+ }
if (dataType == DataTypes.DATE && useDictionaryEncoding) {
- encoders.add(Encoding.DIRECT_DICTIONARY)
- }
- if (dataType == DataTypes.TIMESTAMP &&
- !highCardinalityDims.contains(colName) && useDictionaryEncoding) {
- encoders.add(Encoding.DIRECT_DICTIONARY)
- }
+ encoders.add(Encoding.DIRECT_DICTIONARY)
+ }
+ if (dataType == DataTypes.TIMESTAMP &&
+ !highCardinalityDims.contains(colName) && useDictionaryEncoding) {
+ encoders.add(Encoding.DIRECT_DICTIONARY)
}
columnSchema.setEncodingList(encoders)
val colUniqueIdGenerator = ColumnUniqueIdGenerator.getInstance
@@ -666,28 +659,6 @@ class TableNewProcessor(cm: TableModel) {
if (isVarcharColumn(colName)) {
columnSchema.setDataType(DataTypes.VARCHAR)
}
- if (isParentColumnRelation) {
- val dataMapField = map.get.get(field).get
- columnSchema.setFunction(dataMapField.aggregateFunction)
- val columnRelationList = dataMapField.columnTableRelationList.get
- val parentColumnTableRelationList = new util.ArrayList[ParentColumnTableRelation]
- columnRelationList.foreach {
- columnRelation =>
- val relationIdentifier = new RelationIdentifier(
- columnRelation.parentDatabaseName,
- columnRelation.parentTableName,
- columnRelation.parentTableId)
- if (cm.parentTable.isDefined) {
- relationIdentifier.setTablePath(cm.parentTable.get.getTablePath)
- }
- val parentColumnTableRelation = new ParentColumnTableRelation(
- relationIdentifier,
- columnRelation.parentColumnId,
- columnRelation.parentColumnName)
- parentColumnTableRelationList.add(parentColumnTableRelation)
- }
- columnSchema.setParentColumnTableRelations(parentColumnTableRelationList)
- }
// TODO: Need to fill RowGroupID, converted type
// & Number of Children after DDL finalization
if (field.columnComment.nonEmpty) {
@@ -711,22 +682,14 @@ class TableNewProcessor(cm: TableModel) {
// Sort columns should be at the begin of all columns
cm.sortKeyDims.get.foreach { keyDim =>
val field = cm.dimCols.find(keyDim equals _.column).get
- val encoders = if (getEncoderFromParent(field)) {
- cm.parentTable.get.getColumnByName(
- cm.dataMapRelation.get(field).columnTableRelationList.get.head.parentColumnName
- ).getEncoder
- } else {
- val encoders = new java.util.ArrayList[Encoding]()
- encoders.add(Encoding.DICTIONARY)
- encoders
- }
+ val encoders = new java.util.ArrayList[Encoding]()
+ encoders.add(Encoding.DICTIONARY)
val columnSchema = getColumnSchema(
DataTypeConverterUtil.convertToCarbonType(field.dataType.getOrElse("")),
field.name.getOrElse(field.column),
encoders,
true,
- field,
- cm.dataMapRelation)
+ field)
columnSchema.setSortColumn(true)
allColumns :+= columnSchema
index = index + 1
@@ -735,22 +698,14 @@ class TableNewProcessor(cm: TableModel) {
def addDimensionCol(field: Field): Unit = {
val sortField = cm.sortKeyDims.get.find(field.column equals _)
if (sortField.isEmpty) {
- val encoders = if (getEncoderFromParent(field)) {
- cm.parentTable.get.getColumnByName(
- cm.dataMapRelation.get(field).columnTableRelationList.get.head.parentColumnName
- ).getEncoder
- } else {
- val encoders = new java.util.ArrayList[Encoding]()
- encoders.add(Encoding.DICTIONARY)
- encoders
- }
+ val encoders = new java.util.ArrayList[Encoding]()
+ encoders.add(Encoding.DICTIONARY)
val columnSchema = getColumnSchema(
DataTypeConverterUtil.convertToCarbonType(field.dataType.getOrElse("")),
field.name.getOrElse(field.column),
encoders,
true,
- field,
- cm.dataMapRelation)
+ field)
allColumns :+= columnSchema
index = index + 1
if (field.children.isDefined && field.children.get != null) {
@@ -776,21 +731,8 @@ class TableNewProcessor(cm: TableModel) {
var isAggFunPresent = false
// getting the encoder from maintable so whatever encoding is applied in maintable
// same encoder can be applied on aggregate table
- val encoders = if (getEncoderFromParent(field)) {
- isAggFunPresent =
- cm.dataMapRelation.get(field).aggregateFunction.equalsIgnoreCase("sum") ||
- cm.dataMapRelation.get(field).aggregateFunction.equals("avg") ||
- cm.dataMapRelation.get(field).aggregateFunction.equals("count")
- if (!isAggFunPresent) {
- cm.parentTable.get.getColumnByName(
- cm.dataMapRelation.get(field).columnTableRelationList.get.head.parentColumnName
- ).getEncoder
- } else {
- new java.util.ArrayList[Encoding]()
- }
- } else {
- new java.util.ArrayList[Encoding]()
- }
+ val encoders = new java.util.ArrayList[Encoding]()
+
// check if it can be dimension column
val isDimColumn = !encoders.isEmpty && !isAggFunPresent
val columnSchema = getColumnSchema(
@@ -798,8 +740,7 @@ class TableNewProcessor(cm: TableModel) {
field.name.getOrElse(field.column),
encoders,
isDimColumn,
- field,
- cm.dataMapRelation)
+ field)
allColumns :+= columnSchema
index = index + 1
if (!isDimColumn) {
@@ -816,22 +757,18 @@ class TableNewProcessor(cm: TableModel) {
}
}
- // Setting the boolean value of useInvertedIndex in column schema, if Parent table is defined
- // Encoding is already decided above
- if (cm.parentTable.isEmpty) {
- val noInvertedIndexCols = cm.noInvertedIdxCols.getOrElse(Seq())
- val invertedIndexCols = cm.innvertedIdxCols.getOrElse(Seq())
- for (column <- allColumns) {
- // When the column is measure or the specified no inverted index column in DDL,
- // set useInvertedIndex to false, otherwise true.
- if (invertedIndexCols.contains(column.getColumnName) &&
- !cm.msrCols.exists(_.column.equalsIgnoreCase(column.getColumnName))) {
- column.setUseInvertedIndex(true)
- } else {
- column.setUseInvertedIndex(false)
- }
+ val invertedIndexCols = cm.innvertedIdxCols.getOrElse(Seq())
+ for (column <- allColumns) {
+ // When the column is measure or the specified no inverted index column in DDL,
+ // set useInvertedIndex to false, otherwise true.
+ if (invertedIndexCols.contains(column.getColumnName) &&
+ !cm.msrCols.exists(_.column.equalsIgnoreCase(column.getColumnName))) {
+ column.setUseInvertedIndex(true)
+ } else {
+ column.setUseInvertedIndex(false)
}
}
+
// Adding dummy measure if no measure is provided
if (measureCount == 0) {
val encoders = new java.util.ArrayList[Encoding]()
@@ -846,8 +783,7 @@ class TableNewProcessor(cm: TableModel) {
CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE,
encoders,
false,
- field,
- cm.dataMapRelation)
+ field)
columnSchema.setInvisible(true)
allColumns :+= columnSchema
}
@@ -935,18 +871,6 @@ class TableNewProcessor(cm: TableModel) {
}
}
- /**
- * Method to check to get the encoder from parent or not
- *
- * @param field column field
- * @return get encoder from parent
- */
- private def getEncoderFromParent(field: Field): Boolean = {
- cm.parentTable.isDefined &&
- cm.dataMapRelation.get.get(field).isDefined &&
- cm.dataMapRelation.get(field).columnTableRelationList.size == 1
- }
-
// For checking if the specified col group columns are specified in fields list.
protected def checkColGroupsValidity(colGrps: Seq[String],
allCols: Seq[ColumnSchema],