You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2017/03/22 10:59:42 UTC
[1/2] incubator-carbondata git commit: Select query is throwing
exception when newly column is added without any default value
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 47bdb4d02 -> c2a3e7d9b
Select query is throwing exception when newly column is added without any default value
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c2cdd6e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c2cdd6e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c2cdd6e8
Branch: refs/heads/master
Commit: c2cdd6e80c9a6f9eaa42a48ded7447f4d65ce087
Parents: 47bdb4d
Author: nareshpr <pr...@gmail.com>
Authored: Tue Mar 21 22:16:49 2017 +0530
Committer: nareshpr <pr...@gmail.com>
Committed: Wed Mar 22 15:19:24 2017 +0530
----------------------------------------------------------------------
.../execution/command/carbonTableSchema.scala | 52 ++++++++++++--------
.../sql/parser/CarbonSpark2SqlParser.scala | 19 ++++++-
2 files changed, 48 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2cdd6e8/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 626dc37..c858d0a 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -229,31 +229,41 @@ class AlterTableProcessor(
tablePropertiesMap.put(x._1, x._2)
}
}
- for (elem <- alterTableModel.tableProperties) {
- if (elem._1.toLowerCase.startsWith("default.value.")) {
- val col = newCols.filter(p => p.getColumnName.equalsIgnoreCase(elem._1.substring(14)))
- if (col.size == 1) {
- val data = DataTypeUtil.convertDataToBytesBasedOnDataType(elem._2, col(0).getDataType)
- if (null != data) {
- col(0).setDefaultValue(data)
- } else {
- LOGGER
- .error(
- "Invalid default value for new column " + dbName + "." + alterTableModel.tableName +
- "." + col(0).getColumnName + " : " + elem._2)
- }
- if (col(0).getEncodingList.contains(Encoding.DICTIONARY) &&
- !col(0).getEncodingList.contains(Encoding.DIRECT_DICTIONARY)) {
- GlobalDictionaryUtil
- .loadDefaultDictionaryValueForNewColumn(carbonTablePath,
- col(0),
- tableIdentifier,
- storePath,
- elem._2)
+
+ // This part will create dictionary file for all newly added dictionary columns
+ // if valid default value is provided,
+ // then that value will be included while creating dictionary file
+ val defaultValueString = "default.value."
+ newCols.foreach { col =>
+ var rawData: String = null
+ for (elem <- alterTableModel.tableProperties) {
+ if (elem._1.toLowerCase.startsWith(defaultValueString)) {
+ if (col.getColumnName.equalsIgnoreCase(elem._1.substring(defaultValueString.length))) {
+ rawData = elem._2
+ val data = DataTypeUtil.convertDataToBytesBasedOnDataType(elem._2, col.getDataType)
+ if (null != data) {
+ col.setDefaultValue(data)
+ } else {
+ LOGGER
+ .error(
+ "Invalid default value for new column " + dbName + "." +
+ alterTableModel.tableName +
+ "." + col.getColumnName + " : " + elem._2)
+ }
}
}
}
+ if (col.getEncodingList.contains(Encoding.DICTIONARY) &&
+ !col.getEncodingList.contains(Encoding.DIRECT_DICTIONARY)) {
+ GlobalDictionaryUtil
+ .loadDefaultDictionaryValueForNewColumn(carbonTablePath,
+ col,
+ tableIdentifier,
+ storePath,
+ rawData)
+ }
}
+
tableSchema.setListOfColumns(allColumns.asJava)
tableInfo.setLastUpdatedTime(System.currentTimeMillis())
tableInfo.setFactTable(tableSchema)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2cdd6e8/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index 75ad559..dbc3124 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -166,6 +166,21 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
}
}
val tableProps = if (tblProp.isDefined) {
+ tblProp.get.groupBy(_._1.toLowerCase).foreach(f =>
+ if (f._2.size > 1) {
+ val name = f._1.toLowerCase
+ val colName = name.substring(14)
+ if (name.startsWith("default.value.") &&
+ fields.filter(p => p.column.equalsIgnoreCase(colName)).size == 1) {
+ LOGGER.error(s"Duplicate default value exist for new column: ${ colName }")
+ LOGGER.audit(
+ s"Validation failed for Create/Alter Table Operation " +
+ s"for ${ table }. " +
+ s"Duplicate default value exist for new column: ${ colName }")
+ sys.error(s"Duplicate default value exist for new column: ${ colName }")
+ }
+ }
+ )
// default value should not be converted to lower case
val tblProps = tblProp.get
.map(f => if (CarbonCommonConstants.TABLE_BLOCKSIZE.equalsIgnoreCase(f._1) ||
@@ -210,14 +225,14 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
("(" ~> rep1sep(ident, ",") <~ ")") <~ opt(";") ^^ {
case dbName ~ table ~ drop ~ columns ~ values =>
// validate that same column name is not repeated
- values.groupBy(identity).collect {
+ values.map(_.toLowerCase).groupBy(identity).collect {
case (x, ys) if ys.lengthCompare(1) > 0 =>
throw new MalformedCarbonCommandException(s"$x is duplicate. Duplicate columns not " +
s"allowed")
}
val alterTableDropColumnModel = AlterTableDropColumnModel(convertDbNameToLowerCase(dbName),
table.toLowerCase,
- values)
+ values.map(_.toLowerCase))
AlterTableDropColumns(alterTableDropColumnModel)
}
}
[2/2] incubator-carbondata git commit: [CARBONDATA-802]Select query
is throwing exception when newly column is added without any default value
This closes #681
Posted by ch...@apache.org.
[CARBONDATA-802]Select query is throwing exception when newly column is added without any default value This closes #681
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c2a3e7d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c2a3e7d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c2a3e7d9
Branch: refs/heads/master
Commit: c2a3e7d9befb8fb86d2858fb67cdda844a8fb8a3
Parents: 47bdb4d c2cdd6e
Author: chenliang613 <ch...@huawei.com>
Authored: Wed Mar 22 16:22:01 2017 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Mar 22 16:22:01 2017 +0530
----------------------------------------------------------------------
.../execution/command/carbonTableSchema.scala | 52 ++++++++++++--------
.../sql/parser/CarbonSpark2SqlParser.scala | 19 ++++++-
2 files changed, 48 insertions(+), 23 deletions(-)
----------------------------------------------------------------------