You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/11/18 15:29:59 UTC
[25/28] carbondata git commit: [CARBONDATA-1762] Remove existing
column level dateformat and support dateformat,
timestampformat in the load option
[CARBONDATA-1762] Remove existing column level dateformat and support dateformat, timestampformat in the load option
(1) Remove column level dateformat option
(2) Support dateformat and timestampformat in load options(table level)
This closes #1524
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/85dc4fff
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/85dc4fff
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/85dc4fff
Branch: refs/heads/fgdatamap
Commit: 85dc4fff0ecca160654085379310a1c3096731f7
Parents: 2a12938
Author: akashrn5 <ak...@gmail.com>
Authored: Fri Nov 17 16:55:33 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Nov 18 16:34:43 2017 +0800
----------------------------------------------------------------------
.../constants/CarbonLoadOptionConstants.java | 10 +++++-
.../carbondata/core/util/DataTypeUtil.java | 2 +-
.../TestLoadDataWithDiffTimestampFormat.scala | 31 ++++++++--------
.../carbondata/spark/load/ValidateUtil.scala | 38 +++++++++-----------
.../carbondata/spark/util/DataLoadingUtil.scala | 13 ++++++-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 24 +++++++------
.../processing/loading/DataField.java | 10 ++++++
.../loading/DataLoadProcessBuilder.java | 10 +++---
.../DirectDictionaryFieldConverterImpl.java | 10 +++++-
.../impl/NonDictionaryFieldConverterImpl.java | 8 ++++-
.../loading/model/CarbonLoadModel.java | 13 +++++++
.../util/CarbonDataProcessorUtil.java | 19 ----------
13 files changed, 114 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index ac278d9..e78d125 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -46,12 +46,20 @@ public final class CarbonLoadOptionConstants {
public static final String CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT = "false";
/**
- * option to specify the load option
+ * option to specify the dateFormat in load option for all date columns in table
*/
@CarbonProperty
public static final String CARBON_OPTIONS_DATEFORMAT =
"carbon.options.dateformat";
public static final String CARBON_OPTIONS_DATEFORMAT_DEFAULT = "";
+
+ /**
+ * option to specify the timestampFormat in load option for all timestamp columns in table
+ */
+ @CarbonProperty
+ public static final String CARBON_OPTIONS_TIMESTAMPFORMAT =
+ "carbon.options.timestampformat";
+ public static final String CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT = "";
/**
* option to specify the sort_scope
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 0961a63..3a25988 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -319,7 +319,7 @@ public final class DataTypeUtil {
Date dateToStr = null;
DateFormat dateFormatter = null;
try {
- if (null != dateFormat) {
+ if (null != dateFormat && !dateFormat.trim().isEmpty()) {
dateFormatter = new SimpleDateFormat(dateFormat);
} else {
dateFormatter = timeStampformatter.get();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
index 71d6466..906f05a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
@@ -17,10 +17,12 @@
package org.apache.carbondata.spark.testsuite.dataload
-import java.sql.Timestamp
+import java.sql.{Date, Timestamp}
+import java.text.SimpleDateFormat
import org.apache.spark.sql.Row
import org.scalatest.BeforeAndAfterAll
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
@@ -31,26 +33,25 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
sql("DROP TABLE IF EXISTS t3")
sql("""
CREATE TABLE IF NOT EXISTS t3
- (ID Int, date Timestamp, starttime Timestamp, country String,
+ (ID Int, date date, starttime Timestamp, country String,
name String, phonetype String, serialname String, salary Int)
STORED BY 'carbondata'
""")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
}
test("test load data with different timestamp format") {
sql(s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
- OPTIONS('dateformat' = 'starttime:yyyy-MM-dd HH:mm:ss')
+ OPTIONS('dateformat' = 'yyyy/MM/dd','timestampformat'='yyyy-MM-dd HH:mm:ss')
""")
sql(s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData2.csv' into table t3
- OPTIONS('dateformat' = ' date : yyyy-MM-dd , StartTime : yyyy/MM/dd HH:mm:ss')
+ OPTIONS('dateformat' = 'yyyy-MM-dd','timestampformat'='yyyy/MM/dd HH:mm:ss')
""")
+ val sdf = new SimpleDateFormat("yyyy-MM-dd")
checkAnswer(
sql("SELECT date FROM t3 WHERE ID = 1"),
- Seq(Row(Timestamp.valueOf("2015-07-23 00:00:00.0")))
+ Seq(Row(new Date(sdf.parse("2015-07-23").getTime)))
)
checkAnswer(
sql("SELECT starttime FROM t3 WHERE ID = 1"),
@@ -58,7 +59,7 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
)
checkAnswer(
sql("SELECT date FROM t3 WHERE ID = 18"),
- Seq(Row(Timestamp.valueOf("2015-07-25 00:00:00.0")))
+ Seq(Row(new Date(sdf.parse("2015-07-25").getTime)))
)
checkAnswer(
sql("SELECT starttime FROM t3 WHERE ID = 18"),
@@ -75,19 +76,19 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
assert(false)
} catch {
case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Option DateFormat is not provided for Column date.")
+ assertResult(ex.getMessage)("Error: Wrong option: date is provided for option DateFormat")
case _: Throwable=> assert(false)
}
try {
sql(s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
- OPTIONS('dateformat' = 'fasfdas:yyyy/MM/dd')
+ OPTIONS('timestampformat' = 'timestamp')
""")
assert(false)
} catch {
case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Wrong Column Name fasfdas is provided in Option DateFormat.")
+ assertResult(ex.getMessage)("Error: Wrong option: timestamp is provided for option TimestampFormat")
case _: Throwable => assert(false)
}
@@ -99,7 +100,7 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
assert(false)
} catch {
case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Option DateFormat is not provided for Column date.")
+ assertResult(ex.getMessage)("Error: Wrong option: date: is provided for option DateFormat")
case _: Throwable => assert(false)
}
@@ -111,19 +112,19 @@ class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterA
assert(false)
} catch {
case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Option DateFormat is not provided for Column date .")
+ assertResult(ex.getMessage)("Error: Wrong option: date is provided for option DateFormat")
case _: Throwable => assert(false)
}
try {
sql(s"""
LOAD DATA LOCAL INPATH '$resourcesPath/timeStampFormatData1.csv' into table t3
- OPTIONS('dateformat' = ':yyyy/MM/dd ')
+ OPTIONS('dateformat' = 'fasfdas:yyyy/MM/dd')
""")
assert(false)
} catch {
case ex: MalformedCarbonCommandException =>
- assertResult(ex.getMessage)("Error: Wrong Column Name is provided in Option DateFormat.")
+ assertResult(ex.getMessage)("Error: Wrong option: fasfdas:yyyy/MM/dd is provided for option DateFormat")
case _: Throwable => assert(false)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
index 51e0cc4..dfda92c 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
@@ -17,35 +17,31 @@
package org.apache.carbondata.spark.load
-import scala.collection.JavaConverters._
+import java.text.SimpleDateFormat
-import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.loading.sort.SortScopeOptions
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
object ValidateUtil {
- def validateDateFormat(dateFormat: String, table: CarbonTable, tableName: String): Unit = {
- val dimensions = table.getDimensionByTableName(tableName).asScala
+
+ /**
+ * validates both timestamp and date for illegal values
+ *
+ * @param dateTimeLoadFormat
+ * @param dateTimeLoadOption
+ */
+ def validateDateTimeFormat(dateTimeLoadFormat: String, dateTimeLoadOption: String): Unit = {
// allowing empty value to be configured for dateformat option.
- if (dateFormat != null && dateFormat.trim != "") {
- val dateFormats: Array[String] = dateFormat.split(CarbonCommonConstants.COMMA)
- for (singleDateFormat <- dateFormats) {
- val dateFormatSplits: Array[String] = singleDateFormat.split(":", 2)
- val columnName = dateFormatSplits(0).trim.toLowerCase
- if (!dimensions.exists(_.getColName.equals(columnName))) {
- throw new MalformedCarbonCommandException("Error: Wrong Column Name " +
- dateFormatSplits(0) +
- " is provided in Option DateFormat.")
- }
- if (dateFormatSplits.length < 2 || dateFormatSplits(1).trim.isEmpty) {
- throw new MalformedCarbonCommandException("Error: Option DateFormat is not provided " +
- "for " + "Column " + dateFormatSplits(0) +
- ".")
- }
- }
+ if (dateTimeLoadFormat != null && dateTimeLoadFormat.trim != "") {
+ try {
+ new SimpleDateFormat(dateTimeLoadFormat)
+ } catch {
+ case _: IllegalArgumentException =>
+ throw new MalformedCarbonCommandException(s"Error: Wrong option: $dateTimeLoadFormat is" +
+ s" provided for option $dateTimeLoadOption")
}
+ }
}
def validateSortScope(carbonTable: CarbonTable, sortScope: String): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 84ad85e..271962b 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -104,6 +104,14 @@ object DataLoadingUtil {
CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)))
optionsFinal.put(
+ "timestampformat",
+ options.getOrElse(
+ "timestampformat",
+ carbonProperty.getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT)))
+
+ optionsFinal.put(
"global_sort_partitions",
options.getOrElse(
"global_sort_partitions",
@@ -193,13 +201,15 @@ object DataLoadingUtil {
val bad_records_action = optionsFinal("bad_records_action")
val bad_record_path = optionsFinal("bad_record_path")
val global_sort_partitions = optionsFinal("global_sort_partitions")
+ val timestampformat = optionsFinal("timestampformat")
val dateFormat = optionsFinal("dateformat")
val delimeter = optionsFinal("delimiter")
val complex_delimeter_level1 = optionsFinal("complex_delimiter_level_1")
val complex_delimeter_level2 = optionsFinal("complex_delimiter_level_2")
val all_dictionary_path = optionsFinal("all_dictionary_path")
val column_dict = optionsFinal("columndict")
- ValidateUtil.validateDateFormat(dateFormat, table, table.getTableName)
+ ValidateUtil.validateDateTimeFormat(timestampformat, "TimestampFormat")
+ ValidateUtil.validateDateTimeFormat(dateFormat, "DateFormat")
ValidateUtil.validateSortScope(table, sort_scope)
if (bad_records_logger_enable.toBoolean ||
@@ -242,6 +252,7 @@ object DataLoadingUtil {
}
}
+ carbonLoadModel.setTimestampformat(timestampformat)
carbonLoadModel.setDateFormat(dateFormat)
carbonLoadModel.setDefaultTimestampFormat(carbonProperty.getProperty(
CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index b4cde9b..b7f5def 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -854,7 +854,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
"SERIALIZATION_NULL_FORMAT", "BAD_RECORDS_LOGGER_ENABLE", "BAD_RECORDS_ACTION",
"ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR", "DATEFORMAT", "BAD_RECORD_PATH",
"BATCH_SORT_SIZE_INMB", "GLOBAL_SORT_PARTITIONS", "SINGLE_PASS",
- "IS_EMPTY_DATA_BAD_RECORD", "HEADER"
+ "IS_EMPTY_DATA_BAD_RECORD", "HEADER", "TIMESTAMPFORMAT"
)
var isSupported = true
val invalidOptions = StringBuilder.newBuilder
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 7dad243..38bb936 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -800,18 +800,20 @@ object CarbonDataRDDFactory {
throw new DataLoadingException("Partition column not found.")
}
- val dateFormatMap = CarbonDataProcessorUtil.getDateFormatMap(carbonLoadModel.getDateFormat)
- val specificFormat = Option(dateFormatMap.get(partitionColumn.toLowerCase))
- val timeStampFormat = if (specificFormat.isDefined) {
- new SimpleDateFormat(specificFormat.get)
- } else {
- val timestampFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
- .CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- new SimpleDateFormat(timestampFormatString)
- }
+ val specificTimestampFormat = carbonLoadModel.getTimestampformat
+ val specificDateFormat = carbonLoadModel.getDateFormat
+ val timeStampFormat =
+ if (specificTimestampFormat != null && !specificTimestampFormat.trim.isEmpty) {
+ new SimpleDateFormat(specificTimestampFormat)
+ } else {
+ val timestampFormatString = CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ new SimpleDateFormat(timestampFormatString)
+ }
- val dateFormat = if (specificFormat.isDefined) {
- new SimpleDateFormat(specificFormat.get)
+ val dateFormat = if (specificDateFormat != null && !specificDateFormat.trim.isEmpty) {
+ new SimpleDateFormat(specificDateFormat)
} else {
val dateFormatString = CarbonProperties.getInstance().getProperty(CarbonCommonConstants
.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
index 5c81bb9..fb78deb 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataField.java
@@ -35,6 +35,8 @@ public class DataField implements Serializable {
private String dateFormat;
+ private String timestampFormat;
+
public boolean hasDictionaryEncoding() {
return column.hasEncoding(Encoding.DICTIONARY);
}
@@ -50,4 +52,12 @@ public class DataField implements Serializable {
public void setDateFormat(String dateFormat) {
this.dateFormat = dateFormat;
}
+
+ public String getTimestampFormat() {
+ return timestampFormat;
+ }
+
+ public void setTimestampFormat(String timestampFormat) {
+ this.timestampFormat = timestampFormat;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index 71c3dc2..8118e2f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -20,7 +20,6 @@ package org.apache.carbondata.processing.loading;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -28,6 +27,7 @@ import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -185,8 +185,6 @@ public final class DataLoadProcessBuilder {
carbonTable.getDimensionByTableName(carbonTable.getTableName());
List<CarbonMeasure> measures =
carbonTable.getMeasureByTableName(carbonTable.getTableName());
- Map<String, String> dateFormatMap =
- CarbonDataProcessorUtil.getDateFormatMap(loadModel.getDateFormat());
List<DataField> dataFields = new ArrayList<>();
List<DataField> complexDataFields = new ArrayList<>();
@@ -194,7 +192,11 @@ public final class DataLoadProcessBuilder {
// And then add complex data types and measures.
for (CarbonColumn column : dimensions) {
DataField dataField = new DataField(column);
- dataField.setDateFormat(dateFormatMap.get(column.getColName()));
+ if (column.getDataType() == DataTypes.DATE) {
+ dataField.setDateFormat(loadModel.getDateFormat());
+ } else if (column.getDataType() == DataTypes.TIMESTAMP) {
+ dataField.setTimestampFormat(loadModel.getTimestampformat());
+ }
if (column.isComplex()) {
complexDataFields.add(dataField);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
index 24c2f00..7dcef81 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/DirectDictionaryFieldConverterImpl.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.converter.BadRecordLogHolder;
@@ -42,11 +43,18 @@ public class DirectDictionaryFieldConverterImpl extends AbstractDictionaryFieldC
boolean isEmptyBadRecord) {
this.nullFormat = nullFormat;
this.column = dataField.getColumn();
- if (dataField.getDateFormat() != null && !dataField.getDateFormat().isEmpty()) {
+ if (dataField.getColumn().getDataType() == DataTypes.DATE && dataField.getDateFormat() != null
+ && !dataField.getDateFormat().isEmpty()) {
this.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dataField.getColumn().getDataType(),
dataField.getDateFormat());
+ } else if (dataField.getColumn().getDataType() == DataTypes.TIMESTAMP
+ && dataField.getTimestampFormat() != null && !dataField.getTimestampFormat().isEmpty()) {
+ this.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(dataField.getColumn().getDataType(),
+ dataField.getTimestampFormat());
+
} else {
this.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dataField.getColumn().getDataType());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
index eac9d69..e606cdb 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java
@@ -60,10 +60,16 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter {
} else if (dimensionValue == null || dimensionValue.equals(nullformat)) {
updateWithNullValue(row);
} else {
+ String dateFormat = null;
+ if (dataType == DataTypes.DATE) {
+ dateFormat = dataField.getDateFormat();
+ } else if (dataType == DataTypes.TIMESTAMP) {
+ dateFormat = dataField.getTimestampFormat();
+ }
try {
row.update(DataTypeUtil
.getBytesBasedOnDataTypeForNoDictionaryColumn(dimensionValue, dataType,
- dataField.getDateFormat()), index);
+ dateFormat), index);
} catch (Throwable ex) {
if (dimensionValue.length() > 0 || (dimensionValue.length() == 0 && isEmptyBadRecord)) {
String message = logHolder.getColumnMessageMap().get(column.getColName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index 8c3fe56..326ee62 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -93,6 +93,8 @@ public class CarbonLoadModel implements Serializable {
*/
private String commentChar;
+ private String timestampformat;
+
private String dateFormat;
private String defaultTimestampFormat;
@@ -351,6 +353,7 @@ public class CarbonLoadModel implements Serializable {
copy.escapeChar = escapeChar;
copy.quoteChar = quoteChar;
copy.commentChar = commentChar;
+ copy.timestampformat = timestampformat;
copy.dateFormat = dateFormat;
copy.defaultTimestampFormat = defaultTimestampFormat;
copy.maxColumns = maxColumns;
@@ -399,6 +402,7 @@ public class CarbonLoadModel implements Serializable {
copy.escapeChar = escapeChar;
copy.quoteChar = quoteChar;
copy.commentChar = commentChar;
+ copy.timestampformat = timestampformat;
copy.dateFormat = dateFormat;
copy.defaultTimestampFormat = defaultTimestampFormat;
copy.maxColumns = maxColumns;
@@ -449,6 +453,7 @@ public class CarbonLoadModel implements Serializable {
copyObj.escapeChar = escapeChar;
copyObj.quoteChar = quoteChar;
copyObj.commentChar = commentChar;
+ copyObj.timestampformat = timestampformat;
copyObj.dateFormat = dateFormat;
copyObj.defaultTimestampFormat = defaultTimestampFormat;
copyObj.maxColumns = maxColumns;
@@ -763,4 +768,12 @@ public class CarbonLoadModel implements Serializable {
public void setBadRecordsLocation(String badRecordsLocation) {
this.badRecordsLocation = badRecordsLocation;
}
+
+ public String getTimestampformat() {
+ return timestampformat;
+ }
+
+ public void setTimestampformat(String timestampformat) {
+ this.timestampformat = timestampformat;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/85dc4fff/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 7218a12..b64362e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -21,7 +21,6 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -431,24 +430,6 @@ public final class CarbonDataProcessorUtil {
}
/**
- * Creates map for columns which dateformats mentioned while loading the data.
- * @param dataFormatString
- * @return
- */
- public static Map<String, String> getDateFormatMap(String dataFormatString) {
- Map<String, String> dateformatsHashMap = new HashMap<>();
- if (dataFormatString != null && !dataFormatString.isEmpty()) {
- String[] dateformats = dataFormatString.split(CarbonCommonConstants.COMMA);
- for (String dateFormat : dateformats) {
- String[] dateFormatSplits = dateFormat.split(":", 2);
- dateformatsHashMap
- .put(dateFormatSplits[0].toLowerCase().trim(), dateFormatSplits[1].trim());
- }
- }
- return dateformatsHashMap;
- }
-
- /**
* This method will get the store location for the given path, segment id and partition id
*
* @return data directory path