You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/12/24 13:26:25 UTC
[29/50] [abbrv] carbondata git commit: [CARBONDATA-1258] CarbonData
should not allow loading Date Type values violating the boundary condition
("0001-01-01" through "9999-12-31")
[CARBONDATA-1258] CarbonData should not allow loading Date Type values violating the boundary condition ("0001-01-01" through "9999-12-31")
Spark date type, supporting "0001-01-01" through "9999-12-31". As carbon pushes the data to spark layer so can support beyond this range.
This closes #1126
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/425f820b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/425f820b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/425f820b
Branch: refs/heads/fgdatamap
Commit: 425f820bded52998e2da3bc8712a52a1544e7138
Parents: 71959da
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Mon Jul 3 15:48:40 2017 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Thu Dec 21 12:22:09 2017 +0530
----------------------------------------------------------------------
.../DateDirectDictionaryGenerator.java | 28 +++++++
.../DateDirectDictionaryGeneratorTest.java | 84 ++++++++++++++++++++
2 files changed, 112 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/425f820b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index cd32e97..c49af9c 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -43,11 +43,33 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
private String dateFormat;
/**
+ * min value supported for date type column
+ */
+ private static final long MIN_VALUE;
+ /**
+ * MAx value supported for date type column
+ */
+ private static final long MAX_VALUE;
+ /**
* Logger instance
*/
private static final LogService LOGGER =
LogServiceFactory.getLogService(DateDirectDictionaryGenerator.class.getName());
+ static {
+ SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
+ df.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long minValue = 0;
+ long maxValue = 0;
+ try {
+ minValue = df.parse("0001-01-01").getTime();
+ maxValue = df.parse("9999-12-31").getTime();
+ } catch (ParseException e) {
+ // the Exception will not occur as constant value is being parsed
+ }
+ MIN_VALUE = minValue;
+ MAX_VALUE = maxValue;
+ }
public DateDirectDictionaryGenerator(String dateFormat) {
this.dateFormat = dateFormat;
initialize();
@@ -142,6 +164,12 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
}
private int generateKey(long timeValue) {
+ if (timeValue < MIN_VALUE || timeValue > MAX_VALUE) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Value for date type column is not in valid range. Value considered as null.");
+ }
+ return 1;
+ }
return (int) Math.floor((double) timeValue / MILLIS_PER_DAY) + cutOffDate;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/425f820b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
new file mode 100644
index 0000000..7390556
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.keygenerator.directdictionary;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit test case for the DateDirectDictionaryGenerator
+ */
+public class DateDirectDictionaryGeneratorTest {
+ private DirectDictionaryGenerator dictionaryGenerator = null;
+
+ @Before public void setUp() throws Exception {
+ TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator();
+ dictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(DataTypes.DATE,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
+ }
+
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void lowerBoundaryValueTest() {
+ int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("0001-01-01");
+ Assert.assertTrue(surrogateKey > 1);
+ }
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void lowerBoundaryInvalidValueTest() {
+ int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("0001-01-00");
+ Assert.assertTrue(surrogateKey == 1);
+ }
+
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void upperBoundaryValueTest() {
+ int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("9999-12-31");
+ Assert.assertTrue(surrogateKey > 1);
+ }
+
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void upperBoundaryInvalidValueTest() {
+ int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("10000-12-31");
+ Assert.assertTrue(surrogateKey == 1);
+ }
+
+ @After public void tearDown() throws Exception {
+ dictionaryGenerator = null;
+ }
+}