You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/09/18 10:01:57 UTC
[38/51] [abbrv] carbondata git commit: [CARBONDATA-1452] Issue with
loading timestamp data beyond cutoff
[CARBONDATA-1452] Issue with loading timestamp data beyond cutoff
(1)Removed timeValue>=0 condition => this condition will restrict loading proper data when the CARBON_CUTOFF_TIMESTAMP is set before 1970. In this case timeValue will always be < 0
(2) Added test case for the same
This closes #1355
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8791eabf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8791eabf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8791eabf
Branch: refs/heads/branch-1.2
Commit: 8791eabf0c6f2db385ed9c0886c0aa054a421b2f
Parents: 940f4d5
Author: dhatchayani <dh...@gmail.com>
Authored: Wed Sep 6 15:43:56 2017 +0530
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Fri Sep 15 13:54:21 2017 +0530
----------------------------------------------------------------------
.../TimeStampDirectDictionaryGenerator.java | 13 ++++------
.../TimeStampDirectDictionaryGeneratorTest.java | 26 ++++++++++++++++++++
2 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8791eabf/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index c8b88d8..e0f5d41 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -210,15 +210,12 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
}
private int generateKey(long timeValue) {
- if (timeValue >= 0) {
- long time = (timeValue - cutOffTimeStamp) / granularityFactor;
- int keyValue = -1;
- if (time <= (long) Integer.MAX_VALUE) {
- keyValue = (int) time;
- }
- return keyValue < 0 ? 1 : keyValue + 2;
+ long time = (timeValue - cutOffTimeStamp) / granularityFactor;
+ int keyValue = -1;
+ if (time >= (long) Integer.MIN_VALUE && time <= (long) Integer.MAX_VALUE) {
+ keyValue = (int) time;
}
- return 1;
+ return keyValue < 0 ? 1 : keyValue + 2;
}
public void initialize() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8791eabf/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java b/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
index 7bb433a..bdc4ca5 100644
--- a/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
+++ b/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
@@ -23,6 +23,7 @@ import java.util.TimeZone;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
+import mockit.Deencapsulation;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -77,6 +78,31 @@ public class TimeStampDirectDictionaryGeneratorTest {
*
* @throws Exception
*/
+ @Test public void getSurrogateWithCutoff() throws Exception {
+ SimpleDateFormat timeParser = new SimpleDateFormat(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+ timeParser.setLenient(false);
+ TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator();
+ long cutOffValue = timeParser.parse("1500-01-01 00:00:00").getTime();
+ //setting cutoff time to 1500-01-01 00:00:00 , so we can load data from this time
+ Deencapsulation.setField(generator, "cutOffTimeStamp", cutOffValue);
+ int surrogateFromValue = generator.generateDirectSurrogateKey("1500-01-01 00:00:01");
+ long valueFromSurrogate = (long) generator.getValueFromSurrogate(surrogateFromValue);
+ Date date = new Date(valueFromSurrogate / 1000);
+ Assert.assertEquals("1500-01-01 00:00:01", timeParser.format(date));
+ surrogateFromValue = generator.generateDirectSurrogateKey("1499-12-12 00:00:00");
+ //1499-12-12 00:00:00 is a value before cut off, so it is a bad record and surrogate should be 1
+ Assert.assertEquals(1, surrogateFromValue);
+ //re setting the value to default
+ Deencapsulation.setField(generator, "cutOffTimeStamp", 0L);
+ }
+
+ /**
+ * The memberString should be retrieved from the actual surrogate key
+ *
+ * @throws Exception
+ */
@Test public void lowerBoundaryValueTest() throws Exception {
TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator( );
long valueFromSurrogate = (long) generator.getValueFromSurrogate(2);