[26/51] [abbrv] carbondata git commit: [CARBONDATA-1452] Issue with loading timestamp data beyond cutoff

[ra...@apache.org]

[CARBONDATA-1452] Issue with loading timestamp data beyond cutoff

While generating surrogate for timestamp dictionary column, we are casting the value to int. We are considering only the +ve values for generating dictionary, when the value is out of range,overflow occurs and cyclic rotation happens while casting, in the cyclic rotation there is possibility of getting +ve values in overflow cases too.

Lets say cutoff timestamp is 1970-01-01 05:30:00, so we will be able to load data 68 years from this date, not beyond 68 years

While loading 3007-01-01 00:00:00, dictionary generation will throw bad record exception as converting this data to int is -ve (overflows and cyclic rotation)

But while loading 4016-01-01 00:00:00, dictionary will be generated for this as converting this data to int is +ve (overflows and cyclic rotation) --> This data is loaded but not as actual value. Different timestamp will be loaded.

This PR has,

(1) Refactoring
(2) Checking overflow

This closes #1335


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2176a2f1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2176a2f1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2176a2f1

Branch: refs/heads/branch-1.2
Commit: 2176a2f1d317763f5423dc1a5c254ee29e096c4b
Parents: a8b3fac
Author: dhatchayani <dh...@gmail.com>
Authored: Wed Sep 6 15:43:56 2017 +0530
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Wed Sep 13 15:34:36 2017 +0530

----------------------------------------------------------------------
 .../TimeStampDirectDictionaryGenerator.java     | 23 +++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/2176a2f1/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index 6a0b9e6..c8b88d8 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -89,7 +89,7 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
     }
     long cutOffTimeStampLocal;
     if (null == cutOffTimeStampString) {
-      cutOffTimeStampLocal = -1;
+      cutOffTimeStampLocal = 0;
     } else {
       try {
         SimpleDateFormat timeParser = new SimpleDateFormat(CarbonProperties.getInstance()
@@ -102,7 +102,7 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
         LOGGER.warn("Cannot convert" + cutOffTimeStampString
             + " to Time/Long type value. Value considered for cutOffTimeStamp is -1." + e
             .getMessage());
-        cutOffTimeStampLocal = -1;
+        cutOffTimeStampLocal = 0;
       }
     }
     granularityFactor = granularityFactorLocal;
@@ -187,12 +187,7 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
     if (key == 1) {
       return null;
     }
-    long timeStamp = 0;
-    if (cutOffTimeStamp >= 0) {
-      timeStamp = ((key - 2) * granularityFactor + cutOffTimeStamp);
-    } else {
-      timeStamp = (key - 2) * granularityFactor;
-    }
+    long timeStamp = ((key - 2) * granularityFactor + cutOffTimeStamp);
     return timeStamp * 1000L;
   }
 
@@ -215,13 +210,15 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
   }
 
   private int generateKey(long timeValue) {
-    if (cutOffTimeStamp >= 0) {
-      int keyValue = (int) ((timeValue - cutOffTimeStamp) / granularityFactor);
-      return keyValue < 0 ? 1 : keyValue + 2;
-    } else {
-      int keyValue = (int) (timeValue / granularityFactor);
+    if (timeValue >= 0) {
+      long time = (timeValue - cutOffTimeStamp) / granularityFactor;
+      int keyValue = -1;
+      if (time <= (long) Integer.MAX_VALUE) {
+        keyValue = (int) time;
+      }
       return keyValue < 0 ? 1 : keyValue + 2;
     }
+    return 1;
   }
 
   public void initialize() {