You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2020/09/01 06:10:08 UTC

[carbondata] branch master updated: [CARBONDATA-3963]Fix hive timestamp data mismatch issue and empty data during query issue

This is an automated email from the ASF dual-hosted git repository.

kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 41eadda  [CARBONDATA-3963]Fix hive timestamp data mismatch issue and empty data during query issue
41eadda is described below

commit 41eadda6cc854ec13cd96822b0aa2b92e222e04d
Author: akashrn5 <ak...@gmail.com>
AuthorDate: Wed Aug 26 14:23:58 2020 +0530

    [CARBONDATA-3963]Fix hive timestamp data mismatch issue and empty data during query issue
    
    Why is this PR needed?
    1. when carbon table is queried through beeline, gives empty data.
    2. the timestamp data given by carbon table is different than hive table.
    Because during read we used to send the Timestamp value to WritableTimestampObjectInspector's
    create method, which basically was converting the value to EpochMillli, which resulted in the
    time value go different than the actual input data.
    
    What changes were proposed in this PR?
    1. Pass the carbon storage_handler into the table parameters
    2. Convert the the long value to formatted string and directly form the TimestampWritableV2,
    by passing the Hive's Timestamp object which avoid EpochMilli and gives the proper data as output.
    
    This closes #3903
---
 .../java/org/apache/carbondata/hive/WritableReadSupport.java | 12 ++++++++----
 .../apache/carbondata/hive/CarbonHiveMetastoreListener.scala |  2 ++
 .../test/java/org/apache/carbondata/hive/HiveTestUtils.java  | 11 +++++++++--
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/WritableReadSupport.java b/integration/hive/src/main/java/org/apache/carbondata/hive/WritableReadSupport.java
index b0d87e7..a404b78 100644
--- a/integration/hive/src/main/java/org/apache/carbondata/hive/WritableReadSupport.java
+++ b/integration/hive/src/main/java/org/apache/carbondata/hive/WritableReadSupport.java
@@ -18,12 +18,13 @@
 package org.apache.carbondata.hive;
 
 import java.io.IOException;
-import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -35,7 +36,6 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.serde2.io.*;
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector;
 import org.apache.hadoop.io.ArrayWritable;
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.BytesWritable;
@@ -231,8 +231,12 @@ public class WritableReadSupport<T> implements CarbonReadSupport<T> {
     } else if (dataType == DataTypes.DATE) {
       return new DateWritableV2((Integer) obj);
     } else if (dataType == DataTypes.TIMESTAMP) {
-      WritableTimestampObjectInspector ins = new WritableTimestampObjectInspector();
-      return ins.getPrimitiveWritableObject(ins.create(new Timestamp((long) obj / 1000)));
+      SimpleDateFormat dateFormat =
+          new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+      String formattedTime = dateFormat.format((long) obj / 1000);
+      org.apache.hadoop.hive.common.type.Timestamp t =
+          org.apache.hadoop.hive.common.type.Timestamp.valueOf(formattedTime);
+      return new TimestampWritableV2(t);
     } else if (dataType == DataTypes.STRING) {
       return new Text(obj.toString());
     } else if (DataTypes.isArrayType(dataType)) {
diff --git a/integration/hive/src/main/scala/org/apache/carbondata/hive/CarbonHiveMetastoreListener.scala b/integration/hive/src/main/scala/org/apache/carbondata/hive/CarbonHiveMetastoreListener.scala
index 3f0f4f0..c8e268b 100644
--- a/integration/hive/src/main/scala/org/apache/carbondata/hive/CarbonHiveMetastoreListener.scala
+++ b/integration/hive/src/main/scala/org/apache/carbondata/hive/CarbonHiveMetastoreListener.scala
@@ -51,6 +51,8 @@ class CarbonHiveMetastoreListener(conf: Configuration) extends MetaStorePreEvent
             table.getSd.setCols(hiveSchema)
             table.getSd.setInputFormat("org.apache.carbondata.hive.MapredCarbonInputFormat")
             table.getSd.setOutputFormat("org.apache.carbondata.hive.MapredCarbonOutputFormat")
+            table.getParameters
+              .put("storage_handler", "org.apache.carbondata.hive.CarbonStorageHandler")
             val serdeInfo = table.getSd.getSerdeInfo
             serdeInfo.setSerializationLib("org.apache.carbondata.hive.CarbonHiveSerDe")
             val tablePath = serdeInfo.getParameters.get("tablePath")
diff --git a/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java b/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
index 67c65a4..44c19b7 100644
--- a/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
+++ b/integration/hive/src/test/java/org/apache/carbondata/hive/HiveTestUtils.java
@@ -21,6 +21,9 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 
 import org.apache.carbondata.hive.test.server.HiveEmbeddedServer2;
 
@@ -56,6 +59,8 @@ public abstract class HiveTestUtils {
   public boolean checkAnswer(ResultSet actual, ResultSet expected) throws SQLException {
     Assert.assertEquals("Row Count Mismatch: ", expected.getFetchSize(), actual.getFetchSize());
     int rowCountExpected = 0;
+    List<String> expectedValuesList = new ArrayList<>();
+    List<String> actualValuesList = new ArrayList<>();
     while (expected.next()) {
       rowCountExpected ++;
       if (!actual.next()) {
@@ -65,10 +70,12 @@ public abstract class HiveTestUtils {
       Assert.assertTrue(numOfColumnsExpected > 0);
       Assert.assertEquals(actual.getMetaData().getColumnCount(), numOfColumnsExpected);
       for (int i = 1; i <= numOfColumnsExpected; i++) {
-        Assert.assertEquals(actual.getString(i), actual.getString(i));
+        expectedValuesList.add(expected.getString(i));
+        actualValuesList.add(actual.getString(i));
       }
-      System.out.println();
     }
+    Collections.sort(expectedValuesList);Collections.sort(actualValuesList);
+    Assert.assertArrayEquals(expectedValuesList.toArray(), actualValuesList.toArray());
     Assert.assertTrue(rowCountExpected > 0);
     return true;
   }