You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2017/02/10 12:01:55 UTC

hive git commit: HIVE-15850: Proper handling of timezone in Druid storage handler (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

Repository: hive
Updated Branches:
  refs/heads/master 8f273cc53 -> c0198e529


HIVE-15850: Proper handling of timezone in Druid storage handler (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

Close apache/hive#145


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c0198e52
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c0198e52
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c0198e52

Branch: refs/heads/master
Commit: c0198e5292b6be80eda682d61d2ea80af9cd3e42
Parents: 8f273cc
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Thu Feb 9 09:18:48 2017 +0000
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Fri Feb 10 12:01:44 2017 +0000

----------------------------------------------------------------------
 .../druid/io/DruidQueryBasedInputFormat.java    |  80 +++++----
 .../TestHiveDruidQueryBasedInputFormat.java     | 162 +++++++++++++++++++
 2 files changed, 208 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c0198e52/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java
index d1b2a72..8b37840 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java
@@ -17,24 +17,14 @@
  */
 package org.apache.hadoop.hive.druid.io;
 
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.metamx.common.lifecycle.Lifecycle;
-import com.metamx.http.client.HttpClient;
-import com.metamx.http.client.HttpClientConfig;
-import com.metamx.http.client.HttpClientInit;
-import io.druid.query.Druids;
-import io.druid.query.Druids.SegmentMetadataQueryBuilder;
-import io.druid.query.Druids.SelectQueryBuilder;
-import io.druid.query.Druids.TimeBoundaryQueryBuilder;
-import io.druid.query.Query;
-import io.druid.query.Result;
-import io.druid.query.metadata.metadata.SegmentAnalysis;
-import io.druid.query.metadata.metadata.SegmentMetadataQuery;
-import io.druid.query.select.PagingSpec;
-import io.druid.query.select.SelectQuery;
-import io.druid.query.spec.MultipleIntervalSegmentSpec;
-import io.druid.query.timeboundary.TimeBoundaryQuery;
-import io.druid.query.timeboundary.TimeBoundaryResultValue;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.calcite.adapter.druid.DruidDateTimeUtils;
 import org.apache.calcite.adapter.druid.DruidTable;
 import org.apache.commons.lang3.StringEscapeUtils;
@@ -67,13 +57,28 @@ import org.joda.time.chrono.ISOChronology;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.metamx.common.lifecycle.Lifecycle;
+import com.metamx.http.client.HttpClient;
+import com.metamx.http.client.HttpClientConfig;
+import com.metamx.http.client.HttpClientInit;
+
+import io.druid.query.BaseQuery;
+import io.druid.query.Druids;
+import io.druid.query.Druids.SegmentMetadataQueryBuilder;
+import io.druid.query.Druids.SelectQueryBuilder;
+import io.druid.query.Druids.TimeBoundaryQueryBuilder;
+import io.druid.query.Query;
+import io.druid.query.Result;
+import io.druid.query.metadata.metadata.SegmentAnalysis;
+import io.druid.query.metadata.metadata.SegmentMetadataQuery;
+import io.druid.query.select.PagingSpec;
+import io.druid.query.select.SelectQuery;
+import io.druid.query.spec.MultipleIntervalSegmentSpec;
+import io.druid.query.timeboundary.TimeBoundaryQuery;
+import io.druid.query.timeboundary.TimeBoundaryResultValue;
 
 /**
  * Druid query based input format.
@@ -131,13 +136,19 @@ public class DruidQueryBasedInputFormat extends InputFormat<NullWritable, DruidW
     JobContext jobContext = ShimLoader.getHadoopShims().newJobContext(job);
     Path[] paths = FileInputFormat.getInputPaths(jobContext);
 
+    // We need to deserialize and serialize query so intervals are written in the JSON
+    // Druid query with user timezone, as this is default Hive time semantics.
+    // Then, create splits with the Druid queries.
     switch (druidQueryType) {
       case Query.TIMESERIES:
       case Query.TOPN:
       case Query.GROUP_BY:
-        return new HiveDruidSplit[] { new HiveDruidSplit(address, druidQuery, paths[0]) };
+        return new HiveDruidSplit[] { new HiveDruidSplit(address,
+                deserializeSerialize(druidQuery), paths[0]) };
       case Query.SELECT:
-        return splitSelectQuery(conf, address, druidQuery, paths[0]);
+        SelectQuery selectQuery = DruidStorageHandlerUtils.JSON_MAPPER.readValue(
+                druidQuery, SelectQuery.class);
+        return splitSelectQuery(conf, address, selectQuery, paths[0]);
       default:
         throw new IOException("Druid query type not recognized");
     }
@@ -158,7 +169,7 @@ public class DruidQueryBasedInputFormat extends InputFormat<NullWritable, DruidW
   /* Method that splits Select query depending on the threshold so read can be
    * parallelized */
   private static HiveDruidSplit[] splitSelectQuery(Configuration conf, String address,
-          String druidQuery, Path dummyPath
+          SelectQuery query, Path dummyPath
   ) throws IOException {
     final int selectThreshold = (int) HiveConf.getIntVar(
             conf, HiveConf.ConfVars.HIVE_DRUID_SELECT_THRESHOLD);
@@ -166,12 +177,6 @@ public class DruidQueryBasedInputFormat extends InputFormat<NullWritable, DruidW
             .getIntVar(conf, HiveConf.ConfVars.HIVE_DRUID_NUM_HTTP_CONNECTION);
     final Period readTimeout = new Period(
             HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DRUID_HTTP_READ_TIMEOUT));
-    SelectQuery query;
-    try {
-      query = DruidStorageHandlerUtils.JSON_MAPPER.readValue(druidQuery, SelectQuery.class);
-    } catch (Exception e) {
-      throw new IOException(e);
-    }
 
     final boolean isFetch = query.getContextBoolean(Constants.DRUID_QUERY_FETCH, false);
     if (isFetch) {
@@ -353,6 +358,13 @@ public class DruidQueryBasedInputFormat extends InputFormat<NullWritable, DruidW
     return newIntervals;
   }
 
+  private static String deserializeSerialize(String druidQuery)
+          throws JsonParseException, JsonMappingException, IOException {
+    BaseQuery<?> deserializedQuery = DruidStorageHandlerUtils.JSON_MAPPER.readValue(
+            druidQuery, BaseQuery.class);
+    return DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(deserializedQuery);
+  }
+
   @Override
   public org.apache.hadoop.mapred.RecordReader<NullWritable, DruidWritable> getRecordReader(
           org.apache.hadoop.mapred.InputSplit split, JobConf job, Reporter reporter

http://git-wip-us.apache.org/repos/asf/hive/blob/c0198e52/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java
index 4fde3eb..9b7a1da 100644
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java
+++ b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java
@@ -22,11 +22,17 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.Constants;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat;
+import org.apache.hadoop.hive.druid.io.HiveDruidSplit;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.joda.time.Interval;
 import org.joda.time.chrono.ISOChronology;
 import org.junit.Test;
 
+import io.druid.query.Query;
 import junit.framework.TestCase;
 
 public class TestHiveDruidQueryBasedInputFormat extends TestCase {
@@ -130,4 +136,160 @@ public class TestHiveDruidQueryBasedInputFormat extends TestCase {
     assertEquals(expectedResultList, resultList);
   }
 
+  private static final String TIMESERIES_QUERY =
+      "{  \"queryType\": \"timeseries\", "
+          + " \"dataSource\": \"sample_datasource\", "
+          + " \"granularity\": \"day\", "
+          + " \"descending\": \"true\", "
+          + " \"intervals\": [ \"2012-01-01T00:00:00.000/2012-01-03T00:00:00.000\" ]}";
+  private static final String TIMESERIES_QUERY_SPLIT =
+      "[HiveDruidSplit{localhost:8082, "
+          + "{\"queryType\":\"timeseries\","
+          + "\"dataSource\":{\"type\":\"table\",\"name\":\"sample_datasource\"},"
+          + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2012-01-01T00:00:00.000-08:00/2012-01-03T00:00:00.000-08:00\"]},"
+          + "\"descending\":true,"
+          + "\"filter\":null,"
+          + "\"granularity\":{\"type\":\"duration\",\"duration\":86400000,\"origin\":\"1969-12-31T16:00:00.000-08:00\"},"
+          + "\"aggregations\":[],"
+          + "\"postAggregations\":[],"
+          + "\"context\":null}}]";
+
+  private static final String TOPN_QUERY =
+      "{  \"queryType\": \"topN\", "
+          + " \"dataSource\": \"sample_data\", "
+          + " \"dimension\": \"sample_dim\", "
+          + " \"threshold\": 5, "
+          + " \"metric\": \"count\", "
+          + " \"aggregations\": [  "
+          + "  {   "
+          + "   \"type\": \"longSum\",   "
+          + "   \"name\": \"count\",   "
+          + "   \"fieldName\": \"count\"  "
+          + "  },  "
+          + "  {   "
+          + "   \"type\": \"doubleSum\",   "
+          + "   \"name\": \"some_metric\",   "
+          + "   \"fieldName\": \"some_metric\"  "
+          + "  } "
+          + " ], "
+          + " \"granularity\": \"all\", "
+          + " \"intervals\": [  "
+          + "  \"2013-08-31T00:00:00.000/2013-09-03T00:00:00.000\" "
+          + " ]}";
+  private static final String TOPN_QUERY_SPLIT =
+      "[HiveDruidSplit{localhost:8082, "
+          + "{\"queryType\":\"topN\","
+          + "\"dataSource\":{\"type\":\"table\",\"name\":\"sample_data\"},"
+          + "\"dimension\":{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"sample_dim\",\"outputName\":\"sample_dim\"},"
+          + "\"metric\":{\"type\":\"LegacyTopNMetricSpec\",\"metric\":\"count\"},"
+          + "\"threshold\":5,"
+          + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2013-08-31T00:00:00.000-07:00/2013-09-03T00:00:00.000-07:00\"]},"
+          + "\"filter\":null,"
+          + "\"granularity\":{\"type\":\"all\"},"
+          + "\"aggregations\":[{\"type\":\"longSum\",\"name\":\"count\",\"fieldName\":\"count\"},"
+          + "{\"type\":\"doubleSum\",\"name\":\"some_metric\",\"fieldName\":\"some_metric\"}],"
+          + "\"postAggregations\":[],"
+          + "\"context\":null,"
+          + "\"descending\":false}}]";
+
+  private static final String GROUP_BY_QUERY =
+      "{  \"queryType\": \"groupBy\", "
+          + " \"dataSource\": \"sample_datasource\", "
+          + " \"granularity\": \"day\", "
+          + " \"dimensions\": [\"country\", \"device\"], "
+          + " \"limitSpec\": {"
+          + " \"type\": \"default\","
+          + " \"limit\": 5000,"
+          + " \"columns\": [\"country\", \"data_transfer\"] }, "
+          + " \"aggregations\": [  "
+          + "  { \"type\": \"longSum\", \"name\": \"total_usage\", \"fieldName\": \"user_count\" },  "
+          + "  { \"type\": \"doubleSum\", \"name\": \"data_transfer\", \"fieldName\": \"data_transfer\" } "
+          + " ], "
+          + " \"intervals\": [ \"2012-01-01T00:00:00.000/2012-01-03T00:00:00.000\" ]"
+          + " }";
+  private static final String GROUP_BY_QUERY_SPLIT =
+      "[HiveDruidSplit{localhost:8082, "
+          + "{\"queryType\":\"groupBy\","
+          + "\"dataSource\":{\"type\":\"table\",\"name\":\"sample_datasource\"},"
+          + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2012-01-01T00:00:00.000-08:00/2012-01-03T00:00:00.000-08:00\"]},"
+          + "\"filter\":null,"
+          + "\"granularity\":{\"type\":\"duration\",\"duration\":86400000,\"origin\":\"1969-12-31T16:00:00.000-08:00\"},"
+          + "\"dimensions\":[{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"country\",\"outputName\":\"country\"},"
+          + "{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"device\",\"outputName\":\"device\"}],"
+          + "\"aggregations\":[{\"type\":\"longSum\",\"name\":\"total_usage\",\"fieldName\":\"user_count\"},"
+          + "{\"type\":\"doubleSum\",\"name\":\"data_transfer\",\"fieldName\":\"data_transfer\"}],"
+          + "\"postAggregations\":[],"
+          + "\"having\":null,"
+          + "\"limitSpec\":{\"type\":\"default\",\"columns\":[{\"dimension\":\"country\",\"direction\":\"ascending\",\"dimensionOrder\":{\"type\":\"lexicographic\"}},"
+          + "{\"dimension\":\"data_transfer\",\"direction\":\"ascending\",\"dimensionOrder\":{\"type\":\"lexicographic\"}}],\"limit\":5000},"
+          + "\"context\":null,"
+          + "\"descending\":false}}]";
+
+  private static final String SELECT_QUERY =
+      "{   \"queryType\": \"select\",  "
+          + " \"dataSource\": \"wikipedia\",   \"descending\": \"false\",  "
+          + " \"dimensions\":[\"robot\",\"namespace\",\"anonymous\",\"unpatrolled\",\"page\",\"language\",\"newpage\",\"user\"],  "
+          + " \"metrics\":[\"count\",\"added\",\"delta\",\"variation\",\"deleted\"],  "
+          + " \"granularity\": \"all\",  "
+          + " \"intervals\": [     \"2013-01-01/2013-01-02\"   ],  "
+          + " \"pagingSpec\":{\"pagingIdentifiers\": {}, \"threshold\":5}, "
+          + " \"context\":{\"druid.query.fetch\":true}}";
+  private static final String SELECT_QUERY_SPLIT =
+      "[HiveDruidSplit{localhost:8082, "
+          + "{\"queryType\":\"select\","
+          + "\"dataSource\":{\"type\":\"table\",\"name\":\"wikipedia\"},"
+          + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2013-01-01T00:00:00.000-08:00/2013-01-02T00:00:00.000-08:00\"]},"
+          + "\"descending\":false,"
+          + "\"filter\":null,"
+          + "\"granularity\":{\"type\":\"all\"},"
+          + "\"dimensions\":[{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"robot\",\"outputName\":\"robot\"},"
+          + "{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"namespace\",\"outputName\":\"namespace\"},"
+          + "{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"anonymous\",\"outputName\":\"anonymous\"},"
+          + "{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"unpatrolled\",\"outputName\":\"unpatrolled\"},"
+          + "{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"page\",\"outputName\":\"page\"},"
+          + "{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"language\",\"outputName\":\"language\"},"
+          + "{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"newpage\",\"outputName\":\"newpage\"},"
+          + "{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"user\",\"outputName\":\"user\"}],"
+          + "\"metrics\":[\"count\",\"added\",\"delta\",\"variation\",\"deleted\"],"
+          + "\"pagingSpec\":{\"pagingIdentifiers\":{},\"threshold\":5,\"fromNext\":false},"
+          + "\"context\":{\"druid.query.fetch\":true}}}]";
+
+  @Test
+  public void testTimeZone() throws Exception {
+    DruidQueryBasedInputFormat input = new DruidQueryBasedInputFormat();
+
+    Method method1 = DruidQueryBasedInputFormat.class.getDeclaredMethod(
+            "getInputSplits", Configuration.class);
+    method1.setAccessible(true);
+
+    // Create, initialize, and test
+    Configuration conf = createPropertiesQuery("sample_datasource", Query.TIMESERIES, TIMESERIES_QUERY);
+    HiveDruidSplit[] resultSplits = (HiveDruidSplit[]) method1.invoke(input, conf);
+    assertEquals(TIMESERIES_QUERY_SPLIT, Arrays.toString(resultSplits));
+
+    conf = createPropertiesQuery("sample_datasource", Query.TOPN, TOPN_QUERY);
+    resultSplits = (HiveDruidSplit[]) method1.invoke(input, conf);
+    assertEquals(TOPN_QUERY_SPLIT, Arrays.toString(resultSplits));
+
+    conf = createPropertiesQuery("sample_datasource", Query.GROUP_BY, GROUP_BY_QUERY);
+    resultSplits = (HiveDruidSplit[]) method1.invoke(input, conf);
+    assertEquals(GROUP_BY_QUERY_SPLIT, Arrays.toString(resultSplits));
+
+    conf = createPropertiesQuery("sample_datasource", Query.SELECT, SELECT_QUERY);
+    resultSplits = (HiveDruidSplit[]) method1.invoke(input, conf);
+    assertEquals(SELECT_QUERY_SPLIT, Arrays.toString(resultSplits));
+  }
+
+  private static Configuration createPropertiesQuery(String dataSource, String queryType,
+          String jsonQuery) {
+    Configuration conf = new Configuration();
+    // Set the configuration parameters
+    conf.set(FileInputFormat.INPUT_DIR, "/my/dir");
+    conf.set(HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS.varname, "localhost:8082");
+    conf.set(Constants.DRUID_DATA_SOURCE, dataSource);
+    conf.set(Constants.DRUID_QUERY_JSON, jsonQuery);
+    conf.set(Constants.DRUID_QUERY_TYPE, queryType);
+    return conf;
+  }
+
 }