You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2020/03/31 15:15:47 UTC
[hive] branch master updated: HIVE-22842: Timestamp/date vectors in
Arrow serializer should use correct calendar for value representation
(Shubham Chaurasia, reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new eac25e7 HIVE-22842: Timestamp/date vectors in Arrow serializer should use correct calendar for value representation (Shubham Chaurasia, reviewed by Jesus Camacho Rodriguez)
eac25e7 is described below
commit eac25e711ea750bc52f41da7ed3c32bfe36d4f67
Author: Shubham Chaurasia <sc...@cloudera.com>
AuthorDate: Tue Mar 31 08:11:19 2020 -0700
HIVE-22842: Timestamp/date vectors in Arrow serializer should use correct calendar for value representation (Shubham Chaurasia, reviewed by Jesus Camacho Rodriguez)
Close apache/hive#950
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../jdbc/TestJdbcWithMiniLlapVectorArrowBatch.java | 476 +++++++++++++++++++++
.../apache/hadoop/hive/ql/io/arrow/Serializer.java | 32 +-
.../hadoop/hive/common/type/CalendarUtils.java | 14 +
4 files changed, 522 insertions(+), 3 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 34df01e..73f185a 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4638,6 +4638,9 @@ public class HiveConf extends Configuration {
"If LLAP external clients submits ORDER BY queries, force return a single split to guarantee reading\n" +
"data out in ordered way. Setting this to false will let external clients read data out in parallel\n" +
"losing the ordering (external clients are responsible for guaranteeing the ordering)"),
+ LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR("hive.llap.external.client.use.hybrid.calendar",
+ false,
+ "Whether to use hybrid calendar for parsing of data/timestamps."),
LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap", false,
"Override if grace join should be allowed to run in llap."),
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlapVectorArrowBatch.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlapVectorArrowBatch.java
new file mode 100644
index 0000000..bba599b
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlapVectorArrowBatch.java
@@ -0,0 +1,476 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.jdbc;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.commons.collections4.MultiSet;
+import org.apache.commons.collections4.multiset.HashMultiSet;
+import org.apache.hadoop.hive.common.type.CalendarUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.LlapArrowRowInputFormat;
+import org.apache.hadoop.hive.llap.LlapBaseInputFormat;
+import org.apache.hadoop.hive.llap.Row;
+import org.apache.hadoop.hive.ql.io.arrow.ArrowWrapperWritable;
+import org.apache.hadoop.hive.ql.io.arrow.RootAllocatorFactory;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_PARQUET_DATE_PROLEPTIC_GREGORIAN_DEFAULT;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * TestJdbcWithMiniLlap for Arrow format - uses batch record reader.
+ * We can obtain arrow batches and compare the results.
+ *
+ */
+public class TestJdbcWithMiniLlapVectorArrowBatch extends BaseJdbcWithMiniLlap {
+
+ private final MultiSet<List<Object>> legacyDateExpectedOutput = getLegacyDateExpectedOutput();
+ private final MultiSet<List<Object>> hybridMixedDateExpectedOutput = getHybridMixedDateExpectedOutput();
+
+ private final MultiSet<List<Object>> legacyTimestampExpectedOutput = getLegacyTimestampExpectedOutput();
+ private final MultiSet<List<Object>> convertedLegacyTimestampExpectedOutput =
+ getConvertedLegacyTimestampExpectedOutput();
+ private final MultiSet<List<Object>> hybridMixedTimestampExpectedOutput = getHybridMixedTimestampExpectedOutput();
+
+ @BeforeClass public static void beforeTest() throws Exception {
+ HiveConf conf = defaultConf();
+ conf.setBoolVar(ConfVars.LLAP_OUTPUT_FORMAT_ARROW, true);
+ conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_FILESINK_ARROW_NATIVE_ENABLED, true);
+ BaseJdbcWithMiniLlap.beforeTest(conf);
+ }
+
+ @Override protected InputFormat<NullWritable, Row> getInputFormat() {
+ return new LlapArrowRowInputFormat(Long.MAX_VALUE);
+ }
+
+ private MultiSet<List<Object>> getHybridMixedDateExpectedOutput() {
+ MultiSet<List<Object>> expected = new HashMultiSet<>();
+ expected.add(Lists.newArrayList("2012-02-21"));
+ expected.add(Lists.newArrayList("2014-02-11"));
+ expected.add(Lists.newArrayList("1947-02-11"));
+ expected.add(Lists.newArrayList("8200-02-11"));
+ expected.add(Lists.newArrayList("1012-02-21"));
+ expected.add(Lists.newArrayList("1014-02-11"));
+ expected.add(Lists.newArrayList("0947-02-11"));
+ expected.add(Lists.newArrayList("0200-02-11"));
+ return expected;
+ }
+
+ private MultiSet<List<Object>> getLegacyDateExpectedOutput() {
+ MultiSet<List<Object>> expected = new HashMultiSet<>();
+ expected.add(Lists.newArrayList("2012-02-21"));
+ expected.add(Lists.newArrayList("2014-02-11"));
+ expected.add(Lists.newArrayList("1947-02-11"));
+ expected.add(Lists.newArrayList("8200-02-11"));
+ expected.add(Lists.newArrayList("1012-02-27"));
+ expected.add(Lists.newArrayList("1014-02-17"));
+ expected.add(Lists.newArrayList("0947-02-16"));
+ expected.add(Lists.newArrayList("0200-02-10"));
+ return expected;
+ }
+
+ private MultiSet<List<Object>> getHybridMixedTimestampExpectedOutput() {
+ MultiSet<List<Object>> expected = new HashMultiSet<>();
+ expected.add(Lists.newArrayList("2012-02-21 07:08:09.123"));
+ expected.add(Lists.newArrayList("2014-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("1947-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("8200-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("1012-02-21 07:15:11.123"));
+ expected.add(Lists.newArrayList("1014-02-11 07:15:11.123"));
+ expected.add(Lists.newArrayList("0947-02-11 07:15:11.123"));
+ expected.add(Lists.newArrayList("0200-02-11 07:15:11.123"));
+ return expected;
+ }
+
+ private MultiSet<List<Object>> getLegacyTimestampExpectedOutput() {
+ MultiSet<List<Object>> expected = new HashMultiSet<>();
+ expected.add(Lists.newArrayList("2012-02-21 07:08:09.123"));
+ expected.add(Lists.newArrayList("2014-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("1947-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("8200-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("1012-02-21 07:08:09.123"));
+ expected.add(Lists.newArrayList("1014-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("0947-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("0200-02-11 07:08:09.123"));
+ return expected;
+ }
+
+ private MultiSet<List<Object>> getConvertedLegacyTimestampExpectedOutput() {
+ MultiSet<List<Object>> expected = new HashMultiSet<>();
+ expected.add(Lists.newArrayList("2012-02-21 07:08:09.123"));
+ expected.add(Lists.newArrayList("2014-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("1947-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("8200-02-11 07:08:09.123"));
+ expected.add(Lists.newArrayList("1012-02-27 07:15:11.123"));
+ expected.add(Lists.newArrayList("1014-02-17 07:15:11.123"));
+ expected.add(Lists.newArrayList("0947-02-16 07:15:11.123"));
+ expected.add(Lists.newArrayList("0200-02-10 07:15:11.123"));
+ return expected;
+ }
+
+ // test newly inserted orc records which have calendar info in orc footer.
+ // similar to ql/src/test/queries/clientpositive/orc_hybrid_mixed_date.q
+ @Test public void testOrcHybridMixedDates() throws Exception {
+
+ final String tableName = "testOrcHybridMixedDates";
+ executeSQL("create table " + tableName + " (d date) stored as orc");
+ executeSQL("INSERT INTO " + tableName + " VALUES " + "('2012-02-21'), " + "('2014-02-11'), " + "('1947-02-11'), "
+ + "('8200-02-11'), " + "('1012-02-21'), " + "('1014-02-11'), " + "('0947-02-11'), " + "('0200-02-11')");
+
+ final String query = "select * from " + tableName;
+
+ testDateQueries(query, "orc.proleptic.gregorian.default", hybridMixedDateExpectedOutput,
+ hybridMixedDateExpectedOutput);
+ }
+
+ // test with legacy orc files
+ // similar to ql/src/test/queries/clientpositive/orc_legacy_mixed_date.q
+ @Test public void testOrcLegacyMixedDates() throws Exception {
+
+ final String tableName = "testOrcLegacyMixedDates";
+ executeSQL("create table " + tableName + " (d date) stored as orc");
+ executeSQL("load data local inpath '../../data/files/orc_legacy_mixed_dates.orc' into table " + tableName);
+
+ final String query = "select * from " + tableName;
+
+ // ORC properties (here orc.proleptic.gregorian.default) are not propogated to LLAP as of now
+ // and hence the expected output hybridMixedDateExpectedOutput otherwise it should be hybridMixedDateExpectedOutput ideally
+ testDateQueries(query, "orc.proleptic.gregorian.default", hybridMixedDateExpectedOutput,
+ hybridMixedDateExpectedOutput);
+ }
+
+ // test newly inserted orc records which have calendar info in orc footer.
+ // similar to ql/src/test/queries/clientpositive/orc_hybrid_mixed_timestamp.q
+ @Test public void testOrcHybridMixedTimestamps() throws Exception {
+
+ final String tableName = "testOrcHybridMixedTimestamps";
+ executeSQL("create table " + tableName + " (d timestamp) stored as orc");
+ executeSQL("INSERT INTO " + tableName + " VALUES ('2012-02-21 07:08:09.123')," + "('2014-02-11 07:08:09.123'),"
+ + "('1947-02-11 07:08:09.123')," + "('8200-02-11 07:08:09.123')," + "('1012-02-21 07:15:11.123'),"
+ + "('1014-02-11 07:15:11.123')," + "('0947-02-11 07:15:11.123')," + "('0200-02-11 07:15:11.123')");
+
+ final String query = "select * from " + tableName;
+
+ testTimestampQueries(query, "orc.proleptic.gregorian.default", hybridMixedTimestampExpectedOutput,
+ hybridMixedTimestampExpectedOutput);
+ }
+
+ // test with legacy parquet files
+ // similar to ql/src/test/queries/clientpositive/orc_legacy_mixed_timestamp.q
+ @Test public void testOrcLegacyMixedTimestamps() throws Exception {
+
+ final String tableName = "testOrcLegacyMixedTimestamps";
+ executeSQL("create table " + tableName + " (ts timestamp) stored as orc");
+ executeSQL("load data local inpath '../../data/files/orc_legacy_mixed_timestamps.orc' into table " + tableName);
+
+ final String query = "select * from " + tableName;
+
+ // ORC properties (here orc.proleptic.gregorian.default) are not propogated to LLAP as of now
+ testTimestampQueries(query, "orc.proleptic.gregorian.default", legacyTimestampExpectedOutput,
+ legacyTimestampExpectedOutput);
+ }
+
+ // test with new parquet files
+ // similar to ql/src/test/queries/clientpositive/parquet_hybrid_mixed_date.q
+ @Test public void testParquetHybridMixedDates() throws Exception {
+
+ final String tableName = "testParquetHybrcidMixedDates";
+ executeSQL("create table " + tableName + " (d date) stored as parquet");
+ executeSQL("INSERT INTO " + tableName + " VALUES " + "('2012-02-21'), " + "('2014-02-11'), " + "('1947-02-11'), "
+ + "('8200-02-11'), " + "('1012-02-21'), " + "('1014-02-11'), " + "('0947-02-11'), " + "('0200-02-11')");
+
+ final String query = "select * from " + tableName;
+
+ testDateQueries(query, HIVE_PARQUET_DATE_PROLEPTIC_GREGORIAN_DEFAULT.toString(), hybridMixedDateExpectedOutput,
+ hybridMixedDateExpectedOutput);
+ }
+
+ // test with legacy parquet files
+ // similar to ql/src/test/queries/clientpositive/parquet_legacy_mixed_date.q
+ @Test public void testParquetLegacyMixedDates() throws Exception {
+
+ final String tableName = "testParquetLegacyMixedDates";
+
+ executeSQL("create table " + tableName + " (d date) stored as parquet");
+ executeSQL("load data local inpath '../../data/files/parquet_legacy_mixed_dates.parq' into table " + tableName);
+
+ final String query = "select * from " + tableName;
+
+ testDateQueries(query, HIVE_PARQUET_DATE_PROLEPTIC_GREGORIAN_DEFAULT.toString(), hybridMixedDateExpectedOutput,
+ legacyDateExpectedOutput);
+ }
+
+ // test newly inserted parquet records.
+ // similar to ql/src/test/queries/clientpositive/parquet_hybrid_mixed_timestamp.q
+ @Test public void testParquetHybridMixedTimestamps() throws Exception {
+
+ final String tableName = "testParquetHybridMixedTimestamps";
+ executeSQL("create table " + tableName + " (ts timestamp) stored as parquet");
+ executeSQL("INSERT INTO " + tableName + " VALUES ('2012-02-21 07:08:09.123')," + "('2014-02-11 07:08:09.123'),"
+ + "('1947-02-11 07:08:09.123')," + "('8200-02-11 07:08:09.123')," + "('1012-02-21 07:15:11.123'),"
+ + "('1014-02-11 07:15:11.123')," + "('0947-02-11 07:15:11.123')," + "('0200-02-11 07:15:11.123')");
+
+ final String query = "select * from " + tableName;
+
+ testTimestampQueries(query, HIVE_PARQUET_DATE_PROLEPTIC_GREGORIAN_DEFAULT.toString(),
+ hybridMixedTimestampExpectedOutput, hybridMixedTimestampExpectedOutput);
+ }
+
+ // test with legacy parquet files
+ // similar to ql/src/test/queries/clientpositive/parquet_legacy_mixed_timestamp.q
+ @Test public void testParquetLegacyMixedTimestamps() throws Exception {
+
+ final String tableName = "testParquetLegacyMixedTimestamps";
+ executeSQL("create table " + tableName + " (d timestamp) stored as parquet");
+ executeSQL(
+ "load data local inpath '../../data/files/parquet_legacy_mixed_timestamps.parq' into table " + tableName);
+
+ final String query = "select * from " + tableName;
+
+ testTimestampQueries(query, HIVE_PARQUET_DATE_PROLEPTIC_GREGORIAN_DEFAULT.toString(),
+ hybridMixedTimestampExpectedOutput, hybridMixedTimestampExpectedOutput);
+ }
+
+ // test with new avro files
+ // similar to ql/src/test/queries/clientpositive/avro_hybrid_mixed_date.q
+ @Test public void testAvroHybridMixedDates() throws Exception {
+
+ final String tableName = "testAvroHybridMixedDates";
+ executeSQL("create table " + tableName + " (d date) stored as avro");
+ executeSQL("INSERT INTO " + tableName + " VALUES " + "('2012-02-21'), " + "('2014-02-11'), " + "('1947-02-11'), "
+ + "('8200-02-11'), " + "('1012-02-21'), " + "('1014-02-11'), " + "('0947-02-11'), " + "('0200-02-11')");
+
+ final String query = "select * from " + tableName;
+
+ testDateQueries(query, HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT.toString(), hybridMixedDateExpectedOutput,
+ hybridMixedDateExpectedOutput);
+ }
+
+ // test with legacy parquet files
+ // similar to ql/src/test/queries/clientpositive/avro_legacy_mixed_date.q
+ @Test public void testAvroLegacyMixedDates() throws Exception {
+
+ final String tableName = "testAvroLegacyMixedDates";
+
+ executeSQL("create table " + tableName + " (d date) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' "
+ + "COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' stored as avro");
+ executeSQL("load data local inpath '../../data/files/avro_legacy_mixed_dates.avro' into table " + tableName);
+
+ final String query = "select * from " + tableName;
+
+ testDateQueries(query, HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT.toString(), hybridMixedDateExpectedOutput,
+ legacyDateExpectedOutput);
+ }
+
+ // test newly inserted avro records
+ // similar to ql/src/test/queries/clientpositive/avro_hybrid_mixed_timestamp.q
+ @Test public void testAvroHybridMixedTimestamps() throws Exception {
+
+ final String tableName = "testAvroHybridMixedTimestamps";
+ executeSQL("create table " + tableName + " (d timestamp) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' "
+ + "COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' stored as avro");
+ executeSQL("INSERT INTO " + tableName + " VALUES ('2012-02-21 07:08:09.123')," + "('2014-02-11 07:08:09.123'),"
+ + "('1947-02-11 07:08:09.123')," + "('8200-02-11 07:08:09.123')," + "('1012-02-21 07:15:11.123'),"
+ + "('1014-02-11 07:15:11.123')," + "('0947-02-11 07:15:11.123')," + "('0200-02-11 07:15:11.123')");
+
+ final String query = "select * from " + tableName;
+
+ testTimestampQueries(query, HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT.toString(), hybridMixedTimestampExpectedOutput,
+ hybridMixedTimestampExpectedOutput);
+ }
+
+ // test with legacy avro files
+ // similar to ql/src/test/queries/clientpositive/avro_legacy_mixed_timestamp.q
+ @Test public void testAvroLegacyMixedTimestamps() throws Exception {
+
+ final String tableName = "testAvroLegacyMixedTimestamps";
+ executeSQL("create table " + tableName + "(d timestamp) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'"
+ + "COLLECTION ITEMS TERMINATED BY ',' MAP KEYS TERMINATED BY ':' stored as avro");
+ executeSQL("load data local inpath '../../data/files/avro_legacy_mixed_timestamps.avro' into table " + tableName);
+
+ final String query = "select * from " + tableName;
+
+ testTimestampQueries(query, HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT.toString(), hybridMixedTimestampExpectedOutput,
+ convertedLegacyTimestampExpectedOutput);
+ }
+
+ private void testDateQueries(final String query, final String fileReaderLevelCalendarProp,
+ MultiSet<List<Object>> newHybridExpected, MultiSet<List<Object>> legacyExpected) throws Exception {
+
+ MultiSet<List<Object>> llapResultTransformed = transformLlapDateResultSet(runQueryUsingLlapArrowBatchReader(query,
+ ImmutableMap.of(LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR.toString(), "true")), false);
+ assertEquals(newHybridExpected, llapResultTransformed);
+
+ // test hybrid output with fileReaderLevelCalendarProp
+ llapResultTransformed = transformLlapDateResultSet(runQueryUsingLlapArrowBatchReader(query, ImmutableMap
+ .of(LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR.toString(), "true", fileReaderLevelCalendarProp, "true")), false);
+ assertEquals(legacyExpected, llapResultTransformed);
+ }
+
+ private void testTimestampQueries(final String query, final String fileReaderLevelCalendarProp,
+ MultiSet<List<Object>> newHybridExpected, MultiSet<List<Object>> legacyExpected) throws Exception {
+
+ MultiSet<List<Object>> llapResultTransformed = transformLlapTimestampResultSet(
+ runQueryUsingLlapArrowBatchReader(query,
+ ImmutableMap.of(LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR.toString(), "true")), false);
+ assertEquals(newHybridExpected, llapResultTransformed);
+
+ // test hybrid output with fileReaderLevelCalendarProp
+ llapResultTransformed = transformLlapTimestampResultSet(runQueryUsingLlapArrowBatchReader(query, ImmutableMap
+ .of(LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR.toString(), "true", fileReaderLevelCalendarProp, "true")), false);
+ assertEquals(legacyExpected, llapResultTransformed);
+ }
+
+ private MultiSet<List<Object>> transformLlapDateResultSet(MultiSet<List<Object>> llapResult,
+ final boolean useProleptic) {
+ MultiSet<List<Object>> llapResultTransformed = new HashMultiSet<>();
+ for (List<Object> list : llapResult) {
+ llapResultTransformed.add(
+ list.stream().map(ele -> CalendarUtils.formatDate((int) ele, useProleptic)).collect(Collectors.toList()));
+ }
+ return llapResultTransformed;
+ }
+
+ private MultiSet<List<Object>> transformLlapTimestampResultSet(MultiSet<List<Object>> llapResult,
+ final boolean useProleptic) {
+ MultiSet<List<Object>> llapResultTransformed = new HashMultiSet<>();
+ for (List<Object> list : llapResult) {
+ llapResultTransformed.add(list.stream().map(ele -> CalendarUtils.formatTimestamp((long) ele / 1000, useProleptic))
+ .collect(Collectors.toList()));
+ }
+ return llapResultTransformed;
+ }
+
+ private void executeSQL(String query, String... moreQueries) throws SQLException {
+ try (Statement stmt = hs2Conn.createStatement()) {
+ stmt.execute(query);
+ if (moreQueries != null) {
+ for (String q : moreQueries) {
+ stmt.execute(q);
+ }
+ }
+ }
+ }
+
+ private MultiSet<List<Object>> runQueryUsingLlapArrowBatchReader(String query, Map<String, String> extraHiveConfs)
+ throws Exception {
+ String url = miniHS2.getJdbcURL();
+
+ if (extraHiveConfs != null) {
+ url = url + "?" + extraHiveConfs.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue())
+ .collect(Collectors.joining(";"));
+ }
+
+ String user = System.getProperty("user.name");
+ String pwd = user;
+ String handleId = UUID.randomUUID().toString();
+
+ // Get splits
+ JobConf job = new JobConf(conf);
+ job.set(LlapBaseInputFormat.URL_KEY, url);
+ job.set(LlapBaseInputFormat.USER_KEY, user);
+ job.set(LlapBaseInputFormat.PWD_KEY, pwd);
+ job.set(LlapBaseInputFormat.QUERY_KEY, query);
+ job.set(LlapBaseInputFormat.HANDLE_ID, handleId);
+ job.set(LlapBaseInputFormat.USE_NEW_SPLIT_FORMAT, "false");
+
+ BufferAllocator allocator = RootAllocatorFactory.INSTANCE.getOrCreateRootAllocator(Long.MAX_VALUE)
+ .newChildAllocator(UUID.randomUUID().toString(), 0, Long.MAX_VALUE);
+
+ LlapBaseInputFormat llapBaseInputFormat = new LlapBaseInputFormat(true, allocator);
+ InputSplit[] splits = llapBaseInputFormat.getSplits(job, 1);
+
+ assertTrue(splits.length > 0);
+
+ MultiSet<List<Object>> queryResult = new HashMultiSet<>();
+ for (InputSplit split : splits) {
+ System.out.println("Processing split " + Arrays.toString(split.getLocations()));
+ RecordReader<NullWritable, ArrowWrapperWritable> reader = llapBaseInputFormat.getRecordReader(split, job, null);
+ ArrowWrapperWritable wrapperWritable = new ArrowWrapperWritable();
+ while (reader.next(NullWritable.get(), wrapperWritable)) {
+ queryResult.addAll(collectResultFromArrowVector(wrapperWritable));
+ }
+ reader.close();
+ }
+ LlapBaseInputFormat.close(handleId);
+ return queryResult;
+ }
+
+ private MultiSet<List<Object>> collectResultFromArrowVector(ArrowWrapperWritable wrapperWritable) {
+ List<FieldVector> fieldVectors = wrapperWritable.getVectorSchemaRoot().getFieldVectors();
+ MultiSet<List<Object>> result = new HashMultiSet<>();
+ int valueCount = fieldVectors.get(0).getValueCount();
+ for (int recordIndex = 0; recordIndex < valueCount; recordIndex++) {
+ List<Object> row = new ArrayList<>();
+ for (FieldVector fieldVector : fieldVectors) {
+ row.add(fieldVector.getObject(recordIndex));
+ }
+ result.add(row);
+ }
+ return result;
+ }
+
+ @Override public void testLlapInputFormatEndToEnd() throws Exception {
+ // to be implemented for this reader
+ }
+
+ @Override public void testNonAsciiStrings() throws Exception {
+ // to be implemented for this reader
+ }
+
+ @Override public void testEscapedStrings() throws Exception {
+ // to be implemented for this reader
+ }
+
+ @Override public void testDataTypes() throws Exception {
+ // to be implemented for this reader
+ }
+
+ @Override public void testComplexQuery() throws Exception {
+ // to be implemented for this reader
+ }
+
+ @Override public void testKillQuery() throws Exception {
+ // to be implemented for this reader
+ }
+
+}
+
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
index 12da074..d5a9b2c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DateColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
@@ -87,6 +88,7 @@ import java.util.List;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_ARROW_BATCH_SIZE;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_ARROW_BATCH_ALLOCATOR_LIMIT;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR;
import static org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil.createColumnVector;
import static org.apache.hadoop.hive.ql.io.arrow.ArrowColumnarBatchSerDe.MICROS_PER_MILLIS;
import static org.apache.hadoop.hive.ql.io.arrow.ArrowColumnarBatchSerDe.MILLIS_PER_SECOND;
@@ -111,6 +113,7 @@ public class Serializer {
private List<String> fieldNames;
private int fieldSize;
+ private boolean useHybridCalendar;
private final StructVector rootVector;
private final DecimalHolder decimalHolder = new DecimalHolder();
@@ -119,6 +122,7 @@ public class Serializer {
this.fieldTypeInfos = typeInfos;
this.fieldNames = fieldNames;
long childAllocatorLimit = HiveConf.getLongVar(conf, HIVE_ARROW_BATCH_ALLOCATOR_LIMIT);
+ this.useHybridCalendar = HiveConf.getBoolVar(conf, LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR);
//Use per-task allocator for accounting only, no need to reserve per-task memory
long childAllocatorReservation = 0L;
//Break out accounting of direct memory per-task, so we can check no memory is leaked when task is completed
@@ -136,6 +140,7 @@ public class Serializer {
Serializer(ArrowColumnarBatchSerDe serDe) throws SerDeException {
MAX_BUFFERED_ROWS = HiveConf.getIntVar(serDe.conf, HIVE_ARROW_BATCH_SIZE);
long childAllocatorLimit = HiveConf.getLongVar(serDe.conf, HIVE_ARROW_BATCH_ALLOCATOR_LIMIT);
+ this.useHybridCalendar = HiveConf.getBoolVar(serDe.conf, LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR);
ArrowColumnarBatchSerDe.LOG.info("ArrowColumnarBatchSerDe max number of buffered columns: " + MAX_BUFFERED_ROWS);
String childAllocatorName = Thread.currentThread().getName();
//Use per-task allocator for accounting only, no need to reserve per-task memory
@@ -592,6 +597,9 @@ public class Serializer {
break;
case DATE:
{
+ // and since hive always provides data in proleptic calendar format
+ // set the usingProlepticCalendar flag for conversion to hybrid if required in dateValueSetter
+ ((DateColumnVector) hiveVector).setUsingProlepticCalendar(true);
if(isNative) {
writeGeneric(arrowVector, hiveVector, size, vectorizedRowBatch.selectedInUse, vectorizedRowBatch.selected, dateNullSetter, dateValueSetter, typeInfo);
return;
@@ -607,6 +615,9 @@ public class Serializer {
break;
case TIMESTAMP:
{
+ // and since hive always provides data in proleptic calendar format
+ // set the usingProlepticCalendar flag for conversion to hybrid if required in timestampValueSetter
+ ((TimestampColumnVector) hiveVector).setUsingProlepticCalendar(true);
if(isNative) {
writeGeneric(arrowVector, hiveVector, size, vectorizedRowBatch.selectedInUse, vectorizedRowBatch.selected, timestampNullSetter, timestampValueSetter, typeInfo);
return;
@@ -854,16 +865,31 @@ public class Serializer {
//date
private static final IntAndVectorsConsumer dateNullSetter = (i, arrowVector, hiveVector)
-> ((DateDayVector) arrowVector).setNull(i);
- private static final IntIntAndVectorsConsumer dateValueSetter = (i, j, arrowVector, hiveVector, typeInfo)
- -> ((DateDayVector) arrowVector).set(i, (int) ((LongColumnVector) hiveVector).vector[j]);
+
+ private final IntIntAndVectorsConsumer dateValueSetter = (i, j, arrowVector, hiveVector, typeInfo) -> {
+
+ DateColumnVector dateColumnVector = (DateColumnVector) hiveVector;
+ // useHybridCalendar - means the client wants data in hybrid calendar format
+ if (useHybridCalendar && dateColumnVector.isUsingProlepticCalendar()) {
+ dateColumnVector.changeCalendar(false, true);
+ }
+
+ ((DateDayVector) arrowVector).set(i, (int) (dateColumnVector).vector[j]);
+ };
//timestamp
private static final IntAndVectorsConsumer timestampNullSetter = (i, arrowVector, hiveVector)
-> ((TimeStampMicroTZVector) arrowVector).setNull(i);
- private static final IntIntAndVectorsConsumer timestampValueSetter = (i, j, arrowVector, hiveVector, typeInfo)
+ private final IntIntAndVectorsConsumer timestampValueSetter = (i, j, arrowVector, hiveVector, typeInfo)
-> {
final TimeStampMicroTZVector timeStampMicroTZVector = (TimeStampMicroTZVector) arrowVector;
final TimestampColumnVector timestampColumnVector = (TimestampColumnVector) hiveVector;
+
+ // useHybridCalendar - means the client wants data in hybrid calendar format
+ if (useHybridCalendar && timestampColumnVector.usingProlepticCalendar()) {
+ timestampColumnVector.changeCalendar(false , true);
+ }
+
// Time = second + sub-second
final long secondInMillis = timestampColumnVector.getTime(j);
final long nanos = timestampColumnVector.getNanos(j);
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/CalendarUtils.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/CalendarUtils.java
index 7555d16..51c28f0 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/CalendarUtils.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/CalendarUtils.java
@@ -190,6 +190,20 @@ public class CalendarUtils {
: HYBRID_DATE_FORMAT.get().format(millis);
}
+ /**
+ *
+ * Formats epoch Millis to timestamp according to proleptic or hybrid calendar
+ *
+ * @param epochMillis epoch Millis
+ * @param useProleptic if true - uses proleptic formatter, else uses hybrid formatter
+ * @return formatted timestamp
+ */
+ public static String formatTimestamp(long epochMillis, boolean useProleptic) {
+ final Date date = new Date(epochMillis);
+ return useProleptic ? PROLEPTIC_TIME_FORMAT.get().format(date)
+ : HYBRID_TIME_FORMAT.get().format(date);
+ }
+
private CalendarUtils() {
throw new UnsupportedOperationException();
}