You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/01/05 01:12:46 UTC
hive git commit: HIVE-12706: Incorrect output from
from_utc_timestamp()/to_utc_timestamp when local timezone has DST (Jason Dere,
reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master f7579b282 -> f0faf78dc
HIVE-12706: Incorrect output from from_utc_timestamp()/to_utc_timestamp when local timezone has DST (Jason Dere, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f0faf78d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f0faf78d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f0faf78d
Branch: refs/heads/master
Commit: f0faf78dc79f9b9a6dc8c7fbc9a57ab0f57c6262
Parents: f7579b2
Author: Jason Dere <jd...@hortonworks.com>
Authored: Mon Jan 4 16:12:10 2016 -0800
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Jan 4 16:12:10 2016 -0800
----------------------------------------------------------------------
.../udf/generic/GenericUDFFromUtcTimestamp.java | 59 ++++++++++---
.../generic/TestGenericUDFFromUtcTimestamp.java | 92 ++++++++++++++++++++
2 files changed, 141 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f0faf78d/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
index 331ee6b..33fe507 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.udf.generic;
import java.sql.Timestamp;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.TimeZone;
import org.slf4j.Logger;
@@ -43,6 +45,8 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
private transient PrimitiveObjectInspector[] argumentOIs;
private transient TimestampConverter timestampConverter;
private transient TextConverter textConverter;
+ private transient SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ private transient TimeZone tzUTC = TimeZone.getTimeZone("UTC");
@Override
public ObjectInspector initialize(ObjectInspector[] arguments)
@@ -66,6 +70,26 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector;
}
+ /**
+ * Parse the timestamp string using the input TimeZone.
+ * This does not parse fractional seconds.
+ * @param tsString
+ * @param tz
+ * @return
+ */
+ protected Timestamp timestampFromString(String tsString, TimeZone tz) {
+ dateFormat.setTimeZone(tz);
+ try {
+ java.util.Date date = dateFormat.parse(tsString);
+ if (date == null) {
+ return null;
+ }
+ return new Timestamp(date.getTime());
+ } catch (ParseException err) {
+ return null;
+ }
+ }
+
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
Object o0 = arguments[0].get();
@@ -82,23 +106,38 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
return null;
}
- Timestamp timestamp = ((TimestampWritable) converted_o0).getTimestamp();
+ Timestamp inputTs = ((TimestampWritable) converted_o0).getTimestamp();
String tzStr = textConverter.convert(o1).toString();
TimeZone timezone = TimeZone.getTimeZone(tzStr);
- int offset = timezone.getOffset(timestamp.getTime());
+
+ TimeZone fromTz;
+ TimeZone toTz;
if (invert()) {
- offset = -offset;
+ fromTz = timezone;
+ toTz = tzUTC;
+ } else {
+ fromTz = tzUTC;
+ toTz = timezone;
+ }
+
+ // inputTs is the year/month/day/hour/minute/second in the local timezone.
+ // For this UDF we want it in the timezone represented by fromTz
+ Timestamp fromTs = timestampFromString(inputTs.toString(), fromTz);
+ if (fromTs == null) {
+ return null;
+ }
+
+ // Now output this timestamp's millis value to the equivalent toTz.
+ dateFormat.setTimeZone(toTz);
+ Timestamp result = Timestamp.valueOf(dateFormat.format(fromTs));
+
+ if (inputTs.getNanos() != 0) {
+ result.setNanos(inputTs.getNanos());
}
- return applyOffset(offset, timestamp);
- }
- protected Timestamp applyOffset(long offset, Timestamp t) {
- long newTime = t.getTime() + offset;
- Timestamp t2 = new Timestamp(newTime);
- t2.setNanos(t.getNanos());
+ return result;
- return t2;
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/f0faf78d/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
new file mode 100644
index 0000000..ae4785a
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+
+import junit.framework.TestCase;
+
+public class TestGenericUDFFromUtcTimestamp extends TestCase {
+ public static void runAndVerify(GenericUDF udf,
+ Object arg1, Object arg2, Object expected) throws HiveException {
+ DeferredObject[] args = { new DeferredJavaObject(arg1), new DeferredJavaObject(arg2) };
+ Object result = udf.evaluate(args);
+
+ if (expected == null) {
+ assertNull(result);
+ } else {
+ assertEquals(expected.toString(), result.toString());
+ }
+ }
+
+ public void testFromUtcTimestamp() throws Exception {
+ ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ GenericUDFFromUtcTimestamp udf = new GenericUDFFromUtcTimestamp();
+ ObjectInspector[] args2 = {valueOI, valueOI};
+ udf.initialize(args2);
+
+ runAndVerify(udf,
+ new Text("2015-03-28 17:00:00"), new Text("Europe/London"),
+ Timestamp.valueOf("2015-03-28 17:00:00"));
+ runAndVerify(udf,
+ new Text("2015-03-28 18:00:00"), new Text("Europe/London"),
+ Timestamp.valueOf("2015-03-28 18:00:00"));
+ runAndVerify(udf,
+ new Text("2015-03-28 19:00:00"), new Text("Europe/London"),
+ Timestamp.valueOf("2015-03-28 19:00:00"));
+
+ // Make sure nanos are preserved
+ runAndVerify(udf,
+ new Text("2015-03-28 18:00:00.123456789"), new Text("Europe/London"),
+ Timestamp.valueOf("2015-03-28 18:00:00.123456789"));
+ }
+
+ public void testToUtcTimestamp() throws Exception {
+ ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ GenericUDFToUtcTimestamp udf = new GenericUDFToUtcTimestamp();
+ ObjectInspector[] args2 = {valueOI, valueOI};
+ udf.initialize(args2);
+
+ runAndVerify(udf,
+ new Text("2015-03-28 17:00:00"), new Text("Europe/London"),
+ Timestamp.valueOf("2015-03-28 17:00:00"));
+ runAndVerify(udf,
+ new Text("2015-03-28 18:00:00"), new Text("Europe/London"),
+ Timestamp.valueOf("2015-03-28 18:00:00"));
+ runAndVerify(udf,
+ new Text("2015-03-28 19:00:00"), new Text("Europe/London"),
+ Timestamp.valueOf("2015-03-28 19:00:00"));
+
+ // Make sure nanos are preserved
+ runAndVerify(udf,
+ new Text("2015-03-28 18:00:00.123456789"), new Text("Europe/London"),
+ Timestamp.valueOf("2015-03-28 18:00:00.123456789"));
+ }
+}