You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dm...@apache.org on 2020/03/13 13:42:51 UTC
[hive] branch master updated: HIVE-22762: Leap day is incorrectly
parsed during cast in Hive (Karen Coppage reviewed by David Mollitor)
This is an automated email from the ASF dual-hosted git repository.
dmollitor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 049a588 HIVE-22762: Leap day is incorrectly parsed during cast in Hive (Karen Coppage reviewed by David Mollitor)
049a588 is described below
commit 049a58804c58d208e841c89d221191cab1158438
Author: David Mollitor <dm...@apache.org>
AuthorDate: Fri Mar 13 09:42:18 2020 -0400
HIVE-22762: Leap day is incorrectly parsed during cast in Hive (Karen Coppage reviewed by David Mollitor)
---
.../format/datetime/HiveSqlDateTimeFormatter.java | 101 ++++++++++++++++++---
.../datetime/TestHiveSqlDateTimeFormatter.java | 6 +-
2 files changed, 92 insertions(+), 15 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/common/format/datetime/HiveSqlDateTimeFormatter.java b/common/src/java/org/apache/hadoop/hive/common/format/datetime/HiveSqlDateTimeFormatter.java
index 78e0a71..2b79e03 100644
--- a/common/src/java/org/apache/hadoop/hive/common/format/datetime/HiveSqlDateTimeFormatter.java
+++ b/common/src/java/org/apache/hadoop/hive/common/format/datetime/HiveSqlDateTimeFormatter.java
@@ -24,6 +24,8 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.Timestamp;
@@ -44,12 +46,14 @@ import java.time.temporal.TemporalUnit;
import java.time.temporal.WeekFields;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
/**
* Formatter using SQL:2016 datetime patterns.
@@ -1040,12 +1044,12 @@ public class HiveSqlDateTimeFormatter implements Serializable {
}
public Timestamp parseTimestamp(final String fullInput) {
- LocalDateTime ldt = LocalDateTime.ofInstant(Instant.EPOCH, ZoneOffset.UTC);
String substring;
int index = 0;
int value;
int timeZoneHours = 0, timeZoneMinutes = 0;
int iyyy = 0, iw = 0;
+ List<Integer> temporalValues = new ArrayList<>();
for (Token token : tokens) {
switch (token.type) {
@@ -1058,12 +1062,7 @@ public class HiveSqlDateTimeFormatter implements Serializable {
substring = getNextCharacterSubstring(fullInput, index, token); //e.g. Marcharch -> March
value = parseCharacterTemporal(substring, token); // e.g. July->07
}
- try {
- ldt = ldt.with(token.temporalField, value);
- } catch (DateTimeException e){
- throw new IllegalArgumentException(
- "Value " + value + " not valid for token " + token);
- }
+ temporalValues.add(value);
//update IYYY and IW if necessary
if (token.temporalField == IsoFields.WEEK_BASED_YEAR) {
@@ -1119,14 +1118,23 @@ public class HiveSqlDateTimeFormatter implements Serializable {
}
}
- // anything left unparsed at end of string? throw error
+ checkForLeftoverInput(fullInput, index);
+
+ checkForInvalidIsoWeek(iyyy, iw);
+
+ return getTimestampFromValues(tokens, temporalValues);
+ }
+
+ /**
+ * Anything left unparsed at end of input string? Throw error.
+ * @param fullInput full input String
+ * @param index where we left off parsing
+ */
+ private void checkForLeftoverInput(String fullInput, int index) {
if (!fullInput.substring(index).isEmpty()) {
throw new IllegalArgumentException("Leftover input after parsing: " +
fullInput.substring(index) + " in string " + fullInput);
}
- checkForInvalidIsoWeek(iyyy, iw);
-
- return Timestamp.ofEpochSecond(ldt.toEpochSecond(ZoneOffset.UTC), ldt.getNano());
}
/**
@@ -1139,16 +1147,83 @@ public class HiveSqlDateTimeFormatter implements Serializable {
}
LocalDateTime ldt = LocalDateTime.ofInstant(Instant.EPOCH, ZoneOffset.UTC);
- ldt = ldt.with(IsoFields.WEEK_BASED_YEAR, iyyy);
- ldt = ldt.with(IsoFields.WEEK_OF_WEEK_BASED_YEAR, iw);
+ try {
+ ldt = ldt.with(IsoFields.WEEK_BASED_YEAR, iyyy);
+ ldt = ldt.with(IsoFields.WEEK_OF_WEEK_BASED_YEAR, iw);
+ } catch (DateTimeException e) {
+ throw new IllegalArgumentException(e);
+ }
+
if (ldt.getYear() != iyyy) {
throw new IllegalArgumentException("ISO year " + iyyy + " does not have " + iw + " weeks.");
}
}
+ /**
+ * Create a timestamp from the list of values parsed from the input and the list of tokens
+ * parsed from the pattern input.
+ *
+ * We need to be able to parse input like "29.02.2000" (with pattern "dd.mm.yyyy")
+ * correctly – if we assigned the day value to the timestamp before the year value, then
+ * output would be 2000-02-28. So before creating the Timestamp we have to:
+ *
+ * - Make a list of pairs.
+ * Left value: only the Tokens that represent a temporal value
+ * Right value: their corresponding int values parsed from the input
+ * - Sort this list by length of base unit, in descending order (years before months, etc.).
+ * - Then create the parsed output Timestamp object by creating a LocalDateTime object using the
+ * token's TemporalField and the value.
+ *
+ * @param tokens list of tokens of any type, in order of pattern input
+ * @param temporalValues list of integer values parsed from the input, in order of input
+ * @return the parsed Timestamp
+ * @throws IllegalStateException if temporal values list and tokens in tokens list sizes are not
+ * equal
+ */
+ private Timestamp getTimestampFromValues(List<Token> tokens, List<Integer> temporalValues) {
+
+ // Get list of temporal Tokens
+ List<Token> temporalTokens = tokens.stream()
+ .filter(token-> token.type == TokenType.NUMERIC_TEMPORAL
+ || token.type == TokenType.CHARACTER_TEMPORAL)
+ .collect(Collectors.toList());
+ Preconditions.checkState(temporalTokens.size() == temporalValues.size(),
+ "temporalTokens list length (" + temporalTokens.size()
+ + ") differs from that of temporalValues (length: " + temporalValues.size() + ")");
+
+ // Get sorted list of temporal Token/value Pairs
+ List<ImmutablePair<Token, Integer>> tokenValueList = new ArrayList<>(temporalTokens.size());
+ for (int i = 0; i < temporalTokens.size(); i++) {
+ ImmutablePair<Token, Integer> pair =
+ new ImmutablePair<>(temporalTokens.get(i), temporalValues.get(i));
+ tokenValueList.add(pair);
+ }
+ tokenValueList.sort(((Comparator<ImmutablePair<Token, Integer>>) (o1, o2) -> {
+ Token token1 = o1.left;
+ Token token2 = o2.left;
+ return token1.temporalField.getBaseUnit().getDuration()
+ .compareTo(token2.temporalField.getBaseUnit().getDuration());
+ }).reversed());
+
+ // Create Timestamp
+ LocalDateTime ldt = LocalDateTime.ofInstant(Instant.EPOCH, ZoneOffset.UTC);
+ for (Pair<Token, Integer> pair : tokenValueList) {
+ TemporalField temporalField = pair.getLeft().temporalField;
+ int value = pair.getRight();
+ try {
+ ldt = ldt.with(temporalField, value);
+ } catch (DateTimeException e){
+ throw new IllegalArgumentException(
+ "Value " + value + " not valid for token " + temporalField);
+ }
+ }
+ return Timestamp.ofEpochSecond(ldt.toEpochSecond(ZoneOffset.UTC), ldt.getNano());
+ }
+
public Date parseDate(String input){
return Date.ofEpochMilli(parseTimestamp(input).toEpochMilli());
}
+
/**
* Return the next substring to parse. Length is either specified or token.length, but a
* separator or an ISO-8601 delimiter can cut the substring short. (e.g. if the token pattern is
diff --git a/common/src/test/org/apache/hadoop/hive/common/format/datetime/TestHiveSqlDateTimeFormatter.java b/common/src/test/org/apache/hadoop/hive/common/format/datetime/TestHiveSqlDateTimeFormatter.java
index ff47655..b78e469 100644
--- a/common/src/test/org/apache/hadoop/hive/common/format/datetime/TestHiveSqlDateTimeFormatter.java
+++ b/common/src/test/org/apache/hadoop/hive/common/format/datetime/TestHiveSqlDateTimeFormatter.java
@@ -251,6 +251,8 @@ public class TestHiveSqlDateTimeFormatter {
checkParseTimestamp("YYYY DDD", "2000 60", "2000-02-29 00:00:00");
checkParseTimestamp("YYYY DDD", "2000 61", "2000-03-01 00:00:00");
checkParseTimestamp("YYYY DDD", "2000 366", "2000-12-31 00:00:00");
+ //Leap day, parse day first
+ checkParseTimestamp("dd mm yyyy", "29 02 2000", "2000-02-29 00:00:00");
//Test timezone offset parsing without separators
checkParseTimestamp("YYYYMMDDHH12MIA.M.TZHTZM", "201812310800AM+0515", "2018-12-31 08:00:00");
checkParseTimestamp("YYYYMMDDHH12MIA.M.TZHTZM", "201812310800AM0515", "2018-12-31 08:00:00");
@@ -534,8 +536,8 @@ public class TestHiveSqlDateTimeFormatter {
fail("Parse string to timestamp should have failed.\nString: " + string + "\nPattern: "
+ pattern + ", output = " + output);
} catch (Exception e) {
- assertEquals("Expected IllegalArgumentException, got another exception.",
- e.getClass().getName(), IllegalArgumentException.class.getName());
+ assertEquals("Expected IllegalArgumentException, got another exception:" + e,
+ IllegalArgumentException.class.getName(), e.getClass().getName());
}
}
}