You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by GitBox <gi...@apache.org> on 2020/07/09 11:36:31 UTC
[GitHub] [hudi] nsivabalan commented on a change in pull request #1648: [HUDI-916]: added support for multiple input formats in TimestampBasedKeyGenerator
nsivabalan commented on a change in pull request #1648:
URL: https://github.com/apache/hudi/pull/1648#discussion_r448303029
##########
File path: hudi-spark/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java
##########
@@ -45,52 +45,54 @@
*/
public class TimestampBasedKeyGenerator extends SimpleKeyGenerator {
- enum TimestampType implements Serializable {
+ public enum TimestampType implements Serializable {
UNIX_TIMESTAMP, DATE_STRING, MIXED, EPOCHMILLISECONDS, SCALAR
}
private final TimeUnit timeUnit;
-
private final TimestampType timestampType;
-
- private SimpleDateFormat inputDateFormat;
-
private final String outputDateFormat;
+ private DateTimeFormatter inputFormatter;
+ private final HoodieDateTimeParser parser;
// TimeZone detailed settings reference
// https://docs.oracle.com/javase/8/docs/api/java/util/TimeZone.html
- private final TimeZone timeZone;
+ private final DateTimeZone outputDateTimeZone;
/**
* Supported configs.
*/
- static class Config {
+ public static class Config {
// One value from TimestampType above
- private static final String TIMESTAMP_TYPE_FIELD_PROP = "hoodie.deltastreamer.keygen.timebased.timestamp.type";
- private static final String INPUT_TIME_UNIT =
+ public static final String TIMESTAMP_TYPE_FIELD_PROP = "hoodie.deltastreamer.keygen.timebased.timestamp.type";
+ public static final String INPUT_TIME_UNIT =
"hoodie.deltastreamer.keygen.timebased.timestamp.scalar.time.unit";
- private static final String TIMESTAMP_INPUT_DATE_FORMAT_PROP =
+ //This prop can now accept list of input date formats.
+ public static final String TIMESTAMP_INPUT_DATE_FORMAT_PROP =
"hoodie.deltastreamer.keygen.timebased.input.dateformat";
- private static final String TIMESTAMP_OUTPUT_DATE_FORMAT_PROP =
+ public static final String TIMESTAMP_INPUT_DATE_FORMAT_LIST_DELIMITER_REGEX_PROP = "hoodie.deltastreamer.keygen.timebased.input.dateformat.list.delimiter.regex";
+ public static final String TIMESTAMP_INPUT_TIMEZONE_FORMAT_PROP = "hoodie.deltastreamer.keygen.timebased.input.timezone";
+ public static final String TIMESTAMP_OUTPUT_DATE_FORMAT_PROP =
"hoodie.deltastreamer.keygen.timebased.output.dateformat";
- private static final String TIMESTAMP_TIMEZONE_FORMAT_PROP =
- "hoodie.deltastreamer.keygen.timebased.timezone";
+ //still keeping this prop for backward compatibility so that functionality for existing users does not break.
+ public static final String TIMESTAMP_TIMEZONE_FORMAT_PROP =
+ "hoodie.deltastreamer.keygen.timebased.timezone";
+ public static final String TIMESTAMP_OUTPUT_TIMEZONE_FORMAT_PROP = "hoodie.deltastreamer.keygen.timebased.output.timezone";
+ static final String DATE_TIME_PARSER_CLASS = "hoodie.deltastreamer.keygen.datetime.parser.class";
}
- public TimestampBasedKeyGenerator(TypedProperties config) {
+ public TimestampBasedKeyGenerator(TypedProperties config) throws IOException {
super(config);
- DataSourceUtils.checkRequiredProperties(config,
- Arrays.asList(Config.TIMESTAMP_TYPE_FIELD_PROP, Config.TIMESTAMP_OUTPUT_DATE_FORMAT_PROP));
+ String dateTimeParserClass = config.getString(Config.DATE_TIME_PARSER_CLASS, HoodieDateTimeParserImpl.class.getName());
Review comment:
lets not add "class" in the config name. guess thats not nomenclature.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org