You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2022/01/10 20:15:35 UTC
[hudi] branch release-0.10.1-rc1 updated (3e7d28b -> 78e86a4)
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a change to branch release-0.10.1-rc1
in repository https://gitbox.apache.org/repos/asf/hudi.git.
from 3e7d28b Fixing clustering yaml
add 4e9b428 [MINOR] Remove unused methods in HoodieColumnProjectionUtils (#4408)
add 92093a6 [HUDI-3118] Add default HUDI_DIR in setupKafka.sh (#4460)
add e7241a5 [HUDI-3183] Wrong result of HoodieArchivedTimeline loadInstants with TimeRangeFilter (#4521)
add 063a05e [HUDI-3100] Add config for hive conditional sync (#4440)
add 491194a [HUDI-3188] Update quick start guide for Kafka Connect Sink for Hudi (#4527)
add d93e43b [MINOR] fix typos in DDLExecutor (#4534)
add 82a50d9 [HUDI-2947] Fixing checkpoint fetch in detlastreamer (#4485)
add 37b086f [HUDI-3185] HoodieConfig#getBoolean should return false when default not set (#4536)
add 9bfc50e [HUDI-3192] Spark metastore schema evolution broken (#4533)
add 644f81b [HUDI-2909] Handle logical type in TimestampBasedKeyGenerator (#4203)
add 3c1c1cc [HUDI-3139] Shade htrace and parquet-avro in presto bundle (#4495)
add fc58f4c [HUDI-3178] Fixing metadata table compaction so as to not include uncommitted data (#4530)
add 85487e4 [HUDI-3104] Kafka-connect support of hadoop config environments and properties (#4451)
add 78e86a4 [HUDI-3125] spark-sql write timestamp directly (#4471)
No new revisions were added by this update.
Summary of changes:
.../hudi/cli/commands/TestCommitsCommand.java | 47 ++++++
.../org/apache/hudi/config/HoodieWriteConfig.java | 5 +
.../hudi/keygen/ComplexAvroKeyGenerator.java | 4 +-
.../hudi/keygen/GlobalAvroDeleteKeyGenerator.java | 2 +-
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 18 +-
.../keygen/NonpartitionedAvroKeyGenerator.java | 4 +-
.../apache/hudi/keygen/SimpleAvroKeyGenerator.java | 4 +-
.../keygen/TimestampBasedAvroKeyGenerator.java | 6 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 2 +-
.../metadata/HoodieTableMetadataKeyGenerator.java | 2 +-
.../run/strategy/JavaExecutionStrategy.java | 3 +-
.../JavaCustomColumnsSortPartitioner.java | 8 +-
.../TestJavaBulkInsertInternalPartitioner.java | 4 +-
.../MultipleSparkJobExecutionStrategy.java | 3 +-
.../RDDCustomColumnsSortPartitioner.java | 9 +-
.../apache/hudi/keygen/RowKeyGeneratorHelper.java | 19 ++-
.../SparkHoodieBackedTableMetadataWriter.java | 8 +-
.../org/apache/hudi/AvroConversionHelper.scala | 15 +-
.../functional/TestHoodieBackedMetadata.java | 23 +--
.../TestBulkInsertInternalPartitioner.java | 4 +-
.../hudi/io/TestHoodieTimelineArchiveLog.java | 66 ++++----
.../keygen/TestTimestampBasedKeyGenerator.java | 34 ++++
...dieSparkCopyOnWriteTableArchiveWithReplace.java | 6 +-
.../hudi/keygen/TestRowGeneratorHelper.scala | 102 +++++++++++
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 31 ++--
.../apache/hudi/common/config/HoodieConfig.java | 3 +
.../common/model/DefaultHoodieRecordPayload.java | 13 +-
.../table/timeline/HoodieArchivedTimeline.java | 2 +-
.../org/apache/hudi/keygen/BaseKeyGenerator.java | 7 +
.../hudi/keygen/constant/KeyGeneratorOptions.java | 10 ++
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 10 +-
.../hudi/common/config/TestConfigProperty.java | 19 ++-
.../hudi/common/testutils/SchemaTestUtil.java | 4 +
...st-evolved.avsc => timestamp-logical-type.avsc} | 2 +-
.../apache/hudi/sink/utils/PayloadCreation.java | 2 +-
.../hudi/utils/TestStringToRowDataConverter.java | 2 +-
.../hudi/hadoop/HoodieColumnProjectionUtils.java | 188 ---------------------
hudi-kafka-connect/README.md | 87 +++++++---
hudi-kafka-connect/demo/setupKafka.sh | 4 +
.../hudi/connect/utils/KafkaConnectUtils.java | 68 ++++++++
.../hudi/connect/writers/KafkaConnectConfigs.java | 29 ++++
.../apache/hudi/connect/TestHdfsConfiguration.java | 69 ++++++++
.../src/test/resources/hadoop_conf/core-site.xml | 19 ++-
.../src/test/resources/hadoop_conf/hdfs-site.xml | 18 +-
.../resources/hadoop_home/etc/hadoop/core-site.xml | 19 ++-
.../resources/hadoop_home/etc/hadoop/hdfs-site.xml | 18 +-
.../main/java/org/apache/hudi/DataSourceUtils.java | 2 +
.../scala/org/apache/hudi/DataSourceOptions.scala | 12 ++
.../SparkFullBootstrapDataProviderBase.java | 5 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 4 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 1 +
.../spark/sql/hudi/command/SqlKeyGenerator.scala | 13 +-
.../java/org/apache/hudi/TestDataSourceUtils.java | 16 +-
.../org/apache/hudi/TestDataSourceDefaults.scala | 12 +-
.../hudi/TestGenericRecordAndRowConsistency.scala | 118 +++++++++++++
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 13 +-
.../apache/spark/sql/hudi/TestCreateTable.scala | 27 +++
.../apache/spark/sql/hudi/TestInsertTable.scala | 89 +++++++---
.../java/org/apache/hudi/hive/HiveSyncTool.java | 4 +-
.../java/org/apache/hudi/hive/ddl/DDLExecutor.java | 4 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 84 +++++----
.../functional/HoodieDeltaStreamerTestBase.java | 17 +-
.../functional/TestHoodieDeltaStreamer.java | 18 +-
packaging/hudi-presto-bundle/pom.xml | 11 ++
64 files changed, 1033 insertions(+), 439 deletions(-)
create mode 100644 hudi-client/hudi-spark-client/src/test/scala/org/apache/hudi/keygen/TestRowGeneratorHelper.scala
copy hudi-common/src/test/resources/{timestamp-test-evolved.avsc => timestamp-logical-type.avsc} (89%)
create mode 100644 hudi-kafka-connect/src/test/java/org/apache/hudi/connect/TestHdfsConfiguration.java
copy style/import-control.xml => hudi-kafka-connect/src/test/resources/hadoop_conf/core-site.xml (57%)
copy style/import-control.xml => hudi-kafka-connect/src/test/resources/hadoop_conf/hdfs-site.xml (69%)
copy style/import-control.xml => hudi-kafka-connect/src/test/resources/hadoop_home/etc/hadoop/core-site.xml (57%)
copy style/import-control.xml => hudi-kafka-connect/src/test/resources/hadoop_home/etc/hadoop/hdfs-site.xml (69%)
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestGenericRecordAndRowConsistency.scala