You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by co...@apache.org on 2022/09/01 15:33:30 UTC
[hudi] branch master updated (5d9db864f1 -> f5de4e434b)
This is an automated email from the ASF dual-hosted git repository.
codope pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 5d9db864f1 [HUDI-4742] Fix AWS Glue partition's location is wrong when updatePartition (#6545)
add f5de4e434b [HUDI-4418] Add support for ProtoKafkaSource (#6135)
No new revisions were added by this update.
Summary of changes:
hudi-kafka-connect/pom.xml | 15 -
hudi-utilities/pom.xml | 21 ++
.../deltastreamer/SourceFormatAdapter.java | 37 ++-
.../schema/ProtoClassBasedSchemaProvider.java | 78 +++++
.../hudi/utilities/sources/AvroKafkaSource.java | 43 +--
.../hudi/utilities/sources/JsonKafkaSource.java | 43 +--
.../apache/hudi/utilities/sources/KafkaSource.java | 77 +++++
.../hudi/utilities/sources/ProtoKafkaSource.java | 105 ++++++
.../org/apache/hudi/utilities/sources/Source.java | 2 +-
.../utilities/sources/helpers/AvroConvertor.java | 6 +
.../sources/helpers/ProtoConversionUtil.java | 359 ++++++++++++++++++++
.../schema/TestProtoClassBasedSchemaProvider.java | 61 ++++
...onKafkaSource.java => BaseTestKafkaSource.java} | 213 +++---------
.../utilities/sources/TestJsonKafkaSource.java | 267 +--------------
.../sources/TestJsonKafkaSourcePostProcessor.java | 50 ++-
.../utilities/sources/TestProtoKafkaSource.java | 216 ++++++++++++
.../sources/helpers/TestProtoConversionUtil.java | 367 +++++++++++++++++++++
.../resources/schema-provider/proto/sample.proto | 77 +++++
.../proto/sample_schema_flattened.txt | 1 +
.../schema-provider/proto/sample_schema_nested.txt | 1 +
pom.xml | 49 +++
21 files changed, 1556 insertions(+), 532 deletions(-)
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/ProtoClassBasedSchemaProvider.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KafkaSource.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ProtoKafkaSource.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/ProtoConversionUtil.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestProtoClassBasedSchemaProvider.java
copy hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/{TestJsonKafkaSource.java => BaseTestKafkaSource.java} (51%)
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestProtoKafkaSource.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestProtoConversionUtil.java
create mode 100644 hudi-utilities/src/test/resources/schema-provider/proto/sample.proto
create mode 100644 hudi-utilities/src/test/resources/schema-provider/proto/sample_schema_flattened.txt
create mode 100644 hudi-utilities/src/test/resources/schema-provider/proto/sample_schema_nested.txt