You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2026/04/27 18:11:01 UTC
(hudi) branch master updated (7ae0fd95b3a6 -> 4bdcdf9b52b5)

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


    from 7ae0fd95b3a6 fix(schema): Allow nested projection on BLOB and VARIANT columns in pruneDataSchema (#18566)
     add 4bdcdf9b52b5 feat: Create JsonKinesisSource (#18224)

No new revisions were added by this update.

Summary of changes:
 .github/workflows/bot.yml                          |   2 +-
 hudi-utilities/pom.xml                             |  13 +-
 .../hudi/utilities/config/KinesisSourceConfig.java | 186 ++++++++
 .../hudi/utilities/sources/JsonKinesisSource.java  | 345 ++++++++++++++
 .../hudi/utilities/sources/KinesisSource.java      | 372 +++++++++++++++
 .../sources/helpers/KinesisDeaggregator.java       |  80 ++++
 .../sources/helpers/KinesisOffsetGen.java          | 521 +++++++++++++++++++++
 .../sources/helpers/KinesisReadConfig.java         |  52 ++
 .../deltastreamer/HoodieDeltaStreamerTestBase.java |   4 +
 .../deltastreamer/TestHoodieDeltaStreamer.java     |  12 +-
 .../utilities/sources/TestJsonKinesisSource.java   | 483 +++++++++++++++++++
 .../sources/TestKinesisSourceFiltering.java        | 272 +++++++++++
 .../utilities/sources/TestShardRecordIterator.java | 364 ++++++++++++++
 .../helpers/TestKinesisCheckpointUtils.java        | 165 +++++++
 .../sources/helpers/TestKinesisDataLossChecks.java | 372 +++++++++++++++
 .../sources/helpers/TestKinesisShardRange.java     | 132 ++++++
 packaging/hudi-aws-bundle/pom.xml                  |   3 +
 pom.xml                                            |   1 +
 18 files changed, 3370 insertions(+), 9 deletions(-)
 create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/config/KinesisSourceConfig.java
 create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonKinesisSource.java
 create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KinesisSource.java
 create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KinesisDeaggregator.java
 create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KinesisOffsetGen.java
 create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KinesisReadConfig.java
 create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKinesisSource.java
 create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKinesisSourceFiltering.java
 create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestShardRecordIterator.java
 create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestKinesisCheckpointUtils.java
 create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestKinesisDataLossChecks.java
 create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestKinesisShardRange.java