You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2026/06/04 06:43:11 UTC
(hudi) branch variant-intro-shredded-avro-support updated (4d1e2b8742cd -> 9445ef3575aa)
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a change to branch variant-intro-shredded-avro-support
in repository https://gitbox.apache.org/repos/asf/hudi.git
discard 4d1e2b8742cd feat(schema): Add read + write support for shredded for AVRO
omit 8862585a1b4e feat(schema): Config path implemented for spark record type
add 1582d602b0c5 fix(flink): Reject deferred RLI initialization for flink writer (#18399)
add bc7a766ff3e8 feat: use ScanOperation for Spark 3.3 and 3.4 partition pruning (#17936)
add 0591933bc237 fix: remove unused code (#18473)
add 65f996abe491 feat(lance): throwing exception/guard for users trying to read Lance from non-spark engines (#18481)
add f063aa580771 fix(hfile): use Hadoop WritableUtils VarInt encoding in HFile block index writer (#18465)
add 6310d7030817 fix: avoid duplicate archived timeline instants from leftover merge files (#18408)
add eaa9c8b1b549 fix: fix BufferedReader resource leak in InputStreamConsumer (#18469)
add 5d61a3553aab refactor(flink): Refactor Flink compaction/clean pipeline with composite table service handlers (#18477)
add 8f153b2f9d89 perf(core): optimize rollback listing calls on metadata table (#18279)
add c83ae87941b7 fix(flink): Handle bootstrap write metadata correctly after job resca… (#18485)
add 88c146e8072b chore(ci): Clean up env variable leak in TestSqlConf (#18486)
add 00a406682edb feat(sync): Map VECTOR type to binary for metastore sync support (#18480)
add fc7f30301e94 chore(deps): bump org.apache.logging.log4j:log4j-core (#18490)
add 613fc4995548 feat(common): add log reader scan metrics and logging for log block processing (#18412)
add 0bdab84bc35f feat(flink): Add metrics for RocksDB index backend in bucket assigner (#18484)
add da2667c4c6e6 feat(sync): Map BLOB type to struct in Hive and BigQuery sync (#18482)
add 8cd264825dcc chore: Allow versions to be specified in build_docker_images.sh (#17948)
add 4b15e50e8c95 feat(sync): Map VARIANT type to struct in Hive, Spark, and BigQuery sync (#18483)
add 98f90c14bb95 fix(payload): support sentinel no-op updates in DefaultHoodieRecordPayload (#18413)
add 12b3a06c2cc6 feat: Support to cap max commits to clean in one round of clean commit (#18322)
add d3e020132a91 fix(common): FutureUtils:allOf should always throw root cause exception (#18456)
add 5066fcc46e54 feat: Adding rolling extra metadata support (#18421)
add cad1530b03ca fix: Scanner resource leak in SqlFileBasedSource.fetchNextBatch (#18467)
add 5b6860713e60 fix: fix Scanner file handle leak in HiveIncrementalPuller.executeIncrementalSQL (#18457)
add f144abc5fce0 feat: Include ReverseOrderHoodieRecordPayload (#17928)
add a649188b6460 feat(spark): refresh parquet tools clustering strategy for current master (#18409)
add 2b33f5d407d6 fix: fix BufferedReader resource leak in FileIOUtils.readAsUTFStringLines (#18470)
add c1af4f5f9a96 fix: use forward slash literal and remove unused import in DFSPropertiesConfiguration (#18454)
add a3697738541a chore: cleanup docker-compose files (#17950)
add 0356488034dc fix(docker): fix docker image build with Java 11 and Hive 2.3.10 (#18519)
add 41cfc191f71b chore: Add Java 17 Hadoop base image and Spark 4.0.1 docker compose setup (#18520)
add 7bcb8be7c153 feat(metadata): Allow users to safely execute compaction plans on metadata table concurrently through a table service platform (rather than only inline during write) (#18295)
add 97f9628e038e feat(docker): add --multi-arch flag for cross-platform image builds (#18522)
add 9ddf58223922 chore: add timing logs for file index partition and file listing (#18417)
add 3d0ab800de5e chore(docker): bump integ-test docker-compose to Hive 2.3.10 (#18525)
add eaaae8a4f9ab feat: Add Azure-based storage lock (#17951)
add 55bf91a451bb introduce static helper method to remove clones (#18533)
add 937a64a2f4c5 fix: whitelist Flink _2.12 artifacts in scala-2.13 enforcer rule (#18508)
add cfb98336cf14 chore(docker): Remove duplicate yarn.nodemanager.bind-host in entrypoint.sh (#18527)
add 91dba3e16dec chore(common): Consolidate MapUtils into CollectionUtils (#18529)
add 95199f0da1d1 perf(common): avoid stream allocation in CollectionUtils.createImmutableList (#18530)
add 3a387da0cc7d feat(flink): Implement continuous sorting feature for append write (#18083)
add 0fb445496007 feat(utilities): add external HudiHiveSyncJob for on-demand Hive sync (#18204)
add f35b69cf028e feat(blob): Read Blobs in Spark SQL (#18098)
add adf29acc2edf fix: HoodieStorage resource leak in FileSystemBasedLockProvider.close() (#18461)
add 76a0a27e39be perf(common): Avoid double-iterating log files in file-system-view filters (#18531)
add a83473689eae feat(vector): Add Spark SQL DDL CREATE TABLE support for VECTOR type (#18488)
add 59fee58a71a2 feat(lance): Bump lance to 4.0.0 and lance-spark to 0.4.0 (#18498)
add f9dead0e57a8 feat: Adding support to block archival on last known ECTR for v6 tables (#18380)
add e30357927a5e fix: prevent parseTypeDescriptor crash for VARIANT (#18510)
add 0d5743510b67 fix: VARIANT Hive sync error when performing CREATE table DDL (#18511)
add e4904ba38f28 feat: Add support for exclusive rollbacks with multi writer (#18448)
add 4ef56e4ebd79 feat(blob): followup fixes for blob reader (#18538)
add cd83cf4d4002 chore(docker): add Hadoop 3.4.0 / Hive 2.3.10 / Spark 4.0.2 compose stack (#18550)
add 4260914c8265 fix: Parquet small-precision decimals decode ClassCastException (#18552)
add 86238986b36b fix: JDBC connection leak in HiveIncrementalPuller.saveDelta() (#18460)
add ddbdbb944a21 chore(spark): bump spark4.version to 4.0.2 (#18549)
add 7f4dd3196165 fix(lance): Add Hive InputFormat stubs and fix Spark SQL for Lance file format (#18162)
add ace2871c3718 feat(flink): Introduces dictionary encoding of payload partition path for RocksDBIndexBackend (#18560)
add 1e646621effe feat(lance): round-trip Hudi VECTOR columns as native Lance fixed-size lists (#18497)
add 9d1f81744bd7 fix(vector): Register VECTOR HMS column as BINARY on Spark CREATE (#18545)
add 110b9be76634 fix(variant): allow VariantType writes through Hudi's V1 DataSource on Spark 4 (#18564)
add 2092890af292 fix: ProtoConversionUtil$AvroSupport static init under Avro 1.12 (#18571)
add edaa16820b7b fix: FileGroupReader drops mandatory partition columns from dataSchema (#18570)
add 217e2a7a5400 feat: Adding support to inject custom configs to parquet writer (#18379)
add 2059c112b9af feat(clean): Adding empty clean support to hudi (#18337)
add 436bd66834a5 fix(vector): Pass plain FIXED through to VECTOR projection on Hive read (#18582)
add c1569db7f735 fix(clean): address review comments on empty clean support (#18587)
add 4f3e885f7550 fix(ci): bump surefire test heap from 3g to 4g (#18589)
add 853064454656 feat(lance): support simplified path for lance blob inline reading (#18575)
add 787953fa52dc feat(blob): add support for lance blob inline descriptor reading (#18586)
add 29f9c4034c07 feat(ci): enable auto-merge and require all GitHub Actions checks on master (#18594)
add a8a6917ef5a4 feat(flink): Vendor Flink 2.1 Dremel nested-reader support classes (#18567)
add fdf27db0733e fix(vector): Preserve VECTOR/BLOB metadata on SQL INSERT path (#18540)
add 20a01051ca6d feat(spark): add Spark 4.1 support (#17674)
add 782552a5e024 fix: Curator class conflict in ZookeeperBasedLockProvider (#18593)
add 7ae0fd95b3a6 fix(schema): Allow nested projection on BLOB and VARIANT columns in pruneDataSchema (#18566)
add 4bdcdf9b52b5 feat: Create JsonKinesisSource (#18224)
add 5c73bc0da20d feat(lance): fix lance writer/reader regarding arrow memory limit issue (#18613)
add 1ededfd70605 feat(common): When inferring checkpoint/schema from timeline, check non-ingestion write commits (in case they have metadata rolled-over) (#18576)
add eed9aa7f0639 feat: Add variant support description to RFC-99 (#18274)
add 642e1d33e120 chore(release): Moving to 1.3.0-SNAPSHOT on master branch (#18620)
add 7c2c56ec3186 fix(schema): Handle BLOB and VARIANT in Hive-reader rewriteRecordWithNewSchema (#18580)
add fd63851bba3d feat(flink): extend Flink quickstart example to use source v2 (#18518)
add 426cbb8c232e chore(deps): Pin AWS v1 SDK BOM to short-circuit transitive version-range walk (#18619)
add 8d348cc6b927 feat(examples): Add Hudi Unstructed Demo env (#18643)
add 7b662e891e3e fix: filter spark.sql.* properties in SparkCatalogMetaStoreClient.toCatalogTable (#18654)
add 744385680b11 fix: make SparkCatalogMetaStoreClient.setMetaConf a no-op (#18652)
add 0a28d695eb15 chore(docker): reduce base_java17 and spark_base image size (#18542)
add 38db5ed05e5c chore: Add context7.json with URL and public key (#18662)
add 39797b42cfe9 docs: claim RFC-104 for schema evolution unification (#18660)
add f8d70cb58d7c feat: Introduce a Spark procedure to trigger LSM timeline compaction (#18659)
add f7508ded95a5 test(spark): Add date logical type test to TestAvroConversionUtils (#18584)
add 695294cd1384 fix: Honor SparkSession overrides for rebase mode and timezone in compaction tasks (#18675)
add cde0e3907e17 fix(ci): bump Maven heap to 8g to fix OOM in CI builds (#18618)
add 4d0e9cd47f9e fix(lance): prevent file splitting for Lance base files to avoid duplicate reads (#18678)
add 127c6ee03186 feat(common): roll over commit metadata to clean (#18590)
add 471bb48338bd refactor: move checkpoint metadata lookup helper to hudi-common (#18489)
add 91f341f8fa79 fix: filter EXTERNAL property in SparkCatalogMetaStoreClient.toCatalogTable (#18672)
add c36a5f7f517e fix(flink): Avoid emitting deletes for Flink source v2 batch reads (#18694)
add 40295605aae6 feat(flink): Backport Flink 2.1 nested Parquet column readers and INT64 timestamp dispatch (FLINK-35702) (#18636)
add 87019a332db7 fix(hive): Tolerate pruned ArrayWritable in nested BLOB projection (#18581)
add 63f721ddd077 fix: Fix reflection ctor signature for AwsGlueCatalogSyncTool in HiveSyncContext (#18697)
add 34e9c7c5bbdc test(schema): Add MOR log-only compaction tests for custom types (#18583)
add 47bf4e41342e feat(flink): Wire Flink 2.1 nested Parquet readers into the Hudi read path (FLINK-35702) (#18700)
add 004d15996896 refactor(flink): Remove legacy Parquet nested readers superseded by Flink 2.1 Dremel path (FLINK-35702) (#18701)
add 93e334ce042d feat(flink): Support dynamic bucket for flink streaming with partitio… (#18640)
add bab463aa1ca8 feat(flink): Support data skipping based on column stats for source V2 (#18706)
add f8e4b9f3db52 docs: Document muttley package as internal/optional for OSS users (#18394)
add 1fffd70db1cb perf: Reduce unnecessary FSDataOutputStream#hsync to enhance append performance (#17517)
add f2f6203aa496 fix(flink): add Apache license header to muttley/README.md (#18713)
add 5e72c96fae61 feat: add variant type adapter for Flink (#18702)
add 6bd2ace52ad0 fix(flink): Support read non-VECTOR columns from table containing VEC… (#18712)
add e72b19b622dd feat(flink): Support bucket pruning for source V2 (#18716)
add b6bc1658e8f2 chore: migrate the flink ITs run to flink2.1 (#18717)
add 1f0dfb37d39a fix(flink): Add customized partitioner for global RLI bucket assigner operator (#18725)
add f2fdca26fef4 fix(metadata): Exclude Variant/Blob/Vector from V1 column stats (#18695)
add 0d5b442e266d fix: remove the pk check for Flink append only table (#18738)
add 1b26f2b4e2ba fix(variant): align Spark 4.1 MOR merge with PushVariantIntoScan and restore Spark 4.0 reads (#18674)
add 50efb2edc51b feat(blob): RFC-100: Clarify inline vs out-of-line blob read behavior (#18728)
add e5afbba54f3e fix(flink): shade codehale in flink bundle (#18730)
add 071b3f125b41 fix: clear Hive work map after combine split failures (#18719)
add 21271e4f115d fix: Fix Scanner resource leak in HiveIncrementalPuller (#18441)
add fcc211e0038c fix: Add Apache Licence to hudi-notebooks/requirements.txt (#18745)
add 9230f7c13952 test(spark): Add tests for batch-mode blob reads (#18736)
add ce1f5f0f8c53 fix: Use the latest docker image in hudi-integ-test for Macbook (#18746)
add d50ccf2fd5ab chore: Update scripts to deploy staging jars as part of the release process (#18748)
add 971634bb3e4b chore: Enhance validate_staged_bundles.sh to validate bundles before closing the staging repo (#18749)
add 21cc7c63be9b chore: Add script to copy release artifacts to the same staging repo (#18747)
add 3d32596a741d fix(docker): tag base image per Java version to avoid latest collision (#18663)
add dca76ca279fc fix: Support data pruning using nested partition columns (#18126)
add 6e32f36b6456 fix: Follow-ups to JsonKinesisSource: numeric sequence comparison and call-site fixes (#18689)
add ae9866a16c75 fix: Enable schema merging for incremental and dfs sources (#18385)
add 4035f707f8a1 chore: Fix license for copy_staging_repo.sh (#18753)
add 780595b73c99 fix(flink): fix the write handle close for append write (#18756)
add b934633f066a feat(utilities): add Spark/HoodieStreamer validators for pre-commit validation - Phase 3 (#18405)
add cd2c8b831c8b [MINOR] Fix typos in comments and assertion messages (#18763)
add db460814d365 fix(flink): Fix minibatch lookup for global record level index (#18759)
add 055aa41fd7fa fix(lance): Support Lance file format on Spark 4.1 (#18760)
add 0983c0ddd56a chore: Harden workflow against command injection in PR title validation (#18771)
add 9026c7defd5d test(trino): de-flake TestHudi*FileOperations by polling for span stability (#18766)
add 71aa121035cf fix(flink): enforce Parquet VARIANT annotation in Flink schema conversion for unshredded variant (#18539)
add e406e5d092d0 feat(spark): add restore_to_instant stored procedure (#18696)
add d94b2e2c3162 feat(spark): add show_inflight_commits and cleanup_stale_inflight_com… (#18709)
add 162cac2910ee fix(lance): fail fast when write schema contains VARIANT columns (#18775)
add a5d8fe153067 test(azure): skip ITAzureStorageLockClientAzurite when MCR image pull fails (#18772)
add 58957a7926e2 fix(aws): implement writer-version update in Glue sync client (#18707)
add f67dc268211b fix: Enhance hudi-azure-bundle (#18472)
add 642d88c594ff [MINOR] Reserve RFC-105 for Trino Hudi Connector — Shim/Bundle Refactor (#18781)
add 0b80239a1d61 fix: Disable column stats and partition stats indices for Lance base files (#18588)
add 89f49b18d0af fix: Skip pre-compaction rollback metadata reads in getValidInstantTimestamps (#18544)
add 4ac661a8314c fix: Fix SQL syntax parser for CREATE TABLE on Spark 4.1 (#18779)
add f044d3d6396b fix(spark): handle Avro 1.12 logical type values in Spark 4.1 read path (#18773)
add 990fc29f5609 feat(flink): add metrics for RLI load time for bucket assign functions (#18762)
add 3abe066f3ad5 feat(common): per-partition event-time rollup and decouple watermark tracking from EVENT_TIME_ORDERING (#18778)
add d78b702c64d5 fix(spark): use HoodieStorageUtils factory in Spark 4.1 legacy parquet read (#18785)
add 0138f41bd421 perf: Improve global index performance for commit time ordering (#17797)
add 12e87b80afe8 fix: Fix dangling lock from storage based lock (#18439)
add 9565926dfda9 feat(blob): default blob.inline.mode to DESCRIPTOR for Lance (#18744)
add a687786833a6 chore: add spark4.1 and flink2.1 profile entries to RC bundle validation (#18796)
add 50eb95c7d6f5 fix(flink): fix disable table service not effective in hudi-flink (#13875)
add b82a5b2fe835 feat: add metrics for bucketassign.minibatch cache hit ratio (#18761)
add facb517ef957 feat(spark): Drop Apache Spark 3.3 integration support (#18786)
add e299b84b10b5 chore: fix asf.yaml invalid key (#18808)
add e2e5598838a2 chore: remove outdated required status checks in asf.yaml (#18832)
add 06dc09b1fedd feat(flink): Backport Flink 2.1 Dremel nested Parquet reader rewrite to hudi-flink1.19.x (FLINK-35702) (#18809)
add 7e5b5351bbfa chore: Fix stale zip file for variant backward compatibility test (#18815)
add 62ad88a3dc93 chore: Add release information for Apache Hudi 1.2.0 (#18831)
add b5c5801bf7d5 fix(spark): Add options for archive procedure (#18437)
add 652d95253648 fix: RLI bootstrap fails due to NPE with cleaner table service (#18836)
add 76ddebcc5228 chore: Fix Flink CI Maven profile arguments (#18845)
add 516d9e2093a5 [MINOR] Handle cancellation error with HoodieMetadataTableValidator (#18371)
add b8cab719b205 fix(ci): make required status checks work for PRs (#18598)
add ce29423932ec docs: Update DOAP file to include 0.15.1 (#18838)
add e5ae1115458c feat(trino): [RFC-105] Trino Hudi Connector — Shim/Bundle Refactor (#18782)
add 1bf6b44fcd1f feat(flink): add lance format for Flink append only table (#18741)
add d5c6b4d727e1 feat(flink): Add metrics for remote RLI lookup time in BucketAssign op (#18813)
add 853cbef630b3 feat(vector_search): Implement functionality for pre-filters and maxD… (#18797)
add 588998fc7193 refactor(flink): Refactor RowData writer factory to use HoodieSchema (#18873)
add ff72186936eb fix(cli): Fix the typo in show-inflight CLI command (#18868)
add 4290550378d1 chore: Pin third-party GitHub Actions to commit SHAs (#18872)
add 7af8cdf2c20e perf(streamer): fold validate() error-table WriteStatus sums into one pass (#18871)
add fc85e3ea985c fix(flink): remove pre-seed in metrics in GlobalRecordLevelIndexBackend and RecordLevelIndexBackend (#18875)
add 6e40cffcc92b fix(hudi-sync): Fix Hive test temp directory cleanup with JUnit TempDir (#18883)
add 65562af43371 feat: Enable import order style check and error on violations (#18886)
add da189952aaca fix(flink): set canonical base path in Hive sync config (#18884)
add 314ffaf06e59 feat: add lance format support for Flink COW table (#18862)
add 4e5034d35462 fix(flink): fix data loss in stream read from earliest (#18848)
add 4fdac3d32931 fix(spark): fix MOR bulk insert commit operation error (#18878)
add 8490c968d9dc feat(utilities): migrate HoodieStreamerWriteStatusValidator into pre-commit validator framework (#18765)
add 11f0e7c797ed feat(spark): add input records & bytes metrics (#18882)
add 8259182e7e32 fix: Fix NPE due to race condition while handling rocksdb handles (#18834)
add ed9ea0ead908 fix(flink): Trigger a failover after pending instants recommitted for both global and partitioned RLI (#18793)
add af38b880de5c feat(flink): add shard assign metrics for bucket assign function (#18790)
add 095e80a9c256 refactor: Add Lombok annotations to hudi-utilities (Part 1) (#17823)
add 7964202a751d refactor: Add Lombok Builders (#17781)
add b7adeccfbb5e feat(schema): Config path implemented for spark record type (#18062)
add ba8c4c7b82c7 fix(spark): align CTAS partition fields by table partition order (#18899)
add 8ac3092dc4e3 fix(ci): Flink version corresponds to used in Docker image, resolution for Everit JSON schema (#18905)
add 508e2955ef8b refactor: Add Lombok annotations to hudi-utilities (Part 3) (#17877)
add 438bbdfcaef4 refactor: Add Lombok annotations to hudi-common module (part 3) (#17825)
add 163a15a3c899 refactor: Add Lombok Builder annotation to HoodieLogFormat (#17785)
add 95be21048d69 docs(examples): pin blob.inline.mode=CONTENT after Lance default flip (#18823)
add c40f765ce58d chore(docker): silence Dockerfile lint warnings across image set (#18664)
add 9445ef3575aa feat(schema): Add read + write support for shredded for AVRO
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (4d1e2b8742cd)
\
N -- N -- N refs/heads/variant-intro-shredded-avro-support (9445ef3575aa)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 61 +
.github/workflows/bot.yml | 410 +++++--
.github/workflows/maven_artifact_validation.yml | 4 +-
.github/workflows/pr_title_validation.yml | 8 +-
.github/workflows/release_candidate_validation.yml | 12 +-
.gitignore | 5 +
README.md | 2 +-
azure-pipelines-20230430.yml | 7 +-
context7.json | 4 +
doap_HUDI.rdf | 16 +
docker/README.md | 34 +-
docker/build_docker_images.sh | 125 +-
...-compose_hadoop284_hive2310_spark353_amd64.yml} | 46 +-
...-compose_hadoop284_hive2310_spark353_arm64.yml} | 46 +-
...er-compose_hadoop334_hive313_spark353_amd64.yml | 56 +-
...er-compose_hadoop334_hive313_spark353_arm64.yml | 37 +-
...-compose_hadoop340_hive2310_spark402_amd64.yml} | 74 +-
...-compose_hadoop340_hive2310_spark402_arm64.yml} | 74 +-
...r-compose_hadoop340_hive313_spark401_amd64.yml} | 61 +-
...r-compose_hadoop340_hive313_spark401_arm64.yml} | 61 +-
docker/hoodie/hadoop/base/Dockerfile | 10 +-
docker/hoodie/hadoop/base/entrypoint.sh | 1 -
docker/hoodie/hadoop/base/pom.xml | 2 +-
docker/hoodie/hadoop/base_java11/Dockerfile | 14 +-
docker/hoodie/hadoop/base_java11/entrypoint.sh | 1 -
docker/hoodie/hadoop/base_java11/pom.xml | 2 +-
.../hadoop/{base_java11 => base_java17}/Dockerfile | 51 +-
.../hadoop/{base => base_java17}/entrypoint.sh | 1 -
.../{base => base_java17}/export_container_ip.sh | 0
.../hadoop/{base_java11 => base_java17}/pom.xml | 12 +-
docker/hoodie/hadoop/datanode/Dockerfile | 8 +-
docker/hoodie/hadoop/datanode/pom.xml | 2 +-
docker/hoodie/hadoop/historyserver/Dockerfile | 8 +-
docker/hoodie/hadoop/historyserver/pom.xml | 2 +-
docker/hoodie/hadoop/hive_base/Dockerfile | 19 +-
docker/hoodie/hadoop/hive_base/pom.xml | 2 +-
docker/hoodie/hadoop/namenode/Dockerfile | 8 +-
docker/hoodie/hadoop/namenode/pom.xml | 2 +-
docker/hoodie/hadoop/pom.xml | 3 +-
docker/hoodie/hadoop/prestobase/Dockerfile | 23 +-
docker/hoodie/hadoop/prestobase/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/Dockerfile | 53 +-
docker/hoodie/hadoop/spark_base/pom.xml | 2 +-
docker/hoodie/hadoop/sparkadhoc/Dockerfile | 14 +-
docker/hoodie/hadoop/sparkadhoc/pom.xml | 2 +-
docker/hoodie/hadoop/sparkmaster/Dockerfile | 6 +-
docker/hoodie/hadoop/sparkmaster/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/Dockerfile | 6 +-
docker/hoodie/hadoop/sparkworker/pom.xml | 2 +-
docker/hoodie/hadoop/trinobase/Dockerfile | 11 +-
docker/hoodie/hadoop/trinobase/pom.xml | 2 +-
docker/hoodie/hadoop/trinocoordinator/pom.xml | 2 +-
docker/hoodie/hadoop/trinoworker/pom.xml | 2 +-
hudi-aws/pom.xml | 4 +-
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 23 +-
...amoDBBasedImplicitPartitionKeyLockProvider.java | 3 +-
.../aws/transaction/lock/S3StorageLockClient.java | 1 +
.../hudi/aws/sync/TestAWSGlueSyncClient.java | 51 +
.../apache/hudi/aws/sync/TestAwsGlueSyncTool.java | 31 +
.../lock/TestS3StorageBasedLockProvider.java | 2 +-
.../transaction/lock/TestS3StorageLockClient.java | 8 +-
{hudi-timeline-service => hudi-azure}/pom.xml | 281 ++---
.../azure/credentials/AzureCredentialFactory.java | 94 ++
.../transaction/lock/AzureStorageLockClient.java | 472 ++++++++
.../apache/hudi/config/AzureStorageLockConfig.java | 92 ++
.../lock/ITAzureStorageLockClientAzurite.java | 175 +++
.../lock/TestAzureStorageBasedLockProvider.java | 47 +
.../lock/TestAzureStorageLockClient.java | 604 ++++++++++
.../lock/TestAzureStorageLockClientUriParsing.java | 191 ++++
hudi-cli/pom.xml | 2 +-
.../main/java/org/apache/hudi/cli/HoodieCLI.java | 2 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 2 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 1 -
.../hudi/cli/commands/HoodieLogFileCommand.java | 6 +-
.../hudi/cli/commands/LockAuditingCommand.java | 2 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 2 +-
.../apache/hudi/cli/utils/InputStreamConsumer.java | 5 +-
.../hudi/cli/commands/TestCommitsCommand.java | 8 +-
.../cli/commands/TestHoodieLogFileCommand.java | 36 +-
.../hudi/cli/commands/TestRepairsCommand.java | 2 +-
hudi-client/hudi-client-common/pom.xml | 4 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 156 ++-
.../hudi/client/BaseHoodieTableServiceClient.java | 170 ++-
.../apache/hudi/client/BaseHoodieWriteClient.java | 125 +-
.../timeline/versioning/v1/TimelineArchiverV1.java | 41 +-
.../timeline/versioning/v2/LSMTimelineWriter.java | 2 +-
.../lock/FileSystemBasedLockProvider.java | 13 +-
.../transaction/lock/HoodieInterProcessMutex.java | 3 +-
.../transaction/lock/StorageBasedLockProvider.java | 139 ++-
.../client/transaction/lock/StorageLockClient.java | 2 +-
.../lock/metrics/HoodieLockMetrics.java | 9 +
.../transaction/lock/models/LockUpsertResult.java | 4 +-
.../apache/hudi/client/utils/ArchivalUtils.java | 6 +-
.../hudi/client/utils/LazyIterableIterator.java | 2 +-
.../client/validator/StreamingOffsetValidator.java | 93 +-
.../apache/hudi/config/HoodieArchivalConfig.java | 16 +
.../org/apache/hudi/config/HoodieCleanConfig.java | 39 +-
.../org/apache/hudi/config/HoodieLockConfig.java | 121 ++
.../config/HoodiePreCommitValidatorConfig.java | 10 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 87 +-
.../FileMetadataWriteStatusConverter.java | 106 ++
.../org/apache/hudi/index/HoodieIndexUtils.java | 13 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 2 +-
.../partition/PartitionBucketIndexUtils.java | 2 +-
.../hudi/io/ExternalFileClusteringWriteHandle.java | 118 ++
.../hudi/io/FileGroupReaderBasedAppendHandle.java | 23 +-
.../hudi/io/FileGroupReaderBasedMergeHandle.java | 8 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 4 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 18 +-
.../keygen/TimestampBasedAvroKeyGenerator.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 78 +-
...ieBackedTableMetadataWriterTableVersionSix.java | 13 +-
.../hudi/metadata/HoodieMetadataWriteUtils.java | 47 +-
.../SecondaryIndexRecordGenerationUtils.java | 10 +-
.../hudi/table/action/BaseActionExecutor.java | 2 +-
.../table/action/clean/CleanActionExecutor.java | 68 +-
.../action/clean/CleanPlanActionExecutor.java | 90 +-
.../hudi/table/action/clean/CleanPlanner.java | 27 +-
.../strategy/ClusteringExecutionStrategy.java | 17 +-
.../CommitBasedClusteringPlanStrategy.java | 2 +-
.../rollback/BaseRollbackActionExecutor.java | 2 +-
.../rollback/MarkerBasedRollbackStrategy.java | 122 +-
.../hudi/table/action/rollback/RollbackHelper.java | 89 +-
.../action/rollback/RollbackHelperFactory.java | 2 +-
.../table/action/rollback/RollbackHelperV1.java | 130 ++-
.../table/upgrade/EightToNineUpgradeHandler.java | 2 +-
.../hudi/table/upgrade/UpgradeDowngradeUtils.java | 2 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 4 +-
.../client/TestBaseHoodieTableServiceClient.java | 2 +-
.../hudi/client/TestBaseHoodieWriteClient.java | 2 +-
.../FileSystemBasedLockProviderTestClass.java | 2 +-
.../client/transaction/TestHoodieLockMetrics.java | 45 +-
...TestPreferWriterConflictResolutionStrategy.java | 4 +-
.../lock/TestStorageBasedLockProvider.java | 119 +-
.../utils/TestLegacyArchivedMetaEntryReader.java | 12 +-
.../validator/TestStreamingOffsetValidator.java | 2 +-
.../apache/hudi/config/TestHoodieLockConfig.java | 338 ++++++
.../config/TestHoodieWriteConfigFileStitching.java | 2 +-
.../TestFileMetadataWriteStatusConverter.java | 169 +++
.../hudi/index/simple/TestGlobalSimpleIndex.java | 4 +-
.../apache/hudi/index/simple/TestSimpleIndex.java | 2 +-
.../org/apache/hudi/io/TestHoodieWriteHandle.java | 13 +-
.../TestHoodieBackedTableMetadataWriter.java | 107 ++
.../metadata/TestHoodieMetadataWriteUtils.java | 398 ++++++-
.../org/apache/hudi/table/TestHoodieTable.java | 2 +-
.../apache/hudi/table/action/TestCleanPlanner.java | 12 +-
.../TestCommitBasedClusteringPlanStrategy.java | 44 +-
.../table/action/rollback/TestRollbackHelper.java | 633 ++++++++---
.../GenericRecordValidationTestUtils.java | 2 +-
.../hudi/testutils/HoodieWriteableTestTable.java | 14 +-
.../hudi/utils/TestMetadataConversionUtils.java | 14 +-
hudi-client/hudi-flink-client/pom.xml | 4 +-
.../hudi/client/HoodieFlinkTableServiceClient.java | 4 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 3 +-
.../hudi/client/model/AbstractHoodieRowData.java | 3 +-
.../apache/hudi/client/model/BootstrapRowData.java | 6 +-
.../apache/hudi/index/FlinkHoodieIndexFactory.java | 1 +
.../row/HoodieBloomFilterRowDataWriteSupport.java | 26 +-
.../io/storage/row/HoodieFlinkLanceArrowUtils.java | 305 +++++
.../io/storage/row/HoodieRowDataCreateHandle.java | 12 +-
.../row/HoodieRowDataFileWriterFactory.java | 74 +-
.../io/storage/row/HoodieRowDataLanceWriter.java | 167 +++
.../row/HoodieRowDataParquetWriteSupport.java | 18 +-
.../io/storage/row/RowDataParquetWriteSupport.java | 6 +-
.../row/parquet/ParquetSchemaConverter.java | 76 ++
.../apache/hudi/util/AvroToRowDataConverters.java | 17 +
.../java/org/apache/hudi/util/FlinkClientUtil.java | 2 +-
.../apache/hudi/util/HoodieSchemaConverter.java | 51 +-
.../apache/hudi/util/RowDataToAvroConverters.java | 28 +
.../client/TestHoodieFlinkTableServiceClient.java | 2 +-
.../row/TestHoodieFlinkLanceArrowUtils.java | 95 ++
.../storage/row/TestHoodieRowDataCreateHandle.java | 2 +-
.../TestHoodieRowDataParquetConfigInjector.java | 224 ++++
.../row/parquet/TestParquetSchemaConverter.java | 148 +++
.../testutils/HoodieFlinkWriteableTestTable.java | 14 +-
.../org/apache/hudi/util/TestFlinkClientUtil.java | 3 +-
.../hudi/util/TestHoodieSchemaConverter.java | 106 +-
hudi-client/hudi-java-client/pom.xml | 4 +-
.../run/strategy/JavaExecutionStrategy.java | 2 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 2 +-
.../read/HoodieFileGroupReaderOnJavaTestBase.java | 2 +-
.../hadoop/TestHoodieFileGroupReaderOnHive.java | 2 +-
hudi-client/hudi-spark-client/pom.xml | 6 +-
.../MultipleSparkJobExecutionStrategy.java | 2 +-
...SparkJobConsistentHashingExecutionStrategy.java | 2 +-
...arkExternalFileClusteringExecutionStrategy.java | 104 ++
.../client/common/HoodieSparkEngineContext.java | 2 +-
.../client/utils/SparkMetadataWriterUtils.java | 2 +-
.../hudi/client/utils/SparkValidatorUtils.java | 25 +-
.../org/apache/hudi/data/HoodieJavaPairRDD.java | 2 +-
.../index/SparkMetadataTableRecordLevelIndex.java | 2 +-
.../hudi/io/storage/BlobDescriptorTransform.java | 159 +++
.../io/storage/HoodieSparkFileReaderFactory.java | 7 +-
.../hudi/io/storage/HoodieSparkFileWriter.java | 2 +-
.../io/storage/HoodieSparkFileWriterFactory.java | 31 +-
.../hudi/io/storage/HoodieSparkLanceReader.java | 44 +-
.../hudi/io/storage/HoodieSparkLanceWriter.java | 240 +++-
.../hudi/io/storage/HoodieSparkParquetReader.java | 11 +-
.../hudi/io/storage/LanceRecordIterator.java | 148 +--
.../hudi/io/storage/LanceResourceCloser.java | 80 ++
.../hudi/io/storage/VectorConversionUtils.java | 116 +-
.../row/HoodieInternalRowFileWriterFactory.java | 10 +-
.../storage/row/HoodieRowParquetWriteSupport.java | 48 +-
.../table/HoodieSparkMergeOnReadMetadataTable.java | 2 +-
.../apache/hudi/HoodieSchemaConversionUtils.scala | 99 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 4 +-
.../org/apache/hudi/SparkAdapterSupport.scala | 8 +-
.../SparkFileFormatInternalRowReaderContext.scala | 91 +-
.../spark/sql/HoodieCatalystPlansUtils.scala | 21 +-
.../apache/spark/sql/HoodieInternalRowUtils.scala | 37 -
.../sql/avro/HoodieSparkSchemaConverters.scala | 198 +++-
.../parquet/HoodieParquetReadSupport.scala | 54 +-
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 91 +-
.../ClusteringIdentityTestExecutionStrategy.java | 76 ++
...xternalFileClusteringTestExecutionStrategy.java | 59 +
.../org/apache/hudi/client/TestClientRollback.java | 420 ++++++-
.../hudi/client/TestCoalescingPartitioner.java | 2 +-
.../apache/hudi/client/TestRollingMetadata.java | 630 +++++++++++
...erBasedDetectionStrategyWithZKLockProvider.java | 2 +-
.../client/TestSparkRDDMetadataWriteClient.java | 6 +-
.../functional/TestExternalPathHandling.java | 10 +-
.../functional/TestHoodieBackedTableMetadata.java | 2 +-
.../TestSavepointRestoreCopyOnWrite.java | 89 ++
.../hudi/client/utils/TestSparkValidatorUtils.java | 4 +-
.../hudi/common/table/log/TestLogReaderUtils.java | 2 +-
.../hudi/execution/TestBoundedInMemoryQueue.java | 2 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 231 +++-
.../row/TestHoodieRowParquetWriteSupport.java | 120 ++
.../apache/hudi/keygen/TestSimpleKeyGenerator.java | 2 +-
.../java/org/apache/hudi/table/TestCleaner.java | 97 +-
.../apache/hudi/table/TestHoodieSparkTable.java | 2 +-
.../table/action/compact/CompactionTestBase.java | 4 +-
.../table/functional/TestCleanPlanExecutor.java | 221 ++++
.../TestMarkerBasedRollbackStrategy.java | 21 +-
.../TestTimelineServerBasedWriteMarkers.java | 2 +-
.../hudi/testutils/FunctionalTestHarness.java | 4 +-
.../hudi/testutils/HoodieCleanerTestBase.java | 48 +-
.../testutils/HoodieSparkClientTestHarness.java | 14 +-
.../parquet/TestHoodieParquetReadSupport.scala | 51 +-
hudi-client/pom.xml | 2 +-
hudi-common/pom.xml | 3 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 60 +-
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 145 ++-
.../apache/hudi/avro/HoodieAvroWrapperUtils.java | 26 +-
.../hudi/client/validator/ValidationContext.java | 14 +
.../org/apache/hudi/common/HoodieCleanStat.java | 210 +---
.../org/apache/hudi/common/HoodieJsonPayload.java | 2 +-
.../hudi/common/HoodiePendingRollbackInfo.java | 18 +-
.../org/apache/hudi/common/HoodieRollbackStat.java | 34 +-
.../common/bloom/InternalDynamicBloomFilter.java | 9 +-
.../apache/hudi/common/bloom/InternalFilter.java | 7 +-
.../java/org/apache/hudi/common/bloom/Key.java | 8 +-
.../bootstrap/index/hfile/HFileBootstrapIndex.java | 15 +-
.../index/hfile/HFileBootstrapIndexReader.java | 22 +-
.../index/hfile/HFileBootstrapIndexWriter.java | 17 +-
.../apache/hudi/common/config/ConfigGroups.java | 20 +-
.../apache/hudi/common/config/ConfigProperty.java | 66 +-
.../apache/hudi/common/config/HoodieConfig.java | 12 +-
.../hudi/common/config/HoodieMetadataConfig.java | 87 ++
.../hudi/common/config/HoodieReaderConfig.java | 13 +
.../hudi/common/config/HoodieStorageConfig.java | 70 +-
.../config/HoodieTableServiceManagerConfig.java | 11 +-
.../DirectMarkerBasedDetectionStrategy.java | 21 +-
.../hudi/common/data/HoodieBaseListData.java | 16 +-
.../hudi/common/engine/HoodieEngineContext.java | 22 +-
.../hudi/common/engine/HoodieReaderContext.java | 138 +--
.../apache/hudi/common/engine/RecordContext.java | 12 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 9 +-
.../hudi/common/fs/FailSafeConsistencyGuard.java | 16 +-
.../hudi/common/fs/OptimisticConsistencyGuard.java | 10 +-
.../common/heartbeat/HoodieHeartbeatUtils.java | 7 +-
.../common/model/DefaultHoodieRecordPayload.java | 40 +-
.../org/apache/hudi/common/model/FileSlice.java | 4 +
.../hudi/common/model/HoodieCommitMetadata.java | 38 +-
.../apache/hudi/common/model/HoodieFileFormat.java | 4 +
.../hudi/common/model/HoodiePayloadProps.java | 10 +
.../model/ReverseOrderHoodieRecordPayload.java | 77 ++
.../hudi/common/schema/HoodieProjectionMask.java | 199 ++++
.../apache/hudi/common/schema/HoodieSchema.java | 143 ++-
.../common/schema/HoodieSchemaCompatibility.java | 8 +-
.../hudi/common/schema/HoodieSchemaField.java | 14 +-
.../hudi/common/schema/HoodieSchemaUtils.java | 10 +
.../hudi/common/table/HoodieTableMetaClient.java | 2 +-
.../hudi/common/table/TableSchemaResolver.java | 3 +-
.../table/log/BaseHoodieLogRecordReader.java | 92 +-
.../hudi/common/table/log/HoodieLogFormat.java | 326 +++---
.../common/table/read/HoodieFileGroupReader.java | 296 ++---
.../hudi/common/table/read/HoodieReadStats.java | 8 +-
.../apache/hudi/common/table/read/InputSplit.java | 64 +-
.../hudi/common/table/read/ReaderParameters.java | 82 +-
.../buffer/DefaultFileGroupRecordBufferLoader.java | 6 +-
.../table/read/buffer/FileGroupRecordBuffer.java | 29 +-
.../read/buffer/LogScanningRecordBufferLoader.java | 10 +-
.../buffer/PositionBasedFileGroupRecordBuffer.java | 10 +-
.../ReusableFileGroupRecordBufferLoader.java | 2 +-
.../StreamingFileGroupRecordBufferLoader.java | 4 +-
.../table/timeline/HoodieActiveTimeline.java | 12 +-
.../common/table/timeline/HoodieInstantReader.java | 2 +-
.../hudi/common/table/timeline/HoodieTimeline.java | 5 +
.../hudi/common/table/timeline/LSMTimeline.java | 2 +-
.../hudi/common/table/timeline/TimelineLayout.java | 6 +-
.../hudi/common/table/timeline/TimelineUtils.java | 27 +
.../hudi/common/table/timeline/dto/InstantDTO.java | 2 +-
.../timeline/versioning/v1/ActiveTimelineV1.java | 9 +-
.../versioning/v1/ArchivedTimelineLoaderV1.java | 2 +-
.../timeline/versioning/v1/ArchivedTimelineV1.java | 8 +-
.../versioning/v1/InstantFileNameGeneratorV1.java | 2 +-
.../timeline/versioning/v1/InstantGeneratorV1.java | 4 +-
.../timeline/versioning/v2/ActiveTimelineV2.java | 9 +-
.../versioning/v2/InstantComparatorV2.java | 2 +-
.../versioning/v2/InstantFileNameGeneratorV2.java | 2 +-
.../timeline/versioning/v2/InstantGeneratorV2.java | 4 +-
.../table/view/AbstractTableFileSystemView.java | 14 +-
.../table/view/HoodieTableFileSystemView.java | 9 +-
.../table/view/RocksDbBasedFileSystemView.java | 3 +
.../view/SpillableMapBasedFileSystemView.java | 8 +-
.../org/apache/hudi/common/util/CleanerUtils.java | 66 +-
.../apache/hudi/common/util/CollectionUtils.java | 11 +-
.../org/apache/hudi/common/util/FutureUtils.java | 29 +-
.../hudi/common/util/collection/RocksDBDAO.java | 25 +-
.../MapUtils.java => exception/ExceptionUtil.java} | 36 +-
.../schema/convert/InternalSchemaConverter.java | 51 +
.../hudi/io/HoodieParquetConfigInjector.java | 94 ++
.../apache/hudi/io/storage/HFileReaderFactory.java | 2 +-
.../hudi/io/storage/HoodieAvroFileWriter.java | 2 +-
.../hudi/io/storage/HoodieFileReaderFactory.java | 8 +-
.../hudi/io/storage/HoodieFileWriterFactory.java | 2 +-
.../io/storage/HoodieNativeAvroHFileReader.java | 32 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 2 +-
.../metadata/FileSystemBackedTableMetadata.java | 3 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 8 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 30 +-
.../hudi/metadata/MetadataPartitionType.java | 13 +-
.../apache/hudi/metadata/TableMetadataFactory.java | 1 +
.../java/org/apache/hudi/TestHoodieVersion.java | 4 +-
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 144 +++
.../common/config/TestHoodieMetadataConfig.java | 98 ++
.../common/data/TestHoodieListDataPairData.java | 2 +-
.../model/TestDefaultHoodieRecordPayload.java | 145 ++-
.../common/model/TestHoodieCommitMetadata.java | 85 ++
.../model/TestHoodieReplaceCommitMetadata.java | 1 +
.../model/TestReverseOrderHoodieRecordPayload.java | 130 +++
.../hudi/common/schema/HoodieSchemaTestUtils.java | 29 +
.../hudi/common/schema/TestHoodieSchema.java | 22 +-
.../hudi/common/schema/TestHoodieSchemaUtils.java | 34 +
.../table/log/block/TestHoodieAvroDataBlock.java | 2 +-
.../common/table/log/block/TestHoodieLogBlock.java | 2 +-
.../read/TestFileGroupReaderSchemaHandler.java | 2 +-
.../table/read/TestHoodieFileGroupReaderBase.java | 8 +-
.../common/table/read/TestHoodieReadStats.java | 44 +
.../read/buffer/BaseTestFileGroupRecordBuffer.java | 2 +-
.../buffer/TestFileGroupRecordBufferLoader.java | 4 +-
.../TestSortedKeyBasedFileGroupRecordBuffer.java | 2 +-
.../timeline/TestHoodieInstantTimeGenerator.java | 1 +
.../testutils/DisableDictionaryInjector.java | 40 +
.../hudi/common/testutils/MockHoodieTimeline.java | 4 +-
.../hudi/common/testutils/SchemaTestUtil.java | 9 +-
.../hudi/common/util/TestCheckpointUtils.java | 1 +
.../apache/hudi/common/util/TestCleanerUtils.java | 238 ++++
.../hudi/common/util/TestCollectionUtils.java | 42 +
.../apache/hudi/common/util/TestFutureUtils.java | 128 +++
.../org/apache/hudi/common/util/TestMapUtils.java | 69 --
.../apache/hudi/exception/TestExceptionUtil.java | 57 +
.../hudi/keygen/constant/TestKeyGeneratorType.java | 2 +-
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 21 +
.../hudi/metadata/TestSecondaryIndexKeyUtils.java | 1 +
.../variant_backward_compat/variant_mor_avro.zip | Bin 87511 -> 104069 bytes
.../vector_cross_engine_validation/README.md | 66 ++
.../vector_cross_engine_validation/vector_cow.zip | Bin 0 -> 80614 bytes
.../vector_cross_engine_validation/vector_mor.zip | Bin 0 -> 77871 bytes
hudi-examples/hudi-examples-common/pom.xml | 2 +-
hudi-examples/hudi-examples-flink/pom.xml | 8 +-
.../examples/quickstart/HoodieFlinkQuickstart.java | 58 +-
.../quickstart/TestHoodieFlinkQuickstart.java | 26 +-
hudi-examples/hudi-examples-java/pom.xml | 2 +-
hudi-examples/hudi-examples-k8s/pom.xml | 2 +-
.../k8s/quickstart/HudiDataStreamWriter.java | 14 +-
hudi-examples/hudi-examples-spark/pom.xml | 2 +-
.../src/test/python/HoodiePySparkQuickstart.py | 2 +-
.../hudi-examples-spark/src/test/python/README.md | 6 +-
.../src/test/python/vector_blob_demo/.gitignore | 28 +
.../src/test/python/vector_blob_demo/README.md | 765 +++++++++++++
.../vector_blob_demo/hudi_blob_reader_demo.py | 628 +++++++++++
.../hudi_dataframe_vector_blob_demo.py | 631 +++++++++++
.../vector_blob_demo/hudi_sql_vector_blob_demo.py | 601 ++++++++++
.../vector_blob_demo/notebooks/00_main_demo.ipynb | 563 +++++++++
.../notebooks/01_blob_reader.ipynb | 928 +++++++++++++++
.../notebooks/02_sql_vector_search.ipynb | 912 +++++++++++++++
.../notebooks/03_dataframe_vector_search.ipynb | 826 ++++++++++++++
.../python/vector_blob_demo/notebooks/README.md | 157 +++
.../test/python/vector_blob_demo/requirements.txt | 45 +
.../src/test/python/vector_blob_demo/run_demos.sh | 72 ++
hudi-examples/pom.xml | 2 +-
hudi-flink-datasource/hudi-flink/pom.xml | 4 +-
.../apache/hudi/configuration/FlinkOptions.java | 28 +-
.../apache/hudi/configuration/OptionsResolver.java | 84 +-
.../hudi/metrics/FlinkBucketAssignMetrics.java | 88 ++
.../hudi/metrics/FlinkIndexBackendMetrics.java | 145 +++
.../hudi/metrics/FlinkMdtCompactionMetrics.java | 24 +-
.../FlinkPartitionedIndexBackendMetrics.java | 86 ++
.../hudi/metrics/FlinkRocksDBIndexMetrics.java | 128 +++
.../java/org/apache/hudi/sink/CleanFunction.java | 30 +-
.../hudi/sink/StreamWriteOperatorCoordinator.java | 12 +-
.../hudi/sink/append/AppendWriteFunction.java | 13 +-
.../AppendWriteFunctionWithBIMBufferSort.java | 12 +-
.../AppendWriteFunctionWithContinuousSort.java | 331 ++++++
.../hudi/sink/append/AppendWriteFunctions.java | 11 +-
.../hudi/sink/bootstrap/RLIBootstrapOperator.java | 27 +-
.../org/apache/hudi/sink/buffer/BufferType.java | 6 +-
.../apache/hudi/sink/bulk/AutoRowDataKeyGen.java | 2 +-
.../org/apache/hudi/sink/bulk/RowDataKeyGens.java | 11 +-
.../sink/common/AbstractStreamWriteFunction.java | 26 +-
.../apache/hudi/sink/compact/CompactOperator.java | 42 +-
.../hudi/sink/compact/CompactionCommitSink.java | 45 +-
.../hudi/sink/compact/CompactionPlanOperator.java | 40 +-
.../hudi/sink/compact/FlinkCompactionConfig.java | 2 +-
.../hudi/sink/compact/HoodieFlinkCompactor.java | 2 +-
.../hudi/sink/compact/handler/CleanHandler.java | 98 +-
.../hudi/sink/compact/handler/CompactHandler.java | 177 +--
.../compact/handler/CompactionCommitHandler.java} | 30 +-
.../compact/handler/CompactionPlanHandler.java | 195 +---
.../compact/handler/CompositeCleanHandler.java} | 37 +-
.../compact/handler/CompositeCompactHandler.java | 56 +
.../handler/CompositeCompactionCommitHandler.java | 49 +
.../handler/CompositeCompactionPlanHandler.java | 57 +
.../handler/CompositeTableServiceHandler.java | 49 +
...ctHandler.java => DataTableCompactHandler.java} | 34 +-
....java => DataTableCompactionCommitHandler.java} | 57 +-
...er.java => DataTableCompactionPlanHandler.java} | 31 +-
...{CleanHandler.java => DefaultCleanHandler.java} | 6 +-
...ndler.java => MetadataTableCompactHandler.java} | 23 +-
...a => MetadataTableCompactionCommitHandler.java} | 31 +-
...ava => MetadataTableCompactionPlanHandler.java} | 28 +-
.../handler/TableServiceHandlerFactory.java | 89 ++
.../apache/hudi/sink/event/WriteMetadataEvent.java | 15 +
.../java/org/apache/hudi/sink/muttley/README.md | 42 +
.../sink/partitioner/BucketAssignFunction.java | 159 ++-
.../hudi/sink/partitioner/BucketAssigner.java | 2 +
.../partitioner/DynamicBucketAssignFunction.java | 165 +++
.../partitioner/DynamicBucketAssignOperator.java | 64 ++
...oner.java => GlobalRecordIndexPartitioner.java} | 60 +-
.../partitioner/MinibatchBucketAssignFunction.java | 78 +-
.../sink/partitioner/RecordIndexPartitioner.java | 89 +-
.../index/CodedRecordGlobalLocationSerializer.java | 65 ++
.../index/DummyPartitionedIndexBackend.java | 27 +-
.../partitioner/index/FlinkStateIndexBackend.java | 4 +-
.../{IndexBackend.java => GlobalIndexBackend.java} | 26 +-
...end.java => GlobalRecordLevelIndexBackend.java} | 69 +-
.../hudi/sink/partitioner/index/IndexBackend.java | 32 +-
.../partitioner/index/IndexBackendFactory.java | 22 +-
.../sink/partitioner/index/IndexWriteFunction.java | 15 +-
.../partitioner/index/MinibatchIndexBackend.java | 18 +-
.../partitioner/index/PartitionedIndexBackend.java | 47 +
.../index/RecordGlobalLocationSerializer.java | 26 +-
.../partitioner/index/RecordLevelIndexBackend.java | 393 ++++++-
.../partitioner/index/RocksDBIndexBackend.java | 48 +-
.../apache/hudi/sink/utils/HiveSyncContext.java | 8 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 34 +-
.../hudi/sink/utils/SamplingActionExecutor.java | 30 +-
.../org/apache/hudi/sink/v2/utils/PipelinesV2.java | 10 +-
.../org/apache/hudi/source/HoodieScanContext.java | 8 +-
.../java/org/apache/hudi/source/HoodieSource.java | 14 +-
.../apache/hudi/source/IncrementalInputSplits.java | 35 +-
.../enumerator/AbstractHoodieSplitEnumerator.java | 17 +-
.../HoodieContinuousSplitEnumerator.java | 4 +-
.../hudi/source/prune/PrimaryKeyPruners.java | 2 +-
.../apache/hudi/source/reader/BatchRecords.java | 8 +-
.../hudi/source/reader/HoodieRecordEmitter.java | 3 +-
.../hudi/source/reader/HoodieSourceReader.java | 8 +-
.../source/reader/HoodieSourceSplitReader.java | 12 +-
.../function/AbstractSplitReaderFunction.java | 5 +-
.../reader/function/HoodieSplitReaderFunction.java | 8 +-
.../source/split/DefaultHoodieSplitDiscover.java | 8 +-
.../source/split/DefaultHoodieSplitProvider.java | 1 -
.../hudi/source/split/HoodieCdcSourceSplit.java | 2 +-
.../hudi/source/split/HoodieSourceSplit.java | 2 +-
.../split/assign/DefaultHoodieSplitAssigner.java | 3 +-
.../source/split/assign/HoodieSplitAssigners.java | 3 +-
.../split/assign/HoodieSplitBucketAssigner.java | 3 +-
.../apache/hudi/source/stats/RecordLevelIndex.java | 12 +-
.../apache/hudi/streamer/HoodieFlinkStreamer.java | 6 +-
.../org/apache/hudi/table/HoodieTableFactory.java | 85 +-
.../org/apache/hudi/table/HoodieTableSink.java | 6 +-
.../org/apache/hudi/table/HoodieTableSource.java | 15 +-
.../apache/hudi/table/catalog/HoodieCatalog.java | 6 +-
.../hudi/table/catalog/HoodieHiveCatalog.java | 6 +-
.../table/format/FlinkRowDataReaderContext.java | 18 +-
.../org/apache/hudi/table/format/FormatUtils.java | 8 +-
.../hudi/table/format/HoodieFlinkIOFactory.java | 2 +-
.../format/HoodieRowDataFileReaderFactory.java | 6 +
.../table/format/HoodieRowDataLanceReader.java | 318 ++++++
.../table/format/HoodieRowDataParquetReader.java | 2 +-
.../hudi/table/format/cdc/CdcImageManager.java | 2 +-
.../apache/hudi/table/format/cdc/CdcIterators.java | 2 +-
.../table/format/cow/CopyOnWriteInputFormat.java | 76 +-
.../java/org/apache/hudi/util/FileIndexReader.java | 3 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 25 +-
.../hudi/configuration/TestOptionsResolver.java | 114 ++
.../hudi/metrics/TestFlinkBucketAssignMetrics.java | 172 +++
.../hudi/metrics/TestFlinkCompactionMetrics.java | 59 +
.../hudi/metrics/TestFlinkIndexBackendMetrics.java | 254 +++++
.../TestFlinkPartitionedIndexBackendMetrics.java | 147 +++
.../apache/hudi/sink/ITTestDataStreamWrite.java | 2 +-
.../hudi/sink/TestFlinkCheckpointClient.java | 11 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 25 +-
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 29 +
.../hudi/sink/TestWriteMergeOnReadWithCompact.java | 31 +
...TTestAppendWriteFunctionWithContinuousSort.java | 425 +++++++
.../hudi/sink/append/TestAppendWriteFunction.java | 33 +-
.../TestAppendWriteFunctionWithBIMBufferSort.java | 106 ++
.../sink/bootstrap/TestRLIBootstrapOperator.java | 80 ++
.../apache/hudi/sink/bulk/TestRowDataKeyGens.java | 21 +-
.../compact/handler/TestCompositeHandlers.java | 245 ++++
.../hudi/sink/event/TestWriteMetadataEvent.java | 76 ++
....java => TestGlobalRecordIndexPartitioner.java} | 90 +-
.../TestMinibatchBucketAssignFunction.java | 160 ++-
.../partitioner/TestRecordIndexPartitioner.java | 110 +-
.../TestCodedRecordGlobalLocationSerializer.java | 128 +++
.../index/TestGlobalRecordLevelIndexBackend.java | 300 +++++
.../index/TestRecordLevelIndexBackend.java | 284 ++---
.../partitioner/index/TestRocksDBIndexBackend.java | 139 ++-
.../hudi/sink/utils/BulkInsertFunctionWrapper.java | 4 +-
.../hudi/sink/utils/InsertFunctionWrapper.java | 27 +-
.../hudi/sink/utils/MockKeyedStateStore.java | 12 +-
.../sink/utils/MockStreamingRuntimeContext.java | 24 +-
.../sink/utils/StreamWriteFunctionWrapper.java | 72 +-
.../hudi/sink/utils/TestHiveSyncContext.java | 33 +
.../org/apache/hudi/sink/utils/TestPipelines.java | 91 ++
.../org/apache/hudi/sink/utils/TestWriteBase.java | 19 +-
.../sink/utils/keyedstate/MockKeyedListState.java | 1 +
.../sink/utils/keyedstate/MockKeyedMapState.java | 4 +-
.../sink/utils/keyedstate/MockKeyedValueState.java | 4 +-
.../apache/hudi/source/TestHoodieScanContext.java | 1 -
.../org/apache/hudi/source/TestHoodieSource.java | 64 +-
.../hudi/source/TestIncrementalInputSplits.java | 2 +-
.../TestHoodieContinuousSplitEnumerator.java | 32 +-
.../TestHoodieStaticSplitEnumerator.java | 46 +-
.../source/reader/TestHoodieSourceSplitReader.java | 5 +-
.../hudi/source/reader/TestRecordLimiter.java | 1 -
.../function/TestAbstractSplitReaderFunction.java | 14 +-
.../function/TestHoodieSplitReaderFunction.java | 7 +-
.../split/TestDefaultHoodieSplitProvider.java | 6 +-
.../split/TestHoodieContinuousSplitBatch.java | 1 -
.../split/TestHoodieSourceSplitSerializer.java | 5 +-
.../split/assign/TestHoodieSplitAssigners.java | 2 +-
.../assign/TestHoodieSplitBucketAssigner.java | 2 +-
.../hudi/table/ITTestDynamicBucketStreamWrite.java | 269 +++++
.../apache/hudi/table/ITTestHoodieDataSource.java | 404 ++++++-
.../ITTestVariantCrossEngineCompatibility.java | 30 +-
.../ITTestVectorCrossEngineCompatibility.java | 101 ++
.../table/TestHoodieFileGroupReaderOnFlink.java | 8 +
.../apache/hudi/table/TestHoodieTableFactory.java | 100 ++
.../hudi/table/catalog/TestHoodieCatalog.java | 77 ++
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 13 +
.../org/apache/hudi/utils/TestConfigurations.java | 23 +-
.../test/java/org/apache/hudi/utils/TestData.java | 10 +
.../apache/hudi/utils/TestFlinkWriteClients.java | 2 +-
.../org/apache/hudi/utils/TestStreamerUtil.java | 4 +-
hudi-flink-datasource/hudi-flink1.17.x/pom.xml | 4 +-
.../org/apache/hudi/adapter/DataTypeAdapter.java | 64 ++
.../org/apache/hudi/utils/RuntimeContextUtils.java | 4 +
.../hudi/adapter/DataTypeAdapterTestUtils.java | 21 +-
hudi-flink-datasource/hudi-flink1.18.x/pom.xml | 4 +-
.../org/apache/hudi/adapter/DataTypeAdapter.java | 64 ++
.../table/format/cow/ParquetSplitReaderUtil.java | 685 ++++++-----
.../table/format/cow/utils/BooleanArrayList.java | 69 ++
.../hudi/table/format/cow/utils/IntArrayList.java | 93 ++
.../hudi/table/format/cow/utils/LongArrayList.java | 83 ++
.../table/format/cow/utils/NestedPositionUtil.java | 209 ++++
.../format/cow/vector/ColumnarGroupArrayData.java | 179 ---
.../format/cow/vector/ColumnarGroupMapData.java | 63 --
.../format/cow/vector/ColumnarGroupRowData.java | 138 ---
.../cow/vector/HeapArrayGroupColumnVector.java | 53 -
.../table/format/cow/vector/HeapArrayVector.java | 31 +
.../format/cow/vector/HeapMapColumnVector.java | 74 +-
.../format/cow/vector/HeapRowColumnVector.java | 15 +
.../format/cow/vector/ParquetDecimalVector.java | 197 +++-
.../cow/vector/position/CollectionPosition.java | 57 +
.../cow/vector/position/LevelDelegation.java} | 31 +-
.../format/cow/vector/position/RowPosition.java} | 31 +-
.../cow/vector/reader/ArrayColumnReader.java | 473 --------
.../format/cow/vector/reader/ArrayGroupReader.java | 44 -
.../format/cow/vector/reader/MapColumnReader.java | 56 -
.../cow/vector/reader/NestedColumnReader.java | 277 +++++
.../vector/reader/NestedPrimitiveColumnReader.java | 638 +++++++++++
.../reader/ParquetColumnarRowSplitReader.java | 34 +-
.../reader/ParquetDataColumnReaderFactory.java | 112 +-
.../format/cow/vector/reader/RowColumnReader.java | 63 --
.../table/format/cow/vector/type/ParquetField.java | 72 ++
.../format/cow/vector/type/ParquetGroupField.java | 59 +
.../cow/vector/type/ParquetPrimitiveField.java | 55 +
.../org/apache/hudi/utils/RuntimeContextUtils.java | 4 +
.../hudi/adapter/DataTypeAdapterTestUtils.java | 21 +-
.../cow/vector/TestHeapColumnVectorAccessors.java | 138 +++
.../cow/vector/TestParquetDecimalVector.java | 187 +++
.../reader/TestParquetDataColumnReaderFactory.java | 270 +++++
.../cow/vector/type/TestParquetGroupField.java | 132 +++
hudi-flink-datasource/hudi-flink1.19.x/pom.xml | 4 +-
.../org/apache/hudi/adapter/DataTypeAdapter.java | 64 ++
.../table/format/cow/ParquetSplitReaderUtil.java | 687 ++++++-----
.../table/format/cow/utils/BooleanArrayList.java | 69 ++
.../hudi/table/format/cow/utils/IntArrayList.java | 93 ++
.../hudi/table/format/cow/utils/LongArrayList.java | 83 ++
.../table/format/cow/utils/NestedPositionUtil.java | 209 ++++
.../format/cow/vector/ColumnarGroupArrayData.java | 179 ---
.../format/cow/vector/ColumnarGroupMapData.java | 63 --
.../format/cow/vector/ColumnarGroupRowData.java | 138 ---
.../cow/vector/HeapArrayGroupColumnVector.java | 53 -
.../table/format/cow/vector/HeapArrayVector.java | 31 +
.../format/cow/vector/HeapMapColumnVector.java | 75 +-
.../format/cow/vector/HeapRowColumnVector.java | 15 +
.../format/cow/vector/ParquetDecimalVector.java | 197 +++-
.../cow/vector/position/CollectionPosition.java | 57 +
.../cow/vector/position/LevelDelegation.java} | 31 +-
.../format/cow/vector/position/RowPosition.java} | 31 +-
.../cow/vector/reader/ArrayColumnReader.java | 473 --------
.../format/cow/vector/reader/ArrayGroupReader.java | 45 -
.../format/cow/vector/reader/MapColumnReader.java | 56 -
.../cow/vector/reader/NestedColumnReader.java | 277 +++++
.../vector/reader/NestedPrimitiveColumnReader.java | 638 +++++++++++
.../reader/ParquetColumnarRowSplitReader.java | 34 +-
.../reader/ParquetDataColumnReaderFactory.java | 112 +-
.../format/cow/vector/reader/RowColumnReader.java | 63 --
.../table/format/cow/vector/type/ParquetField.java | 72 ++
.../format/cow/vector/type/ParquetGroupField.java | 59 +
.../cow/vector/type/ParquetPrimitiveField.java | 55 +
.../org/apache/hudi/utils/RuntimeContextUtils.java | 4 +
.../hudi/adapter/DataTypeAdapterTestUtils.java | 21 +-
.../cow/vector/TestHeapColumnVectorAccessors.java | 138 +++
.../cow/vector/TestParquetDecimalVector.java | 187 +++
.../reader/TestParquetDataColumnReaderFactory.java | 270 +++++
.../cow/vector/type/TestParquetGroupField.java | 132 +++
hudi-flink-datasource/hudi-flink1.20.x/pom.xml | 4 +-
.../org/apache/hudi/adapter/DataTypeAdapter.java | 64 ++
.../org/apache/hudi/utils/RuntimeContextUtils.java | 4 +
.../hudi/adapter/DataTypeAdapterTestUtils.java | 21 +-
hudi-flink-datasource/hudi-flink2.0.x/pom.xml | 4 +-
.../org/apache/hudi/adapter/DataTypeAdapter.java | 64 ++
.../org/apache/hudi/utils/RuntimeContextUtils.java | 4 +
.../runtime/kryo/KryoSerializerSnapshot.java | 3 +
.../hudi/adapter/DataTypeAdapterTestUtils.java | 21 +-
hudi-flink-datasource/hudi-flink2.1.x/pom.xml | 4 +-
.../org/apache/hudi/adapter/DataTypeAdapter.java | 92 ++
.../main/java/org/apache/hudi/adapter/Utils.java | 2 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 83 ++
.../org/apache/hudi/utils/RuntimeContextUtils.java | 4 +
.../runtime/kryo/KryoSerializerSnapshot.java | 3 +
.../hudi/adapter/DataTypeAdapterTestUtils.java | 23 +-
hudi-flink-datasource/pom.xml | 4 +-
hudi-gcp/pom.xml | 2 +-
.../hudi/gcp/bigquery/BigQuerySchemaResolver.java | 3 +
.../gcp/transaction/lock/GCSStorageLockClient.java | 1 +
.../gcp/bigquery/TestBigQuerySchemaResolver.java | 120 ++
.../transaction/lock/TestGCSStorageLockClient.java | 5 +-
hudi-hadoop-common/pom.xml | 2 +-
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 45 +-
.../common/config/DFSPropertiesConfiguration.java | 3 +-
.../common/table/log/HoodieLogFormatWriter.java | 131 ++-
.../hadoop/fs/inline/InLineFsDataInputStream.java | 79 +-
.../hudi/io/lance/HoodieBaseLanceWriter.java | 98 +-
.../hadoop/HoodieAvroFileWriterFactory.java | 36 +-
.../avro/AvroSchemaConverterWithTimestampNTZ.java | 11 +-
...TestHoodieAvroWriteSupportVariantShredding.java | 508 ---------
.../common/fs/TestHoodieWrapperFileSystem.java | 2 +-
.../apache/hudi/common/fs/TestStorageSchemes.java | 5 +-
.../common/fs/inline/TestInLineFileSystem.java | 14 +-
.../common/functional/TestHoodieLogFormat.java | 415 ++++---
.../TestHoodieLogFormatAppendFailure.java | 26 +-
.../hudi/common/model/TestHoodieFileGroup.java | 2 +-
.../hudi/common/table/TestHoodieTableConfig.java | 2 +-
.../common/table/TestHoodieTableMetaClient.java | 4 +-
.../hudi/common/table/TestTableSchemaResolver.java | 10 +-
.../hudi/common/table/TestTimelineUtils.java | 88 ++
....java => TestHoodieLogFormatWriterBuilder.java} | 13 +-
.../table/timeline/TestArchivedTimelineV1.java | 57 +-
.../common/table/timeline/TestHoodieInstant.java | 3 +-
.../table/view/TestIncrementalFSViewSync.java | 13 +-
.../common/testutils/HoodieCommonTestHarness.java | 14 +-
.../hudi/common/testutils/HoodieTestTable.java | 24 +-
.../reader/HoodieFileGroupReaderTestHarness.java | 9 +-
.../testutils/reader/HoodieFileSliceTestUtils.java | 7 +-
.../io/hadoop/TestHoodieHFileReaderWriter.java | 6 +-
...oodieAvroFileWriterFactoryVariantShredding.java | 275 -----
.../TestHoodieAvroParquetConfigInjector.java | 169 +++
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 108 ++
.../apache/parquet/schema/TestSchemaRepair.java | 1 +
hudi-hadoop-mr/pom.xml | 2 +-
.../hudi/hadoop/HiveHoodieReaderContext.java | 15 +-
.../hudi/hadoop/HoodieColumnProjectionUtils.java | 99 ++
.../HoodieFileGroupReaderBasedRecordReader.java | 6 +-
.../apache/hudi/hadoop/HoodieLanceInputFormat.java | 67 ++
.../hudi/hadoop/HoodieParquetInputFormat.java | 2 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 2 +-
.../apache/hudi/hadoop/SchemaEvolutionContext.java | 2 +-
.../hadoop/hive/HoodieCombineHiveInputFormat.java | 176 +--
.../realtime/HoodieLanceRealtimeInputFormat.java | 64 ++
.../realtime/RealtimeCompactedRecordReader.java | 2 +-
.../utils/HoodieArrayWritableSchemaUtils.java | 199 +++-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 23 +
.../hadoop/TestHoodieColumnProjectionUtils.java | 92 ++
.../hudi/hadoop/TestHoodieROTablePathFilter.java | 4 +-
.../hive/TestHoodieCombineHiveInputFormat.java | 32 +
.../TestHoodieMergeOnReadTableInputFormat.java | 2 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 2 +-
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 25 +-
.../utils/TestHoodieArrayWritableSchemaUtils.java | 202 +++-
hudi-integ-test/pom.xml | 6 +-
.../hudi/integ/testsuite/converter/Converter.java | 3 +-
.../hudi/integ/testsuite/dag/WorkflowDag.java | 4 +-
.../GenericRecordFullPayloadSizeEstimator.java | 8 +-
.../GenericRecordPartialPayloadGenerator.java | 2 +-
.../generator/LazyRecordGeneratorIterator.java | 6 +-
.../helpers/ZookeeperServiceProvider.java | 5 +-
.../testsuite/reader/DFSAvroDeltaInputReader.java | 11 +-
.../testsuite/reader/DFSDeltaInputReader.java | 15 +-
.../integ/testsuite/reader/DeltaInputReader.java | 3 +-
.../testsuite/writer/DFSSparkAvroDeltaWriter.java | 6 +-
.../integ/testsuite/writer/DeltaWriterAdapter.java | 3 +-
.../writer/SparkAvroDeltaInputWriter.java | 6 +-
.../integ/command/ITTestHoodieSyncCommand.java | 1 -
.../integ/testsuite/TestFileDeltaInputWriter.java | 35 +-
.../testsuite/converter/TestUpdateConverter.java | 16 +-
.../integ/testsuite/dag/ComplexDagGenerator.java | 10 +-
.../integ/testsuite/dag/HiveSyncDagGenerator.java | 11 +-
.../testsuite/dag/HiveSyncDagGeneratorMOR.java | 11 +-
.../TestGenericRecordPayloadGenerator.java | 16 +-
.../reader/TestDFSAvroDeltaInputReader.java | 17 +-
.../hudi/integ/testsuite/utils/TestUtils.java | 6 +-
hudi-io/pom.xml | 2 +-
.../apache/hudi/io/hfile/HFileMetaIndexBlock.java | 12 +-
.../apache/hudi/io/hfile/HFileRootIndexBlock.java | 15 +-
.../java/org/apache/hudi/io/util/FileIOUtils.java | 13 +-
.../main/java/org/apache/hudi/io/util/IOUtils.java | 36 +
.../org/apache/hudi/storage/StorageSchemes.java | 10 +-
.../hudi/io/hfile/TestHFileCompatibility.java | 4 +-
.../hudi/io/hfile/TestHFileReadCompatibility.java | 4 +-
.../org/apache/hudi/io/hfile/TestHFileWriter.java | 47 +
.../java/org/apache/hudi/io/util/TestIOUtils.java | 33 +
hudi-kafka-connect/pom.xml | 4 +-
hudi-notebooks/requirements.txt | 18 +
.../hudi-metaserver/hudi-metaserver-client/pom.xml | 2 +-
.../hudi-metaserver/hudi-metaserver-server/pom.xml | 2 +-
hudi-platform-service/hudi-metaserver/pom.xml | 4 +-
hudi-platform-service/pom.xml | 2 +-
hudi-spark-datasource/README.md | 6 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 4 +-
.../org/apache/hudi/HoodieTimelineCleanupUtil.java | 59 +
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 60 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 122 +-
.../hudi/HoodieHadoopFsRelationFactory.scala | 5 +-
.../org/apache/hudi/HoodieMergeOnReadRDDV2.scala | 53 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 8 +-
.../org/apache/hudi/HoodieStreamingSink.scala | 12 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 165 ++-
.../org/apache/hudi/cdc/CDCFileGroupIterator.scala | 8 +-
.../spark/HoodieSparkInputMetricsUtils.scala | 28 +-
.../HoodieVectorSearchTableValuedFunction.scala | 90 +-
.../spark/sql/catalyst/plans/logical/Index.scala | 12 +-
.../datasources/lance/SparkLanceReaderBase.scala | 138 ++-
.../HoodieFileGroupReaderBasedFileFormat.scala | 60 +-
.../sql/hive/SparkCatalogMetaStoreClient.scala | 23 +-
.../hudi/analysis/HoodieSparkBaseAnalysis.scala | 10 +-
.../analysis/HoodieVectorSearchPlanBuilder.scala | 74 +-
.../spark/sql/hudi/blob/BatchedBlobRead.scala | 52 +
.../spark/sql/hudi/blob/BatchedBlobReadExec.scala | 73 ++
.../spark/sql/hudi/blob/BatchedBlobReader.scala | 798 +++++++++++++
.../sql/hudi/blob/BatchedBlobReaderStrategy.scala | 62 +
.../spark/sql/hudi/blob/ReadBlobExpression.scala | 48 +
.../apache/spark/sql/hudi/blob/ReadBlobRule.scala | 173 +++
.../spark/sql/hudi/blob/ScalarFunctions.scala | 99 ++
.../AlterHoodieTableAddColumnsCommand.scala | 2 +-
.../AlterHoodieTableChangeColumnCommand.scala | 2 +-
.../hudi/command/CreateHoodieTableCommand.scala | 73 +-
.../sql/hudi/streaming/HoodieSourceOffset.scala | 13 +-
.../scala/org/apache/hudi/TestHoodieCLIUtils.scala | 104 ++
hudi-spark-datasource/hudi-spark/pom.xml | 4 +-
.../org/apache/hudi/cli/ArchiveExecutorUtils.java | 17 +-
.../java/org/apache/hudi/cli/SchemaProvider.java | 2 +-
.../sql/hudi/HoodieSparkSessionExtension.scala | 2 +
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 74 +-
.../spark/sql/hudi/command/IndexCommands.scala | 2 +-
.../procedures/ArchiveCommitsProcedure.scala | 99 +-
.../CleanupStaleInflightCommitsProcedure.scala | 224 ++++
.../hudi/command/procedures/HoodieProcedures.scala | 4 +
.../procedures/PartitionBucketIndexManager.scala | 8 +-
.../procedures/RestoreToInstantProcedure.scala | 309 +++++
.../command/procedures/RunCleanProcedure.scala | 2 +-
.../RunTimelineCompactionProcedure.scala | 97 ++
.../command/procedures/ShowCommitsProcedure.scala | 1 -
.../procedures/ShowInflightCommitsProcedure.scala | 113 ++
.../org/apache/hudi/ColumnStatsIndexHelper.java | 2 +-
.../apache/hudi/TestDataSourceReadWithDeletes.java | 2 +-
.../hudi/TestSparkSqlHudiPackageStructure.java | 1 +
.../hudi/client/TestHoodieClientMultiWriter.java | 2 +-
.../hudi/client/TestRollingMetadataSpark.java | 230 ++++
.../TestHoodieClientOnCopyOnWriteStorage.java | 162 ++-
.../TestMetadataUtilRLIandSIRecordGeneration.java | 2 +-
.../hudi/client/model/TestHoodieInternalRow.java | 1 +
.../hudi/functional/TestBufferedRecordMerger.java | 2 +-
.../TestGlobalIndexCommitTimeOrdering.java | 473 ++++++++
.../hudi/functional/TestHoodieBackedMetadata.java | 2 +-
.../apache/hudi/functional/TestHoodieIndex.java | 3 +-
...SparkBinaryCopyClusteringAndValidationMeta.java | 2 +-
.../functional/TestSparkSortAndSizeClustering.java | 2 +-
.../apache/hudi/io/TestMetadataWriterCommit.java | 10 +-
.../storage/TestHoodieParquetConfigInjector.java | 243 ++++
.../io/storage/TestHoodieSparkLanceReader.java | 37 +
.../io/storage/TestHoodieSparkLanceWriter.java | 90 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 2 +-
...dieSparkMergeOnReadTableInsertUpdateDelete.java | 7 +-
.../table/functional/TestHoodieSparkRollback.java | 2 +-
.../apache/hudi/testutils/DataSourceTestUtils.java | 2 +-
.../org/apache/hudi/TestAvroConversionUtils.scala | 27 +
.../org/apache/hudi/TestDataSourceDefaults.scala | 4 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 94 +-
.../hudi/TestHoodieSchemaConversionUtils.scala | 136 +++
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 35 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 28 +-
.../org/apache/hudi/blob/BlobTestHelpers.scala | 6 +-
.../apache/hudi/blob/TestBatchedBlobReader.scala | 457 ++++++++
.../org/apache/hudi/blob/TestBlobSupport.scala | 214 +++-
.../apache/hudi/blob/TestReadBlobBatching.scala | 223 ++++
.../org/apache/hudi/blob/TestReadBlobSQL.scala | 498 ++++++++
.../apache/hudi/functional/TestCOWDataSource.scala | 309 ++++-
.../hudi/functional/TestColumnStatsIndex.scala | 9 +-
.../TestFileGroupReaderPartitionColumn.scala | 158 +++
.../TestHoodieVectorSearchFunction.scala | 520 ++++++++-
.../hudi/functional/TestLanceDataSource.scala | 1190 +++++++++++++++++++-
.../apache/hudi/functional/TestMORDataSource.scala | 18 +-
.../hudi/functional/TestPartitionStatsIndex.scala | 2 +-
.../functional/TestPartitionStatsPruning.scala | 11 +-
.../TestSparkAdapterRebaseModeDefault.scala | 97 ++
.../TestSparkAdapterRebaseModePropagation.scala | 157 +++
.../hudi/functional/TestVectorDataSource.scala | 135 ++-
.../functional/cdc/TestCDCStreamingSuite.scala | 3 +-
.../spark/sql/avro/TestSchemaConverters.scala | 75 +-
.../sql/hive/TestSparkCatalogMetaStoreClient.scala | 26 +
.../sql/hudi/blob/TestBatchedBlobReaderMerge.scala | 183 +++
.../apache/spark/sql/hudi/common/TestSqlConf.scala | 14 +-
.../spark/sql/hudi/ddl/TestCreateTable.scala | 345 ++++++
.../apache/spark/sql/hudi/ddl/TestSpark3DDL.scala | 2 +-
.../sql/hudi/dml/others/TestDeleteFromTable.scala | 73 ++
.../sql/hudi/dml/others/TestMergeIntoTable.scala | 120 ++
.../sql/hudi/dml/others/TestMergeIntoTable2.scala | 7 +-
.../sql/hudi/dml/others/TestUpdateTable.scala | 93 +-
.../sql/hudi/dml/schema/TestBlobDataType.scala | 285 +++++
.../sql/hudi/dml/schema/TestVariantDataType.scala | 653 ++++++-----
.../procedure/TestArchiveCommitsProcedure.scala | 147 ++-
.../TestCleanupStaleInflightCommitsProcedure.scala | 413 +++++++
.../sql/hudi/procedure/TestRestoreProcedure.scala | 258 +++++
.../TestRunTimelineCompactionProcedure.scala | 100 ++
.../TestShowInflightCommitsProcedure.scala | 151 +++
.../procedure/TestShowTimelineTableProcedure.scala | 8 +-
hudi-spark-datasource/hudi-spark3-common/pom.xml | 2 +-
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 17 +-
.../Spark3HoodiePruneFileSourcePartitions.scala | 18 +-
.../command/InsertIntoHoodieTableCommand.scala | 7 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 15 +-
.../hudi/command/UpdateHoodieTableCommand.scala | 9 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 196 ----
.../org/apache/spark/sql/avro/AvroUtils.scala | 227 ----
.../Spark33LegacyHoodieParquetFileFormat.scala | 451 --------
.../datasources/parquet/Spark33ParquetReader.scala | 317 ------
.../sql/hudi/analysis/HoodieSpark33Analysis.scala | 66 --
hudi-spark-datasource/hudi-spark3.4.x/pom.xml | 4 +-
.../src/main/antlr4/imports/SqlBase.g4 | 2 +-
.../spark/sql/HoodieSpark34CatalystPlanUtils.scala | 17 +-
.../apache/spark/sql/adapter/Spark3_4Adapter.scala | 20 +-
.../HoodieSpark3_4ExtendedSqlAstBuilder.scala | 28 +-
.../parser/HoodieSpark3_4ExtendedSqlParser.scala | 3 +-
.../hudi/TestHoodieStreamingSinkConstants.scala} | 21 +-
hudi-spark-datasource/hudi-spark3.5.x/pom.xml | 4 +-
.../src/main/antlr4/imports/SqlBase.g4 | 2 +-
.../spark/sql/HoodieSpark35CatalystPlanUtils.scala | 17 +-
.../apache/spark/sql/adapter/Spark3_5Adapter.scala | 29 +-
.../HoodieSpark3_5ExtendedSqlAstBuilder.scala | 28 +-
.../parser/HoodieSpark3_5ExtendedSqlParser.scala | 3 +-
.../hudi/TestHoodieStreamingSinkConstants.scala} | 21 +-
hudi-spark-datasource/hudi-spark4-common/pom.xml | 5 +-
.../hudi/client/model/Spark4HoodieInternalRow.java | 47 -
...org.apache.spark.sql.sources.DataSourceRegister | 4 +-
.../org/apache/hudi/Spark4DefaultSource.scala | 39 +
.../spark/sql/adapter/BaseSpark4Adapter.scala | 77 +-
.../Spark4HoodiePruneFileSourcePartitions.scala | 16 +-
.../command/InsertIntoHoodieTableCommand.scala | 7 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 15 +-
.../hudi/command/UpdateHoodieTableCommand.scala | 9 +-
.../TestSpark4VariantShreddingProvider.java | 279 -----
hudi-spark-datasource/hudi-spark4.0.x/pom.xml | 4 +-
.../src/main/antlr4/imports/SqlBase.g4 | 2 +-
...Spark40HoodiePartitionCDCFileGroupMapping.scala | 25 +-
.../Spark40HoodiePartitionFileSliceMapping.scala} | 6 +-
.../hudi/Spark40HoodiePartitionValues.scala} | 4 +-
.../client/model/Spark40HoodieInternalRow.scala} | 32 +-
.../spark/sql/HoodieSpark40CatalystPlanUtils.scala | 17 +-
.../apache/spark/sql/adapter/Spark4_0Adapter.scala | 81 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 23 +-
.../parquet/Spark40HoodieParquetReadSupport.scala | 115 ++
.../Spark40LegacyHoodieParquetFileFormat.scala | 4 +-
.../datasources/parquet/Spark40ParquetReader.scala | 2 +-
.../HoodieSpark4_0ExtendedSqlAstBuilder.scala | 28 +-
.../parser/HoodieSpark4_0ExtendedSqlParser.scala | 3 +-
.../hudi/TestHoodieStreamingSinkConstants.scala} | 21 +-
.../TestSpark40HoodieParquetReadSupport.scala | 59 +
.../{hudi-spark3.3.x => hudi-spark4.1.x}/pom.xml | 109 +-
.../src/main/antlr4/imports/SqlBase.g4 | 44 +-
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 0
.../apache/hudi/Spark41HoodieFileScanRDD.scala} | 2 +-
...park41HoodiePartitionCDCFileGroupMapping.scala} | 6 +-
.../Spark41HoodiePartitionFileSliceMapping.scala} | 8 +-
.../hudi/Spark41HoodiePartitionValues.scala} | 14 +-
.../client/model/Spark41HoodieInternalRow.scala | 52 +
.../HoodieSpark41CatalystExpressionUtils.scala} | 28 +-
.../sql/HoodieSpark41CatalystPlanUtils.scala} | 41 +-
.../spark/sql/HoodieSpark41SchemaUtils.scala} | 11 +-
.../apache/spark/sql/adapter/Spark4_1Adapter.scala | 317 ++++++
.../apache/spark/sql/avro/AvroDeserializer.scala | 169 ++-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 33 +-
.../org/apache/spark/sql/avro/AvroUtils.scala | 0
.../sql/avro/HoodieSpark4_1AvroDeserializer.scala} | 7 +-
.../sql/avro/HoodieSpark4_1AvroSerializer.scala} | 2 +-
.../HoodieSpark41PartitionedFileUtils.scala} | 32 +-
.../datasources/Spark41NestedSchemaPruning.scala} | 13 +-
.../datasources/orc/Spark41OrcReader.scala} | 27 +-
.../parquet/Spark41DataSourceUtils.scala} | 9 +-
.../Spark41LegacyHoodieParquetFileFormat.scala} | 410 +++----
.../datasources/parquet/Spark41ParquetReader.scala | 388 +++++++
.../Spark41ResolveHudiAlterTableCommand.scala} | 11 +-
.../sql/hudi/analysis/HoodieSpark41Analysis.scala | 211 ++++
.../HoodieSpark4_1ExtendedSqlAstBuilder.scala} | 107 +-
.../parser/HoodieSpark4_1ExtendedSqlParser.scala} | 37 +-
.../hudi/TestHoodieStreamingSinkConstants.scala} | 24 +-
.../hudi/TestSpark4_1AvroLogicalTypeBytes.scala | 127 +++
hudi-spark-datasource/pom.xml | 2 +-
hudi-sync/hudi-adb-sync/pom.xml | 2 +-
hudi-sync/hudi-datahub-sync/pom.xml | 2 +-
.../sync/datahub/config/TestDataHubSyncConfig.java | 5 +-
.../TestTlsEnabledDataHubEmitterSupplier.java | 7 +-
hudi-sync/hudi-hive-sync/pom.xml | 2 +-
.../java/org/apache/hudi/hive/HiveSyncTool.java | 2 +-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 12 +-
.../org/apache/hudi/hive/util/HiveSchemaUtil.java | 3 +
.../hudi/hive/util/IMetaStoreClientUtil.java | 3 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 16 +-
.../org/apache/hudi/hive/TestSparkSchemaUtils.java | 39 +
.../hudi/hive/testutils/HiveTestCluster.java | 2 +-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 56 +-
.../apache/hudi/hive/util/TestHiveSchemaUtil.java | 26 +-
hudi-sync/hudi-sync-common/pom.xml | 2 +-
.../hudi/sync/common/util/SparkSchemaUtils.java | 2 +
hudi-sync/pom.xml | 2 +-
hudi-tests-common/pom.xml | 2 +-
hudi-timeline-service/pom.xml | 2 +-
.../hudi/TestHudiAlluxioCacheFileOperations.java | 30 +-
.../hudi/TestHudiMemoryCacheFileOperations.java | 30 +-
.../plugin/hudi/TestHudiNoCacheFileOperations.java | 30 +-
hudi-utilities/pom.xml | 16 +-
.../hudi/utilities/HiveIncrementalPuller.java | 100 +-
.../org/apache/hudi/utilities/HoodieCleaner.java | 10 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 36 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 45 +-
.../hudi/utilities/HoodieDataTableValidator.java | 36 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 31 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 36 +-
.../utilities/HoodieMetadataTableValidator.java | 135 +--
.../apache/hudi/utilities/HoodieRepairTool.java | 59 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 21 +-
.../org/apache/hudi/utilities/HoodieTTLJob.java | 9 +-
.../org/apache/hudi/utilities/HudiHiveSyncJob.java | 145 +++
.../org/apache/hudi/utilities/TableSizeStats.java | 58 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 59 +-
.../kafka/HoodieWriteCommitKafkaCallback.java | 18 +-
.../pulsar/HoodieWriteCommitPulsarCallback.java | 14 +-
...heckpointFromAnotherHoodieTimelineProvider.java | 15 +-
.../hudi/utilities/config/CloudSourceConfig.java | 13 +
.../hudi/utilities/config/KinesisSourceConfig.java | 186 +++
...FSSourceConfig.java => ORCDFSSourceConfig.java} | 24 +-
.../utilities/config/ParquetDFSSourceConfig.java | 15 +-
.../deser/KafkaAvroSchemaDeserializer.java | 5 +-
.../ingestion/HoodieIngestionException.java | 6 +-
.../ingestion/HoodieIngestionService.java | 18 +-
.../hudi/utilities/multitable/ArchiveTask.java | 7 +-
.../hudi/utilities/multitable/ClusteringTask.java | 6 +-
.../multitable/HoodieMultiTableServicesMain.java | 12 +-
.../multitable/MultiTableServiceUtils.java | 7 +-
.../hudi/utilities/perf/TimelineServerPerf.java | 9 +-
.../utilities/schema/DelegatingSchemaProvider.java | 10 +-
.../utilities/schema/FilebasedSchemaProvider.java | 9 +-
.../hudi/utilities/schema/HiveSchemaProvider.java | 18 +-
.../utilities/schema/SchemaRegistryProvider.java | 8 +-
.../utilities/schema/SimpleSchemaProvider.java | 7 +-
.../converter/JsonToAvroSchemaConverter.java | 2 +-
.../DeleteSupportSchemaPostProcessor.java | 8 +-
.../DropColumnSchemaPostProcessor.java | 8 +-
.../sources/HoodieRetryingKafkaConsumer.java | 1 +
.../hudi/utilities/sources/JsonKinesisSource.java | 345 ++++++
.../hudi/utilities/sources/KinesisSource.java | 374 ++++++
.../hudi/utilities/sources/ORCDFSSource.java | 6 +-
.../hudi/utilities/sources/ParquetDFSSource.java | 4 +-
.../hudi/utilities/sources/SqlFileBasedSource.java | 3 +-
.../helpers/CloudObjectsSelectorCommon.java | 31 +-
.../sources/helpers/KinesisDeaggregator.java | 80 ++
.../sources/helpers/KinesisOffsetGen.java | 535 +++++++++
.../sources/helpers/KinesisReadConfig.java | 52 +
.../sources/helpers/ProtoConversionUtil.java | 3 +-
.../hudi/utilities/streamer/BootstrapExecutor.java | 16 +-
.../utilities/streamer/DefaultStreamContext.java | 20 +-
.../apache/hudi/utilities/streamer/ErrorEvent.java | 37 +-
.../utilities/streamer/ErrorTableCommitter.java | 86 ++
.../streamer/HoodieMultiTableStreamer.java | 34 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 62 +-
.../utilities/streamer/HoodieStreamerUtils.java | 8 +-
.../streamer/NoNewDataTerminationStrategy.java | 8 +-
.../utilities/streamer/SchedulerConfGenerator.java | 12 +-
.../utilities/streamer/SourceFormatAdapter.java | 23 +-
.../utilities/streamer/SparkSampleWritesUtils.java | 24 +-
.../apache/hudi/utilities/streamer/StreamSync.java | 337 +++---
.../streamer/StreamerCheckpointUtils.java | 31 +-
.../streamer/SuccessfulRecordCounter.java | 106 ++
.../utilities/streamer/TableExecutionContext.java | 58 +-
.../utilities/streamer/WriteErrorReporter.java | 69 ++
.../validator/SparkKafkaOffsetValidator.java | 60 +
.../validator/SparkStreamerValidatorUtils.java | 194 ++++
.../streamer/validator/SparkValidationContext.java | 139 +++
.../validator/SparkWriteErrorValidator.java | 136 +++
.../utilities/transform/ChainedTransformer.java | 8 +-
.../utilities/transform/FlatteningTransformer.java | 7 +-
.../transform/SqlFileBasedTransformer.java | 12 +-
.../transform/SqlQueryBasedTransformer.java | 10 +-
.../hudi/utilities/TestHiveIncrementalPuller.java | 2 +-
.../TestHiveIncrementalPullerExecuteSQL.java | 125 ++
.../TestHoodieMetadataTableValidator.java | 31 +-
.../hudi/utilities/TestHoodieRepairTool.java | 8 +-
.../apache/hudi/utilities/TestHudiHiveSyncJob.java | 165 +++
.../utilities/TestManifestFileWriterSpark.java | 2 +-
...heckpointFromAnotherHoodieTimelineProvider.java | 134 +++
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 38 +-
.../MockConfigurationHotUpdateStrategy.java | 7 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 516 ++++-----
.../TestHoodieDeltaStreamerDAGExecution.java | 2 +-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 22 +-
.../TestHoodieMultiTableDeltaStreamer.java | 12 +-
.../deltastreamer/TestSourceFormatAdapter.java | 2 +-
.../functional/TestHiveSchemaProvider.java | 10 +-
.../functional/TestJdbcbasedSchemaProvider.java | 7 +-
.../TestHoodieMultiTableServicesMain.java | 14 +-
.../offlinejob/TestHoodieClusteringJob.java | 2 +-
.../utilities/sources/BaseTestKafkaSource.java | 22 +-
.../utilities/sources/CheckpointValidator.java | 2 +-
.../sources/S3EventsHoodieIncrSourceHarness.java | 23 +-
.../hudi/utilities/sources/TestAvroDFSSource.java | 103 ++
.../hudi/utilities/sources/TestDataSource.java | 11 +-
.../sources/TestGcsEventsHoodieIncrSource.java | 6 +-
.../utilities/sources/TestHoodieIncrSource.java | 40 +-
.../utilities/sources/TestJsonKinesisSource.java | 482 ++++++++
.../sources/TestKinesisSourceFiltering.java | 271 +++++
.../utilities/sources/TestShardRecordIterator.java | 364 ++++++
.../helpers/TestCloudObjectsSelectorCommon.java | 62 +
.../helpers/TestKinesisCheckpointUtils.java | 166 +++
.../sources/helpers/TestKinesisDataLossChecks.java | 372 ++++++
.../sources/helpers/TestKinesisShardRange.java | 133 +++
.../TestMercifulJsonToRowConverterBase.java | 56 +-
.../sources/helpers/TestProtoConversionUtil.java | 2 +-
.../sources/helpers/TestSanitizationUtils.java | 2 +-
.../helpers/gcs/TestPubsubMessagesFetcher.java | 5 +-
.../streamer/TestErrorTableCommitter.java | 135 +++
.../TestStreamSyncWriteStatusValidation.java | 84 ++
.../streamer/TestSuccessfulRecordCounter.java | 200 ++++
.../utilities/streamer/TestWriteErrorReporter.java | 99 ++
.../validator/TestSparkKafkaOffsetValidator.java | 322 ++++++
.../validator/TestSparkStreamerValidatorUtils.java | 290 +++++
.../validator/TestSparkValidationContext.java | 156 +++
.../validator/TestSparkWriteErrorValidator.java | 198 ++++
.../hudi/utilities/testutils/JdbcTestUtils.java | 18 +-
.../utilities/testutils/UtilitiesTestBase.java | 9 +-
.../testutils/sources/AbstractBaseTestSource.java | 18 +-
.../sources/DistributedTestDataSource.java | 10 +-
packaging/bundle-validation/Dockerfile | 2 -
packaging/bundle-validation/README.md | 12 +-
...h => build_flink1200hive313spark411scala213.sh} | 14 +-
packaging/bundle-validation/ci_run.sh | 43 +-
packaging/bundle-validation/run_docker_java17.sh | 34 +-
packaging/bundle-validation/validate.sh | 8 +-
packaging/hudi-aws-bundle/pom.xml | 5 +-
.../pom.xml | 115 +-
packaging/hudi-cli-bundle/pom.xml | 2 +-
packaging/hudi-datahub-sync-bundle/pom.xml | 2 +-
packaging/hudi-flink-bundle/pom.xml | 6 +-
packaging/hudi-gcp-bundle/pom.xml | 2 +-
packaging/hudi-hadoop-mr-bundle/pom.xml | 2 +-
packaging/hudi-hive-sync-bundle/pom.xml | 2 +-
packaging/hudi-integ-test-bundle/pom.xml | 2 +-
packaging/hudi-kafka-connect-bundle/pom.xml | 2 +-
packaging/hudi-metaserver-server-bundle/pom.xml | 2 +-
packaging/hudi-presto-bundle/pom.xml | 2 +-
packaging/hudi-spark-bundle/pom.xml | 2 +-
packaging/hudi-timeline-server-bundle/pom.xml | 2 +-
packaging/hudi-trino-bundle/pom.xml | 2 +-
packaging/hudi-utilities-bundle/pom.xml | 2 +-
packaging/hudi-utilities-slim-bundle/pom.xml | 2 +-
pom.xml | 201 ++--
release/release_guide.md | 40 +-
rfc/README.md | 4 +-
rfc/rfc-100/rfc-100.md | 82 ++
rfc/rfc-105/rfc-105.md | 225 ++++
rfc/rfc-99/rfc-99.md | 454 +++++++-
rfc/rfc-99/variant-appendix.md | 119 ++
rfc/rfc-99/{appendix.md => vector-appendix.md} | 0
scripts/pr_compliance.py | 2 +-
scripts/release/copy_staging_repo.sh | 175 +++
scripts/release/deploy_staging_jars.sh | 27 +-
scripts/release/deploy_staging_jars_java11.sh | 82 --
scripts/release/deploy_staging_jars_java17.sh | 19 +-
scripts/release/validate_staged_bundles.sh | 107 +-
style/checkstyle.xml | 2 +-
1116 files changed, 60360 insertions(+), 13644 deletions(-)
create mode 100644 context7.json
mode change 100644 => 100755 docker/build_docker_images.sh
rename docker/compose/{docker-compose_hadoop284_hive233_spark353_amd64.yml => docker-compose_hadoop284_hive2310_spark353_amd64.yml} (83%)
rename docker/compose/{docker-compose_hadoop284_hive233_spark353_arm64.yml => docker-compose_hadoop284_hive2310_spark353_arm64.yml} (82%)
copy docker/compose/{docker-compose_hadoop334_hive313_spark353_arm64.yml => docker-compose_hadoop340_hive2310_spark402_amd64.yml} (73%)
copy docker/compose/{docker-compose_hadoop334_hive313_spark353_arm64.yml => docker-compose_hadoop340_hive2310_spark402_arm64.yml} (73%)
copy docker/compose/{docker-compose_hadoop334_hive313_spark353_arm64.yml => docker-compose_hadoop340_hive313_spark401_amd64.yml} (75%)
copy docker/compose/{docker-compose_hadoop334_hive313_spark353_arm64.yml => docker-compose_hadoop340_hive313_spark401_arm64.yml} (75%)
copy docker/hoodie/hadoop/{base_java11 => base_java17}/Dockerfile (60%)
copy docker/hoodie/hadoop/{base => base_java17}/entrypoint.sh (98%)
copy docker/hoodie/hadoop/{base => base_java17}/export_container_ip.sh (100%)
copy docker/hoodie/hadoop/{base_java11 => base_java17}/pom.xml (93%)
copy {hudi-timeline-service => hudi-azure}/pom.xml (51%)
create mode 100644 hudi-azure/src/main/java/org/apache/hudi/azure/credentials/AzureCredentialFactory.java
create mode 100644 hudi-azure/src/main/java/org/apache/hudi/azure/transaction/lock/AzureStorageLockClient.java
create mode 100644 hudi-azure/src/main/java/org/apache/hudi/config/AzureStorageLockConfig.java
create mode 100644 hudi-azure/src/test/java/org/apache/hudi/azure/transaction/lock/ITAzureStorageLockClientAzurite.java
create mode 100644 hudi-azure/src/test/java/org/apache/hudi/azure/transaction/lock/TestAzureStorageBasedLockProvider.java
create mode 100644 hudi-azure/src/test/java/org/apache/hudi/azure/transaction/lock/TestAzureStorageLockClient.java
create mode 100644 hudi-azure/src/test/java/org/apache/hudi/azure/transaction/lock/TestAzureStorageLockClientUriParsing.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/FileMetadataWriteStatusConverter.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/ExternalFileClusteringWriteHandle.java
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieLockConfig.java
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/execution/TestFileMetadataWriteStatusConverter.java
copy hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionException.java => hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieBloomFilterRowDataWriteSupport.java (60%)
create mode 100644 hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieFlinkLanceArrowUtils.java
create mode 100644 hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataLanceWriter.java
create mode 100644 hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieFlinkLanceArrowUtils.java
create mode 100644 hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowDataParquetConfigInjector.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkExternalFileClusteringExecutionStrategy.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/BlobDescriptorTransform.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/LanceResourceCloser.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/ClusteringIdentityTestExecutionStrategy.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/ExternalFileClusteringTestExecutionStrategy.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestRollingMetadata.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieRowParquetWriteSupport.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/ReverseOrderHoodieRecordPayload.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieProjectionMask.java
copy hudi-common/src/main/java/org/apache/hudi/{common/util/MapUtils.java => exception/ExceptionUtil.java} (54%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/io/HoodieParquetConfigInjector.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/model/TestReverseOrderHoodieRecordPayload.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieReadStats.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/testutils/DisableDictionaryInjector.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestFutureUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestMapUtils.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/exception/TestExceptionUtil.java
create mode 100644 hudi-common/src/test/resources/vector_cross_engine_validation/README.md
create mode 100644 hudi-common/src/test/resources/vector_cross_engine_validation/vector_cow.zip
create mode 100644 hudi-common/src/test/resources/vector_cross_engine_validation/vector_mor.zip
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/.gitignore
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/README.md
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/hudi_blob_reader_demo.py
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/hudi_dataframe_vector_blob_demo.py
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/hudi_sql_vector_blob_demo.py
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/notebooks/00_main_demo.ipynb
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/notebooks/01_blob_reader.ipynb
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/notebooks/02_sql_vector_search.ipynb
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/notebooks/03_dataframe_vector_search.ipynb
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/notebooks/README.md
create mode 100644 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/requirements.txt
create mode 100755 hudi-examples/hudi-examples-spark/src/test/python/vector_blob_demo/run_demos.sh
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkBucketAssignMetrics.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkIndexBackendMetrics.java
copy hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/writer/DeltaWriterAdapter.java => hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkMdtCompactionMetrics.java (59%)
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkPartitionedIndexBackendMetrics.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkRocksDBIndexMetrics.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/append/AppendWriteFunctionWithContinuousSort.java
copy hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/{source/reader/HoodieRecordEmitter.java => sink/compact/handler/CompactionCommitHandler.java} (50%)
copy hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/{source/reader/HoodieRecordEmitter.java => sink/compact/handler/CompositeCleanHandler.java} (52%)
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/CompositeCompactHandler.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/CompositeCompactionCommitHandler.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/CompositeCompactionPlanHandler.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/CompositeTableServiceHandler.java
copy hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/{CompactHandler.java => DataTableCompactHandler.java} (89%)
rename hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/{CompactCommitHandler.java => DataTableCompactionCommitHandler.java} (82%)
copy hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/{CompactionPlanHandler.java => DataTableCompactionPlanHandler.java} (90%)
copy hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/{CleanHandler.java => DefaultCleanHandler.java} (96%)
rename hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/{MetadataCompactHandler.java => MetadataTableCompactHandler.java} (85%)
rename hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/{MetadataCompactCommitHandler.java => MetadataTableCompactionCommitHandler.java} (82%)
rename hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/{MetadataCompactionPlanHandler.java => MetadataTableCompactionPlanHandler.java} (87%)
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/compact/handler/TableServiceHandlerFactory.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/muttley/README.md
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/DynamicBucketAssignFunction.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/DynamicBucketAssignOperator.java
copy hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/{RecordIndexPartitioner.java => GlobalRecordIndexPartitioner.java} (51%)
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/CodedRecordGlobalLocationSerializer.java
copy hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java => hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/DummyPartitionedIndexBackend.java (53%)
copy hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/{IndexBackend.java => GlobalIndexBackend.java} (61%)
copy hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/{RecordLevelIndexBackend.java => GlobalRecordLevelIndexBackend.java} (67%)
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/PartitionedIndexBackend.java
copy hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionException.java => hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/SamplingActionExecutor.java (61%)
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieRowDataLanceReader.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkBucketAssignMetrics.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkCompactionMetrics.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkIndexBackendMetrics.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkPartitionedIndexBackendMetrics.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/append/ITTestAppendWriteFunctionWithContinuousSort.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/append/TestAppendWriteFunctionWithBIMBufferSort.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bootstrap/TestRLIBootstrapOperator.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/handler/TestCompositeHandlers.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/event/TestWriteMetadataEvent.java
copy hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/{TestRecordIndexPartitioner.java => TestGlobalRecordIndexPartitioner.java} (50%)
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/index/TestCodedRecordGlobalLocationSerializer.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/index/TestGlobalRecordLevelIndexBackend.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/TestPipelines.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestDynamicBucketStreamWrite.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestVectorCrossEngineCompatibility.java
create mode 100644 hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java
copy hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/writer/DeltaWriterAdapter.java => hudi-flink-datasource/hudi-flink1.17.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java (62%)
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java
copy hudi-flink-datasource/{hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/MinibatchIndexBackend.java => hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java} (55%)
copy hudi-flink-datasource/{hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/MinibatchIndexBackend.java => hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java} (56%)
delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java
copy hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/writer/DeltaWriterAdapter.java => hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java (62%)
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/BooleanArrayList.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/IntArrayList.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/LongArrayList.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/utils/NestedPositionUtil.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupArrayData.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupMapData.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/ColumnarGroupRowData.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayGroupColumnVector.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/CollectionPosition.java
copy hudi-flink-datasource/{hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/MinibatchIndexBackend.java => hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/LevelDelegation.java} (55%)
copy hudi-flink-datasource/{hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/index/MinibatchIndexBackend.java => hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/position/RowPosition.java} (56%)
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayGroupReader.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedColumnReader.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/NestedPrimitiveColumnReader.java
delete mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetField.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetGroupField.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/type/ParquetPrimitiveField.java
copy hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/writer/DeltaWriterAdapter.java => hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java (62%)
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestHeapColumnVectorAccessors.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/TestParquetDecimalVector.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/reader/TestParquetDataColumnReaderFactory.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/test/java/org/apache/hudi/table/format/cow/vector/type/TestParquetGroupField.java
create mode 100644 hudi-flink-datasource/hudi-flink1.20.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java
copy hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/writer/DeltaWriterAdapter.java => hudi-flink-datasource/hudi-flink1.20.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java (62%)
create mode 100644 hudi-flink-datasource/hudi-flink2.0.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java
copy hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/writer/DeltaWriterAdapter.java => hudi-flink-datasource/hudi-flink2.0.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java (62%)
create mode 100644 hudi-flink-datasource/hudi-flink2.1.x/src/main/java/org/apache/hudi/adapter/DataTypeAdapter.java
copy hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/writer/DeltaWriterAdapter.java => hudi-flink-datasource/hudi-flink2.1.x/src/test/java/org/apache/hudi/adapter/DataTypeAdapterTestUtils.java (62%)
delete mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroWriteSupportVariantShredding.java
rename hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/{TestHoodieLogWriterBuilder.java => TestHoodieLogFormatWriterBuilder.java} (87%)
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileSliceTestUtils.java (98%)
delete mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/storage/hadoop/TestHoodieAvroFileWriterFactoryVariantShredding.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/storage/hadoop/TestHoodieAvroParquetConfigInjector.java
create mode 100644 hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieLanceInputFormat.java
create mode 100644 hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieLanceRealtimeInputFormat.java
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/HoodieTimelineCleanupUtil.java
copy hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionException.java => hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/HoodieSparkInputMetricsUtils.scala (59%)
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/blob/BatchedBlobRead.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/blob/BatchedBlobReadExec.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/blob/BatchedBlobReader.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/blob/BatchedBlobReaderStrategy.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/blob/ReadBlobExpression.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/blob/ReadBlobRule.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/blob/ScalarFunctions.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestHoodieCLIUtils.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CleanupStaleInflightCommitsProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RestoreToInstantProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunTimelineCompactionProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInflightCommitsProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestRollingMetadataSpark.java
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestGlobalIndexCommitTimeOrdering.java
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieParquetConfigInjector.java
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/blob/TestBatchedBlobReader.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/blob/TestReadBlobBatching.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/blob/TestReadBlobSQL.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestFileGroupReaderPartitionColumn.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkAdapterRebaseModeDefault.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkAdapterRebaseModePropagation.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/blob/TestBatchedBlobReaderMerge.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/schema/TestBlobDataType.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCleanupStaleInflightCommitsProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRestoreProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRunTimelineCompactionProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowInflightCommitsProcedure.scala
copy hudi-spark-datasource/{hudi-spark-common => hudi-spark3-common}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (98%)
delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_3Adapter.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33LegacyHoodieParquetFileFormat.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33ParquetReader.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark33Analysis.scala
copy hudi-spark-datasource/{hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionCDCFileGroupMapping.scala => hudi-spark3.4.x/src/test/scala/org/apache/hudi/TestHoodieStreamingSinkConstants.scala} (60%)
copy hudi-spark-datasource/{hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionCDCFileGroupMapping.scala => hudi-spark3.5.x/src/test/scala/org/apache/hudi/TestHoodieStreamingSinkConstants.scala} (60%)
delete mode 100644 hudi-spark-datasource/hudi-spark4-common/src/main/java/org/apache/hudi/client/model/Spark4HoodieInternalRow.java
rename hudi-spark-datasource/{hudi-spark-common => hudi-spark4-common}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (94%)
create mode 100644 hudi-spark-datasource/hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4DefaultSource.scala
delete mode 100644 hudi-spark-datasource/hudi-spark4-common/src/test/java/org/apache/hudi/variant/TestSpark4VariantShreddingProvider.java
rename hudi-common/src/main/java/org/apache/hudi/common/util/MapUtils.java => hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionCDCFileGroupMapping.scala (63%)
copy hudi-spark-datasource/{hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionFileSliceMapping.scala => hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionFileSliceMapping.scala} (85%)
copy hudi-spark-datasource/{hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionValues.scala => hudi-spark4.0.x/src/main/scala/org/apache/hudi/Spark40HoodiePartitionValues.scala} (95%)
copy hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala => hudi-spark4.0.x/src/main/scala/org/apache/hudi/client/model/Spark40HoodieInternalRow.scala} (50%)
create mode 100644 hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark40HoodieParquetReadSupport.scala
copy hudi-spark-datasource/{hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionCDCFileGroupMapping.scala => hudi-spark4.0.x/src/test/scala/org/apache/hudi/TestHoodieStreamingSinkConstants.scala} (60%)
create mode 100644 hudi-spark-datasource/hudi-spark4.0.x/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/TestSpark40HoodieParquetReadSupport.scala
rename hudi-spark-datasource/{hudi-spark3.3.x => hudi-spark4.1.x}/pom.xml (80%)
rename hudi-spark-datasource/{hudi-spark3.3.x => hudi-spark4.1.x}/src/main/antlr4/imports/SqlBase.g4 (96%)
rename hudi-spark-datasource/{hudi-spark3.3.x => hudi-spark4.1.x}/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4 (100%)
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/hudi/Spark33HoodieFileScanRDD.scala => hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodieFileScanRDD.scala} (96%)
rename hudi-spark-datasource/{hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionCDCFileGroupMapping.scala => hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionCDCFileGroupMapping.scala} (82%)
copy hudi-spark-datasource/{hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionFileSliceMapping.scala => hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionFileSliceMapping.scala} (82%)
rename hudi-spark-datasource/{hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionValues.scala => hudi-spark4.1.x/src/main/scala/org/apache/hudi/Spark41HoodiePartitionValues.scala} (86%)
create mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/hudi/client/model/Spark41HoodieInternalRow.scala
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystExpressionUtils.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystExpressionUtils.scala} (85%)
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41CatalystPlanUtils.scala} (79%)
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33SchemaUtils.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark41SchemaUtils.scala} (83%)
create mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark4_1Adapter.scala
rename hudi-spark-datasource/{hudi-spark3.3.x => hudi-spark4.1.x}/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala (83%)
rename hudi-spark-datasource/{hudi-spark3.3.x => hudi-spark4.1.x}/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala (93%)
copy hudi-spark-datasource/{hudi-spark3.4.x => hudi-spark4.1.x}/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala (100%)
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroDeserializer.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark4_1AvroDeserializer.scala} (85%)
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_3AvroSerializer.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark4_1AvroSerializer.scala} (94%)
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark33PartitionedFileUtils.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark41PartitionedFileUtils.scala} (62%)
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark33NestedSchemaPruning.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark41NestedSchemaPruning.scala} (86%)
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark33OrcReader.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/orc/Spark41OrcReader.scala} (84%)
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33DataSourceUtils.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41DataSourceUtils.scala} (91%)
copy hudi-spark-datasource/{hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark40LegacyHoodieParquetFileFormat.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41LegacyHoodieParquetFileFormat.scala} (53%)
create mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark41ParquetReader.scala
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/hudi/Spark33ResolveHudiAlterTableCommand.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/Spark41ResolveHudiAlterTableCommand.scala} (89%)
create mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark41Analysis.scala
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_1ExtendedSqlAstBuilder.scala} (97%)
rename hudi-spark-datasource/{hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala => hudi-spark4.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_1ExtendedSqlParser.scala} (87%)
rename hudi-spark-datasource/{hudi-spark4-common/src/main/scala/org/apache/hudi/Spark4HoodiePartitionFileSliceMapping.scala => hudi-spark4.1.x/src/test/scala/org/apache/hudi/TestHoodieStreamingSinkConstants.scala} (56%)
create mode 100644 hudi-spark-datasource/hudi-spark4.1.x/src/test/scala/org/apache/hudi/TestSpark4_1AvroLogicalTypeBytes.scala
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/HudiHiveSyncJob.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/config/KinesisSourceConfig.java
copy hudi-utilities/src/main/java/org/apache/hudi/utilities/config/{ParquetDFSSourceConfig.java => ORCDFSSourceConfig.java} (62%)
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonKinesisSource.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/KinesisSource.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KinesisDeaggregator.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KinesisOffsetGen.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KinesisReadConfig.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/ErrorTableCommitter.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SuccessfulRecordCounter.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/WriteErrorReporter.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkKafkaOffsetValidator.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkStreamerValidatorUtils.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkValidationContext.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/validator/SparkWriteErrorValidator.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHiveIncrementalPullerExecuteSQL.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHudiHiveSyncJob.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestInitialCheckpointFromAnotherHoodieTimelineProvider.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestJsonKinesisSource.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKinesisSourceFiltering.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestShardRecordIterator.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestKinesisCheckpointUtils.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestKinesisDataLossChecks.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestKinesisShardRange.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableCommitter.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncWriteStatusValidation.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestSuccessfulRecordCounter.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestWriteErrorReporter.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkKafkaOffsetValidator.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkStreamerValidatorUtils.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkValidationContext.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/validator/TestSparkWriteErrorValidator.java
rename packaging/bundle-validation/base/{build_flink1171hive313spark334.sh => build_flink1200hive313spark411scala213.sh} (65%)
mode change 100644 => 100755
copy packaging/{hudi-hive-sync-bundle => hudi-azure-bundle}/pom.xml (65%)
create mode 100644 rfc/rfc-105/rfc-105.md
create mode 100644 rfc/rfc-99/variant-appendix.md
rename rfc/rfc-99/{appendix.md => vector-appendix.md} (100%)
create mode 100755 scripts/release/copy_staging_repo.sh
delete mode 100755 scripts/release/deploy_staging_jars_java11.sh