You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by gi...@apache.org on 2022/05/21 13:04:03 UTC
[hudi] branch dependabot/maven/com.fasterxml.jackson.core-jackson-databind-2.12.6.1 updated (bd5c82e48b -> 674e4638e8)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch dependabot/maven/com.fasterxml.jackson.core-jackson-databind-2.12.6.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
omit bd5c82e48b Bump jackson-databind from 2.6.7.3 to 2.12.6.1
add 3195f51562 [HUDI-3748] write and select hudi table when enable hoodie.datasource.write.drop.partition.columns (#5201)
add 92ca426ab7 [HUDI-2319] dbt example models to demonstrate hudi dbt integration (#5220)
add 898be6174a [HUDI-3782] Fixing table config when any of the index is disabled (#5222)
add 8baeb816d5 [HUDI-3723] Fixed stack overflows in Record Iterators (#5235)
add e96f08f355 Moving to 0.12.0-SNAPSHOT on master branch.
add 8683fb1d49 [HUDI-3800] Fixed preserve commit metadata for compaction for untouched records (#5232)
add 7612549bcc [MINOR] Fixing build failure when using flink-1.13 (#5214)
add 9e87d164b3 [HUDI-3760] Adding capability to fetch Metadata Records by prefix (#5208)
add ca273274b0 [HUDI-3340] Fix deploy_staging_jars for different profiles (#5240)
add 939b3d1b07 [HUDI-3726] Switching from non-partitioned to partitioned key gen does not throw any exception (#5205)
add b2f09a1fee [HUDI-3340] Fix deploy_staging_jars command (#5243)
add d43b4cd95e [HUDI-3739] Fix handling of the `isNotNull` predicate in Data Skipping (#5224)
add e33149be9a [HUDI-3808] Flink bulk_insert timestamp(3) can not be read by Spark (#5236)
add 531381faff [HUDI-3096] fixed the bug that the cow table(contains decimalType) write by flink cannot be read by spark. (#4421)
add 9d744bb35c [HUDI-3805] Delete existing corrupted requested rollback plan during rollback (#5245)
add 6a8396420c [HUDI-3643] Fix hive count exception when the table is empty and the path depth is less than 3 (#5051)
add b3c834a242 [HUDI-3571] Spark datasource continuous ingestion tool (#5156)
add cd2c346df6 [HUDI-3637] Exclude uncommitted log files from metadata table validation (#5234)
add ef06e4a526 [HUDI-3810] Fixing lazy read for metadata log record readers (#5241)
add 672974c412 [HUDI-3823] Fix hudi-hive-sync-bundle to include HBase dependencies and shading (#5257)
add df87095ef0 [HUDI-3454] Fix partition name in all code paths for LogRecordScanner (#5252)
add 7a6272fba1 [HUDI-3781] fix spark delete sql can not delete record (#5215)
add 67215abaf0 [HUDI-3827] Promote the inetAddress picking strategy for NetworkUtils#getHostname (#5260)
add d7cc767dbc [HUDI-3825] Fixing non-partitioned table Partition Records persistence in MT (#5259)
add 26eb7b8183 [HUDI-3571] Spark datasource continuous checkpoint should have own fs variable (#5265)
add 1cc7542357 [MINOR] Update README of docker build setup (#5256)
add 81b25c543a [HUDI-3825] Fixing Column Stats Index updating sequence (#5267)
add 5e65aefc61 [HUDI-3837] Fix license and rat check settings (#5273)
add 3e97c88c4f [HUDI-3807] Add a new config to control the use of metadata index in HoodieBloomIndex (#5268)
add 15c264535f [MINOR] Fix typos in the comments of HoodieMergeHandle (#5271)
add 7a9d48d126 [HUDI-3834] Fixing performance hits in reading Column Stats Index (#5266)
add 976840e8eb [HUDI-3812] Fixing Data Skipping configuration to respect Metadata Table configs (#5244)
add 12731f5b89 [HUDI-3842] Integ tests for non partitioned datasets (#5276)
add 63a099c5b7 [HUDI-3847] Fix NPE due to null schema in HoodieMetadataTableValidator (#5284)
add 2245a9515f [HUDI-3798] Fixing ending of a transaction by different owner and removing some extraneous methods in trxn manager (#5255)
add 5c41e30ac5 [HUDI-3817] shade parquet dependency for hudi-hadoop-mr-bundle (#5250)
add 52ea1e4964 [MINOR] fixing timeline server for integ tests (#5289)
add 458fdd5611 [HUDI-3841] Fixing Column Stats in the presence of Schema Evolution (#5275)
add 3d8fc78c66 [HUDI-3844] Update props in indexer based on table config (#5293)
add f91e9e63e1 [HUDI-3799] Fixing not deleting empty instants w/o archiving (#5261)
add 101b82a679 [HUDI-3839] Fixing incorrect selection of MT partitions to be updated (#5274)
add d16740976e [HUDI-3838] Implemented drop partition column feature for delta streamer code path (#5294)
add 84783b9779 [HUDI-3843] Make flink profiles build with scala-2.11 (#5279)
add 25dce94ba2 [MINOR] Integ Test Reducing partitions for log running multi partition yaml (#5300)
add 2d46d5287e [HUDI-3838] Moved the getPartitionColumns logic to driver. (#5303)
add 2e6e302efe [HUDI-3859] Fix spark profiles and utilities-slim dep (#5297)
add 7b78dff45f [HUDI-3855] Fixing `FILENAME_METADATA_FIELD` not being correctly updated in `HoodieMergeHandle` (#5296)
add 434e782b7d [HUDI-3867] Disable Data Skipping by default (#5306)
add 43de2b4702 [HUDI-3868] Disable the sort input for flink streaming append mode (#5309)
add 0281725c6b [MINOR] Inline the partition path logic into the builder (#5310)
add 6f9b02decb [HUDI-3870] Add timeout rollback for flink online compaction (#5314)
add c7f41f9018 [HUDI-3869] Improve error handling of loading Hudi conf (#5311)
add bab691692e [HUDI-3686] Fix inline and async table service check in HoodieWriteConfig (#5307)
add 571cbe4c11 [MINOR] Code cleanup in test utils (#5312)
add a081c2b9b5 [HUDI-3876] Fixing fetching partitions in GlueSyncClient (#5318)
add 44b3630b5d [HUDI-3826] Make truncate partition use delete_partition operation (#5272)
add 6621f3cdbb [HUDI-3845] Fix delete mor table's partition with urlencode's error (#5282)
add f0ab4a6e9e [HUDI-3652] Make ObjectSizeCalculator threadlocal to reduce memory footprint (#5060)
add d6a64f765e Revert "[HUDI-3652] Make ObjectSizeCalculator threadlocal to reduce memory footprint (#5060)" (#5323)
add 9e8664f4d2 [HOTFIX] add missing license (#5322) (#5324)
add 57612c5c32 [HUDI-3848] Fixing restore with cleaned up commits (#5288)
add e8ab915aff [MINOR] Removing invalid code to close parquet reader iterator (#5182)
add 99dd1cb6e6 [HUDI-3835] Add UT for delete in java client (#5270)
add b8e465fdfc [MINOR] Fix typos in log4j-surefire.properties (#5212)
add 05dfc39c29 Fixing async clustering job test in TestHoodieDeltaStreamer (#5317)
add b00d03fd62 [HUDI-3886] Adding default null for some of the fields in col stats in MDT schema (#5329)
add 1718bcab84 [HUDI-3707] Fix target schema handling in HoodieSparkUtils while creating RDD (#5347)
add 7ecb47cd21 [HUDI-3895] Fixing file-partitioning seq for base-file only views to make sure we bucket the files efficiently (#5337)
add ef6c5611dc [HUDI-3894] Fix datahub to include HBase dependencies and shading (#5338)
add 52d878c52b [HUDI-3903] Fix NoClassDefFoundError with Kafka Connect bundle (#5353)
add 4f44e6aeb5 [HUDI-3899] Drop index to delete pending index instants from timeline if applicable (#5342)
add 9af7b09aec [HUDI-3894] Fix gcp bundle to include HBase dependencies and shading (#5349)
add 81bf771e56 [HUDI-3902] Fallback to `HadoopFsRelation` in cases non-involving Schema Evolution (#5352)
add 6f3fe880d2 [HUDI-3905] Add S3 related setup in Kafka Connect quick start (#5356)
add 28fdddfee0 [HUDI-3920] Fix partition path construction in metadata table validator (#5365)
add 7a9e411e9d [HUDI-3917] Flink write task hangs if last checkpoint has no data input (#5360)
add 6a3ce928b1 [HUDI-3904] Claim RFC number for Improve timeline server (#5354)
add 408663c42b [HUDI-3912] Fix lose data when rollback in flink async compact (#5357)
add f7544e23ac [HUDI-3204] Fixing partition-values being derived from partition-path instead of source columns (#5364)
add a9506aa545 [HUDI-3938] Fix default value for num retries to acquire lock (#5380)
add 4b296f79cc [HUDI-3935] Adding config to fallback to enabled Partition Values extraction from Partition path (#5377)
add 4e1ac467da [MINOR] Increase azure CI timeout to 120m (#5384)
add de5fa1fe03 [HUDI-3940] Fix retry count increment in lock manager (#5387)
add 037f89ee7c [HUDI-3921] Fixed schema evolution cannot work with HUDI-3855 (#5376)
add c4bc2deea0 [HUDI-3936] Fix projection for a nested field as pre-combined key (#5379)
add c05a4e7b6f [HUDI-3934] Fix `Spark32HoodieParquetFileFormat` not being compatible w/ Spark 3.2.0 (#5378)
add 20781a5fa6 [DOCS] Add commit activity, twitter badgers, and Hudi logo in README (#5336)
add 7523542c1d [HUDI-3947] Fixing Hive conf usage in HoodieSparkSqlWriter (#5401)
add 505ee672ac [HUDI-3950] add parquet-avro to gcp-bundle (#5399)
add 8633bd6e06 [HUDI-3948] Fix presto bundle missing HBase classes (#5398)
add 5e5c177e4b [HUDI-3923] Fix cast exception while reading boolean type of partitioned field (#5373)
add bda3db078e support generan parameter 'sink.parallelism' for flink-hudi (#5405)
add d994c58cc0 [HUDI-3946] Validate option path in flink hudi sink (#5397)
add 9054b85961 Revert "[HUDI-3951]support generan parameter 'sink.parallelism' for flink-hudi (#5405)" (#5421)
add f2ba0fead2 [HUDI-3085] Improve bulk insert partitioner abstraction (#4441)
add 762623a15c [HUDI-3972] Fixing hoodie.properties/tableConfig for no preCombine field with writes (#5424)
add 77e333298d [HUDI-3478] Claim RFC 51 For CDC (#5437)
add 6ec039ba42 [MINOR] Update alter rename command class type for pattern matching (#5381)
add e1ccf2e00b [HUDI-3977] Flink hudi table with date type partition path throws HoodieNotSupportedException (#5432)
add 924e2e96a6 Claim RFC 52 for Introduce Secondary Index to Improve HUDI Query Performance (#5441)
add cacbd98687 [HUDI-3945] After the async compaction operation is complete, the task should exit. (#5391)
add 52953c8f5e [HUDI-3815] Fix docs description of metadata.compaction.delta_commits default value error (#5368)
add 4e928a6fe1 [HUDI-3943] Some description fixes for 0.10.1 docs (#5447)
add b27e8b51d8 [MINOR] support different cleaning policy for flink (#5459)
add e421d536ea [HUDI-3758] Fix duplicate fileId error in MOR table type with flink bucket hash Index (#5185)
add a1d82b4dc5 [MINOR] Fix CI by ignoring SparkContext error (#5468)
add f492c52ee4 [HUDI-3862] Fix default configurations of HoodieHBaseIndexConfig (#5308)
add 33ff4752ba [HUDI-3978] Fix use of partition path field as hive partition field in flink (#5434)
add 6af1ff7a66 [MINOR] Update DOAP for release 0.11.0 (#5467)
add 9732ba12da [HUDI-3211][RFC-44] Add RFC for Hudi Connector for Presto (#4563)
add 3343cbb47b [MINOR] Update RFC status (#5486)
add 8c9209db28 [HUDI-4005] Update release scripts to help validation (#5479)
add 1562bb658f [HUDI-4031] Avoid clustering update handling when no pending replacecommit (#5487)
add f66e83dc65 [HUDI-3667] Run unit tests of hudi-integ-tests in CI (#5078)
add d794f4fbf9 [MINOR] Optimize code logic (#5499)
add abb4893b25 [HUDI-2875] Make HoodieParquetWriter Thread safe and memory executor exit gracefully (#4264)
add 248b0591b0 [HUDI-4042] Support truncate-partition for Spark-3.2 (#5506)
add c319ee9cea [HUDI-4017] Improve spark sql coverage in CI (#5512)
add 52fe1c9fae [HUDI-3675] Adding post write termination strategy to deltastreamer continuous mode (#5073)
add 9625d16937 [HUDI-3849] AvroDeserializer supports AVRO_REBASE_MODE_IN_READ configuration (#5287)
add 80f99893a0 [MINOR] Fixing class not found when using flink and enable metadata table (#5527)
add 569a76a9a5 [MINOR] fixing flaky tests in deltastreamer tests (#5521)
add 75eaa0bffe [HUDI-4055]refactor ratelimiter to avoid stack overflow (#5530)
add 4c70840275 [MINOR] Fixing close for HoodieCatalog's test (#5531)
add 6b47ef6ed2 [HUDI-4053] Flaky ITTestHoodieDataSource.testStreamWriteBatchReadOpti… (#5526)
add 6285a239a3 [HUDI-3995] Making perf optimizations for bulk insert row writer path (#5462)
add 6fd21d0f10 [HUDI-4044] When reading data from flink-hudi to external storage, the … (#5516)
add 4258a71517 [HUDI-4003] Try to read all the log file to parse schema (#5473)
add 4a8589f222 [HUDI-4038] Avoid calling `getDataSize` after every record written (#5497)
add 7f0c1f3ddf [HUDI-4079] Supports showing table comment for hudi with spark3 (#5546)
add b10ca7e69f [HUDI-4085] Fixing flakiness with parquet empty batch tests in TestHoodieDeltaStreamer (#5559)
add ecd47e7aae [HUDI-3963][Claim RFC number 53] Use Lock-Free Message Queue Improving Hoodie Writing Efficiency. (#5562)
add 0cec955fa2 [HUDI-4018][HUDI-4027] Adding integ test yamls for immutable use-cases. Added delete partition support to integ tests (#5501)
add 701f8c039d [HUDI-3336][HUDI-FLINK]Support custom hadoop config for flink (#5528)
add 8ad0bb9745 [MINOR] Fix a NPE for Option (#5461)
add 7fb436d3cf [HUDI-4078][HUDI-FLINK]BootstrapOperator contains the pending compact… (#5545)
add a704e3740c [HUDI-3336][HUDI-FLINK]Support custom hadoop config for flink (#5574)
add 5c4813f101 [HUDI-4072] Fix NULL schema for empty batches in deltastreamer (#5543)
add 52e63b39d6 [HUDI-4097] add table info to jobStatus (#5529)
add 6e16e719cd [HUDI-3980] Suport kerberos hbase index (#5464)
add 75f847691f [HUDI-4001] Filter the properties should not be used when create table for Spark SQL (#5495)
add 1fded18dff fix hive sync no partition table error (#5585)
add 61030d8e7a [HUDI-3123] consistent hashing index: basic write path (upsert/insert) (#4480)
add 43e08193ef [HUDI-4098] Metadata table heartbeat for instant has expired, last heartbeat 0 (#5583)
add a7a42e4490 [HUDI-4103] [HUDI-4001] Filter the properties should not be used when create table for Spark SQL
add ad773b3d96 [HUDI-3654] Preparations for hudi metastore. (#5572)
add fdd96cc97e [HUDI-4104] DeltaWriteProfile includes the pending compaction file slice when deciding small buckets (#5594)
add d52d13302d [HUDI-4101] BucketIndexPartitioner should take partition path for better dispersion (#5590)
add d422f69a0d [HUDI-4087] Support dropping RO and RT table in DropHoodieTableCommand (#5564)
add 99555c897a [HUDI-4110] Clean the marker files for flink compaction (#5604)
add f8b9399615 [MINOR] Fixing spark long running yaml for non-partitioned (#5607)
add ebbe56e862 [minor] Some code refactoring for LogFileComparator and Instant instantiation (#5600)
add f1f8a1abb7 [HUDI-4109] Copy the old record directly when it is chosen for merging (#5603)
add a1017c66aa Clean the marker files for flink compaction (#5611)
add 008616c4f6 [HUDI-3942] [RFC-50] Improve Timeline Server (#5392)
add 199f64255e [HUDI-4111] Bump ANTLR runtime version in Spark 3.x (#5606)
add 551aa959c5 Revert "[HUDI-3870] Add timeout rollback for flink online compaction (#5314)" (#5622)
add 6573469e73 [HUDI-4116] Unify clustering/compaction related procedures' output type (#5620)
add 6f37863ba8 [HUDI-4114] Remove the unnecessary fs view sync for BaseWriteClient#initTable (#5617)
add 1da0b21edd [HUDI-4119] the first read result is incorrect when Flink upsert- Kafka connector is used in HUDi (#5626)
add c7576f7613 [HUDI-4130] Remove the upgrade/downgrade for flink #initTable (#5642)
add 85b146d3d5 [HUDI-3985] Refactor DLASyncTool to support read hoodie table as spark datasource table (#5532)
add 7d02b1fd3c [MINOR] Minor fixes to exception log and removing unwanted metrics flush in integ test (#5646)
add 2af98303d3 [HUDI-4122] Fix NPE caused by adding kafka nodes (#5632)
add b5adba3e55 [MINOR] remove unused gson test dependency (#5652)
add 674e4638e8 Bump jackson-databind from 2.6.7.3 to 2.12.6.1
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (bd5c82e48b)
\
N -- N -- N refs/heads/dependabot/maven/com.fasterxml.jackson.core-jackson-databind-2.12.6.1 (674e4638e8)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/bot.yml | 45 +-
README.md | 57 +-
azure-pipelines.yml | 28 +-
doap_HUDI.rdf | 5 +
docker/README.md | 18 +-
.../docker-compose_hadoop284_hive233_spark244.yml | 33 +-
...e.yaml => deltastreamer-immutable-dataset.yaml} | 49 +-
...mer-long-running-multi-partitions-metadata.yaml | 12 +-
...eltastreamer-long-running-multi-partitions.yaml | 12 +-
.../deltastreamer-medium-clustering.yaml | 8 +-
...ltastreamer-medium-full-dataset-validation.yaml | 8 +-
...-ds.yaml => deltastreamer-non-partitioned.yaml} | 32 +-
.../deltastreamer-pure-bulk-inserts.yaml | 37 +-
...unning.yaml => deltastreamer-pure-inserts.yaml} | 37 +-
.../config/test-suite/insert-overwrite-table.yaml | 2 +-
.../demo/config/test-suite/insert-overwrite.yaml | 3 +-
.../demo/config/test-suite/multi-writer-1-ds.yaml | 2 +-
...i-writer-2-sds.yaml => multi-writer-1-sds.yaml} | 8 +-
.../demo/config/test-suite/multi-writer-2-sds.yaml | 4 +-
...w-spark-simple.yaml => multi-writer-3-sds.yaml} | 24 +-
...i-writer-2-sds.yaml => multi-writer-4-sds.yaml} | 10 +-
....properties => multi-writer-local-3.properties} | 23 +-
....properties => multi-writer-local-4.properties} | 23 +-
...ark-simple.yaml => spark-delete-partition.yaml} | 43 +-
...rk-simple.yaml => spark-immutable-dataset.yaml} | 49 +-
...aml => spark-long-running-non-partitioned.yaml} | 23 +-
.../test-suite/spark-non-core-operations.yaml | 204 ++++++
.../config/test-suite/spark-pure-bulk-inserts.yaml | 37 +-
...w-spark-simple.yaml => spark-pure-inserts.yaml} | 40 +-
...ssive-clean-archival-inline-compact.properties} | 11 +-
.../test-aggressive-clean-archival.properties | 9 +-
...ssive-clean-archival-inline-compact.properties} | 13 +-
...clustering-aggressive-clean-archival.properties | 9 +-
...s => test-clustering-inline-compact.properties} | 11 +-
...ssive-clean-archival-inline-compact.properties} | 10 +-
...g-metadata-aggressive-clean-archival.properties | 9 +-
.../config/test-suite/test-clustering.properties | 10 +-
...l.properties => test-inline-compact.properties} | 14 +-
...ssive-clean-archival-inline-compact.properties} | 10 +-
...t-metadata-aggressive-clean-archival.properties | 9 +-
...ies => test-metadata-inline-compact.properties} | 10 +-
.../config/test-suite/test-metadata.properties | 9 +-
... test-nonpartitioned-inline-compact.properties} | 16 +-
...partitioned-metadata-inline-compact.properties} | 16 +-
...ies => test-nonpartitioned-metadata.properties} | 15 +-
...g.properties => test-nonpartitioned.properties} | 15 +-
docker/demo/config/test-suite/test.properties | 9 +-
docker/hoodie/hadoop/base/pom.xml | 2 +-
docker/hoodie/hadoop/base_java11/pom.xml | 2 +-
docker/hoodie/hadoop/datanode/pom.xml | 2 +-
docker/hoodie/hadoop/historyserver/pom.xml | 2 +-
docker/hoodie/hadoop/hive_base/pom.xml | 2 +-
docker/hoodie/hadoop/namenode/pom.xml | 2 +-
docker/hoodie/hadoop/pom.xml | 2 +-
docker/hoodie/hadoop/prestobase/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/pom.xml | 2 +-
docker/hoodie/hadoop/sparkadhoc/pom.xml | 2 +-
docker/hoodie/hadoop/sparkmaster/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/pom.xml | 2 +-
docker/hoodie/hadoop/trinobase/pom.xml | 2 +-
docker/hoodie/hadoop/trinocoordinator/pom.xml | 2 +-
docker/hoodie/hadoop/trinoworker/pom.xml | 2 +-
docker/{ => images}/push_to_docker_hub.png | Bin
hudi-aws/pom.xml | 4 +-
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 22 +-
hudi-cli/pom.xml | 2 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 4 +-
.../apache/hudi/cli/commands/ExportCommand.java | 2 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 8 +-
.../scala/org/apache/hudi/cli/SparkHelpers.scala | 5 +-
.../hudi/cli/commands/TestRepairsCommand.java | 4 +-
.../hudi/cli/integ/ITTestClusteringCommand.java | 15 +-
.../hudi/cli/integ/ITTestCommitsCommand.java | 10 +-
.../hudi/cli/integ/ITTestCompactionCommand.java | 15 +-
hudi-client/hudi-client-common/pom.xml | 4 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 3 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 67 +-
.../apache/hudi/client/CompactionAdminClient.java | 6 +-
.../apache/hudi/client/HoodieTimelineArchiver.java | 35 +-
.../client/transaction/TransactionManager.java | 27 +-
.../hudi/client/transaction/lock/LockManager.java | 3 +-
.../hudi/client/utils/LazyIterableIterator.java | 4 +-
.../hudi/client/utils/MetadataConversionUtils.java | 40 ++
.../apache/hudi/config/HoodieClusteringConfig.java | 2 +-
.../apache/hudi/config/HoodieHBaseIndexConfig.java | 52 ++
.../org/apache/hudi/config/HoodieIndexConfig.java | 59 +-
.../org/apache/hudi/config/HoodieLockConfig.java | 2 +-
.../org/apache/hudi/config/HoodieMemoryConfig.java | 6 +-
.../apache/hudi/config/HoodieStorageConfig.java | 16 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 55 +-
.../java/org/apache/hudi/index/HoodieIndex.java | 6 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 2 +-
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 7 +-
.../apache/hudi/index/bucket/BucketIdentifier.java | 49 +-
.../index/bucket/BucketIndexLocationMapper.java | 35 +
.../index/bucket/ConsistentBucketIdentifier.java | 104 +++
.../hudi/index/bucket/HoodieBucketIndex.java | 119 ++--
.../hudi/index/bucket/HoodieSimpleBucketIndex.java | 99 +++
.../org/apache/hudi/io/HoodieAppendHandle.java | 14 +-
.../org/apache/hudi/io/HoodieConcatHandle.java | 7 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 5 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 7 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 40 +-
.../apache/hudi/io/HoodieSortedMergeHandle.java | 9 +-
.../hudi/io/HoodieUnboundedCreateHandle.java | 3 +
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 12 +-
.../org/apache/hudi/io/WriteHandleFactory.java | 3 +-
.../hudi/io/storage/HoodieAvroParquetWriter.java | 86 +++
.../hudi/io/storage/HoodieBaseParquetWriter.java | 87 +++
.../apache/hudi/io/storage/HoodieFileWriter.java | 13 +-
.../hudi/io/storage/HoodieFileWriterFactory.java | 2 +-
.../apache/hudi/io/storage/HoodieHFileWriter.java | 31 +-
.../apache/hudi/io/storage/HoodieOrcWriter.java | 41 +-
.../hudi/io/storage/HoodieParquetWriter.java | 110 ----
.../keygen/TimestampBasedAvroKeyGenerator.java | 4 +
.../metadata/HoodieBackedTableMetadataWriter.java | 198 ++++--
.../apache/hudi/table/BulkInsertPartitioner.java | 28 +-
.../java/org/apache/hudi/table/HoodieTable.java | 94 ++-
.../table/action/clean/CleanActionExecutor.java | 2 +-
.../action/clean/CleanPlanActionExecutor.java | 4 +-
.../table/action/commit/BaseBulkInsertHelper.java | 2 +-
.../action/commit/BaseCommitActionExecutor.java | 2 +-
.../hudi/table/action/commit/BaseWriteHelper.java | 2 +-
.../table/action/commit/HoodieMergeHelper.java | 19 +-
.../hudi/table/action/compact/CompactHelpers.java | 3 +-
.../hudi/table/action/compact/HoodieCompactor.java | 4 +-
.../compact/RunCompactionActionExecutor.java | 2 +-
.../compact/ScheduleCompactionActionExecutor.java | 2 +-
.../table/action/index/RunIndexActionExecutor.java | 4 +-
.../action/index/ScheduleIndexActionExecutor.java | 2 +-
.../table/action/rollback/BaseRollbackHelper.java | 4 +-
.../rollback/ListingBasedRollbackStrategy.java | 12 +-
.../action/savepoint/SavepointActionExecutor.java | 2 +-
.../org/apache/hudi/table/marker/WriteMarkers.java | 2 +-
...yout.java => HoodieConsistentBucketLayout.java} | 44 +-
.../hudi/table/storage/HoodieDefaultLayout.java | 7 +-
.../hudi/table/storage/HoodieLayoutFactory.java | 9 +-
...etLayout.java => HoodieSimpleBucketLayout.java} | 32 +-
.../hudi/table/storage/HoodieStorageLayout.java | 2 +-
.../client/transaction/TestTransactionManager.java | 74 ++-
.../apache/hudi/config/TestHoodieWriteConfig.java | 74 ++-
.../hudi/index/bucket/TestBucketIdentifier.java | 122 ++++
.../bucket/TestConsistentBucketIdIdentifier.java | 79 +++
.../io/storage/TestHoodieHFileReaderWriter.java | 123 +++-
.../io/storage/TestHoodieReaderWriterBase.java | 3 +-
.../hudi/testutils/HoodieWriteableTestTable.java | 9 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-client/hudi-flink-client/pom.xml | 15 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 19 +-
.../hudi/execution/FlinkLazyInsertIterable.java | 1 +
.../row/parquet/ParquetSchemaConverter.java | 19 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 5 +
.../org/apache/hudi/table/HoodieFlinkTable.java | 3 +
.../hudi/table/action/commit/FlinkMergeHelper.java | 5 +-
.../src/main/resources/log4j-surefire.properties | 4 +-
.../row/parquet/TestParquetSchemaConverter.java | 25 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-client/hudi-java-client/pom.xml | 4 +-
.../run/strategy/JavaExecutionStrategy.java | 11 +-
.../hudi/execution/JavaLazyInsertIterable.java | 1 +
.../table/action/commit/JavaBulkInsertHelper.java | 17 +-
.../hudi/table/action/commit/JavaMergeHelper.java | 5 +-
.../table/action/commit/JavaUpsertPartitioner.java | 2 +-
.../commit/TestJavaCopyOnWriteActionExecutor.java | 91 ++-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-client/hudi-spark-client/pom.xml | 4 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 12 +-
.../MultipleSparkJobExecutionStrategy.java | 5 +-
.../SparkSingleFileSortExecutionStrategy.java | 1 +
.../java/org/apache/hudi/data/HoodieJavaRDD.java | 5 +
.../hudi/execution/SparkLazyInsertIterable.java | 1 +
.../bulkinsert/BulkInsertMapFunction.java | 11 +-
.../bulkinsert/RDDSpatialCurveSortPartitioner.java | 11 +-
.../apache/hudi/index/SparkHoodieIndexFactory.java | 16 +-
.../index/bloom/SparkHoodieBloomIndexHelper.java | 13 +-
.../bucket/HoodieSparkConsistentBucketIndex.java | 210 ++++++
.../hudi/index/hbase/SparkHoodieHBaseIndex.java | 27 +-
.../row/HoodieInternalRowParquetWriter.java | 33 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 14 +-
.../apache/hudi/keygen/BuiltinKeyGenerator.java | 103 +--
.../apache/hudi/keygen/ComplexKeyGenerator.java | 8 +-
.../hudi/keygen/GlobalDeleteKeyGenerator.java | 4 +-
.../hudi/keygen/NonpartitionedKeyGenerator.java | 6 +
.../apache/hudi/keygen/RowKeyGeneratorHelper.java | 57 +-
.../org/apache/hudi/keygen/SimpleKeyGenerator.java | 8 +-
.../hudi/keygen/TimestampBasedKeyGenerator.java | 16 +-
.../org/apache/hudi/table/HoodieSparkTable.java | 6 +-
.../bootstrap/OrcBootstrapMetadataHandler.java | 3 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 8 +-
.../SparkBootstrapCommitActionExecutor.java | 2 +-
.../commit/BaseSparkCommitActionExecutor.java | 16 +-
.../table/action/commit/SparkBulkInsertHelper.java | 24 +-
.../table/action/commit/UpsertPartitioner.java | 2 +-
.../org/apache/hudi/HoodieConversionUtils.scala | 8 +
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 27 +-
.../spark/sql/HoodieCatalystExpressionUtils.scala | 20 +-
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 4 +-
.../org/apache/hudi/client/TestClientRollback.java | 106 +++
.../functional/TestConsistentBucketIndex.java | 250 ++++++++
.../functional/TestHoodieBackedMetadata.java | 323 ++++++++--
.../functional/TestHoodieBackedTableMetadata.java | 12 +-
.../hudi/client/functional/TestHoodieIndex.java | 3 +
.../client/functional/TestHoodieMetadataBase.java | 4 +
.../TestBoundedInMemoryExecutorInSpark.java | 45 ++
.../apache/hudi/index/TestHoodieIndexConfigs.java | 14 +-
...Index.java => TestHoodieSimpleBucketIndex.java} | 17 +-
.../org/apache/hudi/io/TestHoodieMergeHandle.java | 26 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 20 +-
.../io/storage/TestHoodieFileWriterFactory.java | 2 +-
.../java/org/apache/hudi/table/TestCleaner.java | 2 +-
.../commit/TestCopyOnWriteActionExecutor.java | 10 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 88 +++
.../hudi/testutils/HoodieClientTestHarness.java | 10 +-
.../hudi/testutils/HoodieClientTestUtils.java | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../hudi/keygen/TestRowGeneratorHelper.scala | 24 +-
hudi-client/pom.xml | 2 +-
hudi-common/pom.xml | 2 +-
hudi-common/src/main/avro/HoodieMetadata.avsc | 15 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 2 +-
.../java/org/apache/hudi/avro/AvroSchemaUtils.java | 112 ++++
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 201 +++---
.../common/config/DFSPropertiesConfiguration.java | 12 +-
.../apache/hudi/common/config/HoodieConfig.java | 2 +
.../hudi/common/config/HoodieMetadataConfig.java | 37 +-
.../hudi/common/config/HoodieMetastoreConfig.java | 93 +++
.../apache/hudi/common/config/TypedProperties.java | 4 +-
.../org/apache/hudi/common/data/HoodieData.java | 9 +
.../org/apache/hudi/common/data/HoodieList.java | 5 +
.../java/org/apache/hudi/common/fs/FSUtils.java | 30 +-
.../hudi/common/fs/HoodieWrapperFileSystem.java | 4 +-
.../org/apache/hudi/common/model/BaseFile.java | 33 +-
.../hudi/common/model/ConsistentHashingNode.java | 78 +++
.../hudi/common/model/HoodieCommitMetadata.java | 23 +-
.../model/HoodieConsistentHashingMetadata.java | 142 ++++
.../apache/hudi/common/model/HoodieLogFile.java | 7 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 5 +-
.../org/apache/hudi/common/model/HoodieRecord.java | 6 +-
.../common/model/HoodieReplaceCommitMetadata.java | 17 +-
.../common/model/HoodieRollingStatMetadata.java | 4 +-
.../hudi/common/table/HoodieTableConfig.java | 28 +-
.../hudi/common/table/HoodieTableMetaClient.java | 176 +++--
.../hudi/common/table/TableSchemaResolver.java | 121 +++-
.../table/log/AbstractHoodieLogRecordReader.java | 69 +-
.../common/table/log/HoodieLogFormatReader.java | 12 +-
.../table/log/HoodieMergedLogRecordScanner.java | 28 +-
.../table/log/block/HoodieAvroDataBlock.java | 2 +-
.../common/table/log/block/HoodieDataBlock.java | 28 +-
.../table/log/block/HoodieHFileDataBlock.java | 23 +-
.../table/timeline/HoodieActiveTimeline.java | 10 +-
.../table/timeline/HoodieArchivedTimeline.java | 2 +-
.../table/view/AbstractTableFileSystemView.java | 6 +-
.../common/table/view/FileSystemViewManager.java | 14 +
.../table/view/FileSystemViewStorageConfig.java | 2 +-
.../common/table/view/TableFileSystemView.java | 2 +-
.../apache/hudi/common/util/CollectionUtils.java | 13 +
.../org/apache/hudi/common/util/Functions.java | 8 +-
.../org/apache/hudi/common/util/JsonUtils.java | 38 ++
.../org/apache/hudi/common/util/NetworkUtils.java | 60 +-
.../org/apache/hudi/common/util/RateLimiter.java | 17 +-
.../org/apache/hudi/common/util/StringUtils.java | 33 +-
.../org/apache/hudi/common/util/hash/HashID.java | 9 +
.../common/util/queue/BoundedInMemoryExecutor.java | 24 +-
.../HoodieIncompatibleSchemaException.java | 2 +-
.../{FileNameCachingPath.java => CachingPath.java} | 44 +-
.../org/apache/hudi/hadoop/SerializablePath.java | 69 ++
.../schema/action/InternalSchemaMerger.java | 26 +-
.../internal/schema/utils/InternalSchemaUtils.java | 16 +
.../apache/hudi/io/storage/HoodieFileReader.java | 42 +-
.../apache/hudi/io/storage/HoodieHFileReader.java | 712 ++++++++++++---------
.../apache/hudi/io/storage/HoodieOrcReader.java | 11 +-
.../hudi/io/storage/HoodieParquetReader.java | 19 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 2 +-
.../metadata/FileSystemBackedTableMetadata.java | 7 +
.../hudi/metadata/HoodieBackedTableMetadata.java | 329 +++++++---
.../HoodieMetadataMergedLogRecordReader.java | 99 ++-
.../hudi/metadata/HoodieMetadataPayload.java | 60 +-
.../apache/hudi/metadata/HoodieTableMetadata.java | 13 +
.../hudi/metadata/HoodieTableMetadataUtil.java | 154 +++--
.../metadata/MetadataRecordsGenerationParams.java | 31 +-
.../src/main/java/org/apache/hudi/util/Lazy.java | 77 +++
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 37 ++
.../common/functional/TestHoodieLogFormat.java | 2 +-
.../TestHoodieConsistentHashingMetadata.java} | 18 +-
.../hudi/common/table/TestHoodieTableConfig.java | 10 +-
.../hudi/common/table/TestTableSchemaResolver.java | 71 ++
.../hudi/common/testutils/FileCreateUtils.java | 24 +-
.../common/testutils/HoodieCommonTestHarness.java | 4 +
.../common/testutils/HoodieTestDataGenerator.java | 10 +-
.../hudi/common/testutils/HoodieTestTable.java | 14 +-
.../apache/hudi/common/util/TestStringUtils.java | 21 +-
.../schema/utils/TestAvroSchemaEvolutionUtils.java | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-examples/hudi-examples-common/pom.xml | 2 +-
hudi-examples/hudi-examples-dbt/.gitignore | 8 +
hudi-examples/hudi-examples-dbt/README.md | 134 ++++
hudi-examples/hudi-examples-dbt/dbt_project.yml | 49 ++
.../models/example/hudi_insert_overwrite_table.sql | 37 ++
.../models/example/hudi_insert_table.sql | 42 ++
.../example/hudi_upsert_partitioned_cow_table.sql | 40 ++
.../example/hudi_upsert_partitioned_mor_table.sql | 40 ++
.../models/example/hudi_upsert_table.sql | 38 ++
.../hudi-examples-dbt/models/example/schema.yml | 104 +++
hudi-examples/hudi-examples-flink/pom.xml | 2 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-examples/hudi-examples-java/pom.xml | 2 +-
hudi-examples/hudi-examples-spark/pom.xml | 2 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-examples/pom.xml | 2 +-
hudi-flink-datasource/hudi-flink/pom.xml | 6 +-
.../apache/hudi/configuration/FlinkOptions.java | 45 +-
.../hudi/configuration/HadoopConfigurations.java | 54 ++
.../hudi/schema/FilebasedSchemaProvider.java | 6 +-
.../hudi/sink/StreamWriteOperatorCoordinator.java | 33 +-
.../hudi/sink/append/AppendWriteFunction.java | 2 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 5 +-
.../sink/bucket/BucketStreamWriteFunction.java | 12 +-
.../hudi/sink/bulk/BulkInsertWriteFunction.java | 2 +-
.../org/apache/hudi/sink/bulk/RowDataKeyGen.java | 5 +
.../sink/common/AbstractStreamWriteFunction.java | 9 +-
.../hudi/sink/compact/CompactionCommitSink.java | 17 +-
.../hudi/sink/compact/CompactionPlanOperator.java | 4 +
.../org/apache/hudi/sink/meta/CkpMetadata.java | 12 +-
.../sink/partitioner/BucketAssignFunction.java | 3 +-
.../sink/partitioner/BucketIndexPartitioner.java | 8 +-
.../partitioner/profile/DeltaWriteProfile.java | 2 +-
.../apache/hudi/sink/utils/HiveSyncContext.java | 10 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 21 +-
.../java/org/apache/hudi/source/FileIndex.java | 3 +-
.../apache/hudi/source/IncrementalInputSplits.java | 2 +-
.../hudi/source/StreamReadMonitoringFunction.java | 3 +-
.../apache/hudi/streamer/FlinkStreamerConfig.java | 12 +
.../org/apache/hudi/table/HoodieTableFactory.java | 25 +-
.../org/apache/hudi/table/HoodieTableSink.java | 2 +-
.../org/apache/hudi/table/HoodieTableSource.java | 25 +-
.../apache/hudi/table/catalog/HoodieCatalog.java | 3 +-
.../apache/hudi/table/format/FilePathUtils.java | 13 +
.../org/apache/hudi/table/format/FormatUtils.java | 12 -
.../table/format/mor/MergeOnReadInputFormat.java | 6 +-
.../table/format/mor/MergeOnReadInputSplit.java | 13 +-
.../org/apache/hudi/util/AvroSchemaConverter.java | 15 +-
.../java/org/apache/hudi/util/FlinkTables.java | 4 +-
.../apache/hudi/util/RowDataToAvroConverters.java | 7 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 28 +-
.../apache/hudi/util/ViewStorageProperties.java | 11 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 50 +-
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 6 +-
.../org/apache/hudi/sink/TestWriteMergeOnRead.java | 23 +
.../apache/hudi/sink/bulk/TestRowDataKeyGen.java | 38 ++
.../org/apache/hudi/sink/meta/TestCkpMetadata.java | 3 +-
.../hudi/sink/partitioner/TestBucketAssigner.java | 3 +-
.../hudi/sink/utils/TestHiveSyncContext.java | 62 ++
.../org/apache/hudi/sink/utils/TestWriteBase.java | 3 +-
.../apache/hudi/source/TestStreamReadOperator.java | 3 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 41 +-
.../apache/hudi/table/TestHoodieTableFactory.java | 1 -
.../hudi/table/catalog/TestHoodieCatalog.java | 8 +
.../apache/hudi/table/format/TestInputFormat.java | 3 +-
.../org/apache/hudi/utils/TestConfigurations.java | 19 +
.../test/java/org/apache/hudi/utils/TestData.java | 22 +-
.../test/java/org/apache/hudi/utils/TestSQL.java | 10 +
.../org/apache/hudi/utils/TestStreamerUtil.java | 2 +-
.../test/java/org/apache/hudi/utils/TestUtils.java | 11 +-
.../hudi/utils/TestViewStorageProperties.java | 10 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-flink-datasource/hudi-flink1.13.x/pom.xml | 4 +-
hudi-flink-datasource/hudi-flink1.14.x/pom.xml | 4 +-
hudi-flink-datasource/pom.xml | 4 +-
hudi-gcp/pom.xml | 2 +-
hudi-hadoop-mr/pom.xml | 2 +-
.../apache/hudi/hadoop/utils/HoodieHiveUtils.java | 1 -
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 18 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 29 +-
.../hudi/hadoop/TestHoodieHFileInputFormat.java | 15 +
.../hudi/hadoop/TestHoodieParquetInputFormat.java | 15 +
.../apache/hudi/hadoop/TestInputPathHandler.java | 18 +
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-integ-test/README.md | 72 +++
hudi-integ-test/pom.xml | 2 +-
.../testsuite/HoodieMultiWriterTestSuiteJob.java | 13 +-
.../SparkDataSourceContinuousIngestTool.java | 167 +++++
.../integ/testsuite/configuration/DeltaConfig.java | 16 +-
.../apache/hudi/integ/testsuite/dag/DagUtils.java | 40 +-
.../dag/nodes/BaseValidateDatasetNode.java | 11 +-
.../testsuite/dag/nodes/ValidateDatasetNode.java | 5 +-
.../testsuite/dag/scheduler/DagScheduler.java | 3 -
.../integ/testsuite/generator/DeltaGenerator.java | 2 +-
.../GenericRecordFullPayloadGenerator.java | 2 +-
.../SparkDataSourceContinuousIngest.scala | 97 +++
.../testsuite/dag/nodes/SparkBulkInsertNode.scala | 39 +-
.../dag/nodes/SparkDeletePartitionNode.scala | 70 ++
.../TestDFSHoodieTestSuiteWriterAdapter.java | 27 +-
.../testsuite/converter/TestDeleteConverter.java | 3 +-
.../hudi/integ/testsuite/dag/TestDagUtils.java | 13 +-
.../TestGenericRecordPayloadEstimator.java | 11 +-
.../testsuite/job/TestHoodieTestSuiteJob.java | 3 +
.../reader/TestDFSHoodieDatasetInputReader.java | 19 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-kafka-connect/README.md | 21 +-
hudi-kafka-connect/pom.xml | 4 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 4 +-
.../apache/hudi/HoodieDatasetBulkInsertHelper.java | 71 +-
.../BulkInsertDataInternalWriterHelper.java | 5 +-
.../internal/DataSourceInternalWriterHelper.java | 2 +-
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../org/apache/hudi/BaseFileOnlyRelation.scala | 106 ++-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 212 ++++--
.../scala/org/apache/hudi/DataSourceOptions.scala | 45 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 36 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 267 ++++++--
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 15 +-
.../org/apache/hudi/HoodieDataSourceHelper.scala | 21 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 57 +-
.../org/apache/hudi/HoodieMergeOnReadRDD.scala | 34 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 42 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 33 +-
.../org/apache/hudi/IncrementalRelation.scala | 5 +-
.../hudi/MergeOnReadIncrementalRelation.scala | 14 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 17 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 5 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 3 +
...eFormat.scala => HoodieParquetFileFormat.scala} | 34 +-
.../apache/spark/sql/hudi/DataSkippingUtils.scala | 30 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 7 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 4 +-
.../AlterHoodieTableDropPartitionCommand.scala | 27 +-
.../command/AlterHoodieTableRenameCommand.scala | 6 +-
.../hudi/command/CreateHoodieTableCommand.scala | 6 +-
.../sql/hudi/command/DropHoodieTableCommand.scala | 79 +--
.../hudi/command/TruncateHoodieTableCommand.scala | 125 ++--
.../sql/hudi/streaming/HoodieStreamSource.scala | 1 +
hudi-spark-datasource/hudi-spark/pom.xml | 4 +-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 4 +-
.../hudi/command/CompactionHoodiePathCommand.scala | 11 +-
.../command/CompactionHoodieTableCommand.scala | 13 +-
.../command/CompactionShowHoodiePathCommand.scala | 12 +-
.../command/CompactionShowHoodieTableCommand.scala | 12 +-
.../command/CreateHoodieTableAsSelectCommand.scala | 25 +-
.../procedures/RunClusteringProcedure.scala | 34 +-
.../procedures/RunCompactionProcedure.scala | 29 +-
.../procedures/ShowClusteringProcedure.scala | 37 +-
.../procedures/ShowCompactionProcedure.scala | 16 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 2 +-
.../TestHoodieDatasetBulkInsertHelper.java | 53 +-
.../hudi/index/bucket/TestBucketIdentifier.java | 67 --
.../hudi/keygen/TestComplexKeyGenerator.java | 2 +-
.../keygen/TestGlobalDeleteRecordGenerator.java | 4 +-
.../keygen/TestNonpartitionedKeyGenerator.java | 2 +-
.../apache/hudi/keygen/TestSimpleKeyGenerator.java | 4 +-
...-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json | 0
...-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json | 0
...-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json | 0
...-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json | 0
.../index/colstats/column-stats-index-table.json | 4 +
...-4468afca-8a37-4ae8-a150-0c2fd3361080-c000.json | 0
...-4468afca-8a37-4ae8-a150-0c2fd3361080-c000.json | 0
...-4468afca-8a37-4ae8-a150-0c2fd3361080-c000.json | 0
...-4468afca-8a37-4ae8-a150-0c2fd3361080-c000.json | 0
...-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json | 10 +
...-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json | 10 +
...-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json | 10 +
...-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json | 10 +
.../colstats/partial-column-stats-index-table.json | 4 +
.../colstats/updated-column-stats-index-table.json | 8 +
.../updated-partial-column-stats-index-table.json | 8 +
.../index/zorder/column-stats-index-table.json | 4 -
.../zorder/updated-column-stats-index-table.json | 8 -
.../src/test/resources/log4j-surefire.properties | 4 +-
.../scala/org/apache/hudi/SparkDatasetMixin.scala | 43 ++
.../org/apache/hudi/TestDataSkippingUtils.scala | 314 ++++-----
.../org/apache/hudi/TestHoodieFileIndex.scala | 79 ++-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 145 ++++-
.../apache/hudi/functional/TestCOWDataSource.scala | 113 +++-
.../hudi/functional/TestCOWDataSourceStorage.scala | 2 +-
.../hudi/functional/TestColumnStatsIndex.scala | 241 +++++--
.../hudi/functional/TestLayoutOptimization.scala | 10 +-
.../apache/hudi/functional/TestMORDataSource.scala | 42 +-
.../hudi/functional/TestMORDataSourceStorage.scala | 28 +-
.../TestMetadataTableWithSparkDataSource.scala | 17 +-
.../functional/TestParquetColumnProjection.scala | 13 +-
.../apache/hudi/functional/TestSqlStatement.scala | 4 +-
.../benchmark/SpaceCurveOptimizeBenchmark.scala | 4 +-
...eSqlBase.scala => HoodieSparkSqlTestBase.scala} | 2 +-
.../org/apache/spark/sql/hudi/TestAlterTable.scala | 2 +-
.../sql/hudi/TestAlterTableDropPartition.scala | 41 +-
.../spark/sql/hudi/TestCompactionTable.scala | 2 +-
.../apache/spark/sql/hudi/TestCreateTable.scala | 247 +++++--
.../apache/spark/sql/hudi/TestDeleteTable.scala | 96 ++-
.../org/apache/spark/sql/hudi/TestDropTable.scala | 168 ++++-
.../spark/sql/hudi/TestHoodieOptionConfig.scala | 20 +-
.../apache/spark/sql/hudi/TestInsertTable.scala | 35 +-
.../spark/sql/hudi/TestMergeIntoLogOnlyTable.scala | 2 +-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 2 +-
.../spark/sql/hudi/TestMergeIntoTable2.scala | 2 +-
.../sql/hudi/TestPartialUpdateForMergeInto.scala | 2 +-
.../apache/spark/sql/hudi/TestShowPartitions.scala | 2 +-
.../org/apache/spark/sql/hudi/TestSpark3DDL.scala | 33 +-
.../org/apache/spark/sql/hudi/TestSqlConf.scala | 2 +-
.../spark/sql/hudi/TestTimeTravelTable.scala | 2 +-
.../apache/spark/sql/hudi/TestTruncateTable.scala | 2 +-
.../apache/spark/sql/hudi/TestUpdateTable.scala | 2 +-
.../sql/hudi/procedure/TestCallCommandParser.scala | 4 +-
.../sql/hudi/procedure/TestCallProcedure.scala | 4 +-
.../hudi/procedure/TestClusteringProcedure.scala | 107 +++-
.../hudi/procedure/TestCompactionProcedure.scala | 82 ++-
.../hudi/procedure/TestSavepointsProcedure.scala | 4 +-
hudi-spark-datasource/hudi-spark2-common/pom.xml | 20 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 4 +-
.../apache/spark/sql/adapter/Spark2Adapter.scala | 6 +-
.../parquet/Spark24HoodieParquetFileFormat.scala | 229 +++++++
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-spark-datasource/hudi-spark3-common/pom.xml | 2 +-
.../datasources/Spark3ParsePartitionUtil.scala | 3 +-
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 4 +-
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 22 +-
.../parquet/Spark312HoodieParquetFileFormat.scala | 365 -----------
.../parquet/Spark31HoodieParquetFileFormat.scala | 411 ++++++++++++
hudi-spark-datasource/hudi-spark3/pom.xml | 4 +-
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 13 +-
.../sql/avro/HoodieSpark3_2AvroDeserializer.scala | 5 +-
.../parquet/Spark32DataSourceUtils.scala | 77 +++
.../parquet/Spark32HoodieParquetFileFormat.scala | 524 +++++++++------
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 23 +-
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 37 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-spark-datasource/pom.xml | 2 +-
hudi-sync/{hudi-dla-sync => hudi-adb-sync}/pom.xml | 4 +-
.../src/assembly/src.xml | 0
.../hudi/sync/adb/AbstractAdbSyncHoodieClient.java | 128 ++++
.../org/apache/hudi/sync/adb/AdbSyncConfig.java | 240 +++++++
.../java/org/apache/hudi/sync/adb/AdbSyncTool.java | 283 ++++++++
.../apache/hudi/sync/adb/HoodieAdbJdbcClient.java | 440 +++++++++++++
.../hudi/sync/adb/HoodieAdbSyncException.java | 16 +-
.../apache/hudi/sync/adb/TestAdbSyncConfig.java | 65 ++
.../test/resources/log4j-surefire-quiet.properties | 0
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-sync/hudi-datahub-sync/pom.xml | 2 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
.../java/org/apache/hudi/dla/DLASyncConfig.java | 111 ----
.../main/java/org/apache/hudi/dla/DLASyncTool.java | 213 ------
.../java/org/apache/hudi/dla/HoodieDLAClient.java | 428 -------------
.../main/java/org/apache/hudi/dla/util/Utils.java | 77 ---
.../org/apache/hudi/dla/TestDLASyncConfig.java | 55 --
hudi-sync/hudi-hive-sync/pom.xml | 2 +-
.../java/org/apache/hudi/hive/HiveSyncTool.java | 84 +--
.../org/apache/hudi/hive/TestHiveSyncTool.java | 2 +-
.../hudi/hive/TestParquet2SparkSchemaUtils.java | 2 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-sync/hudi-sync-common/pom.xml | 2 +-
.../apache/hudi/sync/common/AbstractSyncTool.java | 82 +++
.../apache/hudi/sync/common}/util/ConfigUtils.java | 2 +-
.../common}/util/Parquet2SparkSchemaUtils.java | 2 +-
.../hudi/sync/common/util/SyncUtilHelpers.java | 19 +
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-sync/pom.xml | 4 +-
hudi-timeline-service/pom.xml | 2 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
hudi-utilities/pom.xml | 2 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 2 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 4 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 20 +-
.../utilities/HoodieMetadataTableValidator.java | 172 ++++-
.../hudi/utilities/HoodieSnapshotCopier.java | 2 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 2 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 1 +
.../hudi/utilities/deltastreamer/DeltaSync.java | 55 +-
.../deltastreamer/HoodieDeltaStreamer.java | 20 +-
.../NoNewDataTerminationStrategy.java | 56 ++
.../PostWriteTerminationStrategy.java | 39 ++
.../deltastreamer/TerminationStrategyUtils.java | 45 ++
.../apache/hudi/utilities/sources/InputBatch.java | 3 +-
.../utilities/sources/helpers/KafkaOffsetGen.java | 47 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 239 ++++++-
.../functional/HoodieDeltaStreamerTestBase.java | 2 +
.../functional/TestHoodieDeltaStreamer.java | 128 +++-
.../hudi/utilities/sources/TestDataSource.java | 9 +-
.../sources/TestParquetDFSSourceEmptyBatch.java | 14 +-
.../indexer-only-bloom.properties | 25 +
.../schema_registry.source_schema_tab.sql | 18 +-
.../schema_registry.target_schema_tab.sql | 18 +-
.../src/test/resources/log4j-surefire.properties | 4 +-
packaging/hudi-datahub-sync-bundle/pom.xml | 120 +++-
packaging/hudi-flink-bundle/pom.xml | 9 +-
packaging/hudi-gcp-bundle/pom.xml | 120 +++-
packaging/hudi-hadoop-mr-bundle/pom.xml | 20 +-
packaging/hudi-hive-sync-bundle/pom.xml | 119 +++-
packaging/hudi-integ-test-bundle/pom.xml | 2 +-
packaging/hudi-kafka-connect-bundle/pom.xml | 4 +-
packaging/hudi-presto-bundle/pom.xml | 9 +-
packaging/hudi-spark-bundle/pom.xml | 2 +-
packaging/hudi-timeline-server-bundle/pom.xml | 2 +-
packaging/hudi-trino-bundle/pom.xml | 3 +-
packaging/hudi-utilities-bundle/pom.xml | 2 +-
packaging/hudi-utilities-slim-bundle/pom.xml | 10 +-
pom.xml | 35 +-
rfc/README.md | 11 +-
rfc/rfc-34/rfc-34.md | 16 +
rfc/rfc-44/presto-connector.png | Bin 0 -> 182080 bytes
rfc/rfc-44/rfc-44.md | 158 +++++
rfc/rfc-50/ComparisonDiagram.png | Bin 0 -> 200212 bytes
rfc/rfc-50/CurrentDesign.png | Bin 0 -> 120217 bytes
rfc/rfc-50/Design.png | Bin 0 -> 114358 bytes
rfc/rfc-50/SchematicDiagram.png | Bin 0 -> 133584 bytes
rfc/rfc-50/rfc-50.md | 93 +++
scripts/release/create_source_release.sh | 1 +
scripts/release/deploy_staging_jars.sh | 72 ++-
scripts/release/validate_staged_bundles.sh | 66 ++
scripts/release/validate_staged_release.sh | 27 +-
609 files changed, 14756 insertions(+), 5632 deletions(-)
copy docker/demo/config/test-suite/{cow-spark-simple.yaml => deltastreamer-immutable-dataset.yaml} (66%)
copy docker/demo/config/test-suite/{multi-writer-1-ds.yaml => deltastreamer-non-partitioned.yaml} (80%)
copy hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire.properties => docker/demo/config/test-suite/deltastreamer-pure-bulk-inserts.yaml (58%)
copy docker/demo/config/test-suite/{cow-spark-long-running.yaml => deltastreamer-pure-inserts.yaml} (60%)
copy docker/demo/config/test-suite/{multi-writer-2-sds.yaml => multi-writer-1-sds.yaml} (93%)
copy docker/demo/config/test-suite/{cow-spark-simple.yaml => multi-writer-3-sds.yaml} (80%)
copy docker/demo/config/test-suite/{multi-writer-2-sds.yaml => multi-writer-4-sds.yaml} (91%)
copy docker/demo/config/test-suite/{test-metadata-aggressive-clean-archival.properties => multi-writer-local-3.properties} (77%)
copy docker/demo/config/test-suite/{test-metadata-aggressive-clean-archival.properties => multi-writer-local-4.properties} (77%)
copy docker/demo/config/test-suite/{cow-spark-simple.yaml => spark-delete-partition.yaml} (66%)
copy docker/demo/config/test-suite/{cow-spark-simple.yaml => spark-immutable-dataset.yaml} (67%)
rename docker/demo/config/test-suite/{cow-spark-long-running.yaml => spark-long-running-non-partitioned.yaml} (79%)
create mode 100644 docker/demo/config/test-suite/spark-non-core-operations.yaml
copy hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire.properties => docker/demo/config/test-suite/spark-pure-bulk-inserts.yaml (59%)
rename docker/demo/config/test-suite/{cow-spark-simple.yaml => spark-pure-inserts.yaml} (62%)
copy docker/demo/config/test-suite/{test-metadata-aggressive-clean-archival.properties => test-aggressive-clean-archival-inline-compact.properties} (93%)
copy docker/demo/config/test-suite/{test-clustering-metadata-aggressive-clean-archival.properties => test-clustering-aggressive-clean-archival-inline-compact.properties} (94%)
copy docker/demo/config/test-suite/{test-clustering.properties => test-clustering-inline-compact.properties} (93%)
copy docker/demo/config/test-suite/{test-clustering-metadata-aggressive-clean-archival.properties => test-clustering-metadata-aggressive-clean-archival-inline-compact.properties} (94%)
copy docker/demo/config/test-suite/{test-aggressive-clean-archival.properties => test-inline-compact.properties} (91%)
copy docker/demo/config/test-suite/{test-metadata-aggressive-clean-archival.properties => test-metadata-aggressive-clean-archival-inline-compact.properties} (93%)
copy docker/demo/config/test-suite/{test-metadata.properties => test-metadata-inline-compact.properties} (93%)
copy docker/demo/config/test-suite/{test-clustering.properties => test-nonpartitioned-inline-compact.properties} (90%)
copy docker/demo/config/test-suite/{test-clustering.properties => test-nonpartitioned-metadata-inline-compact.properties} (90%)
copy docker/demo/config/test-suite/{test-clustering.properties => test-nonpartitioned-metadata.properties} (91%)
copy docker/demo/config/test-suite/{test-clustering.properties => test-nonpartitioned.properties} (91%)
rename docker/{ => images}/push_to_docker_hub.png (100%)
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/BucketIndexLocationMapper.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/ConsistentBucketIdentifier.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/HoodieSimpleBucketIndex.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroParquetWriter.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieBaseParquetWriter.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java
copy hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/storage/{HoodieBucketLayout.java => HoodieConsistentBucketLayout.java} (56%)
rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/storage/{HoodieBucketLayout.java => HoodieSimpleBucketLayout.java} (71%)
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/index/bucket/TestBucketIdentifier.java
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/index/bucket/TestConsistentBucketIdIdentifier.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bucket/HoodieSparkConsistentBucketIndex.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestConsistentBucketIndex.java
rename hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/bucket/{TestHoodieBucketIndex.java => TestHoodieSimpleBucketIndex.java} (91%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetastoreConfig.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/ConsistentHashingNode.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/HoodieConsistentHashingMetadata.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/JsonUtils.java
rename hudi-common/src/main/java/org/apache/hudi/hadoop/{FileNameCachingPath.java => CachingPath.java} (54%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/hadoop/SerializablePath.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/util/Lazy.java
copy hudi-common/src/{main/java/org/apache/hudi/exception/HoodieIncompatibleSchemaException.java => test/java/org/apache/hudi/common/model/TestHoodieConsistentHashingMetadata.java} (64%)
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
create mode 100644 hudi-examples/hudi-examples-dbt/.gitignore
create mode 100644 hudi-examples/hudi-examples-dbt/README.md
create mode 100644 hudi-examples/hudi-examples-dbt/dbt_project.yml
create mode 100644 hudi-examples/hudi-examples-dbt/models/example/hudi_insert_overwrite_table.sql
create mode 100644 hudi-examples/hudi-examples-dbt/models/example/hudi_insert_table.sql
create mode 100644 hudi-examples/hudi-examples-dbt/models/example/hudi_upsert_partitioned_cow_table.sql
create mode 100644 hudi-examples/hudi-examples-dbt/models/example/hudi_upsert_partitioned_mor_table.sql
create mode 100644 hudi-examples/hudi-examples-dbt/models/example/hudi_upsert_table.sql
create mode 100644 hudi-examples/hudi-examples-dbt/models/example/schema.yml
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/HadoopConfigurations.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/TestHiveSyncContext.java
create mode 100644 hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngestTool.java
create mode 100644 hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/SparkDataSourceContinuousIngest.scala
create mode 100644 hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/dag/nodes/SparkDeletePartitionNode.scala
rename hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/{SparkHoodieParquetFileFormat.scala => HoodieParquetFileFormat.scala} (50%)
delete mode 100644 hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/index/bucket/TestBucketIdentifier.java
rename hudi-spark-datasource/hudi-spark/src/test/resources/index/{zorder => colstats}/another-input-table-json/part-00000-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json (100%)
rename hudi-spark-datasource/hudi-spark/src/test/resources/index/{zorder => colstats}/another-input-table-json/part-00001-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json (100%)
rename hudi-spark-datasource/hudi-spark/src/test/resources/index/{zorder => colstats}/another-input-table-json/part-00002-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json (100%)
rename hudi-spark-datasource/hudi-spark/src/test/resources/index/{zorder => colstats}/another-input-table-json/part-00003-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json (100%)
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/column-stats-index-table.json
rename hudi-spark-datasource/hudi-spark/src/test/resources/index/{zorder => colstats}/input-table-json/part-00000-4468afca-8a37-4ae8-a150-0c2fd3361080-c000.json (100%)
rename hudi-spark-datasource/hudi-spark/src/test/resources/index/{zorder => colstats}/input-table-json/part-00001-4468afca-8a37-4ae8-a150-0c2fd3361080-c000.json (100%)
rename hudi-spark-datasource/hudi-spark/src/test/resources/index/{zorder => colstats}/input-table-json/part-00002-4468afca-8a37-4ae8-a150-0c2fd3361080-c000.json (100%)
rename hudi-spark-datasource/hudi-spark/src/test/resources/index/{zorder => colstats}/input-table-json/part-00003-4468afca-8a37-4ae8-a150-0c2fd3361080-c000.json (100%)
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/partial-another-input-table-json/part-00000-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/partial-another-input-table-json/part-00001-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/partial-another-input-table-json/part-00002-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/partial-another-input-table-json/part-00003-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/partial-column-stats-index-table.json
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/updated-column-stats-index-table.json
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/updated-partial-column-stats-index-table.json
delete mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/index/zorder/column-stats-index-table.json
delete mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/index/zorder/updated-column-stats-index-table.json
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/SparkDatasetMixin.scala
rename hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/{TestHoodieSqlBase.scala => HoodieSparkSqlTestBase.scala} (98%)
create mode 100644 hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark24HoodieParquetFileFormat.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark312HoodieParquetFileFormat.scala
create mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31HoodieParquetFileFormat.scala
create mode 100644 hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala
rename hudi-sync/{hudi-dla-sync => hudi-adb-sync}/pom.xml (98%)
rename hudi-sync/{hudi-dla-sync => hudi-adb-sync}/src/assembly/src.xml (100%)
create mode 100644 hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/AbstractAdbSyncHoodieClient.java
create mode 100644 hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/AdbSyncConfig.java
create mode 100644 hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/AdbSyncTool.java
create mode 100644 hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/HoodieAdbJdbcClient.java
copy hudi-common/src/main/java/org/apache/hudi/exception/HoodieIncompatibleSchemaException.java => hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/HoodieAdbSyncException.java (72%)
create mode 100644 hudi-sync/hudi-adb-sync/src/test/java/org/apache/hudi/sync/adb/TestAdbSyncConfig.java
rename hudi-sync/{hudi-dla-sync => hudi-adb-sync}/src/test/resources/log4j-surefire-quiet.properties (100%)
rename hudi-sync/{hudi-dla-sync => hudi-adb-sync}/src/test/resources/log4j-surefire.properties (91%)
delete mode 100644 hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncConfig.java
delete mode 100644 hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncTool.java
delete mode 100644 hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/HoodieDLAClient.java
delete mode 100644 hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/util/Utils.java
delete mode 100644 hudi-sync/hudi-dla-sync/src/test/java/org/apache/hudi/dla/TestDLASyncConfig.java
rename hudi-sync/{hudi-hive-sync/src/main/java/org/apache/hudi/hive => hudi-sync-common/src/main/java/org/apache/hudi/sync/common}/util/ConfigUtils.java (98%)
rename hudi-sync/{hudi-hive-sync/src/main/java/org/apache/hudi/hive => hudi-sync-common/src/main/java/org/apache/hudi/sync/common}/util/Parquet2SparkSchemaUtils.java (99%)
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/NoNewDataTerminationStrategy.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/PostWriteTerminationStrategy.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/TerminationStrategyUtils.java
create mode 100644 hudi-utilities/src/test/resources/delta-streamer-config/indexer-only-bloom.properties
create mode 100644 rfc/rfc-44/presto-connector.png
create mode 100644 rfc/rfc-44/rfc-44.md
create mode 100644 rfc/rfc-50/ComparisonDiagram.png
create mode 100644 rfc/rfc-50/CurrentDesign.png
create mode 100644 rfc/rfc-50/Design.png
create mode 100644 rfc/rfc-50/SchematicDiagram.png
create mode 100644 rfc/rfc-50/rfc-50.md
create mode 100755 scripts/release/validate_staged_bundles.sh