You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by gi...@apache.org on 2023/02/26 18:35:19 UTC
[hudi] branch dependabot/maven/packaging/hudi-cli-bundle/com.google.code.gson-gson-2.8.9 updated (35662a898bc -> b5389217bd1)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch dependabot/maven/packaging/hudi-cli-bundle/com.google.code.gson-gson-2.8.9
in repository https://gitbox.apache.org/repos/asf/hudi.git
discard 35662a898bc Bump gson from 2.6.2 to 2.8.9 in /packaging/hudi-cli-bundle
add fd62a1413e7 [HUDI-5414] No need to guard the table initialization by lock for HoodieFlinkWriteClient (#7509)
add 73490ececa4 [MINOR] Improve flink bundle validation stability (#7538)
add bdfaa4e1167 HUDI-5398. Fix Typo in hudi-integ-test#README.md. (#7477)
add 2a472f4c436 [HUDI-5456] Flink streaming read skips uncommitted instants (#7540)
add 2567ada6d56 Revert "[HUDI-5409] Avoid file index and use fs view cache in COW input format (#7493)" (#7526)
add 64b814ea237 [HUDI-5400] Fix read issues when Hudi-FULL schema evolution is not enabled (#7480)
add a882f440d37 [HUDI-5411] Avoid virtual key info for COW table in the input format (#7527)
add 7ae627179e7 [HUDI-4827] Upgrade Azure CI to Ubuntu 22.04 and scalatest-maven-plugin (#7521)
add 8130415b457 [HUDI-5476] Fix utilities test failure with zookeeper (#7560)
add 0b3f8c1cda3 [MINOR] Update DOAP with 0.12.2 release (#7558)
add 60cabeb208c Remove minlog.Log (#7441)
add bd57282f248 [HUDI-5482] Nulls should be counted in the value count stats for mor table (#7482)
add f2b2ec9539d [HUDI-5343] HoodieFlinkStreamer supports async clustering for append mode (#7403)
add 495b6fbb062 [HUDI-5332] HiveSyncTool can avoid initializing all permanent custom functions of Hive (#7385)
add fb28ad8f737 [HUDI-5420] Fix metadata table validator to exclude uncommitted log files due to retry (#7517)
add f5b1b496023 [MINOR] Fix doap file syntax and date (#7586)
add 870b05e1fcd [MINOR] Filter out empty GCS objects in events table. (#7592)
add 41bea2fec54 [HUDI-3572] support DAY_ROLLING strategy in ClusteringPlanPartitionFilterMode (#4966)
add c501d11eba5 [HUDI-5489] Flink offline compactor throws exception in service mode (#7588)
add a0df6ecfb47 [HUDI-5483] Make RetryHelper more commonly usable (#7572)
add 2f5e487445a [HUDI-5492] spark call command 'show_compaction' doesn't return the completed compaction (#7593)
add a80bb4f717a [HUDI-5205] Support flink 1.16.0 (#7584)
add ab61f61df96 [HUDI-5477] Optimize timeline loading in Hudi sync client (#7561)
add 2a486770cc1 [HUDI-5502] Support insert overwrite for flink mor table with bucket index (#7610)
add e0fe1612ebf [HUDI-5160] Fix data source write save as table (#7448)
add a090ad049e2 [MINOR] Fix parameter parsing in hudi utilities (#7435)
add f745e645735 [HUDI-5341] CleanPlanner retains earliest commits must not be later than earliest pending commit (#7568)
add 3e49e4c26da [HUDI-5506] StreamWriteOperatorCoordinator may not recommit with partial uncommitted write metadata event (#7611)
add cf20c95f48d [MINOR] fix typo (#7616)
add f75671454c9 [HUDI-5192] add non-code file extensions to ignore list (#7597)
add 4e2a5bb488b [MINOR] fixed spelling and syntax (#7535)
add 8ae1e8c6761 [HUDI-5231] suppress checkstyle warnings (#7473)
add 633fe09514e [MINOR] Set engine when creating meta write config (#7575)
add d426ea610df [HUDI-5394] Fix RowCustomColumnsSortPartitioner (#7468)
add ff403c82d8a [HUDI-5511] Do not clean the CkpMetadata dir when restart the job (#7620)
add dc0f880c1b2 [HUDI-5504] Fix concurrency conflict for flink async compaction with latency marker (#7609)
add 672751975d0 [HUDI-5484] Avoid using `GenericRecord` in `HoodieColumnStatMetadata` (#7573)
add aa945b84e63 [HUDI-5519] Fix LogFileComparator with suffix for CDC (#7629)
add 2d1dd2a8ab1 [HUDI-5515] Fix concurrency conflict in ClusteringOperator with latency marker (#7625)
add b9c32fb9435 [HUDI-5326] Fix clustering group building in SparkSizeBasedClusteringPlanStrategy (#7372)
add d5bc4d9a180 [MINOR] fix typo in hudi-client (#7606)
add 9a0e1ce4887 [HUDI-5349] Clean up partially failed restore (#7605)
add 7a9aabd9b25 [HUDI-5434] Fix archival in metadata table to not rely on completed rollback or clean in data table (#7580)
add e15a3cdc02b [HUDI-5381] Fix for class cast exception when running with Flink 1.15 (#7651)
add a06556334f8 [HUDI-5528] HiveSyncProcedure & HiveSyncTool also needs to add HIVE_SYNC_TABLE_STRATEGY. (#7644)
add 2450b19dcdd [MINOR] Remove useless RollbackTimeline (#7631)
add d65668611e3 [HUDI-5317] Fix insert overwrite table for partitioned table (#7365)
add f18e9284208 [MINOR] Fix flaky tests in ITTestHoodieDataSource caused by unordered hashmap (#7641)
add eacae1e0dc4 [HUDI-5514] Add in support for a keyless workflow by building an ID based off of values within the record (#7640)
add 00332f14653 [MINOR] Add metastore_db/ into gitignore file (#7648)
add 6c0323963c5 [MINOR] Polish README and template.md in RFC (#7618)
add 70d450e8389 [HUDI-5545] Extending support to other special characters (#7585)
add 669e5676725 [HUDI-5543] Description of clustering.plan.partition.filter.mode supports DAY_ROLLING strategy (#7656)
add b6db9b1ff87 [MINOR] Fix flaky testStructuredStreamingWithCompaction (#7485)
add 6aa8e9681b2 [HUDI-5538] Fix ContinuousFileSource and ITTestDataStreamWrite for flink 1.16 support (#7637)
add febc55c7792 [HUDI-5544] Improve log msgs during bulk insert (#7658)
add 83dc08b1f65 [HUDI-5542] Added javadocs for InternalSchemaManager (#7646)
add fcc508c8c0a [MINOR] Fix minor issues in HoodieMetadataTableValidator docs (#7518)
add 202ae47b473 [HUDI-5391] Modify the default value of parameter `hoodie.write.lock.client` (#7460)
add 7c213d5ff72 [HUDI-5554] Add UT TestHiveSyncTool#testSyncMergeOnReadWithStrategy for parameter HIVE_SYNC_TABLE_STRATEGY. (#7671)
add 502933d2f70 [HUDI-5512] fix spark call procedure run_bootstrap missing conf and conf can not take effect (#7621)
add f750773109a [HUDI-5275] Fix reading data using the HoodieHiveCatalog will cause the Spark write to fail (#7666)
add 16d33ba3cb9 [HUDI-3654] Add new module `hudi-metaserver` (#5064)
add d2a3d11977d [MINOR] Add database config for flink (#7682)
add 6cb4580defc [HUDI-4710] Fix flaky: TestKeyRangeLookupTree#testFileGroupLookUpManyEntriesWithSameStartValue (#7571)
add c9bc03ed868 [HUDI-4148] Add client for Hudi table service manager (TSM) (#6732)
add ec5022b4fdd [MINOR] Unify naming for record merger (#7660)
add 27a8866a1e1 [HUDI-5433] Fix the way we deduce the pending instants for MDT writes (#7544)
add 124ab5fda66 [HUDI-5488] Make sure Disrupt queue start first, then insert records (#7582)
add 43d23a49a02 [HUDI-5577] Validate option catalog.path in dfs mode (#7698)
add 9ee36de98ab [Minor] add missing link and fix typo (#7696)
add f8028a400eb [HUDI-4911][HUDI-3301] Fixing `HoodieMetadataLogRecordReader` to avoid flushing cache for every lookup (#6782)
add 86be8554820 [HUDI-5336] Fixing log file pattern match to ignore extraneous files (#7612)
add 0e9bb024fb9 [HUDI-5559] Support CDC for flink bounded source (#7677)
add e62b9da66b6 [HUDI-5516] Reduce memory footprint on workload with thousand active partitions (#7626)
add b1552eff7af [HUDI-5384] Adding optimization rule to appropriately push down filters into the `HoodieFileIndex` (#7423)
add 6f4c60e1835 [HUDI-5589] Fix Hudi config inference (#7713)
add f0f8d618b30 [HUDI-5499] Fixing Spark SQL configs not being properly propagated for CTAS and other commands (#7607)
add d03c8f9c155 [MINOR] Disable async clean in testCleanerDeleteReplacedDataWithArchive (#7721)
add 6f6652a84aa [HUDI-5407][HUDI-5408] Fixing rollback in MDT to be eager (#7490)
add febff4afd2a [HUDI-5417] support to read avro from non-legacy map/list in parquet log (#7512)
add a70355f4457 [HUDI-5579] Fixing Kryo registration to be properly wired into Spark sessions (#7702)
add 1d28d51d00c [HUDI-5596] Fix hudi-cli-bundle startup issue with gson (#7728)
add 6593e8ba9d9 [minor] Fix flink 1.15 build profile (#7731)
add c18d6153e10 [HUDI-1575] Early Conflict Detection For Multi-writer (#6133)
add 811dcc591cd [MINOR] Eliminating Kryo from `hudi-integ-test-bundle` since it's being used w/ Spark (#7735)
add 8917971e9f7 [HUDI-5555] Set class-loader in parquet data block (#7670)
add 4f6b831ea11 [HUDI-5235] Clustering target size should larger than small file limit (#7232)
add fc1831b22c3 Fixing FS `InputStream` leaks (#7741)
add 5e3ca834366 [MINOR] Fixing `TestStructuredStreaming` test (#7736)
add 146f39d49e5 [HUDI-5593] Fixing deadlocks due to async cleaner awaiting for lock while main thread is acquired the lock and awaiting for async cleaner to finish (#7739)
add d439fab2421 [HUDI-3673] Clean up hbase shading dependencies (#7371)
add 2fc20c186b7 [HUDI-5575] Adding/Fixing auto generation of record keys w/ hudi (#7726)
add 25afb357df9 [HUDI-5401] Ensure user-provided hive metastore uri is set in HiveConf if not already set (#7543)
add 20969c26059 [HUDI-5392] Fixing Bootstrapping flow handling of arrays (#7461)
add 26b719a7fba [HUDI-2608] Support json schema in SchemaRegistryProvider (#7727)
add 1769ff8fb90 [HUDI-5443] Fixing exception trying to read MOR table after `NestedSchemaPruning` rule has been applied (#7528)
add a79f8093755 [HUDI-5582] Do not let users override internal metadata configs (#7709)
add 65044d38fbe [HUDI-2118] Skip checking corrupt log blocks for transactional write file systems (#6830)
add f49b5d34342 [HUDI-4991] Allow kafka-like configs to set truststore and keystore for the SchemaProvider
add 2f22b07385f [HUDI-5276] Fix getting partition paths under relative paths (#7744)
add c95abd3213f Revert "[HUDI-5575] Adding/Fixing auto generation of record keys w/ hudi (#7726)" (#7747)
add 8c640b51331 [HUDI-5610] Fix hudi-cli-bundle startup conflict for spark 3.2.0 (#7746)
add 31b02d1798e [HUDI-5594] Add metaserver bundle validation (#7722)
add e00ee2d88e2 [HUDI-5620] Fix metaserver bundle validation (#7749)
add 7e35874c7ba [HUDI-5617] Rename configs for async conflict detector for clarity (#7750)
add d4dcb3d1190 [HUDI-5618] Add `since version` to new configs for 0.13.0 release (#7751)
add 3a08bdc3f97 [HUDI-5363] Removing default value for shuffle parallelism configs (#7723)
add 2e59c8e6b9d [MINOR] Correct RFC numbering (#7754)
add 98643c63056 [HUDI-5380] Fixing change table path but table location in metastore … (#7445)
add 45da30dc3ec [HUDI-5485] Add File System View API for batch listing and improve savepoint performance with metadata table (#7690)
add e969a4c7848 [HUDI-5592] Fixing some of the flaky tests in CI (#7720)
add 5ea2f8487c9 Moving to 0.14.0-SNAPSHOT on master branch. (#7755)
add e270924c5eb [HUDI-5534] Optimizing Bloom Index lookup when using Bloom Filters from Metadata Table (#7642)
add a62f0f684fa [HUDI-5627] Improve the usability of Hudi CLI bundle (#7762)
add 1ecc0401eef [HUDI-5623] Increase default time to wait between retries by lock provider client (#7758)
add ff590c6d72c [HUDI-5023] Switching default Write Executor type to `SIMPLE` (#7476)
add 67d661c2952 [HUDI-5630] Fixing flaky parquet projection tests (#7768)
add 6011de44d47 [HUDI-5629] Clean CDC log files for enable/disable scenario (#7767)
add 7352661283e [MINOR] Add `hudi-platform-service` and `hudi-metaserver-server-bundle` to root pom (#7774)
add 50b2cb5d8f7 [HUDI-5635] Fix release scripts (#7775)
add 1cc6442781e [MINOR] Add missing Apache License in source files (#7779)
add aebe3cc53e5 [MINOR] Fix validate_staged_release.sh (#7780)
add c21eca564c6 [HUDI-5626] Rename CDC logging mode options (#7760)
add 5d8c3a98189 [MINOR] Skip docs generation for table service manager (#7773)
add d25058b36b2 Revert "[MINOR] Add `hudi-platform-service` and `hudi-metaserver-server-bundle` to root pom (#7774)" (#7782)
add 2c56aa4ce99 [HUDI-5631] Improve defaults of early conflict detection configs (#7770)
add 9bbd71eb67a [HUDI-5624] Fix HoodieAvroRecordMerger to use new precombine API (#7759)
add 3979848a499 [HUDI-5628] Fixing log record reader scan V2 config name (#7764)
add 4a9ae5c4091 [HUDI-5638] Fix metaserver test setup (#7785)
add 3b301aa7f03 [HUDI-5640] Add missing profiles in `deploy_staging_jars.sh` (#7784)
add a1ba929b5e4 [HUDI-5639] Fixing stream identifier for single writer with spark streaming ingest (#7783)
add 44b245dc820 [MINOR] Fix `HoodieCDCRDD` setting flag `usesVirtualKeys` (#7777)
add b00dac54642 [HUDI-5503] Optimize flink table factory option check (#7608)
add 88d8e5e96d5 [MINOR] Cleaning up recently introduced configs (#7772)
add 53b813a565b [HUDI-5629] Clean CDC log files for enable/disable scenario (#7786)
add e6c0bd63b2f [HUDI-5637] Add Kryo for hive sync bundle (#7781)
add 32f45f0f7d6 [HUDI-5634] Rename CDC related classes (#7410)
add 22eab392fc7 [HUDI-5632] Fix failure launching Spark jobs from hudi-cli-bundle (#7790)
add d968f393bd2 [MINOR] Make `data_before_after` the default cdc logging mode (#7797)
add 1a72f504eef [HUDI-5563] Check table exist before drop table (#7679)
add d90f286a197 [HUDI-5568] Fix the BucketStreamWriteFunction to rebase the local filesystem instance instead (#7685)
add 9906df48e7c [HUDI-5655] Closing write client for spark ds writer in all cases (including exception) (#7799)
add 5acc6fe51ac [HUDI-5654] Fixing read of an empty rollback completed meta files from data table timeline w/ metadata reads (#7798)
add 1377143656a [HUDI-5487] Reduce duplicate logs in ExternalSpillableMap (#7579)
add 252c4033010 [MINOR] Standardise schema concepts on Flink Engine (#7761)
add d55a1cef330 [HUDI-5567] Make the bootstrapping exception message more clear (#7684)
add d8576933a9b [HUDI-5553] Prevent partition(s) from being dropped if there are pending… (#7669)
add 628dc8cd851 [HUDI-5633] Fixing performance regression in `HoodieSparkRecord` (#7769)
add 9469882d80f [HUDI-5585][flink] Fix flink creates and writes the table, the spark alter table reports an error (#7706)
add 255d40c5fb5 [HUDI-5540] Close write client after usage of DeleteMarker/RollbackToInstantTime/RunClean/RunCompactionProcedure (#7655)
add 0a9a6d20471 [HUDI-5317] Fix insert overwrite table for partitioned table (#7793)
add 5e616ab115c [HUDI-5646] Guard dropping columns by a config, do not allow by default (#7787)
add 7064c380506 [MINOR] Restoring existing behavior for `DeltaStreamer` Incremental Source (#7810)
add e93fbeee4ac [HUDI-5681] Fixing Kryo being instantiated w/ invalid `SparkConf` (#7821)
add abe26d4169c [HUDI-5676] Fix BigQuerySyncTool standalone mode (#7816)
add 6fbf9d4f840 [HUDI-5647] Automate savepoint and restore tests (#7796)
add 1459eddb0e7 [HUDI-5684] Fix CTAS and Insert Into to avoid combine-on-insert by default (#7813)
add e3b95e88a76 [HUDI-5678] Fix `deduceShuffleParallelism` in row-writing Bulk Insert helper (#7818)
add 9c969380233 [HUDI-5691] Fixing `HoodiePruneFileSourcePartitions` to properly handle non-partitioned tables (#7833)
add 3282caa2242 [HUDI-5671] BucketIndexPartitioner partition algorithm skew (#7815)
add 0ea3c44110c [HUDI-5551] support seconds unit on event_time metrics (#7664)
add fc031b9d3b0 [HUDI-5682] Bucket index does not work correctly for multi-writer scenarios (#7838)
add 3e1171cf70d [HUDI-5653] Reset TestDataSource for TestHoodieDeltaStreamerWithMultiWriter (#7831)
add f3ac50935bf [MINOR] Fixing CTAS configuration not propagated properly (#7832)
add 5cdfacba1bd [HUDI-5496] Avoid unnecessary file system parsing to initialize metadata table for a new data table (#7841)
add 6424d4ddb57 [HUDI-5689] Make CDC config working for DeltaStreamer path (#7830)
add ef3a17e3d97 [HUDI-5692] SpillableMapBasePath should be lazily loaded (#7837)
add ff832f4d860 [MINOR] Validate configs for OCC early conflict detection (#7848)
add 13ed95a9fd2 Make most of the Spark SQL DML operations configs overridable (#7850)
add 352445fb965 [HUDI-5700] Annotate config classes for docs generation (#7853)
add 52e41052217 [HUDI-5701] Remove meta fields from cdc new record in CDCLogger (#7852)
add 1d305c9f5da [HUDI-5702] Avoid to write useless change-data when compaction (#7849)
add ea36857035f [HUDI-5703] Improve `ConfigProperty` APIs for docs generation (#7854)
add 0c9465f2ab6 [MINOR] Improve configuration configs (#7855)
add 33dd8ae5153 [HUDI-5638] Activate platform service profile when deploy (#7858)
add 21f58999a88 [HUDI-5704] De-coupling column drop flag and schema validation flag (#7857)
add 2c38af5e627 [HUDI-5651] Sort the inputs by record keys for bulk insert tasks (#7795)
add 48381df096a [HUDI-5717] Precision field is not need in FixedLenBytesColumnReader (#7866)
add 6436ef3ee2c [HUDI-5329] Spark reads hudi table error when flink creates the table without preCombine fields (#7378)
add 4609ed156b7 [MINOR] Added safety-net check to catch any potential issue to deduce parallelism from the incoming `Dataset` appropriately (#7873)
add 5976a519303 [HUDI-5718] Unsupported Operation Exception for compaction (#7874)
add 33212ccabb5 [HUDI-5720] Improve the validation script of the staged source release (#7882)
add 08b47fcd9fa [MINOR] Fix bootstrap typos (#7676)
add dee020e18ce [HUDI-5657] Fix NPE if filters condition contains null literal when using column stats data skipping for flink (#7801)
add 380d648d085 [MINOR] Fix indexer typos (#7863)
add 53fca761be1 [MINOR] fixed docs for WRITE_EXECUTOR_TYPE (#7880)
add a9f1be52ffc [HUDI-5721] Add Github actions on source validation (#7883)
add 8329414edfa [MINOR] Remove useless code (#7756)
add a1da9ec2e4f [HUDI-5270] Duplicate key error when insert_overwrite the same partition in multi writer (#7281)
add 0dbc3450a95 [HUDI-4968] Update misleading read.streaming.skip_compaction/skip_clustering config (#6856)
add b2e3805b3fb [MINOR] improve RunClusteringProcedure with partition selected (#7876)
add 2b0615c52ba [HUDI-5725] Creating Hudi table with misspelled table type in Flink leads to Flink cluster crash (#7888)
add 088d9a061e5 [MINOR] Description update for Flink Clustering and Compaction configuration (#7887)
add 60dfe4d766d Cleaning up unnecessary relocation for com.google.common packages (#7900)
add a7151ef53c2 [HUDI-5734] Fix flink batch read skip clustering data lost (#7903)
add 653aa86145f [MINOR] Fix wrong assertion in TestHoodieTableFactory.java (#7916)
add 67f4c78c5a0 [HUDI-5758] Restoring state of `HoodieKey` to make sure it's binary compatible w/ its state in 0.12 (#7917)
add 3e31ca73828 [MINOR] Remove unnecessary TestCallExpressions which are adapters for CallExpression (#7911)
add 1cb8ffe7264 [HUDI-5768] Fix Spark Datasource read of metadata table (#7924)
add e25381c6966 [HUDI-5764] Rollback delta commits from `HoodieIndexer` lazily in metadata table (#7921)
add a932e482408 [HUDI-5771] Improve deploy script of release artifacts (#7927)
add d395f058183 [HUDI-3580] [RFC-48] Create RFC for LogCompaction support to Hudi (#5041)
add 71a62627cdd [HUDI-5772] Align Flink clustering configuration with HoodieClusteringConfig (#7928)
add ed6b7f6aedc [HUDI-4406] Support Flink compaction/clustering write error resolvement to avoid data loss (#6121)
add 4f8f2d8dc5c [HUDI-5737] Fix Deletes issued without any prior commits (#7633)
add 87b97369b68 [HUDI-5773] Support archive command for spark sql (#7931)
add 5d7edd5097e [MINOR] Fixing RFC 48 title (#7937)
add b59fa1bd7fd [HUDI-5792] Disabling flaky tests: TestDisruptorExecutionInSpark (#7946)
add af61dea6f98 [MINOR] Enable Azure CI to publish test results (#7943)
add 25f6927b47d [HUDI-5787] HMSDDLExecutor should set table type to EXTERNAL_TABLE when hoodie.datasource.hive_sync.create_managed_table of sync config is false (#7940)
add da80b8c8d52 [HUDI-5673] Support multi writer for bucket index with guarded lock (#7860)
add 91505853144 [MINOR] Fix spark sql run clean do not exit (#7918)
add c9395adeb21 [MINOR] Fix format name and remove redundant line in examples (#7952)
add a584c6f97f4 [MINOR] Improving logging and tracking of failures for metadata validator (#7792)
add e5ee6576cba [HUDI-5814] Disable flaky tests to unblock CI (#7986)
add 45017036ecf [HUDI-1593] Add support for "show restores" and "show restore" commands in hudi-cli (#7868)
add 6356dbdca35 [HUDI-5809][RFC-56] Keep RFC-56 early conflict detection update to date (#7972)
add aad6516da96 [HUDI-5664] Improve SqlQueryPreCommitValidator#queries Parallelism. (#7809)
add c173edfaf0d [HUDI-5811] Specify the artifact of protoc to solve the problem that cannot be compiled on m1 mac (#7977)
add 68135ad2296 [HUDI-5800] Fix test failure in TestHoodieMergeOnReadTable (#7959)
add 73ca223e2e5 [HUDI-5799] Fix Spark partition validation in TestBulkInsertInternalPartitionerForRows (#7958)
add 96032b9b768 [HUDI-5557]Avoid converting columns that are not indexed in CSI (#7672)
add 25fc4f2ab52 [MINOR] De-duplicating Iterator implementations (#7752)
add 28d6bfef58c [HUDI-5795] Ensure we only have a single push gw client and hence a single connection pool
add be92be657a3 [HUDI-5786] Add a new config to specific spark write rdd storage level (#7941)
add d3840a0c02f [HUDI-5058] HoodieCatalog#getTable sets primary key with hoodie.datasource.write.recordkey.field for table intialized via Spark (#7981)
add 688d947e44b [HUDI-5729] Fix RowDataKeyGen method getRecordKey (#7894)
add dfac12385bd [HUDI-5802] Allow configuration for deletes in DefaultHoodieRecordPayload (#7961)
add 0c84482aa91 [HUDI-5774] Fix prometheus configs for metadata table and support metric labels (#7933)
add 767da51baeb [HUDI-5821] Improve the resource release for HoodieTableFileSystemView (#7995)
add 33987b1bc27 [HUDI-5792] fix CI test:TestDisruptorExecutionInSpark timeout problem (#8000)
add 8ee354fcaf6 [HUDI-5778] support absolute path names for hierarchical configs (#7920)
add 0f99315ab84 Handle empty payloads for AbstractDebeziumAvroPayload (#7944)
add 9c8144045de [HUDI-5823] Claim RFC-65 for Partition TTL Management (#8006)
add 6430e9d82a2 [MINOR] Close InflaterInputStream in finally when calling decompressBytes in BitCaskDiskMap (#7840)
add 018ecbd1fa6 update hudi-flink exception string (#8003)
add ad2a37d6796 [HUDI-5817] Fix async indexer metadata writer to avoid eager rollback and failed write cleaning (#8001)
add 24c9a86dae2 Fix usage of SchemaRegistryProvider with Row source (#7601)
add 812950bc9ea [HUDI-4690][HUDI-4503] Cleaning up Hudi custom Spark `Rule`s (#7871)
add d705dcc4188 [HUDI-5173] Skip if there is only one file in clusteringGroup (#7159)
add 265aac89584 [HUDI-5825] disable Spark UI in tests if SPARK_EVLOG_DIR not set (#8005)
add 2770ff50714 [HUDI-915][HUDI-5656] Rebased `HoodieBootstrapRelation` onto `HoodieBaseRelation` (#7804)
add b39d99c7e67 [MINOR] Update DOAP with 0.13.0 Release (#8012)
add 654600990b8 [HUDI-5455] Add commons-configuration2 in hudi cli bundle (#7536)
add b5389217bd1 Bump gson from 2.6.2 to 2.8.9 in /packaging/hudi-cli-bundle
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (35662a898bc)
\
N -- N -- N refs/heads/dependabot/maven/packaging/hudi-cli-bundle/com.google.code.gson-gson-2.8.9 (b5389217bd1)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/bot.yml | 41 +-
.gitignore | 3 +-
README.md | 4 +-
azure-pipelines.yml | 23 +-
doap_HUDI.rdf | 10 +
docker/demo/config/spark-defaults.conf | 2 +
.../templates/spark_command.txt.template | 1 +
docker/hoodie/hadoop/base/pom.xml | 2 +-
docker/hoodie/hadoop/base_java11/pom.xml | 2 +-
docker/hoodie/hadoop/datanode/pom.xml | 2 +-
docker/hoodie/hadoop/historyserver/pom.xml | 2 +-
docker/hoodie/hadoop/hive_base/pom.xml | 2 +-
docker/hoodie/hadoop/namenode/pom.xml | 2 +-
docker/hoodie/hadoop/pom.xml | 2 +-
docker/hoodie/hadoop/prestobase/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/pom.xml | 2 +-
docker/hoodie/hadoop/sparkadhoc/pom.xml | 2 +-
docker/hoodie/hadoop/sparkmaster/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/pom.xml | 2 +-
docker/hoodie/hadoop/trinobase/pom.xml | 2 +-
docker/hoodie/hadoop/trinocoordinator/pom.xml | 2 +-
docker/hoodie/hadoop/trinoworker/pom.xml | 2 +-
hudi-aws/pom.xml | 4 +-
.../hudi/config/DynamoDbBasedLockConfig.java | 11 +-
hudi-cli/pom.xml | 2 +-
.../apache/hudi/cli/HoodieTableHeaderFields.java | 6 +
.../src/main/java/org/apache/hudi/cli/Main.java | 2 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 2 +-
.../apache/hudi/cli/commands/ExportCommand.java | 2 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 6 +-
.../apache/hudi/cli/commands/RestoresCommand.java | 172 ++++
.../apache/hudi/cli/commands/RollbacksCommand.java | 18 +-
.../hudi/cli/utils/SparkTempViewProvider.java | 5 +-
.../java/org/apache/hudi/cli/utils/SparkUtil.java | 15 +-
.../cli/commands/TestHoodieLogFileCommand.java | 7 +-
.../hudi/cli/commands/TestRestoresCommand.java | 207 +++++
.../hudi/cli/commands/TestRollbacksCommand.java | 4 +-
.../cli/functional/CLIFunctionalTestHarness.java | 4 +-
.../hudi/cli/integ/ITTestCommitsCommand.java | 7 +-
.../hudi/cli/integ/ITTestRepairsCommand.java | 5 +-
hudi-client/hudi-client-common/pom.xml | 10 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 70 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 871 +++++++++++++++++++++
.../apache/hudi/client/BaseHoodieWriteClient.java | 704 ++++-------------
.../apache/hudi/client/CompactionAdminClient.java | 5 -
.../client/HoodieTableServiceManagerClient.java | 169 ++++
.../apache/hudi/client/HoodieTimelineArchiver.java | 62 +-
.../org/apache/hudi/client/RunsTableService.java | 5 +
.../java/org/apache/hudi/client/WriteStatus.java | 13 +-
.../client/embedded/EmbeddedTimelineService.java | 11 +
.../hudi/client/heartbeat/HeartbeatUtils.java | 9 +-
.../client/heartbeat/HoodieHeartbeatClient.java | 25 +-
...urrentFileWritesConflictResolutionStrategy.java | 55 ++
.../DirectMarkerTransactionManager.java | 93 +++
.../client/transaction/TransactionManager.java | 26 +-
.../hudi/client/transaction/lock/LockManager.java | 7 +-
.../hudi/client/utils/DeletePartitionUtils.java | 77 ++
.../apache/hudi/client/utils/MergingIterator.java | 12 +-
.../common/table/log/HoodieFileSliceReader.java | 2 +-
.../apache/hudi/config/HoodieBootstrapConfig.java | 2 +-
.../apache/hudi/config/HoodieClusteringConfig.java | 15 +-
.../apache/hudi/config/HoodieCompactionConfig.java | 14 +-
.../apache/hudi/config/HoodieHBaseIndexConfig.java | 1 +
.../org/apache/hudi/config/HoodieIndexConfig.java | 19 +-
.../org/apache/hudi/config/HoodieLockConfig.java | 21 +-
.../org/apache/hudi/config/HoodieMemoryConfig.java | 16 +-
.../config/HoodieWriteCommitCallbackConfig.java | 5 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 289 +++++--
.../hudi/config/metrics/HoodieMetricsConfig.java | 1 +
.../metrics/HoodieMetricsPrometheusConfig.java | 11 +
.../hudi/execution/HoodieLazyInsertIterable.java | 30 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 5 +-
.../index/bloom/BaseHoodieBloomIndexHelper.java | 5 +-
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 77 +-
...ion.java => HoodieBloomIndexCheckFunction.java} | 59 +-
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 14 +-
.../bloom/ListBasedHoodieBloomIndexHelper.java | 26 +-
.../apache/hudi/index/bucket/BucketIdentifier.java | 6 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 32 +-
.../org/apache/hudi/io/HoodieBootstrapHandle.java | 24 +-
.../java/org/apache/hudi/io/HoodieCDCLogger.java | 9 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 27 +-
.../java/org/apache/hudi/io/HoodieIOHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 20 +-
.../apache/hudi/io/HoodieMergeHandleFactory.java | 18 +-
.../java/org/apache/hudi/io/HoodieReadHandle.java | 2 +-
.../apache/hudi/io/HoodieSortedMergeHandle.java | 3 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 28 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 144 ++--
.../prometheus/PushGatewayMetricsReporter.java | 31 +
.../metrics/prometheus/PushGatewayReporter.java | 30 +-
.../java/org/apache/hudi/table/HoodieTable.java | 70 +-
.../action/clean/CleanPlanActionExecutor.java | 12 +-
.../hudi/table/action/clean/CleanPlanner.java | 57 +-
.../cluster/ClusteringPlanActionExecutor.java | 1 +
.../cluster/ClusteringPlanPartitionFilter.java | 23 +
.../cluster/ClusteringPlanPartitionFilterMode.java | 3 +-
.../PartitionAwareClusteringPlanStrategy.java | 2 +
.../table/action/commit/BaseBulkInsertHelper.java | 7 +-
.../hudi/table/action/commit/BaseDeleteHelper.java | 7 +-
.../hudi/table/action/commit/BaseWriteHelper.java | 14 +-
.../table/action/commit/HoodieDeleteHelper.java | 35 +-
.../table/action/commit/HoodieMergeHelper.java | 69 +-
.../table/action/commit/HoodieWriteHelper.java | 3 +-
.../table/action/commit/ParallelismHelper.java | 41 +-
.../action/compact/CompactionExecutionHelper.java | 4 +-
.../hudi/table/action/compact/HoodieCompactor.java | 2 +-
.../compact/LogCompactionExecutionHelper.java | 2 +-
.../compact/ScheduleCompactionActionExecutor.java | 14 +-
.../BaseHoodieCompactionPlanGenerator.java | 3 +
.../HoodieLogCompactionPlanGenerator.java | 5 +-
.../table/action/index/RunIndexActionExecutor.java | 5 +-
.../action/restore/BaseRestoreActionExecutor.java | 10 +-
.../restore/CopyOnWriteRestoreActionExecutor.java | 4 +-
.../restore/MergeOnReadRestoreActionExecutor.java | 4 +-
.../hudi/table/action/restore/RestoreUtils.java | 27 +
.../rollback/BaseRollbackActionExecutor.java | 4 +-
.../action/rollback/RestorePlanActionExecutor.java | 15 +-
.../action/savepoint/SavepointActionExecutor.java | 55 +-
.../hudi/table/marker/ConflictDetectionUtils.java | 42 +
.../hudi/table/marker/DirectWriteMarkers.java | 24 +
.../SimpleDirectMarkerBasedDetectionStrategy.java | 85 ++
...nsactionDirectMarkerBasedDetectionStrategy.java | 65 ++
.../marker/TimelineServerBasedWriteMarkers.java | 75 +-
.../org/apache/hudi/table/marker/WriteMarkers.java | 55 +-
.../hudi/table/marker/WriteMarkersFactory.java | 5 +-
.../storage/HoodieConsistentBucketLayout.java | 1 +
.../table/storage/HoodieSimpleBucketLayout.java | 1 +
.../java/org/apache/hudi/util/ExecutorFactory.java | 52 +-
...urrentFileWritesConflictResolutionStrategy.java | 150 ++++
.../TestZookeeperBasedLockProvider.java | 19 +-
.../client/utils/TestDeletePartitionUtils.java | 110 +++
.../prometheus/TestPushGateWayReporter.java | 49 ++
hudi-client/hudi-flink-client/pom.xml | 4 +-
.../hudi/client/HoodieFlinkTableServiceClient.java | 222 ++++++
.../apache/hudi/client/HoodieFlinkWriteClient.java | 269 +++----
.../FlinkSizeBasedClusteringPlanStrategy.java | 32 +-
.../hudi/execution/FlinkLazyInsertIterable.java | 2 +-
.../apache/hudi/index/FlinkHoodieIndexFactory.java | 4 +-
.../apache/hudi/io/FlinkWriteHandleFactory.java | 25 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 21 +-
.../hudi/table/HoodieFlinkCopyOnWriteTable.java | 6 +-
.../org/apache/hudi/table/HoodieFlinkTable.java | 18 +-
.../table/action/commit/FlinkDeleteHelper.java | 1 +
.../FlinkDeletePartitionCommitActionExecutor.java | 3 +
.../hudi/table/action/commit/FlinkWriteHelper.java | 3 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 8 +-
.../TestFlinkSizeBasedClusteringPlanStrategy.java | 96 +++
hudi-client/hudi-java-client/pom.xml | 10 +-
.../hudi/client/HoodieJavaTableServiceClient.java | 65 ++
.../apache/hudi/client/HoodieJavaWriteClient.java | 23 +-
.../hudi/execution/JavaLazyInsertIterable.java | 2 +-
.../hudi/table/HoodieJavaCopyOnWriteTable.java | 12 +-
.../org/apache/hudi/table/HoodieJavaTable.java | 4 +-
.../table/action/commit/JavaBulkInsertHelper.java | 12 +-
.../hudi/table/action/commit/JavaDeleteHelper.java | 1 +
.../hudi/table/action/commit/JavaWriteHelper.java | 1 +
hudi-client/hudi-spark-client/pom.xml | 11 +-
.../hudi/client/HoodieSparkClusteringClient.java | 2 +-
.../hudi/client/SparkRDDTableServiceClient.java | 295 +++++++
.../apache/hudi/client/SparkRDDWriteClient.java | 234 +-----
.../HoodieSparkBootstrapSchemaProvider.java | 4 +-
.../SparkSizeBasedClusteringPlanStrategy.java | 24 +-
.../MultipleSparkJobExecutionStrategy.java | 8 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 6 +-
.../SparkSortAndSizeExecutionStrategy.java | 6 +-
.../client/common/HoodieSparkEngineContext.java | 10 +-
.../validator/SqlQueryPreCommitValidator.java | 6 +-
.../hudi/common/model/HoodieSparkRecord.java | 104 +--
.../org/apache/hudi/data/HoodieJavaPairRDD.java | 5 +
.../java/org/apache/hudi/data/HoodieJavaRDD.java | 14 +-
.../hudi/execution/SparkLazyInsertIterable.java | 16 +-
.../RowCustomColumnsSortPartitioner.java | 5 +-
.../bloom/BucketizedBloomCheckPartitioner.java | 18 +-
.../bloom/HoodieBloomFilterProbingResult.java | 35 +
.../index/bloom/HoodieBloomIndexCheckFunction.java | 120 ---
.../index/bloom/HoodieFileProbingFunction.java | 143 ++++
.../HoodieMetadataBloomFilterProbingFunction.java | 157 ++++
.../HoodieMetadataBloomIndexCheckFunction.java | 154 ----
.../index/bloom/SparkHoodieBloomIndexHelper.java | 229 +++++-
.../hudi/io/storage/HoodieSparkFileWriter.java | 17 -
.../hudi/io/storage/HoodieSparkParquetReader.java | 9 +-
.../hudi/io/storage/HoodieSparkParquetWriter.java | 46 +-
.../apache/hudi/keygen/BuiltinKeyGenerator.java | 2 +-
.../SparkHoodieBackedTableMetadataWriter.java | 31 +-
.../hudi/table/HoodieSparkCopyOnWriteTable.java | 12 +-
.../hudi/table/HoodieSparkMergeOnReadTable.java | 4 +-
.../org/apache/hudi/table/HoodieSparkTable.java | 17 +-
.../bootstrap/BaseBootstrapMetadataHandler.java | 9 +-
.../bootstrap/OrcBootstrapMetadataHandler.java | 5 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 87 +-
.../commit/BaseSparkCommitActionExecutor.java | 4 +-
.../table/action/commit/SparkBulkInsertHelper.java | 24 +-
.../SparkDeletePartitionCommitActionExecutor.java | 3 +
.../apache/hudi/util/HoodieSparkRecordUtils.java | 69 --
.../org/apache/hudi/AvroConversionUtils.scala | 27 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 29 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 4 +-
.../scala/org/apache/hudi/util/JFunction.scala | 9 +-
...ovider.scala => HoodieSparkKryoRegistrar.scala} | 57 +-
.../spark/sql/HoodieCatalystExpressionUtils.scala | 74 +-
.../spark/sql/HoodieCatalystPlansUtils.scala | 42 +-
.../apache/spark/sql/HoodieInternalRowUtils.scala | 595 ++++++++------
.../apache/spark/sql/HoodieUnsafeRowUtils.scala | 33 +-
.../org/apache/spark/sql/HoodieUnsafeUtils.scala | 25 +
.../sql/catalyst/expressions/AttributeEq.scala | 32 +-
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 75 +-
.../sql/parser/HoodieExtendedParserInterface.scala | 24 +-
.../org/apache/hudi/client/TestClientRollback.java | 94 ++-
.../hudi/client/TestHoodieClientMultiWriter.java | 193 ++++-
.../java/org/apache/hudi/client/TestSavepoint.java | 152 ++++
.../hudi/client/TestTableSchemaEvolution.java | 125 +--
.../org/apache/hudi/client/TestWriteStatus.java | 92 +++
.../TestSparkSizeBasedClusteringPlanStrategy.java | 94 +++
.../functional/TestHoodieBackedMetadata.java | 219 +++++-
.../functional/TestHoodieBackedTableMetadata.java | 22 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 44 +-
.../TestHoodieClientOnMergeOnReadStorage.java | 95 +--
.../TestSavepointRestoreCopyOnWrite.java | 173 ++++
.../TestSavepointRestoreMergeOnRead.java | 248 ++++++
.../TestBoundedInMemoryExecutorInSpark.java | 28 +-
.../hudi/execution/TestBoundedInMemoryQueue.java | 23 +-
.../execution/TestDisruptorExecutionInSpark.java | 52 +-
.../hudi/execution/TestDisruptorMessageQueue.java | 34 +-
.../hudi/execution/TestSimpleExecutionInSpark.java | 51 +-
.../TestBulkInsertInternalPartitionerForRows.java | 12 +-
.../bloom/TestBucketizedBloomCheckPartitioner.java | 43 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 10 +-
.../index/bloom/TestHoodieGlobalBloomIndex.java | 10 +-
.../hudi/index/bloom/TestKeyRangeLookupTree.java | 2 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 27 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 6 +
...TestSparkBuildClusteringGroupsForPartition.java | 93 +++
.../TestSparkClusteringPlanPartitionFilter.java | 29 +
.../table/action/compact/TestHoodieCompactor.java | 118 ++-
.../hudi/testutils/FunctionalTestHarness.java | 4 +-
.../hudi/testutils/HoodieClientTestBase.java | 64 +-
.../hudi/testutils/HoodieClientTestHarness.java | 9 +-
.../hudi/testutils/HoodieClientTestUtils.java | 29 +-
.../SparkClientFunctionalTestHarness.java | 4 +-
.../hudi/testutils/providers/SparkProvider.java | 1 +
.../spark/sql/TestHoodieUnsafeRowUtils.scala | 36 +-
hudi-client/pom.xml | 2 +-
hudi-common/pom.xml | 4 +-
hudi-common/src/main/avro/HoodieRestorePlan.avsc | 10 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 5 +-
.../apache/hudi/avro/AvroSchemaCompatibility.java | 15 +-
.../java/org/apache/hudi/avro/AvroSchemaUtils.java | 32 +-
.../org/apache/hudi/common/HoodieJsonPayload.java | 4 +-
.../bootstrap/index/HFileBootstrapIndex.java | 10 +-
.../hudi/common/config/ConfigClassProperty.java | 6 +-
.../apache/hudi/common/config/ConfigGroups.java | 60 +-
.../apache/hudi/common/config/ConfigProperty.java | 47 +-
.../common/config/DFSPropertiesConfiguration.java | 27 +-
.../apache/hudi/common/config/HoodieConfig.java | 33 +-
.../hudi/common/config/HoodieMetadataConfig.java | 28 +-
.../hudi/common/config/HoodieMetaserverConfig.java | 117 +++
.../hudi/common/config/HoodieMetastoreConfig.java | 97 ---
.../config/HoodieTableServiceManagerConfig.java | 183 +++++
.../common/config/SerializableConfiguration.java | 1 +
.../DirectMarkerBasedDetectionStrategy.java | 111 +++
.../detection/EarlyConflictDetectionStrategy.java | 48 ++
.../TimelineServerBasedDetectionStrategy.java | 64 ++
.../org/apache/hudi/common/data/HoodieData.java | 17 +-
.../apache/hudi/common/data/HoodieListData.java | 10 +
.../hudi/common/data/HoodieListPairData.java | 23 +
.../apache/hudi/common/data/HoodiePairData.java | 11 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 11 +-
.../common/fs/HoodieRetryWrapperFileSystem.java | 71 +-
.../org/apache/hudi/common/fs/StorageSchemes.java | 67 +-
.../ThrowingConsumer.java} | 18 +-
.../common/heartbeat/HoodieHeartbeatUtils.java | 75 ++
.../common/model/DefaultHoodieRecordPayload.java | 36 +-
.../org/apache/hudi/common/model/DeleteRecord.java | 18 +-
.../hudi/common/model/HoodieAvroIndexedRecord.java | 34 +-
.../apache/hudi/common/model/HoodieAvroRecord.java | 42 +-
.../hudi/common/model/HoodieAvroRecordMerger.java | 15 +-
.../hudi/common/model/HoodieEmptyRecord.java | 9 +-
.../org/apache/hudi/common/model/HoodieKey.java | 28 +-
.../apache/hudi/common/model/HoodieLogFile.java | 14 +-
.../org/apache/hudi/common/model/HoodieRecord.java | 20 +-
.../apache/hudi/common/model/MetadataValues.java | 76 +-
.../debezium/AbstractDebeziumAvroPayload.java | 14 +-
.../hudi/common/table/HoodieTableConfig.java | 53 +-
.../hudi/common/table/HoodieTableMetaClient.java | 134 ++--
.../hudi/common/table/TableSchemaResolver.java | 36 -
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 20 +-
.../hudi/common/table/cdc/HoodieCDCFileSplit.java | 22 +-
...CInferCase.java => HoodieCDCInferenceCase.java} | 8 +-
.../cdc/HoodieCDCSupplementalLoggingMode.java | 37 +-
.../hudi/common/table/cdc/HoodieCDCUtils.java | 14 +-
.../table/log/AbstractHoodieLogRecordReader.java | 252 +++---
.../table/log/HoodieCDCLogRecordIterator.java | 6 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 22 +-
.../table/log/HoodieMergedLogRecordScanner.java | 209 +++--
.../table/log/HoodieUnMergedLogRecordScanner.java | 23 +-
.../table/log/block/HoodieAvroDataBlock.java | 9 +-
.../common/table/log/block/HoodieDataBlock.java | 2 +-
.../common/table/log/block/HoodieDeleteBlock.java | 2 +
.../table/log/block/HoodieHFileDataBlock.java | 6 +-
.../table/log/block/HoodieParquetDataBlock.java | 5 +-
.../hudi/common/table/marker/MarkerOperation.java | 1 +
.../table/timeline/HoodieActiveTimeline.java | 2 +-
.../table/timeline/HoodieArchivedTimeline.java | 2 +-
.../hudi/common/table/timeline/TimelineUtils.java | 92 ++-
.../table/view/AbstractTableFileSystemView.java | 199 ++++-
.../common/table/view/FileSystemViewManager.java | 18 +-
.../table/view/FileSystemViewStorageConfig.java | 1 +
.../table/view/HoodieTableFileSystemView.java | 3 +-
.../table/view/PriorityBasedFileSystemView.java | 7 +
.../view/RemoteHoodieTableFileSystemView.java | 47 +-
.../common/table/view/TableFileSystemView.java | 11 +
.../org/apache/hudi/common/util/BaseFileUtils.java | 1 +
.../hudi/common/util/BufferedRandomAccessFile.java | 2 +-
.../apache/hudi/common/util/ClusteringUtils.java | 37 +
.../apache/hudi/common/util/CollectionUtils.java | 24 +-
...rovider.java => HoodieCommonKryoRegistrar.java} | 13 +-
.../apache/hudi/common/util/HoodieRecordUtils.java | 4 +-
.../org/apache/hudi/common/util/MarkerUtils.java | 116 ++-
.../apache/hudi/common/util/OrcReaderIterator.java | 1 +
.../java/org/apache/hudi/common/util/OrcUtils.java | 1 +
.../hudi/common/util/ParquetReaderIterator.java | 1 +
.../org/apache/hudi/common/util/ParquetUtils.java | 35 +-
.../apache/hudi/common/util/ReflectionUtils.java | 34 +
.../org/apache/hudi/common/util/RetryHelper.java | 32 +-
.../hudi/common/util/SerializationUtils.java | 4 +-
.../org/apache/hudi/common/util/StringUtils.java | 16 +
.../apache/hudi/common/util/TablePathUtils.java | 4 +
.../apache/hudi/common/util/ValidationUtils.java | 9 +
.../common/util/collection/BitCaskDiskMap.java | 4 +-
.../util/{ => collection}/ClosableIterator.java | 23 +-
.../util/collection/CloseableMappingIterator.java | 9 +-
.../util/collection/ExternalSpillableMap.java | 11 +-
.../common/util/collection/FlatteningIterator.java | 55 ++
.../common/util/collection/LazyFileIterable.java | 1 -
.../common/util/collection/MappingIterator.java | 4 +-
.../util/queue/BaseHoodieQueueBasedExecutor.java | 3 +
.../common/util/queue/BoundedInMemoryExecutor.java | 2 +-
.../hudi/common/util/queue/DisruptorExecutor.java | 28 +-
.../common/util/queue/DisruptorMessageQueue.java | 49 +-
.../hudi/common/util/queue/ExecutorType.java | 15 +-
.../util/queue/IteratorBasedQueueProducer.java | 2 +-
...mpleHoodieExecutor.java => SimpleExecutor.java} | 51 +-
.../common/util/queue/WaitStrategyFactory.java | 8 +-
.../HoodieEarlyConflictDetectionException.java} | 35 +-
.../hudi/internal/schema/InternalSchema.java | 27 +-
.../hudi/internal/schema/action/TableChange.java | 6 +
.../hudi/internal/schema/action/TableChanges.java | 23 +-
.../schema/utils/AvroSchemaEvolutionUtils.java | 66 +-
.../internal/schema/utils/InternalSchemaUtils.java | 8 +-
.../internal/schema/visitor/NameToIDVisitor.java | 2 +-
.../hudi/io/storage/HoodieAvroFileReaderBase.java | 6 +-
.../hudi/io/storage/HoodieAvroHFileReader.java | 8 +-
.../hudi/io/storage/HoodieAvroOrcReader.java | 2 +-
.../hudi/io/storage/HoodieAvroParquetReader.java | 10 +-
.../apache/hudi/io/storage/HoodieFileReader.java | 2 +-
.../hudi/io/storage/HoodieSeekingFileReader.java | 2 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 53 +-
.../metadata/FileSystemBackedTableMetadata.java | 15 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 161 ++--
.../metadata/HoodieMetadataFileSystemView.java | 30 +-
.../metadata/HoodieMetadataLogRecordReader.java | 238 ++++++
.../HoodieMetadataMergedLogRecordReader.java | 254 ------
.../hudi/metadata/HoodieMetadataPayload.java | 7 +-
.../apache/hudi/metadata/HoodieTableMetadata.java | 10 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 24 +-
.../hudi/parquet/io/ByteBufferBackedInputFile.java | 4 -
.../main/java/org/apache/hudi/util/Transient.java | 108 +++
.../avro/HoodieAvroParquetReaderBuilder.java | 79 ++
.../apache/parquet/avro/HoodieAvroReadSupport.java | 133 ++++
.../hudi/common/config/HoodieTestFakeConfig.java | 96 +++
.../hudi/common/config/TestConfigProperty.java | 44 +-
.../hudi/common/config/TestHoodieConfig.java | 70 ++
.../org/apache/hudi/common/fs/TestFSUtils.java | 24 +
.../common/functional/TestHoodieLogFormat.java | 457 +++++++++--
.../TestHoodieLogFormatAppendFailure.java | 4 +-
.../model/TestDefaultHoodieRecordPayload.java | 65 ++
.../apache/hudi/common/model/TestHoodieRecord.java | 3 +-
.../debezium/TestPostgresDebeziumAvroPayload.java | 19 +
.../hudi/common/table/TestTimelineUtils.java | 184 ++++-
.../table/view/TestHoodieTableFileSystemView.java | 4 +-
.../common/testutils/HoodieTestDataGenerator.java | 9 +-
.../hudi/common/testutils/SchemaTestUtil.java | 25 +-
.../hudi/common/util/TestClusteringUtils.java | 69 ++
.../util/TestDFSPropertiesConfiguration.java | 4 +-
.../hudi/common/util/TestReflectionUtils.java | 45 ++
.../apache/hudi/common/util/TestRetryHelper.java | 52 ++
.../common/util/collection/TestBitCaskDiskMap.java | 26 +-
.../util/collection/TestExternalSpillableMap.java | 27 +-
.../hudi/common/util/collection/TestIterators.java | 48 ++
.../util/collection/TestRocksDbBasedMap.java | 3 +-
.../common/util/collection/TestRocksDbDiskMap.java | 14 +-
hudi-examples/bin/hudi-delta-streamer | 2 +
hudi-examples/hudi-examples-common/pom.xml | 8 +-
hudi-examples/hudi-examples-flink/pom.xml | 2 +-
.../quickstart/source/ContinuousFileSource.java | 18 +-
hudi-examples/hudi-examples-java/pom.xml | 8 +-
hudi-examples/hudi-examples-spark/pom.xml | 2 +-
.../examples/common/HoodieExampleSparkUtils.java | 3 +-
.../examples/quickstart/HoodieSparkQuickstart.java | 18 +-
.../examples/spark/HoodieDataSourceExample.scala | 54 +-
.../examples/spark/HoodieMorCompactionJob.scala | 4 +-
.../quickstart/TestHoodieSparkQuickstart.java | 4 +-
.../src/test/python/HoodiePySparkQuickstart.py | 1 +
hudi-examples/pom.xml | 2 +-
hudi-flink-datasource/hudi-flink/pom.xml | 28 +-
.../apache/hudi/configuration/FlinkOptions.java | 109 ++-
.../apache/hudi/configuration/OptionsResolver.java | 60 +-
.../org/apache/hudi/sink/StreamWriteFunction.java | 10 +-
.../hudi/sink/StreamWriteOperatorCoordinator.java | 26 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 11 +-
.../sink/bucket/BucketStreamWriteFunction.java | 12 +-
.../hudi/sink/bulk/BulkInsertWriteFunction.java | 1 -
.../hudi/sink/bulk/BulkInsertWriterHelper.java | 5 +-
.../org/apache/hudi/sink/bulk/RowDataKeyGen.java | 52 +-
.../hudi/sink/bulk/sort/SortOperatorGen.java | 4 +-
.../hudi/sink/clustering/ClusteringCommitSink.java | 11 +
.../hudi/sink/clustering/ClusteringOperator.java | 17 +-
.../sink/clustering/FlinkClusteringConfig.java | 55 +-
.../sink/clustering/HoodieFlinkClusteringJob.java | 16 +-
.../sink/common/AbstractStreamWriteFunction.java | 2 +-
.../apache/hudi/sink/compact/CompactOperator.java | 7 +-
.../hudi/sink/compact/CompactionCommitSink.java | 11 +
.../hudi/sink/compact/FlinkCompactionConfig.java | 34 +-
.../hudi/sink/compact/HoodieFlinkCompactor.java | 13 +-
.../org/apache/hudi/sink/meta/CkpMetadata.java | 28 +-
.../sink/partitioner/BucketIndexPartitioner.java | 5 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 18 +-
.../apache/hudi/source/IncrementalInputSplits.java | 180 +++--
.../hudi/source/stats/ExpressionEvaluator.java | 107 ++-
.../apache/hudi/streamer/FlinkStreamerConfig.java | 84 +-
.../apache/hudi/streamer/HoodieFlinkStreamer.java | 21 +-
.../org/apache/hudi/table/HoodieTableFactory.java | 72 +-
.../org/apache/hudi/table/HoodieTableSource.java | 38 +-
.../apache/hudi/table/catalog/HiveSchemaUtils.java | 8 +-
.../apache/hudi/table/catalog/HoodieCatalog.java | 42 +-
.../hudi/table/catalog/HoodieHiveCatalog.java | 38 +-
.../hudi/table/catalog/TableOptionProperties.java | 25 +-
.../hudi/table/format/InternalSchemaManager.java | 83 +-
.../table/format/ParquetSplitRecordIterator.java | 2 +-
.../apache/hudi/table/format/RecordIterators.java | 10 +-
.../table/format/SchemaEvolvedRecordIterator.java | 2 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 8 +-
.../table/format/cow/CopyOnWriteInputFormat.java | 2 +-
.../table/format/mor/MergeOnReadInputFormat.java | 4 +-
.../java/org/apache/hudi/util/ClusteringUtil.java | 4 +-
.../org/apache/hudi/util/FlinkWriteClients.java | 26 +-
.../hudi/util/JsonDeserializationFunction.java | 73 ++
.../java/org/apache/hudi/util/StreamerUtil.java | 35 +-
.../apache/hudi/util/ViewStorageProperties.java | 15 +-
.../apache/hudi/sink/ITTestDataStreamWrite.java | 53 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 16 +
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 80 +-
.../org/apache/hudi/sink/TestWriteMergeOnRead.java | 10 +
.../hudi/sink/TestWriteMergeOnReadWithCompact.java | 11 +
.../apache/hudi/sink/bulk/TestRowDataKeyGen.java | 43 +
.../sink/cluster/ITTestHoodieFlinkClustering.java | 3 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 3 +-
.../org/apache/hudi/sink/meta/TestCkpMetadata.java | 21 +-
.../utils/BucketStreamWriteFunctionWrapper.java | 212 +++++
.../org/apache/hudi/sink/utils/TestWriteBase.java | 68 +-
.../hudi/source/TestIncrementalInputSplits.java | 51 ++
.../hudi/source/stats/TestExpressionEvaluator.java | 43 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 170 +++-
.../apache/hudi/table/TestHoodieTableFactory.java | 122 ++-
.../hudi/table/catalog/TestHoodieCatalog.java | 43 +-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 33 +
.../apache/hudi/table/format/TestInputFormat.java | 100 ++-
.../test/java/org/apache/hudi/utils/TestData.java | 72 ++
.../hudi/utils/TestViewStorageProperties.java | 10 +-
.../utils/factory/CollectSinkTableFactory.java | 11 +-
.../hudi/utils/source/ContinuousFileSource.java | 18 +-
hudi-flink-datasource/hudi-flink1.13.x/pom.xml | 16 +-
.../hudi/adapter/OperatorCoordinatorAdapter.java | 29 +-
.../hudi/adapter/SortCodeGeneratorAdapter.java | 33 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 2 +-
.../vector/reader/FixedLenBytesColumnReader.java | 2 +-
.../reader/ParquetColumnarRowSplitReader.java | 3 +-
hudi-flink-datasource/hudi-flink1.14.x/pom.xml | 16 +-
.../hudi/adapter/OperatorCoordinatorAdapter.java | 13 +-
.../hudi/adapter/SortCodeGeneratorAdapter.java | 33 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 2 +-
.../vector/reader/FixedLenBytesColumnReader.java | 2 +-
.../reader/ParquetColumnarRowSplitReader.java | 3 +-
hudi-flink-datasource/hudi-flink1.15.x/pom.xml | 16 +-
.../hudi/adapter/OperatorCoordinatorAdapter.java | 13 +-
.../hudi/adapter/SortCodeGeneratorAdapter.java | 33 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 2 +-
.../vector/reader/FixedLenBytesColumnReader.java | 2 +-
.../reader/ParquetColumnarRowSplitReader.java | 3 +-
.../{hudi-flink1.15.x => hudi-flink1.16.x}/pom.xml | 34 +-
.../adapter/AbstractStreamOperatorAdapter.java | 0
.../AbstractStreamOperatorFactoryAdapter.java | 0
.../adapter/DataStreamScanProviderAdapter.java | 0
.../adapter/DataStreamSinkProviderAdapter.java | 0
.../hudi/adapter/MailboxExecutorAdapter.java | 0
.../hudi/adapter/OperatorCoordinatorAdapter.java | 50 ++
.../apache/hudi/adapter/RateLimiterAdapter.java | 0
.../hudi/adapter/SortCodeGeneratorAdapter.java | 36 +-
.../main/java/org/apache/hudi/adapter/Utils.java | 0
.../table/format/cow/ParquetSplitReaderUtil.java | 2 +-
.../table/format/cow/vector/HeapArrayVector.java | 0
.../format/cow/vector/HeapMapColumnVector.java | 0
.../format/cow/vector/HeapRowColumnVector.java | 0
.../format/cow/vector/ParquetDecimalVector.java | 0
.../cow/vector/reader/AbstractColumnReader.java | 0
.../cow/vector/reader/ArrayColumnReader.java | 0
.../vector/reader/BaseVectorizedColumnReader.java | 0
.../vector/reader/FixedLenBytesColumnReader.java | 2 +-
.../vector/reader/Int64TimestampColumnReader.java | 0
.../format/cow/vector/reader/MapColumnReader.java | 0
.../reader/ParquetColumnarRowSplitReader.java | 0
.../cow/vector/reader/ParquetDataColumnReader.java | 0
.../reader/ParquetDataColumnReaderFactory.java | 0
.../format/cow/vector/reader/RowColumnReader.java | 0
.../format/cow/vector/reader/RunLengthDecoder.java | 0
.../org/apache/hudi/adapter/OutputAdapter.java | 0
.../adapter/StateInitializationContextAdapter.java | 0
.../adapter/StreamingRuntimeContextAdapter.java | 0
.../org/apache/hudi/adapter/TestTableEnvs.java | 0
hudi-flink-datasource/pom.xml | 5 +-
hudi-gcp/pom.xml | 2 +-
.../hudi/gcp/bigquery/BigQuerySyncConfig.java | 46 +-
.../gcp/bigquery/TestBigQuerySyncToolArgs.java | 70 ++
hudi-hadoop-mr/pom.xml | 8 +-
.../hadoop/HoodieCopyOnWriteTableInputFormat.java | 142 +---
.../HoodieMergeOnReadTableInputFormat.java | 42 +-
.../realtime/RealtimeCompactedRecordReader.java | 2 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 2 +-
.../apache/hudi/hadoop/TestInputPathHandler.java | 2 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 1 +
hudi-integ-test/README.md | 18 +-
hudi-integ-test/pom.xml | 2 +-
.../reader/DFSHoodieDatasetInputReader.java | 8 +-
.../TestDFSHoodieTestSuiteWriterAdapter.java | 2 +-
.../integ/testsuite/TestFileDeltaInputWriter.java | 2 +-
.../testsuite/job/TestHoodieTestSuiteJob.java | 2 +-
.../reader/TestDFSAvroDeltaInputReader.java | 2 +-
.../reader/TestDFSHoodieDatasetInputReader.java | 2 +-
hudi-kafka-connect/pom.xml | 11 +-
hudi-platform-service/hudi-metaserver/README.md | 91 +++
.../hudi-metaserver/hudi-metaserver-client/pom.xml | 78 ++
.../common/table/HoodieTableMetaserverClient.java | 162 ++++
.../timeline/HoodieMetaserverBasedTimeline.java | 85 ++
.../table/view/HoodieMetaserverFileSystemView.java | 44 ++
.../metaserver/client/HoodieMetaserverClient.java | 55 ++
.../client/HoodieMetaserverClientImp.java | 188 +++++
.../client/HoodieMetaserverClientProxy.java | 61 ++
.../hudi/metaserver/util/EntityConversions.java | 110 +++
.../client/TestHoodieMetaserverClient.java | 51 ++
.../bin/start_hudi_metaserver.sh | 26 +
.../hudi-metaserver/hudi-metaserver-server/pom.xml | 150 ++++
.../apache/hudi/metaserver/HoodieMetaserver.java | 127 +++
.../metaserver/HoodieMetaserverPreparations.java | 25 +-
.../service/HoodieMetaserverGateway.java | 91 +++
.../service/HoodieMetaserverProxyHandler.java | 63 ++
.../hudi/metaserver/service/TableService.java | 83 ++
.../hudi/metaserver/service/TimelineService.java | 155 ++++
.../hudi/metaserver/store/MetaserverStorage.java | 75 ++
.../metaserver/store/RelationalDBBasedStorage.java | 236 ++++++
.../hudi/metaserver/store/bean/InstantBean.java | 104 +++
.../hudi/metaserver/store/bean/TableBean.java | 135 ++++
.../hudi/metaserver/store/jdbc/BasicDao.java | 91 +++
.../metaserver/store/jdbc/BatchDaoOperation.java | 70 ++
.../store/jdbc/HikariDataSourceFactory.java | 23 +-
.../store/jdbc/SqlSessionFactoryUtils.java | 61 ++
.../hudi/metaserver/store/jdbc/WrapperDao.java | 113 +++
.../hudi/metaserver/util/MetaserverTableUtils.java | 23 +-
.../hudi/metaserver/util/TServerSocketWrapper.java | 51 ++
.../src/main/resources/hikariPool.properties | 16 +-
.../src/main/resources/mybatis-config.xml | 46 ++
.../src/main/resources/mybatis/DDLMapper.xml | 126 +++
.../src/main/resources/mybatis/TableMapper.xml | 60 ++
.../src/main/resources/mybatis/TimelineMapper.xml | 146 ++++
.../hudi/metaserver/TestHoodieMetaserver.java | 20 +-
.../store/TestRelationalDBBasedStore.java | 112 +++
hudi-platform-service/hudi-metaserver/pom.xml | 217 +++++
.../src/main/thrift/bin/thrift_binary.sh | 31 +
.../src/main/thrift/bin/thrift_in_docker.sh | 31 +
.../src/main/thrift/bin/thrift_in_mac_m1.sh | 34 +
.../src/main/thrift/hudi-metaserver.thrift | 100 +++
.../pom.xml | 8 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 4 +-
.../scala/org/apache/hudi/AvroProjection.scala | 40 +-
.../org/apache/hudi/BaseFileOnlyRelation.scala | 46 +-
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 35 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 26 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 27 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 261 +++---
.../scala/org/apache/hudi/HoodieBootstrapRDD.scala | 103 ++-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 259 +++---
.../scala/org/apache/hudi/HoodieFileIndex.scala | 25 +-
.../org/apache/hudi/HoodieMergeOnReadRDD.scala | 65 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 292 ++++---
.../org/apache/hudi/HoodieStreamingSink.scala | 26 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 5 +-
.../src/main/scala/org/apache/hudi/Iterators.scala | 109 +--
.../hudi/MergeOnReadIncrementalRelation.scala | 20 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 72 +-
.../scala/org/apache/hudi/SafeAvroProjection.scala | 73 --
.../apache/hudi/SparkHoodieTableFileIndex.scala | 6 +-
.../scala/org/apache/hudi/cdc/CDCRelation.scala | 21 +-
.../scala/org/apache/hudi/cdc/HoodieCDCRDD.scala | 131 ++--
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 20 +-
.../plans/logical/HoodieUnaryLikeSham.scala | 5 +-
.../parquet/HoodieParquetFileFormatHelper.scala | 72 ++
.../apache/spark/sql/hudi/HoodieOptionConfig.scala | 77 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 78 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 352 +++++----
.../AlterHoodieTableChangeColumnCommand.scala | 2 +-
.../hudi/command/HoodieLeafRunnableCommand.scala | 16 +-
.../sql/hudi/streaming/HoodieStreamSource.scala | 2 +-
hudi-spark-datasource/hudi-spark/pom.xml | 4 +-
.../org/apache/hudi/cli/ArchiveExecutorUtils.java | 69 ++
.../apache/hudi/cli/BootstrapExecutorUtils.java | 89 ++-
.../apache/hudi/cli/HDFSParquetImporterUtils.java | 2 +-
.../sql/hudi/HoodieSparkSessionExtension.scala | 15 +-
.../org/apache/spark/sql/hudi/HoodieSqlUtils.scala | 51 --
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 765 ++++++++----------
.../analysis/HoodiePruneFileSourcePartitions.scala | 124 +++
.../command/CreateHoodieTableAsSelectCommand.scala | 7 +-
.../hudi/command/DeleteHoodieTableCommand.scala | 47 +-
.../spark/sql/hudi/command/IndexCommands.scala | 43 +-
.../command/InsertIntoHoodieTableCommand.scala | 32 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 850 +++++++++++---------
.../hudi/command/UpdateHoodieTableCommand.scala | 78 +-
.../hudi/command/payload/ExpressionPayload.scala | 152 +++-
.../procedures/ArchiveCommitsProcedure.scala | 79 ++
.../command/procedures/DeleteMarkerProcedure.scala | 8 +-
.../command/procedures/HiveSyncProcedure.scala | 8 +-
.../hudi/command/procedures/HoodieProcedures.scala | 1 +
.../RollbackToInstantTimeProcedure.scala | 50 +-
.../command/procedures/RunBootstrapProcedure.scala | 3 +
.../command/procedures/RunCleanProcedure.scala | 27 +-
.../procedures/RunClusteringProcedure.scala | 31 +-
.../procedures/RunCompactionProcedure.scala | 126 +--
.../procedures/ShowCompactionProcedure.scala | 2 +-
.../ShowHoodieLogFileRecordsProcedure.scala | 2 +-
.../procedures/ShowRollbacksProcedure.scala | 17 +-
.../spark/sql/parser/HoodieCommonSqlParser.scala | 25 +-
.../hudi-spark/src/test/java/HoodieJavaApp.java | 7 +-
.../src/test/java/HoodieJavaGenerateApp.java | 8 +-
.../src/test/java/HoodieJavaStreamingApp.java | 15 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 7 +-
.../java/org/apache/hudi/TestDataSourceUtils.java | 29 +
.../functional/SparkRDDWriteClientOverride.java | 38 -
.../org/apache/hudi/functional/TestBootstrap.java | 17 +-
.../functional/TestHiveTableSchemaEvolution.java | 20 +-
.../TestHoodieDatasetBulkInsertHelper.java | 4 +-
.../apache/hudi/functional/TestOrcBootstrap.java | 24 +-
.../keygen/TestTimestampBasedKeyGenerator.java | 6 +-
.../hudi/testutils/HoodieSparkClientTestBase.java | 20 +-
.../src/test/resources/sql-statements.sql | 8 +-
.../hudi/TestAvroSchemaResolutionSupport.scala | 811 +++++++++++++++++++
.../org/apache/hudi/TestDataSkippingUtils.scala | 6 +-
.../hudi/TestGenericRecordAndRowConsistency.scala | 4 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 146 +++-
.../apache/hudi/TestHoodieInternalRowUtils.scala | 89 ---
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 63 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 4 +
.../hudi/TestTableSchemaResolverWithSparkSQL.scala | 6 +-
.../hudi/functional/TestBasicSchemaEvolution.scala | 25 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 9 +-
.../hudi/functional/TestCOWDataSourceStorage.scala | 10 +-
.../hudi/functional/TestColumnStatsIndex.scala | 176 ++++-
.../functional/TestDataSourceForBootstrap.scala | 237 +++---
.../apache/hudi/functional/TestEmptyCommit.scala | 4 +-
.../hudi/functional/TestHoodieActiveTimeline.scala | 4 +-
.../TestIncrementalReadWithFullTableScan.scala | 4 +-
.../hudi/functional/TestLayoutOptimization.scala | 7 +-
.../apache/hudi/functional/TestMORDataSource.scala | 66 +-
.../TestMORDataSourceWithBucketIndex.scala | 4 +-
.../functional/TestParquetColumnProjection.scala | 17 +-
.../functional/TestPartialUpdateAvroPayload.scala | 125 +++
.../hudi/functional/TestStreamingSource.scala | 5 +-
.../hudi/functional/TestStructuredStreaming.scala | 141 ++--
.../hudi/functional/TestTimeTravelQuery.scala | 4 +-
.../hudi/functional/cdc/HoodieCDCTestBase.scala | 36 +-
.../functional/cdc/TestCDCDataFrameSuite.scala | 51 +-
.../functional/cdc/TestCDCStreamingSuite.scala | 16 +-
.../benchmark/BoundInMemoryExecutorBenchmark.scala | 1 +
.../benchmark/CowTableReadBenchmark.scala | 1 +
.../ReadAndWriteWithoutAvroBenchmark.scala | 7 +-
.../spark/sql/hudi/HoodieSparkSqlTestBase.scala | 51 +-
.../sql/hudi/TestAlterTableDropPartition.scala | 129 ++-
.../apache/spark/sql/hudi/TestCDCForSparkSQL.scala | 27 +-
.../spark/sql/hudi/TestCompactionTable.scala | 6 +-
.../apache/spark/sql/hudi/TestCreateTable.scala | 48 +-
.../org/apache/spark/sql/hudi/TestDropTable.scala | 11 +
.../sql/hudi/TestHoodieInternalRowUtils.scala} | 99 ++-
.../spark/sql/hudi/TestHoodieOptionConfig.scala | 6 +-
.../apache/spark/sql/hudi/TestInsertTable.scala | 95 ++-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 128 ++-
.../spark/sql/hudi/TestMergeIntoTable2.scala | 55 +-
.../hudi/TestNestedSchemaPruningOptimization.scala | 130 ++-
.../sql/hudi/TestPartialUpdateForMergeInto.scala | 23 +-
.../org/apache/spark/sql/hudi/TestSpark3DDL.scala | 2 +-
.../org/apache/spark/sql/hudi/TestSqlConf.scala | 2 +-
.../TestHoodiePruneFileSourcePartitions.scala | 240 ++++++
.../sql/hudi/command/index/TestIndexSyntax.scala | 8 +-
.../procedure/TestArchiveCommitsProcedure.scala | 71 ++
.../hudi/procedure/TestBootstrapProcedure.scala | 53 +-
.../hudi/procedure/TestClusteringProcedure.scala | 14 +-
.../hudi/procedure/TestCompactionProcedure.scala | 38 +-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 5 +-
hudi-spark-datasource/hudi-spark2-common/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 4 +-
.../internal/HoodieDataSourceInternalWriter.java | 4 +-
...Spark24HoodieVectorizedParquetRecordReader.java | 185 +++++
.../sql/HoodieSpark2CatalystExpressionUtils.scala | 60 +-
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 35 +-
.../apache/spark/sql/adapter/Spark2Adapter.scala | 48 +-
.../catalyst/analysis/HoodieSpark2Analysis.scala | 208 +++++
.../sql/catalyst/plans/logical/MergeInto.scala | 7 +
.../parquet/Spark24HoodieParquetFileFormat.scala | 62 +-
.../parser/HoodieSpark2ExtendedSqlParser.scala | 3 +-
hudi-spark-datasource/hudi-spark3-common/pom.xml | 2 +-
.../sql/HoodieSpark3CatalystExpressionUtils.scala} | 32 +-
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 57 +-
.../spark/sql/adapter/BaseSpark3Adapter.scala | 49 +-
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 4 +-
.../sql/HoodieSpark31CatalystExpressionUtils.scala | 10 +-
.../spark/sql/HoodieSpark31CatalystPlanUtils.scala | 14 +-
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 42 +-
.../parquet/Spark31HoodieParquetFileFormat.scala | 12 +-
...lder.scala => HoodieSpark31SqlAstBuilder.scala} | 2 +-
.../parser/HoodieSpark3_1ExtendedSqlParser.scala | 10 +-
hudi-spark-datasource/hudi-spark3.2.x/pom.xml | 4 +-
.../sql/HoodieSpark32CatalystExpressionUtils.scala | 11 +-
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 16 +-
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 28 +-
.../HoodieSpark3_2ExtendedSqlAstBuilder.scala | 3 +-
.../parser/HoodieSpark3_2ExtendedSqlParser.scala | 2 +-
.../hudi-spark3.2plus-common/pom.xml | 2 +-
.../sql/catalyst/plans/logcal/HoodieQuery.scala | 46 +-
.../Spark32PlusHoodieParquetFileFormat.scala | 10 +-
.../hudi/analysis/HoodieSpark32PlusAnalysis.scala | 167 ++++
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 175 -----
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 6 +-
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 31 +-
.../sql/hudi/logical/TimeTravelRelation.scala | 16 +-
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 4 +-
.../sql/HoodieSpark33CatalystExpressionUtils.scala | 9 +-
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 16 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 39 +-
.../parser/HoodieSpark3_3ExtendedSqlParser.scala | 6 +-
hudi-spark-datasource/pom.xml | 2 +-
hudi-sync/hudi-adb-sync/pom.xml | 2 +-
hudi-sync/hudi-datahub-sync/pom.xml | 2 +-
.../sync/datahub/config/DataHubSyncConfig.java | 24 +-
hudi-sync/hudi-hive-sync/pom.xml | 2 +-
.../java/org/apache/hudi/hive/HiveSyncConfig.java | 15 +-
.../org/apache/hudi/hive/HiveSyncConfigHolder.java | 7 +-
.../java/org/apache/hudi/hive/HiveSyncTool.java | 17 +-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 7 +
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 13 +-
.../apache/hudi/hive/ddl/HiveQueryDDLExecutor.java | 11 +-
.../hive/replication/GlobalHiveSyncConfig.java | 8 +
.../org/apache/hudi/hive/TestHiveSyncTool.java | 139 ++++
.../apache/hudi/hive/testutils/HiveTestUtil.java | 10 +-
hudi-sync/hudi-sync-common/pom.xml | 8 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 12 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 9 +
.../apache/hudi/sync/common/util/ConfigUtils.java | 6 +
.../common/util/SparkDataSourceTableUtils.java | 3 +-
hudi-sync/pom.xml | 2 +-
hudi-tests-common/pom.xml | 2 +-
hudi-timeline-service/pom.xml | 8 +-
.../hudi/timeline/service/RequestHandler.java | 12 +-
.../hudi/timeline/service/TimelineService.java | 81 +-
.../timeline/service/handlers/BaseFileHandler.java | 11 +
.../timeline/service/handlers/MarkerHandler.java | 115 ++-
.../AsyncTimelineServerBasedDetectionStrategy.java | 86 ++
.../MarkerBasedEarlyConflictDetectionRunnable.java | 124 +++
.../service/handlers/marker/MarkerDirState.java | 69 +-
.../TestRemoteHoodieTableFileSystemView.java | 4 +-
...tMarkerBasedEarlyConflictDetectionRunnable.java | 133 ++++
hudi-utilities/pom.xml | 6 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 9 +-
.../org/apache/hudi/utilities/HoodieIndexer.java | 4 +-
.../utilities/HoodieMetadataTableValidator.java | 200 +++--
.../hudi/utilities/HoodieSnapshotCopier.java | 5 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 5 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 29 +-
.../HoodieWriteCommitKafkaCallbackConfig.java | 1 +
.../HoodieWriteCommitPulsarCallbackConfig.java | 1 +
.../hudi/utilities/deltastreamer/DeltaSync.java | 22 +-
.../deltastreamer/SourceFormatAdapter.java | 3 +-
.../utilities/schema/SchemaRegistryProvider.java | 92 ++-
.../converter/JsonToAvroSchemaConverter.java | 254 ++++++
.../hudi/utilities/sources/HoodieIncrSource.java | 36 +-
.../sources/S3EventsHoodieIncrSource.java | 3 +-
.../sources/helpers/S3EventsMetaSelector.java | 4 +-
.../sources/helpers/gcs/MetadataMessage.java | 30 +
.../hudi/utilities/TestHiveIncrementalPuller.java | 38 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 170 +++-
.../hudi/utilities/TestHoodieRepairTool.java | 4 +-
.../HoodieDeltaStreamerTestBase.java | 39 +-
.../TestHoodieDeltaStreamer.java | 103 ++-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 21 +-
.../TestHoodieMultiTableDeltaStreamer.java | 19 +-
.../multisync/TestMultipleMetaSync.java | 25 +-
.../schema/TestSchemaRegistryProvider.java | 60 +-
.../converter/TestJsonToAvroSchemaConverter.java | 63 ++
.../hudi/utilities/sources/TestSqlSource.java | 2 +-
.../debezium/TestAbstractDebeziumSource.java | 2 +-
.../sources/helpers/TestS3EventsMetaSelector.java | 5 +-
.../utilities/testutils/UtilitiesTestBase.java | 2 +-
.../AbstractCloudObjectsSourceTestBase.java | 2 +-
.../transform/TestSqlFileBasedTransformer.java | 2 +-
.../short_trip_uber_config.properties | 4 +-
.../json/enum-properties/expected.json | 24 +
.../json/enum-properties/input.json | 19 +
.../json/example-address/expected.json | 39 +
.../json/example-address/input.json | 34 +
.../json/example-calendar/expected.json | 60 ++
.../json/example-calendar/input.json | 47 ++
.../json/example-card/expected.json | 162 ++++
.../schema-provider/json/example-card/input.json | 99 +++
.../example-geographical-location/expected.json | 15 +
.../json/example-geographical-location/input.json | 20 +
.../json/multiple-properties/expected.json | 15 +
.../json/multiple-properties/input.json | 18 +
.../json/nested-properties/expected.json | 70 ++
.../json/nested-properties/input.json | 63 ++
.../json/single-properties/expected.json | 11 +
.../json/single-properties/input.json | 14 +
.../schema-provider/proto/oneof_schema.avsc | 17 +
.../parent_schema_recursive_default_limit.avsc | 17 +
.../proto/parent_schema_recursive_depth_2.avsc | 17 +
.../proto/sample_schema_defaults.avsc | 17 +
...ple_schema_wrapped_and_timestamp_as_record.avsc | 17 +
packaging/bundle-validation/Dockerfile | 2 +-
packaging/bundle-validation/ci_run.sh | 1 +
.../bundle-validation/conf/spark-defaults.conf | 1 +
packaging/bundle-validation/flink/compact.sh | 2 +
.../bundle-validation/service/read.scala | 36 +-
.../{spark_hadoop_mr => service}/write.scala | 14 +-
.../bundle-validation/spark_hadoop_mr/write.scala | 1 -
packaging/bundle-validation/validate.sh | 65 +-
packaging/hudi-aws-bundle/pom.xml | 98 +--
packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh | 27 +-
packaging/hudi-cli-bundle/pom.xml | 61 +-
packaging/hudi-datahub-sync-bundle/pom.xml | 98 +--
packaging/hudi-flink-bundle/pom.xml | 166 ++--
packaging/hudi-gcp-bundle/pom.xml | 106 +--
packaging/hudi-hadoop-mr-bundle/pom.xml | 134 +---
packaging/hudi-hive-sync-bundle/pom.xml | 111 +--
packaging/hudi-integ-test-bundle/pom.xml | 99 +--
packaging/hudi-kafka-connect-bundle/pom.xml | 131 +---
packaging/hudi-metaserver-server-bundle/pom.xml | 152 ++++
packaging/hudi-presto-bundle/pom.xml | 129 +--
packaging/hudi-spark-bundle/pom.xml | 141 +---
packaging/hudi-timeline-server-bundle/pom.xml | 98 +--
packaging/hudi-trino-bundle/pom.xml | 129 +--
packaging/hudi-utilities-bundle/pom.xml | 98 +--
packaging/hudi-utilities-slim-bundle/README.md | 5 +
packaging/hudi-utilities-slim-bundle/pom.xml | 98 +--
pom.xml | 166 +++-
rfc/README.md | 148 ++--
rfc/rfc-41/rfc-41.md | 2 +-
rfc/rfc-46/rfc-46.md | 2 +-
rfc/rfc-48/async_logcompaction_issues.jpeg | Bin 0 -> 67412 bytes
rfc/rfc-48/base_case.jpeg | Bin 0 -> 53991 bytes
...r_logcompaction_written_to_multiple_blocks.jpeg | Bin 0 -> 100262 bytes
rfc/rfc-48/rfc-48.md | 174 ++++
rfc/rfc-48/rollback_block_scenario.jpeg | Bin 0 -> 150939 bytes
rfc/rfc-51/rfc-51.md | 4 +-
rfc/rfc-56/rfc-56.md | 78 +-
rfc/template.md | 2 -
.../release/create_source_directory.sh | 24 +-
scripts/release/create_source_release.sh | 9 +-
scripts/release/cut_release_branch.sh | 20 +-
scripts/release/deploy_staging_jars.sh | 60 +-
.../release/validate_source_binary_files.sh | 23 +-
scripts/release/validate_source_copyright.sh | 55 ++
.../release/validate_source_rat.sh | 19 +-
scripts/release/validate_staged_bundles.sh | 5 +
scripts/release/validate_staged_release.sh | 76 +-
879 files changed, 28929 insertions(+), 10960 deletions(-)
create mode 100644 hudi-cli/src/main/java/org/apache/hudi/cli/commands/RestoresCommand.java
create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRestoresCommand.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTableServiceManagerClient.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/BucketIndexConcurrentFileWritesConflictResolutionStrategy.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/DirectMarkerTransactionManager.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/DeletePartitionUtils.java
rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/{HoodieBaseBloomIndexCheckFunction.java => HoodieBloomIndexCheckFunction.java} (61%)
rename hudi-common/src/main/java/org/apache/hudi/common/util/ClosableIteratorWithSchema.java => hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/ParallelismHelper.java (51%)
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/ConflictDetectionUtils.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleDirectMarkerBasedDetectionStrategy.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/marker/SimpleTransactionDirectMarkerBasedDetectionStrategy.java
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestBucketIndexConcurrentFileWritesConflictResolutionStrategy.java
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/utils/TestDeletePartitionUtils.java
create mode 100644 hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkTableServiceClient.java
create mode 100644 hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestFlinkSizeBasedClusteringPlanStrategy.java
create mode 100644 hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaTableServiceClient.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDTableServiceClient.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomFilterProbingResult.java
delete mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieFileProbingFunction.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieMetadataBloomFilterProbingFunction.java
delete mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieMetadataBloomIndexCheckFunction.java
delete mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/util/HoodieSparkRecordUtils.java
rename hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/{HoodieSparkKryoProvider.scala => HoodieSparkKryoRegistrar.scala} (50%)
copy hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/SerDeUtils.scala => hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/catalyst/expressions/AttributeEq.scala (53%)
copy hudi-cli/src/main/java/org/apache/hudi/cli/Main.java => hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/parser/HoodieExtendedParserInterface.scala (55%)
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSavepoint.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/clustering/plan/strategy/TestSparkSizeBasedClusteringPlanStrategy.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreCopyOnWrite.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestSavepointRestoreMergeOnRead.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/cluster/strategy/TestSparkBuildClusteringGroupsForPartition.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetaserverConfig.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetastoreConfig.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/config/HoodieTableServiceManagerConfig.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/DirectMarkerBasedDetectionStrategy.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/EarlyConflictDetectionStrategy.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/conflict/detection/TimelineServerBasedDetectionStrategy.java
copy hudi-common/src/main/java/org/apache/hudi/common/{util/ClosableIterator.java => function/ThrowingConsumer.java} (73%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/heartbeat/HoodieHeartbeatUtils.java
rename hudi-common/src/main/java/org/apache/hudi/common/table/cdc/{HoodieCDCInferCase.java => HoodieCDCInferenceCase.java} (91%)
rename hudi-common/src/main/java/org/apache/hudi/common/util/{HoodieCommonKryoProvider.java => HoodieCommonKryoRegistrar.java} (93%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/{ => collection}/ClosableIterator.java (70%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/collection/FlatteningIterator.java
rename hudi-common/src/main/java/org/apache/hudi/common/util/queue/{SimpleHoodieExecutor.java => SimpleExecutor.java} (53%)
copy hudi-common/src/main/java/org/apache/hudi/{common/util/collection/MappingIterator.java => exception/HoodieEarlyConflictDetectionException.java} (58%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataLogRecordReader.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMergedLogRecordReader.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/util/Transient.java
create mode 100644 hudi-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetReaderBuilder.java
create mode 100644 hudi-common/src/main/java/org/apache/parquet/avro/HoodieAvroReadSupport.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/config/HoodieTestFakeConfig.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieConfig.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestReflectionUtils.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestRetryHelper.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestIterators.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/JsonDeserializationFunction.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/BucketStreamWriteFunctionWrapper.java
copy hudi-common/src/main/java/org/apache/hudi/common/util/IdentityIterator.java => hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/OperatorCoordinatorAdapter.java (66%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/IdentityIterator.java => hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/hudi/adapter/SortCodeGeneratorAdapter.java (61%)
copy hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanPartitionFilterMode.java => hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/OperatorCoordinatorAdapter.java (73%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/IdentityIterator.java => hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/SortCodeGeneratorAdapter.java (60%)
copy hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/ClusteringPlanPartitionFilterMode.java => hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/OperatorCoordinatorAdapter.java (73%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/IdentityIterator.java => hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/SortCodeGeneratorAdapter.java (60%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/pom.xml (89%)
copy hudi-flink-datasource/{hudi-flink1.14.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.14.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/adapter/AbstractStreamOperatorFactoryAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.14.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/adapter/MailboxExecutorAdapter.java (100%)
create mode 100644 hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/adapter/OperatorCoordinatorAdapter.java
copy hudi-flink-datasource/{hudi-flink1.14.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/adapter/RateLimiterAdapter.java (100%)
rename hudi-common/src/main/java/org/apache/hudi/common/util/MappingIterator.java => hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/adapter/SortCodeGeneratorAdapter.java (56%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/adapter/Utils.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java (99%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java (97%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.13.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.13.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java (100%)
copy hudi-flink-datasource/{hudi-flink1.14.x => hudi-flink1.16.x}/src/test/java/org/apache/hudi/adapter/OutputAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.16.x}/src/test/java/org/apache/hudi/adapter/StateInitializationContextAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.14.x => hudi-flink1.16.x}/src/test/java/org/apache/hudi/adapter/StreamingRuntimeContextAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.14.x => hudi-flink1.16.x}/src/test/java/org/apache/hudi/adapter/TestTableEnvs.java (100%)
create mode 100644 hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java
create mode 100644 hudi-platform-service/hudi-metaserver/README.md
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-client/pom.xml
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/timeline/HoodieMetaserverBasedTimeline.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/view/HoodieMetaserverFileSystemView.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/metaserver/client/HoodieMetaserverClient.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/metaserver/client/HoodieMetaserverClientImp.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/metaserver/client/HoodieMetaserverClientProxy.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/metaserver/util/EntityConversions.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/test/java/org/apache/hudi/metaserver/client/TestHoodieMetaserverClient.java
create mode 100755 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/bin/start_hudi_metaserver.sh
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/HoodieMetaserver.java
copy hudi-cli/src/main/java/org/apache/hudi/cli/Main.java => hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/HoodieMetaserverPreparations.java (54%)
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/HoodieMetaserverGateway.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/HoodieMetaserverProxyHandler.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/TableService.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/service/TimelineService.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/MetaserverStorage.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/RelationalDBBasedStorage.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/bean/InstantBean.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/bean/TableBean.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/jdbc/BasicDao.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/jdbc/BatchDaoOperation.java
copy hudi-cli/src/main/java/org/apache/hudi/cli/Main.java => hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/jdbc/HikariDataSourceFactory.java (54%)
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/jdbc/SqlSessionFactoryUtils.java
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/store/jdbc/WrapperDao.java
copy hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/RunsTableService.java => hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/util/MetaserverTableUtils.java (56%)
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/java/org/apache/hudi/metaserver/util/TServerSocketWrapper.java
copy hudi-utilities/src/test/resources/delta-streamer-config/short_trip_uber_config.properties => hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/hikariPool.properties (53%)
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis-config.xml
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/DDLMapper.xml
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/TableMapper.xml
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/main/resources/mybatis/TimelineMapper.xml
rename hudi-common/src/main/java/org/apache/hudi/common/util/ClosableIterator.java => hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/test/java/org/apache/hudi/metaserver/TestHoodieMetaserver.java (70%)
create mode 100644 hudi-platform-service/hudi-metaserver/hudi-metaserver-server/src/test/java/org/apache/hudi/metaserver/store/TestRelationalDBBasedStore.java
create mode 100644 hudi-platform-service/hudi-metaserver/pom.xml
create mode 100755 hudi-platform-service/hudi-metaserver/src/main/thrift/bin/thrift_binary.sh
create mode 100755 hudi-platform-service/hudi-metaserver/src/main/thrift/bin/thrift_in_docker.sh
create mode 100755 hudi-platform-service/hudi-metaserver/src/main/thrift/bin/thrift_in_mac_m1.sh
create mode 100644 hudi-platform-service/hudi-metaserver/src/main/thrift/hudi-metaserver.thrift
copy {hudi-spark-datasource => hudi-platform-service}/pom.xml (89%)
copy hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieIOHandle.java => hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/AvroProjection.scala (50%)
delete mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SafeAvroProjection.scala
rename hudi-spark-datasource/{hudi-spark3-common => hudi-spark-common}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HoodieUnaryLikeSham.scala (94%)
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieParquetFileFormatHelper.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/cli/ArchiveExecutorUtils.java
delete mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlUtils.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodiePruneFileSourcePartitions.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ArchiveCommitsProcedure.scala
delete mode 100644 hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/SparkRDDWriteClientOverride.java
copy hudi-common/src/main/java/org/apache/hudi/common/util/collection/CloseableMappingIterator.java => hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestBase.java (63%)
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroSchemaResolutionSupport.scala
delete mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieInternalRowUtils.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartialUpdateAvroPayload.scala
rename hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/{hudi/TestStructTypeSchemaEvolutionUtils.scala => spark/sql/hudi/TestHoodieInternalRowUtils.scala} (77%)
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodiePruneFileSourcePartitions.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestArchiveCommitsProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark2/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark24HoodieVectorizedParquetRecordReader.java
create mode 100644 hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/analysis/HoodieSpark2Analysis.scala
rename hudi-spark-datasource/{hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/SerDeUtils.scala => hudi-spark3-common/src/main/scala/org/apache/spark/sql/HoodieSpark3CatalystExpressionUtils.scala} (52%)
rename hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/parser/{HoodieSpark312SqlAstBuilder.scala => HoodieSpark31SqlAstBuilder.scala} (98%)
create mode 100644 hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark32PlusAnalysis.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
create mode 100644 hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/AsyncTimelineServerBasedDetectionStrategy.java
create mode 100644 hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/marker/MarkerBasedEarlyConflictDetectionRunnable.java
create mode 100644 hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/handlers/marker/TestMarkerBasedEarlyConflictDetectionRunnable.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/converter/JsonToAvroSchemaConverter.java
rename hudi-utilities/src/test/java/org/apache/hudi/utilities/{functional => deltastreamer}/HoodieDeltaStreamerTestBase.java (96%)
rename hudi-utilities/src/test/java/org/apache/hudi/utilities/{functional => deltastreamer}/TestHoodieDeltaStreamer.java (98%)
rename hudi-utilities/src/test/java/org/apache/hudi/utilities/{functional => deltastreamer}/TestHoodieDeltaStreamerWithMultiWriter.java (97%)
rename hudi-utilities/src/test/java/org/apache/hudi/utilities/{functional => deltastreamer}/TestHoodieMultiTableDeltaStreamer.java (96%)
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/converter/TestJsonToAvroSchemaConverter.java
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/enum-properties/expected.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/enum-properties/input.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/example-address/expected.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/example-address/input.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/example-calendar/expected.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/example-calendar/input.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/example-card/expected.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/example-card/input.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/example-geographical-location/expected.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/example-geographical-location/input.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/multiple-properties/expected.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/multiple-properties/input.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/nested-properties/expected.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/nested-properties/input.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/single-properties/expected.json
create mode 100644 hudi-utilities/src/test/resources/schema-provider/json/single-properties/input.json
rename hudi-common/src/main/java/org/apache/hudi/common/util/IdentityIterator.java => packaging/bundle-validation/service/read.scala (63%)
copy packaging/bundle-validation/{spark_hadoop_mr => service}/write.scala (78%)
create mode 100644 packaging/hudi-metaserver-server-bundle/pom.xml
create mode 100644 rfc/rfc-48/async_logcompaction_issues.jpeg
create mode 100644 rfc/rfc-48/base_case.jpeg
create mode 100644 rfc/rfc-48/huge_data_after_logcompaction_written_to_multiple_blocks.jpeg
create mode 100644 rfc/rfc-48/rfc-48.md
create mode 100644 rfc/rfc-48/rollback_block_scenario.jpeg
copy packaging/bundle-validation/flink/compact.sh => scripts/release/create_source_directory.sh (65%)
copy packaging/bundle-validation/flink/compact.sh => scripts/release/validate_source_binary_files.sh (51%)
create mode 100755 scripts/release/validate_source_copyright.sh
copy packaging/bundle-validation/flink/compact.sh => scripts/release/validate_source_rat.sh (65%)