You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by gi...@apache.org on 2024/09/19 19:54:24 UTC
(hudi) branch dependabot/maven/org.xerial.snappy-snappy-java-1.1.10.4 updated (43623c0a860 -> 8143527fe0c)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch dependabot/maven/org.xerial.snappy-snappy-java-1.1.10.4
in repository https://gitbox.apache.org/repos/asf/hudi.git
discard 43623c0a860 Bump org.xerial.snappy:snappy-java from 1.1.8.3 to 1.1.10.4
add ddaef8feddb [HUDI-5101] Adding spark-structured streaming test support via spark-submit job (#7074)
add e2dfb465f13 [HUDI-7495] Bump mysql-connector-java from 8.0.22 to 8.0.28 in /hudi-platform-service/hudi-metaserver/hudi-metaserver-server (#7674)
add e6664159bed [HUDI-7163] Fix not parsable text DateTimeParseException when compact (#10220)
add 3698d49383b [HUDI-7496] Bump mybatis from 3.4.6 to 3.5.6 in /hudi-platform-service/hudi-metaserver/hudi-metaserver-server (#7673)
add 819788f8651 [MINOR] Remove repetitive words in docs (#10844)
add ee11b9c951c [HUDI-7489] Avoid collecting WriteStatus to driver in row writer code path (#10836)
add 130498708bb add job context (#10848)
add 8bc9a4bc875 [HUDI-7478] Fix max delta commits guard check w/ MDT (#10820)
add f8f12ba9ef3 [MINOR] Fix and enable test TestHoodieDeltaStreamer.testJdbcSourceIncrementalFetchInContinuousMode (#10867)
add aac6b2e5486 [HUDI-7382] Get partitions from active timeline instead of listing when building clustering plan (#10621)
add ca2140e2003 [MINOR] rename KeyGenUtils#enableAutoGenerateRecordKeys (#10871)
add 3c8488b831c [HUDI-7506] Compute offsetRanges based on eventsPerPartition allocated in each range (#10869)
add e726306cf09 [HUDI-7466] Add parallel listing of existing partitions in Glue Catalog sync (#10460)
add 2dcdd311245 [HUDI-7421] Build HoodieDeltaWriteStat using HoodieDeltaWriteStat#copy (#10870)
add b7ccecf3205 [HUDI-7492] Fix the incorrect keygenerator specification for multi partition or multi primary key tables creation (#10840)
add 7631e0dcb89 [MINOR] Add Hudi icon for idea (#10880)
add 784af0e1786 [HUDI-7514] Update Manifest file after the parquet writer closed in LSMTimelineWriter (#10883)
add 7c55ac35ba1 [HUDI-7516] Put jdbc-h2 creds into static variables for hudi-utilities tests (#10889)
add 135db099afc [MINOR] Remove redundant fileId from HoodieAppendHandle (#10901)
add 5a21a1dd260 [HUDI-7529] Resolve hotspots in stream read (#10911)
add 47151f653d8 [HUDI-7487] Fixed test with in-memory index by proper heap clearing (#10910)
add 6be7205a1e3 [MINOR] Refactored `@Before*` and `@After*` in `HoodieDeltaStreamerTestBase` (#10912)
add a8e9db446c3 [HUDI-7530] Refactoring of handleUpdateInternal in CommitActionExecutors and HoodieTables (#10908)
add f98a40bd369 [HUDI-7499] Support FirstValueAvroPayload for Hudi (#10857)
add da9660bf38a checkstyle (#10919)
add d749457f9d5 [HUDI-7513] Add jackson-module-scala to spark bundle (#10877)
add d22bfba08fd [MINOR] Restore the setMaxParallelism setting for HoodieTableSource.produceDataStream (#10925)
add 5e4a6c650e0 [HUDI-7531] Consider pending clustering when scheduling a new clustering plan (#10923)
add 8a137631da8 [HUDI-7518] Fix HoodieMetadataPayload merging logic around repeated deletes (#10913)
add 136d0755ad7 [HUDI-7500] fix gaps with deduce schema and null schema (#10858)
add 28f67ff3561 [HUDI-7551] Avoid loading all partitions in CleanPlanner when MDT is enabled (#10928)
add 4741ba06462 [HUDI-6317] Streaming read should skip compaction and clustering instants to avoid duplicates (#8884)
add ae4f46874a9 [MINOR} When M3 metrics reporter type is used HoodieMetricsConfig should create default values for HoodieMetricsM3Config (#10936)
add 06d3bb8cfbd [HUDI-6884] hudi-cli should generate correct HoodieTimeGeneratorConfig (#10941)
add 26c00a3adef [HUDI-7187] Fix integ test props to honor new streamer properties (#10866)
add 9b094e628d6 [HUDI-7510] Loosen the compaction scheduling and rollback check for MDT (#10874)
add 44ab6f32bff [HUDI-6538] Refactor methods in TimelineDiffHelper class (#10938)
add 9efced37f81 [HUDI-7557] Fix incremental cleaner when commit for savepoint removed (#10946)
add bb51aca75d0 [MINOR] Upgrade mockito to 3.12.4 (#10953)
add 8bb6bee6234 [HUDI-7564] Fix HiveSyncConfig inconsistency (#10951)
add 59e32b7e686 [HUDI-7569] [RLI] Fix wrong result generated by query (#10955)
add bf723f56cd0 [HUDI-7486] Classify schema exceptions when converting from avro to spark row representation (#10778)
add 398c9a23c84 [HUDI-7564] Revert hive sync inconsistency and reason for it (#10959)
add 8b61696f158 [HUDI-7556] Fixing MDT validator and adding tests (#10939)
add 19c20e4dd93 [HUDI-7571] Add api to get exception details in HoodieMetadataTableValidator with ignoreFailed mode (#10960)
add bac6ea7b26b [MINOR] Removed FSUtils.makeBaseFileName without fileExt param (#10963)
add d41541cb9f8 [MINOR] Handle cases of malformed records when converting to json (#10943)
add 56142a0ff61 [MINOR] use Temurin jdk (#10948)
add b487e9826d0 [MINOR] Removed FSUtils.makeBaseFileName without fileExt param (#10967)
add 5519e9c13b3 [HUDI-6854] Change default payload type to HOODIE_AVRO_DEFAULT (#10949)
add 6f117f22c23 [HUDI-7230] Flink stream read supports skipping insert overwrite instant (#10328)
add 27217621a37 [HUDI-7572] Avoid to schedule empty compaction plan without log files (#10974)
add be7120d6d91 [HUDI-7480] Fix functional index and avoid multiple initializations (#10860)
add 984a248de4c [HUDI-7559] [1/n] Fix RecordLevelIndexSupport::filterQueryWithRecordKey (#10947)
add 016bcf769b6 [MINOR] Optimize print write error msg in StreamWriteOperatorCoordinator#doCommit (#10809)
add a23c5b783e7 [HUDI-7556] Fixing false positive validation with MDT validator (#10986)
add c2f2e615106 [HUDI-7583] Read log block header only for the schema and instant time (#10984)
add ee0bcc417be [HUDI-7597] Add logs of Kafka offsets when the checkpoint is out of bound (#10987)
add 913c320e4a5 [MINOR] Fix BUG: HoodieLogFormatWriter: unable to close output stream for log file HoodieLogFile{xxx} (#10989)
add 8b8b2241d17 [HUDI-7552] Remove the suffix for MDT table service instants (#10945)
add c2b5cc91d8e [HUDI-7600] Shutdown ExecutorService when HiveMetastoreBasedLockProvider is closed (#10993)
add 2a0d2aeef6a [HUDI-7391] HoodieMetadataMetrics should use Metrics instance for metrics registry (#10635)
add c870da2f375 [HUDI-6441] Passing custom Headers with Hudi Callback URL (#10970)
add a41d7aeafed [HUDI-7605] Allow merger strategy to be set in spark sql writer (#10999)
add c9256e5e784 [HUDI-7290] Don't assume ReplaceCommits are always Clustering (#10479)
add 4fbf5c52f19 [HUDI-7601] Add heartbeat mechanism to refresh lock (#10994)
add f715e8a02e8 [HUDI-7565] Create spark file readers to read a single file instead of an entire partition (#10954)
add 56aded81287 [HUDI-7599] add bootstrap mor legacy reader back to default source (#10990)
add 17ea14ab6d6 [HUDI-7378] Fix Spark SQL DML with custom key generator (#10615)
add 9e7a42a128e [HUDI-7616] Avoid multiple cleaner plans and deprecate hoodie.clean.allow.multiple (#11013)
add 48952ae5dfd [HUDI-7606] Unpersist RDDs after table services, mainly compaction and clustering (#11000)
add 28fd4bfe956 [HUDI-7615] Mark a few write configs with the correct sinceVersion (#11012)
add ad493d2ab3c [HUDI-7584] Always read log block lazily and remove readBlockLazily argument (#11015)
add 9cfc543bf3b [HUDI-7493] Consistent naming of Cleaner configuration parameters (#10851)
add 2a0969c9972 [HUDI-7619] Removed code duplicates in HoodieTableMetadataUtil (#11022)
add db7e93e712c [HUDI-3907] Claim RFC-77 for secondary idnex design (#11025)
add 7c12decc86c [HUDI-6762] Removed usages of MetadataRecordsGenerationParams (#10962)
add c71ac326b79 [HUDI-7566] Add schema evolution to spark file readers (#10956)
add 40d4f489389 [HUDI-7577] Avoid MDT compaction instant time conflicts (#10992)
add c54be848f96 [MINOR] Remove redundant lines in StreamSync and TestStreamSyncUnitTests (#11027)
add b4451c05ce2 [HUDI-7582] Fix functional index lookup (#11021)
add 7f20e108897 [MINOR] Rename location to path in `makeQualified` (#11037)
add 876c4e26ecf [HUDI-7578] Avoid unnecessary rewriting to improve performance (#11028)
add 5d63616a99f [HUDI-7625] Avoid unnecessary rewrite for metadata table (#11038)
add 6c6bddcef3e [HUDI-7626] Propagate UserGroupInformation from the main thread to the new thread of timeline service threadpool (#11039)
add 4c04216d5ca [HUDI-4228] Clean up literal usage in Hudi CLI argument check (#11042)
add 78fc62554ce [HUDI-7633] Use try with resources for AutoCloseable (#11045)
add df5cdb7f290 [MINOR] Remove redundant TestStringUtils in hudi-common (#11046)
add 999007b39d5 [HUDI-7636] Make StoragePath Serializable (#11049)
add b3773657f69 [HUDI-7635] Add default block size and openSeekable APIs to HoodieStorage (#11048)
add b5b14f7d4fa [HUDI-7637] Make StoragePathInfo Comparable (#11050)
add 8a0ce61b2fd [HUDI-6497] Replace FileSystem, Path, and FileStatus usage in hudi-common module (#10591)
add caa1bef75c3 [HUDI-7640] Uses UUID as temporary file suffix for HoodieStorage.createImmutableFileInPath (#11052)
add ca77fda51fe [HUDI-7618] Add ability to ignore checkpoints in delta streamer (#11018)
add 7ac26bce3f3 [HUDI-7643] Fix test by using the right StreamSync constructor (#11056)
add 7a44b1ebc41 [HUDI-7515] Fix partition metadata write failure (#10886)
add 60424c5f998 [MINOR] Added configurations of Hudi table, file-based SQL source, Hudi error table, and timestamp key generator to configuration listing (#11057)
add 683c4998d6d [HUDI-7628] Rename FSUtils.getPartitionPath to constructAbsolutePath (#11054)
add 95cdfa010a9 [HUDI-7631] Clean up usage of CachingPath outside hudi-common module (#11059)
add 08564b387fd [HUDI-7623] Refactoring of RemoteHoodieTableFileSystemView and RequestHandler (#11032)
add 0dbeb77b92a [HUDI-7655] Minor fix to rli validation with MDT validator (#11060)
add 6d645e4fe40 [MINOR] Reuse MetadataPartitionType enum to get all partition paths (#11068)
add ffbc0356f7e [HUDI-7608] Fix Flink table creation configuration not taking effect when writing to Spark (#11005)
add 172fefa709d [MINOR] Fix incorrect catch of ClassCastException using HoodieSparkKeyGeneratorFactory (#11062)
add cb6eb6785fd [HUDI-7653] Refactor HoodieFileIndex for more flexibility (#11074)
add c17f50dbbfc [MINOR] Fixe naming of methods in HoodieMetadataConfig (#11076)
add ce0c2671a0a [HUDI-7647] READ_UTC_TIMEZONE doesn't affect log files for MOR tables (#11066)
add b45a87e19f7 [HUDI-6386] Enable testArchivalWithMultiWriters back as they are passing (#9085)
add f710206531b [MINOR] Fix LoggerName for JDBCExecutor (#11063)
add 7c0a1c1475b [HUDI-7651] Add util methods for creating meta client (#11081)
add cab0e2bbb9a [HUDI-7632] Remove FileSystem usage in HoodieLogFormatWriter (#11082)
add 0e35678120a [HUDI-7650] Remove FileSystem argument in TestHelpers methods (#11072)
add 7f0a88ce905 [MINOR] Remove unused util methods in LogReaderUtils (#11086)
add f8e559f5e6f [HUDI-7660] Fix excessive object creation in RowDataKeyGen (#11084)
add fb022491b2b [HUDI-7235] Fix checkpoint bug for S3/GCS Incremental Source (#10336)
add 9d689fd68f5 [HUDI-7648] Refactor MetadataPartitionType so as to enahance reuse (#11067)
add 835d47367e3 [HUDI-7645] Optimize BQ sync tool for MDT (#11065)
add 9a8751e7fe9 [HUDI-7666] Fix serializable implementation of StorageConfiguration class (#11091)
add 0babc947c96 [MINOR] Make KafkaSource abstraction public and more flexible (#11093)
add 98a75b04a95 [HUDI-7658] Add time to meta sync failure log (#11080)
add fffba243506 [HUDI-7511] Fixing offset range calculation for kafka (#10875)
add e5e635fb27b [HUDI-7672] Fix the Hive server scratch dir for tests in hudi-utilities (#11097)
add 9c61c4c3cac [HUDI-7575] Avoid repeated fetching of pending replace instants (#10976)
add 93f753d159d [HUDI-7676] Fix serialization in Spark DAG in HoodieBackedTableMetadataWriter (#11103)
add 550367cdbda [HUDI-7664] Remove Hadoop dependency from hudi-io module (#11089)
add 6911abf1755 [MINOR] Streamer test setup performance (#10806)
add 3b613ffec62 [HUDI-7670] Return StorageConfiguration from getConf() in HoodieStorage (#11096)
add 658e964cf77 [HUDI-7668] Add and rename APIs in StorageConfiguration (#11102)
add 32ecf248e5f [HUDI-7675] Don't set default value for primary key when get schema from hms (#11101)
add 2c4192fa0f8 [HUDI-7674] Fix Hudi CLI Command "metadata validate-files" to use file listing to validate (#11100)
add dffde114570 [MINOR] Fix build on master (#11109)
add f35ae58089f [HUDI-7681] Remove Hadoop Path usage in a few classes in hudi-common module (#11108)
add 5bde36d904f [HUDI-7683] Make HoodieMetadataMetrics log level debug ro reduce noise (#11114)
add 60134eb0796 [HUDI-7682] Remove the files copy in Azure CI tests report (#11110)
add d286ae44c0f [MINOR] Remove the redundant log in HFileBootstrapIndex (#11115)
add fef14db3d9b [HUDI-7667] Created util method to get offset range for fetching new data (#11092)
add 9d1aaf2d103 [HUDI-7684] Sort the records for Flink metadata table bulk_insert (#11116)
add f99b181a04e [HUDI-7588] Replace hadoop Configuration with StorageConfiguration in meta client (#11071)
add a29fe277df8 [HUDI-7694] Unify bijection-avro dependency version (#11132)
add f553ba25fe3 [HUDI-7144] Build storage partition stats index and use it for data skipping (#10352)
add c20c01978b4 [HUDI-7685] Fix delete partition instant commit in partition TTL (#11117)
add 1ec7e631c38 [HUDI-7702] Remove unused method in ReflectUtil (#11135)
add 156e7604f8d [HUDI-4372] Enable matadata table by default for flink (#11124)
add 65f4b594c28 [HUDI-7706] Improve validation in PARTITION_STATS index test (#11141)
add 3930119544d [HUDI-6296] Add Scala 2.13 support for Spark 3.5 integration (#11130)
add aff11ac2c2f [HUDI-7688] Stop retry inflate if encounter InterruptedIOException (#11125)
add a0ea78fdd8d [MINOR] remove unnecessary lines from java test (#11139)
add feb82e61a06 [HUDI-7686] Add tests on the util methods for type cast of configuration instances (#11121)
add 5e2290e260a [MINOR] Fix flaky test testMetadataRollbackWithCompaction (#11143)
add 8911aa2d3c7 [HUDI-7576] Improve efficiency of getRelativePartitionPath, reduce computation of partitionPath in AbstractTableFileSystemView (#11001)
add 1c7f8376ade [HUDI-7710] Remove compaction.inflight from conflict resolution (#11148)
add d4f55f193cd [HUDI-7703] Clean plan to exclude partitions with no deleting file (#11136)
add be3f8fcd000 [HUDI-7701] Metadata table initailization with pending instants (#11137)
add fda52a0cfd3 [HUDI-7706] Improve validation of partition stats index include key (#11158)
add 7cbb72786eb [HUDI-7641] Adding metadata enablement metrics and index type metrics (#11053)
add 5f35be01606 Fixing deltastreamer tests for auto record key gen (#11099)
add bc07e3ed252 [HUDI-7710] Use compaction.requested during conflict resolution (#11151)
add fdb94192508 [HUDI-7715] Partition TTL for Flink (#11156)
add c359ecc971a [HUDI-7721] Fix broken build on master (#11164)
add f35bfa4b69e [HUDI-7720] Fix HoodieTableFileSystemView NPE in fetchAllStoredFileGroups (#11161)
add 113233f97ba [MINOR] Do not force setting spark conf in UtilHelpers (#11166)
add 5453ce75aaf [MINOR] Remove duplicate settings (#11167)
add 519b78d3acd [MINOR] Use parent as the glob path when full file path specified (#11150)
add 91aa38712e8 [HUDI-7522] Support find out the conflict instants in bucket partition when bucket id multiple (#10898)
add ace3789e2d5 [HUDI-7727] Avoid constructAbsolutePathInHadoopPath in hudi-common module (#11172)
add 5d66a5e4e12 [HUDI-7728] Use StorageConfiguration in LockProvider constructors (#11173)
add 9860e943c63 [HUDI-7699] Support STS external ids and configurable session names in the AWS StsAssumeRoleCredentialsProvider (#11134)
add 769d77aed2b [HUDI-7734] Remove unused FSPermissionDTO (#11176)
add be8d5256a82 [HUDI-7735] Remove usage of SerializableConfiguration (#11177)
add a60266142dd [MINOR] Cosmetic changes for names and log msgs (#11179)
add cdd146b2c73 [HUDI-7737] Bump Spark 3.4 version to Spark 3.4.3 (#11180)
add 008804c2ea0 [HUDI-7587] Make hudi-hadoop-common module dependent on hudi-common module (#11131)
add e42217d2368 [HUDI-7350] Make Hudi reader and writer factory APIs Hadoop-independent (#11163)
add b1496068b3a [HUDI-7725] Restructure HFileBootstrapIndex to separate Hadoop-dependent logic (#11171)
add 8f5f5470b61 [HUDI-7729] Move ParquetUtils to hudi-hadoop-common module (#11186)
add 0a6338b86e7 [HUDI-7738] Set FileStreamReader Charset as UTF-8 (#11181)
add c75ae7f9ba2 [HUDI-7654] Optimizing BQ sync for MDT (#11061)
add 23b283acf3e [HUDI-7726] Restructure TableSchemaResolver to separate Hadoop logic and use BaseFileUtils (#11185)
add badcca2ebe8 [HUDI-7742] Move Hadoop-dependent reader util classes to hudi-hadoop-common module (#11190)
add 0d0e27e2b9b [HUDI-7673] Fixing false positive validation failure for RLI with MDT validation tool (#11098)
add c7c636c2d18 [HUDI-7731] Fix usage of new Configuration() in production code (#11191)
add 86f7a6554df [HUDI-7739] Shudown asyncDetectorExecutor in AsyncTimelineServerBasedDetectionStrategy (#11182)
add 49072d1e2e7 [HUDI-7508] Avoid collecting records in HoodieStreamerUtils.createHoodieRecords and JsonKafkaSource mapPartitions (#10872)
add 61f54a0dcc2 [HUDI-7745] Move Hadoop-dependent util methods to hudi-hadoop-common (#11193)
add aa5bb0dda34 [HUDI-4732] Add support for confluent schema registry with proto (#11070)
add be0a6604b12 [HUDI-7501] Use source profile for S3 and GCS sources (#10861)
add ce08875a0d7 [HUDI-7523] Add HOODIE_SPARK_DATASOURCE_OPTIONS to be used in HoodieIncrSource (#10900)
add 4b59b202f49 [HUDI-7743] Improve StoragePath usages (#11189)
add ea4f14c2851 [HUDI-7744] Introduce IOFactory and a config to set the factory (#11192)
add 6627218f71f [HUDI-7750] Move HoodieLogFormatWriter class to hoodie-hadoop-common module (#11207)
add c15bdb34f89 remove a few classes from hudi-common (#11209)
add 076a9adcf78 [HUDI-7589] Add API to create HoodieStorage in HoodieIOFactory (#11208)
add 45ad35b960c [HUDI-7549] Reverting spurious log block deduction with LogRecordReader (#10922)
add bbf028027ba [HUDI-7617] Fix issues for bulk insert user defined partitioner in StreamSync (#11014)
add d9eb7b469bf [HUDI-7535] Add metrics for sourceParallelism and Refresh profile in S3/GCS (#10918)
add 7f837d0d0a4 [HUDI-7749] Bump Spark version 3.3.1 to 3.3.4 (#11198)
add d74e642b5be [HUDI-7712] Fixing RLI initialization to account for file slices instead of just base files while initializing (#11153)
add 8d103ad87d8 [HUDI-7624] Fixing index tagging duration (#11035)
add a8da2fdd1c0 [HUDI-7752] Abstract serializeRecords for log writing (#11210)
add c8dec0ef523 [HUDI-7429] Fixing average record size estimation for delta commits (#10763)
add 7e5ff1ec89f [HUDI-7759] Remove Hadoop dependencies in hudi-common module (#11220)
add 9e9bb9006b7 [HUDI-7532] Include only compaction instants for lastCompaction in getDeltaCommitsSinceLatestCompaction (#10915)
add 5bf509014a6 [MINOR] Rebalance CI with tests in hudi-hadoop-common module (#11230)
add 7fc5adad7aa [HUDI-7717] Disable row writer for bulk insert if combining before insert is set (#11216)
add d93e4eb9d70 [MINOR] Remove legacy code and add try catch to listStatus of partition (#11250)
add e0ca6dd0d52 [HUDI-7652] Add new `HoodieMergeKey` API to support simple and composite keys (#11077)
add e4b56b090fd [MINOR] Added condition to check default value to fix extracting password from credential store (#11246)
add 0804a79f20c [HUDI-7761] Make the ManifestWriter Extendable (#11253)
add 761af879198 [HUDI-6563] Supports flink lookup join (#9228)
add 1bc0c5a84ad [HUDI-7775] Remove unused APIs in HoodieStorage (#11255)
add 74308cdbd24 [HUDI-6207] Spark bucket index query pruning (#10191)
add c7d2fc05fd7 [HUDI-7781] Filter wrong partitions when using hoodie.datasource.write.partitions.to.delete (#11260)
add e1aa1bcb4af [HUDI-7784] Fix serde of HoodieHadoopConfiguration in Spark (#11269)
add 94b564151a2 [MINOR] LSMTimeline needs to handle case for tables which has not performed first archived yet (#11271)
add 8d4a35b1f2e [HUDI-4491] Re-enable TestHoodieFlinkQuickstart (#11272)
add f2e276a38e5 [MINOR] Query index warning check (#11276)
add bcc1f8de4d9 [HUDI-7785] Keep public APIs in utilities module the same as before HoodieStorage abstraction (#11279)
add 40b23b327de [HUDI-7788] Fixing exception handling in AverageRecordSizeUtils (#11289)
add edb02d4c450 [HUDI-7776] Simplify HoodieStorage instance fetching (#11259)
add 376571af371 [HUDI-7787] Ignore loading data for look up function when the commit has not changed (#11287)
add 660c6c7a5b0 [MINOR] Fix the look up warning msg (#11286)
add 70a04c6c782 Bump h2 in /packaging/hudi-metaserver-server-bundle (#9148)
add a0d60d98976 Bump h2 in /hudi-platform-service/hudi-metaserver (#9147)
add eba95af7ee6 [HUDI-7790] Revert changes in DFSPathSelector and UtilHelpers.readConfig (#11293)
add 54871a42905 [HUDI-7794] Bump org.apache.hive:hive-service from 2.3.1 to 2.3.4 (#11295)
add 8547a7ef343 [HUDI-7777] Allow HoodieTableMetaClient to take HoodieStorage instance directly (#11266)
add b82df80cb6d [HUDI-7796] Gracefully cast file system instance in Avro writers (#11301)
add ddf63b59434 [HUDI-7778] Fixing global index for duplicate updates (#11256)
add 4b6df31a09f [HUDI-7798] Mark configs included in 0.15.0 release (#11306)
add e80d50065cc [HUDI-7799] Optimize the access modifier of AbstractHoodieLogRecordReader#processNextRecord (#11308)
add 63ad11f54bb [HUDI-7797] Use HoodieIOFactory to return pluggable FileFormatUtils implementation (#11302)
add b1ebcb7b95b [HUDI-7801] Directly pass down HoodieStorage instance instead of recreation (#11309)
add b51d61ad7ce [HUDI-7707] Enable bundle validation on Java 8 and 11 (#11142)
add 0c7f5c2191d [HUDI-7802] Fix bundle validation scripts (#11314)
add 0d863f84a96 [MINOR] Updating logging usage and level for noisier logs (#11345)
add 777c3aadf4d [MINOR] Disable release candidate validation by default (#11338)
add cb9daa6c227 [MINOR] Improve TestHoodieHadoopIOFactory (#11342)
add 2291e995feb [HUDI-7795] Fix loading of input splits from look up table reader (#11297)
add 42243862f02 [HUDI-5505] Fix counting of delta commits since last compaction in ScheduleCompactionActionExecutor.getLatestDeltaCommitInfo (#11251)
add 2ab11a1f8e6 [HUDI-7804] Improve flink bucket index paritioner (#11346)
add 014f2d789dd [MINOR] Archive operation only releases lock when holds it (#11160)
add 4a530837f1b [HUDI-7809] Use Spark SerializableConfiguration to avoid NPE in Kryo serde (#11355)
add 2de1e4ae1fb [HUDI-7808] Upgrade io.acryl:datahub-client from 0.8.31 to 0.8.45 (#11357)
add 0c3e9b56952 [HUDI-7807] Fixing spark-sql for pk less tables (#11352)
add 0e6e49abeac [HUDI-7770] Parse partition path from hudi directory for bootstrap tables (#11237)
add e8a36ce3a3e [HUDI-7655] Ensuring clean action executor cleans up all intended files (#11343)
add cd62c31f368 [HUDI-7146] Implement secondary index write path (#11146)
add f0c1a88f8d0 [HUDI-7407] Making clean optional in standalone compaction and clustering jobs (#10668)
add c758508b62f [HUDI-7769] Fix Hudi CDC read on Spark 3.3.4 and 3.4.3 (#11242)
add db7480820e3 [MINOR] Fix GitHub CI concurrency (#11361)
add 0e55f0900d8 [HUDI-7817] Use Jackson Core instead of org.codehaus.jackson for JSON encoding (#11369)
add 130ea1a3142 [HUDI-7762] Optimizing Hudi Table Check with Delta Lake by Refining Class Name Checks In Spark3.5 (#11224)
add 9536d40f75d [MINOR] Avoid logging full commit metadata at info level (#11372)
add 7f8da18e550 [HUDI-7766] Adding staging jar deployment command for Spark 3.5 and Scala 2.13 profile (#11234)
add acecf304254 [MINOR] Avoid listing files for empty tables (#11155)
add 16e1adb5b3c [HUDI-7822] Resolve the conflicts between mixed hdfs and local path in Flink tests (#10931)
add 6ca7efa2a3a [HUDI-7007] Add bloom_filters index support on read side (#11043)
add 92eab8d6868 [HUDI-7816] Provide SourceProfileSupplier option into the SnapshotLoadQuerySplitter (#11368)
add e131a049de2 [MINOR] Fix LoggerName for BaseHoodieTableServiceClient (#11384)
add d0c7de050a8 [HUDI-7822] Bump io.airlift:aircompressor from 0.25 to 0.27 (#11380)
add ffd4f52b9ab [HUDI-7824] Fixing incr cleaner with savepoint removal (#11375)
add 06457c0aff2 [HUDI-7704] Unify test client storage classes with duplicate code (#11152)
add 9246fd7128b [MINOR] Correct order of test services start in `UtilitiesTestBase` (#11387)
add eb63e1ffa1a [HUDI-7782] Task not serializable due to DynamoDBBasedLockProvider and HiveMetastoreBasedLockProvider in clean action (#11389)
add d964895cf0b [HUDI-7713] Enforce ordering of fields during schema reconciliation (#11154)
add dfbbd1890a0 [HUDI-7828] Support Flink 1.18.1 (#11394)
add a1ba9728310 [HUDI-7414] Remove redundant base path config in BQ sync (#11395)
add 44922f160bd [MINOR] Allow recreation of metrics instance for base path (#11400)
add 4224d872719 [MINOR] Update DOAP with 0.15.0 Release (#11409)
add b5913c65147 [MINOR] Add logs whenever fallback to fs listing even with MDT enabled (#11407)
add e0cf4b67a4e [HUDI-7146] Integrate secondary index on reader path (#11162)
add 52775b89127 [MINOR] Remove calcite dependency (#11411)
add f4be74c2947 [HUDI-7567] Add schema evolution to the filegroup reader (#10957)
add 0d7cc87d687 [HUDI-7840] Add position merging to the new file group reader (#11413)
add a33b2a5e03f [HUDI-7834] Create placeholder table versions and introduce new hoodie table property to track initial table version (#11406)
add a31fda59555 [HUDI-7844] Fix HoodieSparkSqlTestBase to throw error upon test failure (#11416)
add 90011bf6314 [HUDI-7846] Bump apache-rat-plugin to 0.16.1 to eliminate thread-safe warning in maven parallel build (#11420)
add 37564b4fd68 [HUDI-7845] Call show_fsview_latest procedure support path_regex (#11418)
add 9f9064761ba [HUDI-7390] HoodieStreamer no longer works without --props being supplied (#11414)
add 0abc00df841 [HUDI-6787] Implement the HoodieFileGroupReader API for Hive (#10422)
add c0576131759 [HUDI-6798] Add record merging mode and implement event-time ordering in the new file group reader (#9894)
add 7783306398b [HUDI-7851] Fix java doc of DeltaWriteProfile (#11422)
add e63722d24b5 [HUDI-7852] Constrain the comparison of different types of ordering values to limited cases (#11424)
add 9804318ecc0 [HUDI-7786] Fix roaring bitmap dependency in hudi-integ-test-bundle (#11283) (#11428)
add 2a56065c854 [HUDI-7853] Fix missing serDe properties post migration from hiveSync to glueSync (#11404)
add 51ef7097cb4 [HUDI-7826] Make column nullable when setNullForMissingColumns is true (#11381)
add 5143a982bca [HUDI-7269] Fallback to key based merge if positions are missing from log block (#11415)
add 8eed6aefbc4 [MINOR] Update release guide (#11433)
add 3415a4a1a2c [HUDI-7819] Fix OptionsResolver#allowCommitOnEmptyBatch default value (#11370)
add e38f78cedb9 [MINOR] Fix README for Slack link update and Linkedin badge (#11442)
add 6f3539268f8 [HUDI-7854] Bump AWS SDK v2 version to 2.25.69 (#11426)
add eb20273cf66 Ensure properties are copied when modifying schema (#11441)
add 35d79270205 [MINOR] add `ad1happy2go` to github collaborators (#11447)
add 8f0467b7229 remove tableconfig from filegroup reader params (#11449)
add 35cdac0a645 [HUDI-7747] In MetaClient remove getBasePathV2() and return StoragePath from getBasePath() (#11385)
add 6aea47a13f5 [HUDI-7671] Make Hudi timeline backward compatible (#11443)
add 84381afe7aa [HUDI-7872] Recreate and sync glue and hive table when meta sync fails (#11451)
add 9b77eb1f5c6 [HUDI-7879] Optimize the redundant creation of HoodieTable in DataSourceInternalWriterHelper and the unnecessary parameters in createTable within BaseHoodieWriteClien (#11456)
add ec5244d5fe6 [HUDI-7880] Support extraMetadata in Spark SQL Insert Into (#11458)
add b236396d877 [HUDI-7891] Fix HoodieActiveTimeline#deleteCompletedRollback missing check for Action type (#11462)
add ed0295f8fbd [HUDI-7847] Infer record merge mode during table upgrade (#11439)
add 662f0822bd4 [HUDI-7838] Remove the option hoodie.schema.cache.enable and always do the cache (#11444)
add bf1df335442 [HUDI-7876] use properties to store log spill map configs for fg reader (#11455)
add 9f0130442a5 [HUDI-7874] Fix Hudi being able to read 2-level structure (#11450)
add 1ce97bae116 [MINOR][DNM] Test disabling new HFile reader (#11488)
add 51c9c0e226a [HUDI-7906] Improve the parallelism deduce in rdd write (#11470)
add 8a4bed03fa7 [HUDI-7849] Reduce time spent on running testFiltersInFileFormat (#11423)
add 2cc45cc228a [HUDI-7881] Verify table base path as well for syncing table in bigquery metastore (#11460)
add bb76de48e9f [HUDI-6508] Support compilation on Java 11 (#11479)
add b0580ef56ca [HUDI-5956] Fix spark DAG ui when write (#11376)
add 05a07cf76a0 [HUDI-7909] Add Comment to the FieldSchema returned by Aws Glue Client (#11474)
add 7567eaef2c0 [HUDI-4123] Enchancing deltastreamer sql source tests (#6781)
add dbc6ac50aec [HUDI-7395] Fix computation for metrics in HoodieMetadataMetrics (#10641)
add 58b53f05980 [HUDI-4945] Add a test case for batch clean (#6845)
add 7178fa6d497 [MINOR] Adding tests for streaming read mor with compaction (#6695)
add 6192cfb0e95 [MINOR] Reduce logging volume (#11505)
add c5ff6a2113f [MINOR] Removed useless checks from SqlBasedTransformers (#11499)
add 4370178eb0b [HUDI-7927] Lazy init secondary view in FS view (#10652)
add 3152e47876f [MINOR] Bump JUnit version to 5.8.2 (#11511)
add 4b7e6e41573 [HUDI-7922] Add Hudi CLI bundle for Scala 2.13 (#11495)
add 1c731769d60 [HUDI-7882] Picking RFC-78 for bridge release (#11515)
add 7c2480f903b [HUDI-7403] Support Filter/Transformer to Hudi Exporter Utility (non-hudi export) (#11509)
add ae1ee05ab8c [HUDI-7709] ClassCastException while reading the data using `TimestampBasedKeyGenerator` (#11501)
add eb0725d1ef5 [HUDI-7877] Add record position to record index metadata payload (#11467)
add c80b5596c1d [HUDI-7932] Fix Import ordering (#11524)
add ea6a6145223 [HUDI-7763] Fix that multiple jmx reporter can exist if metadata enables (#11226)
add 8e36fe91715 [HUDI-7924] Capture Latency and Failure Metrics For Hive Table recreation (#11498)
add 63b7b15fff2 [MINOR] doap: don't nest multiple versions in a release (#11533)
add 4c0fb6d951e [HUDI-7914] Use internal schema without metadata fields for delete-partitions OP (#11487)
add 107cffca4af [HUDI-7908] HoodieFileGroupReader fails if preCombine and partition fields are the same (#11473)
add 78d7ee7cba5 Closing fsview in HoodieTableFileIndex (#11497)
add eeafa734a6c [HUDI-7903] Fix storage partition stats index to skip data (#11472)
add 9f66ac0fb4b [HUDI-7942] Fix SmallFileAssignState#totalUnassigned naming error (#11544)
add e942a27d446 [MINOR] Fix NPE after a clustering plan is finished (#11550)
add 990c0c565b7 [HUDI-7933] Sync table in Glue/HMS if table base path is updated (#11529)
add 9bc0164a387 [HUDI-7841] RLI and secondary index should consider only pruned partitions for file skipping (#11434)
add a9a75f43a1c [HUDI-7941] add show_file_status procedure & run_rollback_inflight_tableservice procedure (#11538)
add 29e2f6cd0f0 [HUDI-7926] Data skipping failure mode should be strict in query test (#11502)
add be0068065d6 [MINOR] If sync mode is glue, fix the sync tool class (#11543)
add ddfa681b6ea [HUDI-7758] Only consider files in Hudi partitions when initializing MDT (#11219)
add d91fd2b3bb3 [HUDI-7951] Fix conflict caused by classes using avro in hudi-aws-bundle (#11563)
add 70f44efe298 [HUDI-7945] Fix partition pruning using PARTITION_STATS index in Spark (#11556)
add 4b52e27eb3e [HUDI-7954] Fix data skipping with secondary index when there are no log files (#11575)
add dbfe8b23c0b [HUDI-7953] Improved the variable naming and formatting of HoodieActiveTimeline and HoodieIndex (#11574)
add ef83379d38a [HUDI-7949] insert into hudi table with columns specified (#11568)
add 1f6520b1b26 [HUDI-7965] Clean up SchemaTestUtil code (#11584)
add 36a9113aca2 [HUDI-7929] Create k8s example for flink hudi integration (#11570)
add 968da86593f [HUDI-7962] Add show create table command (#11471)
add 6d01bcfa6a2 [HUDI-7966] Handle NPE from AvroSchemaUtils.createNewSchemaFromFieldsWithReference (#11585)
add 26ac119ee25 Revert "[HUDI-7709] ClassCastException while reading the data using `TimestampBasedKeyGenerator` (#11501)" (#11586)
add b0e0efcc5f9 [HUDI-7955] Account for WritableTimestampObjectInspector#getPrimitiveJavaObject discrepancies in Hive3 and Hive2 (#11576)
add 87f9fdaee90 [HUDI-7911] Enable cdc log for MOR table (#11490)
add 0cd429153ef [MINOR] Improve release docs and update some links (#11591)
add b0ee6152d99 [HUDI-7929] Fix file name in k8s example (#11603)
add dbcd089b679 [HUDI-7968] Claiming rfc for robust spark writes (#11592)
add bb0621edee9 [HUDI-7957] Fix data skew when writing with bulk_insert + bucket_index enabled (#11578)
add 16ee6bc4e32 [HUDI-7977] Improve bucket index partitioner algorithm (#11608)
add 01d905ff46b [HUDI-7961] Optimizing upsert partitioner for prepped write operations (#11581)
add 3789840be3d [HUDI-7921] Fixing file system view closures in MDT (#11496)
add c51f4675e2d [MINOR] Codestyle of license and package statement (#11617)
add 9ee96655094 [HUDI-7979] Adjusting defaults with spillable map memory (#11611)
add e2860cddf54 [HUDI-7709] Pass partition paths as partition column values if `TimestampBasedKeyGenerator` is used (#11615)
add 11304fd93ba [HUDI-7950] Shade roaring bitmap dependency in root POM (#11561)
add afc91515983 [MINOR] Update DOAP with 1.0.0-beta2 Release (#11627)
add 98b3d3bac0f [HUDI-7980] Optimize the configuration content when performing clustering with row writer (#11614)
add 918c2e0009c [HUDI-7976] Fix BUG introduced in HUDI-7955 due to usage of wrong class (#11612)
add 2286589de31 [HUDI-7938] Broadcast `SerializableConfiguration` to avoid NullPointerException in Kryo SerDe (#11626)
add 32650ef6995 [MINOR] Remove repeated look up candidate (#11569)
add c67cb42846b [HUDI-7934] Fix RocksDBDAO prefixDelete to delete last entry properly (#11531)
add 6456d779600 [HUDI-7905] Use cluster action for clustering pending instants (#11553)
add e97aedc0ecb [HUDI-7859] Rename instant files to be consistent with 0.x naming format when downgrade table (#11545)
add 103b8222887 [HUDI-7988] ListingBasedRollbackStrategy support log compact (#11631)
add 37ca7b694a2 [HotFix] Fix wrong skipClean description in HoodieClusteringJob.java (#11641)
add 587cf33509c [MINOR] Fixed checkstyle in k8s example code (HUDI-7929) (#11645)
add 3d1fbf74bbc [HUDI-4822] Extract the baseFile and logFiles from HoodieDeltaWriteStat in the right way (#11518)
add ac5e85587ce [HUDI-7999] Disable CI for spark < 3.3 (#11644)
add bf5ec3503a6 [HUDI-7779] Guard archival on savepoint removal until cleaner is able to clean it up (#11440)
add 4588c35ec00 [HUDI-7998] Fix insert overwrite hudi table with static partition whe… (#11643)
add 3933cdeb315 [MINOR] Improve error logging in JSON Avro conversion (#11654)
add 79f1768e5ac [HUDI-7997] Fix hivesync to support promotion type correctly (#11642)
add 5915cc9f2be [MINOR] Update how RLI validation filters out new entries (#11658)
add e4b2067064b [HUDI-7908] HoodieFileGroupReader fails if preCombine and partition fields are the same (#11656)
add bada8110120 [HUDI-7494]: multi writer sync partition to glue will missing some partitions (#10841)
add 3349839f029 [HUDI-8001] Insert overwrite failed due to missing 'path' property when using Spark 3.5.1 and Hudi 1.0.0 (#11646)
add 2af5f67bac3 [HUDI-8007] ShowInvalidParquetProcedure support delete by parameter (#11657)
add 19dcc8d9ec6 [HUDI-6510] Support compilation on Java 17 (#11604)
add abed00a1f89 [HUDI-7821] Handle case where older proto message is read with new schema (#11373)
add ad10bd4e040 [HUDI-712] Revert back to overwrite=true in HoodieSnapshotExporter (#11668)
add edb1c758d8e [MINOR] Fix SyntaxWarning in pr_compliance.py (#11669)
add ccb8f1d6945 [HUDI-8008] Handle proto schema references in returned schema (#11660)
add a0bae464592 [HUDI-8006] Fix glue sync async call to update table (#11659)
add 4f50fde3da3 [HUDI-8009] Optimize the code of HoodieTable#getPendingCommitTimeline (#11665)
add 54700538ab2 [HUDI-8011] allow schema.on.read with positional merging (#11671)
add 65e68cb1709 [HUDI-7025] Consolidate index and functional index config groups (#11596)
add c9c8ae4e879 [HUDI-7964] Error out if partition schema and table schema do not match in create table command (#11650)
add 5baaa87521d [HUDI-7943] Resolve version conflict of fasterxml on spark3.2 (#11548)
add d2304c559ae [HUDI-7963] Minor enhancement to RLI flow with MDT (#11582)
add 628236e2e8e [MINOR] Add build scripts for new bundle validation images (#11683)
add 43f04a8c1d7 [HUDI-7774] Add Avro Logical type support for Merciful Java convertor (#11265)
add d45970cdd88 [HUDI-8002] Added back Flink 1.15 and 1.14 bundle validation (#11653)
add 7232d2461f4 [HUDI-8005] Add a new DynamoDB based lock provider with auto-generated partition key (#11667)
add 23a8b602475 [HUDI-7993] Apply record index if _hoodie_record_key meta field in filter (#11678)
add f54b9bb8dd7 [HUDI-6191] Improve passing the debezium checkpoint values to start job from offset (#11686)
add 00ebe1bf7aa [HUDI-8030] Fix for add missing table configurations to payloadProps (fixes #11685) (#11698)
add 6485fa5c8a7 [HUDI-4625] Clean up KafkaOffsetGen: introduce retrying KafkaConsumer (#11664)
add e315b16d9fb [HUDI-7948] Claiming RFC-80 for Column families (#11699)
add 2024d17704e [HUDI-6453] Cascade Glue schema changes to partitions (#11670)
add 2aa60f201ba [HUDI-8029] Enforce s3 base path format for partition key calculation (#11693)
add 9f21c2d9d9b [MINOR] Support MOR table type for HoodieJavaWriteClientExample (#11700)
add 4bbc04ba41c [HUDI-8033] Claiming RFC-81 for Log Compaction with Merge Sort (#11703)
add 4dbe5b80d39 [HUDI-8035] Fetching commit metadata from timeline fails during upgrade (#11711)
add c858d56862a [HUDI-7948] RFC-80: Support column families for wide tables (#11559)
add 0fbf3128196 [HUDI-8040] Fix SimpleConcurrentFileWritesConflictResolutionStrategy get clustering instant contains insert overwrite (#11691)
add 06d3fad0194 [HUDI-8041] Support projection push down for lookup join (#11722)
add af8080991d5 [HUDI-7996] Store partition type with partition fields in table configs (#11638)
add c12bac435a8 [HUDI-7692] Extract metadata record type to MetadataPartitionType enum (#11597)
add 7f60243009a [MINOR] Reduce unnecessary code paths (#11729)
add 5da405ec8c8 [MINOR] don't run scheduled workflow on forks (#11736)
add fb2361f78dc [HUDI-7881] Compare bigquery table base path via source-uris if hive partitioning options are not available (#11679)
add 5da41a1dac9 [HUDI-8043] Fix DDB Lock provider bug (#11728)
add 36242ff516d [HUDI-8015] Fix Glue Meta Sync Failure with recreating the table (#11677)
add f1ecdf6e080 [HUDI-6948] Fix schema evolution for non row writer (#11731)
add cbd2573bdeb [MINOR] Improve performance of generateBucketKey (#11748)
add 0cd28e52873 [MINOR] HoodieAvroUtils performance optimization of createFullName method (#11747)
add b7e11b1d018 [HUDI-8067] Use exec to run the IT (#11751)
add c71b2daa105 [HUDI-8012] Update checkstyle.xml and fix checkstyle errors (#11672)
add ae40574ef6d [MINOR] Avoid repeated write behavior in ut (#11755)
add 410b4f5060a [MINOR] Fixed checkstyle in HoodieJavaWriteClientExample (#11756)
add 9db0a60e677 [MINOR] Update Azure CI links in README (#11771)
add 35c00daaf87 [HUDI-8074] Improve comaction operator shuffle rebanlance (#11757)
add 64f546b8f0c [HUDI-7873] Remove method from reader context (#11452)
add 367ca8c8595 [HUDI-7920] Make Spark 3.5 the default build profile for Spark integration (#11493)
add 31eea3b9a85 [HUDI-8079] Get rid of other count case for fg reader (#11774)
add d4a4d9c43e4 use parquet reader for cdc reading (#11775)
add ef8a7ad4f17 [HUDI-7918] Remove support of Spark 3.0, 3.1, and 3.2 (#11692)
add db64cf2a7ac [HUDI-8088] Fix documentation: filename of Externalized Config file (#11786)
add 97b4d591aa0 [HUDI-8073] Add hosts to storage path info and use it if present (#11761)
add db5c2d97dc9 [HUDI-8096] Improve OverwriteNonDefaultsWithLatestAvroPayload Java Doc (#11795)
add e4e5bf02320 [HUDI-8097] Fix getting Schema evolution setting from hudi-defaults.conf on altering columns (#11796)
add 9f345342df9 [HUDI-8093] Replace Hadoop Configuration with StorageConfiguration in hudi-client-common module (#11792)
add b6bad403fa9 [HUDI-8104] Fix hive on spark select mor rt error: HoodieCombineRealtimeFileSplit cannot be cast to org.apache.hadoop.hive.shims.HadoopShimsSecure (#11799)
add 4b5c22339c7 [MINOR] Fix the log offset for merged write stat (#11806)
add 5b31ef20c1e [HUDI-8108] Optimize HoodieTimeline#filterPendingExcludingMajorAndMinorCompaction function naming (#11808)
add e1f70fda589 [HUDI-8112] Fix TestHoodieActiveTimeline#testTimelineGetOperations test logic error (#11814)
add d550b46d724 [HUDI-8070] Support Flink 1.19 (#11779)
add cb2499cc252 [HUDI-7970] Move methods from HoodieTableConfigUtils to HoodieTableConfig (#11601)
add e0ef8642199 [HUDI-8078] Persisting writestatus to optimize the writes (#11811)
add 8335ce2e96a [HUDI-8105] Refactor MercifulJsonConverter to be extendible for Json to Row conversion (#11800)
add 8720c87061d [HUDI-8133] allow the ability to add additional jars to execute the run_sync_tool (#11848)
add 6334bf19569 [HUDI-6791] Make some comments look better (#11854)
add b15a4d632e5 [HUDI-7696] Refactoring functions in HoodieTableMetadataUtil (#11823)
add 0cc89702140 [MINOR] Removing unused function convertMetadataToRecords (#11822)
add 3cb1dba99ce [HUDI-8134] Prevent mac m1 activation of profile without specifying spark version (#11849)
add 52fd36d0e11 [HUDI-8109] Fix the error related to locking during clustering/compaction when calling the file system which is not atomic (#11809)
add f64ff333369 [HUDI-8125] Update handling of non-string inputs for string fields in MercifulJsonConverter (#11825)
add b1ac8e97ed9 [MINOR] Add backward compatibility test for HoodieTableConfig partition field APIs (#11824)
add 5a30d5b8af3 [HUDI-8112] Improve HoodieActiveTimeline#revertCompleteToInflight parameter check (#11815)
add 0ed4e11a87b [HUDI-5807] Read partition values from file and create infra to support reading only a subset of columns (#11770)
add bc8fd9b8bf5 [HUDI-8137] Support time travel query on MDT in Spark Datasource (#11860)
add 7137bc6096d [MINOR] Validate that file group reader reads entire files for Bootstrap and MOR queries (#11789)
add 08f9e6b926b [HUDI-7989] Fix secondary index updates (#11634)
add 8d884345ef2 [HUDI-8036] Handle partition schema for custom key gen in SparkHoodieTableFileIndex (#11710)
add 012c86a38db [HUDI-8175] Fix LongWritable cannot be cast to TimestampWritable for MOR table with timestamp column and schema evolution enabled (#11890)
add d7eb992f97b [HUDI-8174] Update GcsEventsSource messages info logging to debug to reduce noise (#11891)
add 46d2a30dffe [HUDI-7958] Create partition stats index for all columns when no cols specified (#11579)
add 6bc093ebdc4 [HUDI-8159] Use SerializableConfiguration with Spark broadcast (#11868)
add 4437495d39b [HUDI-8083] Read precombine value from both table write config (#11777)
add ee67e3f82f5 [HUDI-8111] Adding support to validate last N file slices with HoodieMetadataValidator (#11864)
add df5c59b3812 [HUDI-8180] Delete the caches with workflow (#11904)
add 3162691a8d1 [HUDI-8180] use individual cache key when deleting caches (#11905)
add 4ea31d16e93 [HUDI-8180] Give all permissions to scheduled workflow (#11906)
add 5e575769d54 [HUDI-8180] Make permissions more strict for scheduled workflow (#11907)
add b69480aecd9 [MINOR] Replace hard-coded path by temp dir in tests (#11912)
add 845885a271d [HUDI-8071] Handle skew for user defined sort columns in BULK_INSERT (#11758)
add 61d1a7a5673 [MINOR] Add remote file system props to meta sync properties (#11846)
add 33948280a09 [HUDI-5583] Add UT on multiple Java client writers while cleaning to validate no deadlock (#7704)
add be080a87ef4 [HUDI-8098] With custom keygen, only read timestamp partition columns from the file (#11895)
add cedc58c5ee3 [MINOR] Fix empty metasync class issue (#11915)
add 9bb8dcdf116 [HUDI-8092] Replace FileSystem and related classes to dehadoop hudi-client-common (#11805)
add 703980da181 [HUDI-8184] Fix Hudi CLI 'version' command to return Hudi version (#10973)
add ad236232414 Remove hudi-hadoop-common compile dependency from hudi-client-common (#11916)
add 75ec4597657 [HUDI-7723] DayBasedCompactionStrategy supports io bounded (#11169)
add f389b9ddfbe [HUDI-1441] Unify HoodieAvroUtils.rewriteRecord with HoodieAvroUtils.rewriteRecordWithNewSchema (#11893)
add 1e5d65a1beb [MINOR] Exclude slf4j-reload4j dependency from hudi-tests-common (#10643)
add 54250705ecc [HUDI-7867] Ensuring delete invalid all files during finalizing the writer (#11445)
add 4f44eeda5bc [HUDI-8018] Improve Spark SQL tests (#11859)
add eb88b8f2f76 [HUDI-8185] Handle Avro GenericData byte array comparison (#11920)
add 276133b814e [HUDI-8003] Add OverwriteWithLatestHiveRecordMerger and restructure existing merger classes (#11649)
add ac79b148eaf [MINOR] Improve code quality of ComparableVersion class (#10934)
add 4abedd66b28 [MINOR] Fix the target location for auxlib download in hudi CLI (#11931)
add 7dad9139653 [HUDI-8197] Get rid of set sqlconf in Filegroupreader parquet file format (#11928)
add a1e0123f4a1 [HUDI-5128] Use HadoopFSUtils.getFs to get filesystem (#11937)
add 3c0d93ed075 [MINOR] Have BaseWriterCommitMessage implement Serializable (#11938)
add d4784c42d8d [HUDI-7928] Remove shared HFile reader in HoodieNativeAvroHFileReader (#11935)
add f605f28d989 [HUDI-8102] Turn on data skipping and position-based record merging by default (#11620)
add cecfbbddffd [HUDI-8116] Coalesce Row Source Aliases with Schema Fields in S3/GCS (#11817)
add aa922ed48fb [HUDI-8090] Add new Zookeeper-based lock provider with automatically derived base path and lock key (#11790)
add 75ab91c58e5 [HUDI-5530] Fix WARNING during compile (#7647)
add d15bdfcebd4 [HUDI-7937] Handle legacy writer requirements in StreamSync and Clustering (#11534)
add fe8760104f3 [HUDI-8124] Allow reuse of meta client in Meta Sync path (#11832)
add 6d6d7783225 [HUDI-8034] Support custom key generator with HoodieCatalogTable (#11816)
add 461e58b0d92 [MINOR] pass the avro exception for better information (#11925)
add c87663d085b [HUDI-8023] Add multiwriter test for secondary and partition stats index (#11948)
add 5c9ea4d22ed [HUDI-7341] Support unmerged log reader for HoodieFileGroupReader (#11881)
add ec32e035b39 [HUDI-7940] Pass HoodieIngestionMetrics to Error Table Writer to be able to emit metrics for Error Table Writer (#11541)
add 34261f0d502 [HUDI-6909] Add logic to check operation metadata field for Spark Record (#11930)
add b9e40df9b44 [HUDI-8191] Fix MockStateInitializationContext#getKeyedStateStore to use non-KeyedStateStore (#11952)
add 73e6ab076f5 [HUDI-7718] Use source profile in HoodieIncrSource (#11175)
add 4d74acef8a1 [HUDI-7900] Delete unsupported metadata partitions when downgrade (#11889)
add f6a2ef4cd65 [HUDI-4240] Use a different nested field in tests to be compatible across Avro versions (#5868)
add f71a47f2ee8 [HUDI-7236] Allow MIT to change partition path when using global index (#10337)
add 8143527fe0c Bump org.xerial.snappy:snappy-java from 1.1.8.3 to 1.1.10.4
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (43623c0a860)
\
N -- N -- N refs/heads/dependabot/maven/org.xerial.snappy-snappy-java-1.1.10.4 (8143527fe0c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 2 +
.github/workflows/bot.yml | 401 ++-
.github/workflows/release_candidate_validation.yml | 71 +-
.github/workflows/scheduled_workflow.yml | 24 +
.gitignore | 1 +
.idea/icon.png | Bin 0 -> 14245 bytes
README.md | 30 +-
azure-pipelines-20230430.yml | 38 +-
doap_HUDI.rdf | 54 +
.../config/test-suite/multi-writer-1.properties | 2 +-
.../config/test-suite/multi-writer-2.properties | 2 +-
.../test-suite/multi-writer-local-1.properties | 2 +-
.../test-suite/multi-writer-local-2.properties | 2 +-
.../test-suite/multi-writer-local-3.properties | 2 +-
.../test-suite/multi-writer-local-4.properties | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
.../test-aggressive-clean-archival.properties | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
...clustering-aggressive-clean-archival.properties | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
...g-metadata-aggressive-clean-archival.properties | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
...t-metadata-aggressive-clean-archival.properties | 2 +-
docker/setup_demo.sh | 6 +-
docker/stop_demo.sh | 2 +-
hudi-aws/pom.xml | 5 +
.../hudi/aws/cloudwatch/CloudWatchReporter.java | 12 +-
...dieConfigAWSAssumedRoleCredentialsProvider.java | 6 +-
.../HoodieConfigAWSCredentialsProvider.java | 8 +-
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 591 +++-
.../hudi/aws/sync/AwsGlueCatalogSyncTool.java | 18 +-
...amoDBBasedImplicitPartitionKeyLockProvider.java | 69 +
.../lock/DynamoDBBasedLockProvider.java | 193 +-
.../lock/DynamoDBBasedLockProviderBase.java | 225 ++
.../hudi/config/DynamoDbBasedLockConfig.java | 16 +-
.../hudi/config/GlueCatalogSyncClientConfig.java | 38 +-
.../org/apache/hudi/config/HoodieAWSConfig.java | 53 +-
.../aws/cloudwatch/TestCloudWatchReporter.java | 10 +-
.../hudi/aws/sync/ITTestGluePartitionPushdown.java | 59 +-
.../hudi/aws/sync/MockAwsGlueCatalogSyncTool.java | 44 +
.../hudi/aws/sync/TestAWSGlueSyncClient.java | 290 ++
.../apache/hudi/aws/sync/TestAwsGlueSyncTool.java | 102 +
.../hudi/aws/sync/TestGluePartitionPushdown.java | 1 +
.../apache/hudi/aws/testutils/GlueTestUtil.java | 138 +
.../integ/ITTestDynamoDBBasedLockProvider.java | 229 +-
.../lock/DynamoDBBasedLockProviderBaseTest.java | 83 +
.../org/apache/hudi/aws/utils/TestS3Utils.java | 59 +
hudi-cli/pom.xml | 9 +
.../main/java/org/apache/hudi/cli/HoodieCLI.java | 27 +-
.../org/apache/hudi/cli/HoodiePrintHelper.java | 1 -
.../java/org/apache/hudi/cli/HoodiePrompt.java | 3 +-
.../hudi/cli/commands/ArchivedCommitsCommand.java | 141 +-
.../apache/hudi/cli/commands/BootstrapCommand.java | 6 +-
.../apache/hudi/cli/commands/CleansCommand.java | 6 +-
.../hudi/cli/commands/ClusteringCommand.java | 8 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 6 +-
.../hudi/cli/commands/CompactionCommand.java | 77 +-
.../apache/hudi/cli/commands/ExportCommand.java | 127 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 20 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 149 +-
.../cli/commands/HoodieSyncValidateCommand.java | 1 +
.../apache/hudi/cli/commands/MarkersCommand.java | 5 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 140 +-
.../apache/hudi/cli/commands/RepairsCommand.java | 74 +-
.../apache/hudi/cli/commands/RestoresCommand.java | 1 +
.../apache/hudi/cli/commands/RollbacksCommand.java | 3 +-
.../hudi/cli/commands/SavepointsCommand.java | 13 +-
.../apache/hudi/cli/commands/SparkEnvCommand.java | 1 +
.../org/apache/hudi/cli/commands/SparkMain.java | 223 +-
.../org/apache/hudi/cli/commands/StatsCommand.java | 29 +-
.../org/apache/hudi/cli/commands/TableCommand.java | 51 +-
.../apache/hudi/cli/commands/TempViewCommand.java | 1 +
.../apache/hudi/cli/commands/TimelineCommand.java | 35 +-
.../cli/commands/UpgradeOrDowngradeCommand.java | 9 +-
.../org/apache/hudi/cli/commands/UtilsCommand.java | 1 +
.../java/org/apache/hudi/cli/utils/CommitUtil.java | 2 +-
.../apache/hudi/cli/utils/InputStreamConsumer.java | 3 +-
hudi-cli/src/main/resources/application.yml | 6 +-
hudi-cli/src/main/resources/version.txt | 1 +
.../java/org/apache/hudi/cli/TestSparkUtil.java | 4 +-
.../hudi/cli/commands/TestArchiveCommand.java | 4 +-
.../cli/commands/TestArchivedCommitsCommand.java | 11 +-
.../hudi/cli/commands/TestCleansCommand.java | 17 +-
.../hudi/cli/commands/TestCommitsCommand.java | 31 +-
.../hudi/cli/commands/TestCompactionCommand.java | 7 +-
.../apache/hudi/cli/commands/TestDiffCommand.java | 13 +-
.../cli/commands/TestFileSystemViewCommand.java | 18 +-
.../cli/commands/TestHoodieLogFileCommand.java | 34 +-
.../hudi/cli/commands/TestMetadataCommand.java | 10 +-
.../hudi/cli/commands/TestRepairsCommand.java | 21 +-
.../hudi/cli/commands/TestRestoresCommand.java | 2 +-
.../hudi/cli/commands/TestRollbacksCommand.java | 2 +-
.../hudi/cli/commands/TestSavepointsCommand.java | 4 +-
.../hudi/cli/commands/TestSparkEnvCommand.java | 2 +-
.../apache/hudi/cli/commands/TestStatsCommand.java | 4 +-
.../apache/hudi/cli/commands/TestTableCommand.java | 39 +-
.../cli/commands/TestUpgradeDowngradeCommand.java | 19 +-
.../cli/functional/CLIFunctionalTestHarness.java | 9 +-
.../hudi/cli/integ/ITTestBootstrapCommand.java | 3 +-
.../hudi/cli/integ/ITTestClusteringCommand.java | 8 +-
.../hudi/cli/integ/ITTestCommitsCommand.java | 4 +-
.../hudi/cli/integ/ITTestCompactionCommand.java | 3 +-
.../cli/integ/ITTestHDFSParquetImportCommand.java | 26 +-
.../hudi/cli/integ/ITTestRepairsCommand.java | 66 +-
.../hudi/cli/integ/ITTestSavepointsCommand.java | 37 +-
.../apache/hudi/cli/integ/ITTestTableCommand.java | 4 +-
.../testutils/HoodieCLIIntegrationTestBase.java | 1 +
.../testutils/HoodieCLIIntegrationTestHarness.java | 1 +
.../HoodieTestCommitMetadataGenerator.java | 16 +-
.../cli/testutils/ShellEvaluationResultUtil.java | 4 +-
hudi-cli/src/test/resources/clean.properties | 4 +-
hudi-client/hudi-client-common/pom.xml | 72 +-
.../http/HoodieWriteCommitHttpCallbackClient.java | 46 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 61 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 194 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 91 +-
.../apache/hudi/client/CompactionAdminClient.java | 51 +-
.../client/HoodieTableServiceManagerClient.java | 2 +-
.../client/embedded/EmbeddedTimelineService.java | 19 +-
.../hudi/client/heartbeat/HeartbeatUtils.java | 37 +-
.../client/heartbeat/HoodieHeartbeatClient.java | 24 +-
.../client/timeline/HoodieTimelineArchiver.java | 29 +-
.../hudi/client/timeline/LSMTimelineWriter.java | 58 +-
.../client/transaction/ConcurrentOperation.java | 9 +-
.../DirectMarkerTransactionManager.java | 7 +-
.../PreferWriterConflictResolutionStrategy.java | 3 +-
...urrentFileWritesConflictResolutionStrategy.java | 10 +-
.../client/transaction/TransactionManager.java | 6 +-
.../lock/BaseZookeeperBasedLockProvider.java | 199 ++
.../lock/FileSystemBasedLockProvider.java | 47 +-
.../hudi/client/transaction/lock/LockInfo.java | 3 +-
.../hudi/client/transaction/lock/LockManager.java | 19 +-
...ZookeeperBasedImplicitBasePathLockProvider.java | 71 +
.../lock/ZookeeperBasedLockProvider.java | 184 +-
.../lock/metrics/HoodieLockMetrics.java | 13 +-
.../hudi/client/utils/DeletePartitionUtils.java | 4 +-
.../hudi/client/utils/LazyIterableIterator.java | 6 +-
.../utils/LegacyArchivedMetaEntryReader.java | 44 +-
.../apache/hudi/client/utils/TransactionUtils.java | 2 +-
.../common/table/log/HoodieFileSliceReader.java | 98 -
.../apache/hudi/config/HoodieBootstrapConfig.java | 2 +-
.../org/apache/hudi/config/HoodieCleanConfig.java | 40 +-
.../apache/hudi/config/HoodieErrorTableConfig.java | 5 +-
.../org/apache/hudi/config/HoodieIndexConfig.java | 7 +-
.../org/apache/hudi/config/HoodieLockConfig.java | 13 +
.../apache/hudi/config/HoodiePayloadConfig.java | 4 +-
.../config/HoodieWriteCommitCallbackConfig.java | 15 +
.../org/apache/hudi/config/HoodieWriteConfig.java | 210 +-
.../hudi/config/metrics/HoodieMetricsConfig.java | 227 --
.../hudi/config/metrics/HoodieMetricsM3Config.java | 126 -
...eletePartitionPendingTableServiceException.java | 28 +
.../hudi/execution/CopyOnWriteInsertHandler.java | 1 +
.../hudi/execution/HoodieLazyInsertIterable.java | 9 +-
.../java/org/apache/hudi/index/HoodieIndex.java | 1 +
.../org/apache/hudi/index/HoodieIndexUtils.java | 189 +-
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 1 +
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 2 +-
.../index/bucket/ConsistentBucketIndexUtils.java | 85 +-
.../hudi/index/bucket/HoodieSimpleBucketIndex.java | 76 +-
.../hudi/index/simple/HoodieGlobalSimpleIndex.java | 7 +-
.../hudi/index/simple/HoodieSimpleIndex.java | 10 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 61 +-
.../org/apache/hudi/io/HoodieBootstrapHandle.java | 5 +-
.../java/org/apache/hudi/io/HoodieCDCLogger.java | 19 +-
.../org/apache/hudi/io/HoodieConcatHandle.java | 3 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 23 +-
.../java/org/apache/hudi/io/HoodieIOHandle.java | 9 +-
.../hudi/io/HoodieKeyLocationFetchHandle.java | 12 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 3 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 58 +-
.../apache/hudi/io/HoodieMergeHandleFactory.java | 7 +
.../hudi/io/HoodieMergeHandleWithChangeLog.java | 6 +-
.../org/apache/hudi/io/HoodieMergedReadHandle.java | 24 +-
.../java/org/apache/hudi/io/HoodieReadHandle.java | 20 +-
.../apache/hudi/io/HoodieSortedMergeHandle.java | 7 +-
.../io/HoodieSortedMergeHandleWithChangeLog.java | 2 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 31 +-
.../hudi/keygen/ComplexAvroKeyGenerator.java | 3 +-
.../apache/hudi/keygen/CustomAvroKeyGenerator.java | 46 +-
.../hudi/keygen/GlobalAvroDeleteKeyGenerator.java | 3 +-
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 39 +-
.../apache/hudi/keygen/SimpleAvroKeyGenerator.java | 3 +-
.../factory/HoodieAvroKeyGeneratorFactory.java | 4 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 1012 +++---
.../hudi/metadata/HoodieMetadataWriteUtils.java | 19 +-
.../metadata/HoodieTableMetadataKeyGenerator.java | 3 +-
.../hudi/metadata/HoodieTableMetadataWriter.java | 3 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 49 +-
.../hudi/metrics/InMemoryMetricsReporter.java | 36 -
.../apache/hudi/metrics/JmxMetricsReporter.java | 123 -
.../java/org/apache/hudi/metrics/MetricUtils.java | 81 -
.../main/java/org/apache/hudi/metrics/Metrics.java | 179 -
.../hudi/metrics/MetricsGraphiteReporter.java | 93 -
.../hudi/metrics/MetricsReporterFactory.java | 102 -
.../cloudwatch/CloudWatchMetricsReporter.java | 29 +-
.../metrics/datadog/DatadogMetricsReporter.java | 87 -
.../apache/hudi/metrics/m3/M3MetricsReporter.java | 120 -
.../hudi/metrics/m3/M3ScopeReporterAdaptor.java | 145 -
.../metrics/prometheus/PrometheusReporter.java | 132 -
.../prometheus/PushGatewayMetricsReporter.java | 99 -
.../table/BucketIndexBulkInsertPartitioner.java | 1 +
.../apache/hudi/table/BulkInsertPartitioner.java | 29 -
.../java/org/apache/hudi/table/HoodieTable.java | 73 +-
.../hudi/table/action/BaseActionExecutor.java | 10 +-
.../hudi/table/action/HoodieWriteMetadata.java | 12 +
.../table/action/bootstrap/BootstrapUtils.java | 82 +-
.../table/action/clean/CleanActionExecutor.java | 30 +-
.../action/clean/CleanPlanActionExecutor.java | 13 +-
.../hudi/table/action/clean/CleanPlanner.java | 104 +-
.../action/clean/CleaningTriggerStrategy.java | 2 +-
.../cluster/ClusteringPlanActionExecutor.java | 7 +-
.../cluster/strategy/ClusteringPlanStrategy.java | 2 +-
.../PartitionAwareClusteringPlanStrategy.java | 9 +-
.../util/ConsistentHashingUpdateStrategyUtils.java | 6 +-
.../action/commit/BaseCommitActionExecutor.java | 9 +-
.../hudi/table/action/commit/BaseWriteHelper.java | 6 -
.../table/action/commit/HoodieDeleteHelper.java | 2 +-
.../table/action/commit/HoodieMergeHelper.java | 34 +-
.../table/action/commit/HoodieWriteHelper.java | 2 +-
.../apache/hudi/table/action/commit/SmallFile.java | 3 +-
.../hudi/table/action/compact/CompactHelpers.java | 4 +-
.../hudi/table/action/compact/HoodieCompactor.java | 17 +-
.../hudi/table/action/compact/OperationResult.java | 3 +-
.../compact/RunCompactionActionExecutor.java | 3 +-
.../compact/ScheduleCompactionActionExecutor.java | 12 +-
.../BaseHoodieCompactionPlanGenerator.java | 16 +-
.../HoodieLogCompactionPlanGenerator.java | 2 +-
.../compact/strategy/CompactionStrategy.java | 2 +-
.../strategy/DayBasedCompactionStrategy.java | 2 +-
.../index/RecordBasedIndexingCatchupTask.java | 3 +-
.../table/action/index/RunIndexActionExecutor.java | 23 +-
.../action/index/ScheduleIndexActionExecutor.java | 4 +-
.../BaseHoodieFunctionalIndexClient.java | 5 +-
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../restore/CopyOnWriteRestoreActionExecutor.java | 3 +-
.../restore/MergeOnReadRestoreActionExecutor.java | 1 +
.../rollback/BaseRollbackActionExecutor.java | 24 +-
.../table/action/rollback/BaseRollbackHelper.java | 32 +-
.../rollback/ListingBasedRollbackStrategy.java | 148 +-
.../rollback/MarkerBasedRollbackStrategy.java | 8 +-
.../action/rollback/RestorePlanActionExecutor.java | 4 +-
.../hudi/table/action/rollback/RollbackUtils.java | 4 +-
.../action/rollback/SerializablePathFilter.java | 26 -
.../action/savepoint/SavepointActionExecutor.java | 3 +-
.../ttl/strategy/KeepByCreationTimeStrategy.java | 4 +-
.../action/ttl/strategy/KeepByTimeStrategy.java | 2 +-
.../action/ttl/strategy/PartitionTTLStrategy.java | 3 +-
.../hudi/table/marker/DirectWriteMarkers.java | 87 +-
.../table/marker/MarkerBasedRollbackUtils.java | 18 +-
.../SimpleDirectMarkerBasedDetectionStrategy.java | 6 +-
...nsactionDirectMarkerBasedDetectionStrategy.java | 8 +-
.../marker/TimelineServerBasedWriteMarkers.java | 14 +-
.../org/apache/hudi/table/marker/WriteMarkers.java | 42 +-
.../hudi/table/marker/WriteMarkersFactory.java | 4 +-
.../org/apache/hudi/table/repair/RepairUtils.java | 21 +-
.../upgrade/EightToSevenDowngradeHandler.java | 166 +
.../table/upgrade/FiveToSixUpgradeHandler.java | 8 +-
.../table/upgrade/FourToFiveUpgradeHandler.java | 9 +-
.../table/upgrade/OneToZeroDowngradeHandler.java | 2 +-
.../table/upgrade/SevenToEightUpgradeHandler.java | 84 +
.../table/upgrade/SevenToSixDowngradeHandler.java | 40 +
.../table/upgrade/SixToFiveDowngradeHandler.java | 60 +-
.../table/upgrade/SixToSevenUpgradeHandler.java | 42 +
.../table/upgrade/TwoToOneDowngradeHandler.java | 35 +-
.../hudi/table/upgrade/UpgradeDowngrade.java | 30 +-
.../hudi/table/upgrade/UpgradeDowngradeUtils.java | 92 +
.../table/upgrade/ZeroToOneUpgradeHandler.java | 13 +-
.../java/org/apache/hudi/DummyActiveAction.java | 2 +-
.../org/apache/hudi/DummyTaskContextSupplier.java | 5 -
.../org/apache/hudi/HoodieTestCommitGenerator.java | 41 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 26 +-
.../client/http/TestCallbackHttpClient.java | 202 ++
.../hudi/callback/http/TestCallbackHttpClient.java | 143 -
.../embedded/TestEmbeddedTimelineService.java | 13 +-
.../heartbeat/TestHoodieHeartbeatClient.java | 35 +-
.../timeline/TestCompletionTimeQueryView.java | 15 +-
.../FileSystemBasedLockProviderTestClass.java | 30 +-
.../InProcessLockProviderWithRuntimeError.java | 7 +-
.../TestConflictResolutionStrategyUtil.java | 69 +-
.../hudi/client/transaction/TestLockManager.java | 2 +-
...TestPreferWriterConflictResolutionStrategy.java | 11 +-
...urrentFileWritesConflictResolutionStrategy.java | 49 +-
...itesConflictResolutionStrategyWithMORTable.java | 89 +
.../client/transaction/TestTransactionManager.java | 5 +-
.../TestZookeeperBasedLockProvider.java | 106 +-
.../utils/TestLegacyArchivedMetaEntryReader.java | 10 +-
.../table/timeline/TestHoodieArchivedTimeline.java | 6 +-
.../common/testutils/HoodieMetadataTestTable.java | 53 +-
.../apache/hudi/config/TestHoodieWriteConfig.java | 181 +-
.../apache/hudi/index/TestHoodieIndexUtils.java | 76 -
.../org/apache/hudi/keygen/TestKeyGenUtils.java | 57 +
...estCreateAvroKeyGeneratorByTypeWithFactory.java | 1 +
.../hudi/metrics/TestHoodieConsoleMetrics.java | 17 +-
.../hudi/metrics/TestHoodieGraphiteMetrics.java | 23 +-
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 74 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 57 +-
.../hudi/metrics/TestMetricsReporterFactory.java | 32 +-
.../cloudwatch/TestCloudWatchMetricsReporter.java | 27 +-
.../datadog/TestDatadogMetricsReporter.java | 61 +-
.../datadog/TestHoodieMetricsDatadogConfig.java | 2 +-
.../org/apache/hudi/metrics/m3/TestM3Metrics.java | 67 +-
.../metrics/prometheus/TestPrometheusReporter.java | 20 +-
.../prometheus/TestPushGateWayReporter.java | 59 +-
.../hudi/table/TestBulkInsertPartitioner.java | 20 -
.../apache/hudi/table/action/TestCleanPlanner.java | 255 +-
.../table/action/commit/TestWriterHelperBase.java | 20 -
.../hudi/table/marker/TestWriteMarkersFactory.java | 15 +-
.../apache/hudi/table/repair/TestRepairUtils.java | 27 +-
.../upgrade/TestEightToSevenDowngradeHandler.java | 116 +
.../java/org/apache/hudi/testutils/Assertions.java | 97 +
.../GenericRecordValidationTestUtils.java | 37 +-
.../hudi/testutils/HoodieMergeOnReadTestUtils.java | 68 +-
.../hudi/testutils/HoodieWriteableTestTable.java | 53 +-
.../hudi/testutils/providers/DFSProvider.java | 5 +-
.../providers/HoodieMetaClientProvider.java | 13 +-
.../hudi/utils/HoodieWriterClientTestHarness.java | 1291 +++++++-
.../hudi/utils/TestConcatenatingIterator.java | 1 +
hudi-client/hudi-flink-client/pom.xml | 13 +
.../hudi/client/FlinkTaskContextSupplier.java | 5 -
.../hudi/client/HoodieFlinkTableServiceClient.java | 23 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 9 +-
...inkSizeBasedClusteringPlanStrategyRecently.java | 133 +
.../client/common/HoodieFlinkEngineContext.java | 19 +-
.../hudi/execution/FlinkLazyInsertIterable.java | 3 +-
.../java/org/apache/hudi/io/FlinkAppendHandle.java | 8 +-
.../hudi/io/FlinkConcatAndReplaceHandle.java | 10 +-
.../java/org/apache/hudi/io/FlinkConcatHandle.java | 6 +-
.../java/org/apache/hudi/io/FlinkCreateHandle.java | 22 +-
.../apache/hudi/io/FlinkMergeAndReplaceHandle.java | 27 +-
.../FlinkMergeAndReplaceHandleWithChangeLog.java | 8 +-
.../java/org/apache/hudi/io/FlinkMergeHandle.java | 36 +-
.../hudi/io/FlinkMergeHandleWithChangeLog.java | 4 +-
.../apache/hudi/io/FlinkWriteHandleFactory.java | 17 +-
.../java/org/apache/hudi/io/MiniBatchHandle.java | 4 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 29 +-
.../row/HoodieRowDataFileWriterFactory.java | 25 +-
.../io/storage/row/HoodieRowDataParquetWriter.java | 9 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 76 +-
.../hudi/table/HoodieFlinkCopyOnWriteTable.java | 22 +-
.../org/apache/hudi/table/HoodieFlinkTable.java | 5 +-
.../commit/BaseFlinkCommitActionExecutor.java | 16 +-
.../FlinkDeletePartitionCommitActionExecutor.java | 24 +-
.../commit/FlinkPartitionTTLActionExecutor.java | 74 +
.../hudi/table/action/commit/FlinkWriteHelper.java | 3 +-
.../java/org/apache/hudi/util/FlinkClientUtil.java | 4 +-
.../java/org/apache/hudi/util/WriteStatMerger.java | 2 +-
.../apache/hudi/client/TestFlinkWriteClient.java | 2 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 4 +-
.../TestFlinkSizeBasedClusteringPlanStrategy.java | 5 +-
.../testutils/HoodieFlinkClientTestHarness.java | 11 +-
.../testutils/HoodieFlinkWriteableTestTable.java | 26 +-
hudi-client/hudi-java-client/pom.xml | 21 +
.../hudi/client/HoodieJavaTableServiceClient.java | 8 +-
.../apache/hudi/client/HoodieJavaWriteClient.java | 33 +-
.../run/strategy/JavaExecutionStrategy.java | 16 +-
.../client/common/HoodieJavaEngineContext.java | 16 +-
.../client/common/JavaTaskContextSupplier.java | 6 -
.../hudi/execution/JavaLazyInsertIterable.java | 3 +-
.../JavaCustomColumnsSortPartitioner.java | 27 +-
.../JavaHoodieBackedTableMetadataWriter.java | 41 +-
.../hudi/table/HoodieJavaCopyOnWriteTable.java | 19 +-
.../org/apache/hudi/table/HoodieJavaTable.java | 10 +-
.../JavaExecuteClusteringCommitActionExecutor.java | 4 +-
.../commit/BaseJavaCommitActionExecutor.java | 19 +-
...vaInsertOverwriteTableCommitActionExecutor.java | 2 +-
.../JavaInsertPreppedCommitActionExecutor.java | 1 -
.../table/action/commit/JavaUpsertPartitioner.java | 2 +-
.../JavaUpsertPreppedCommitActionExecutor.java | 1 -
.../hudi/table/action/commit/JavaWriteHelper.java | 3 +-
.../table/upgrade/JavaUpgradeDowngradeHelper.java | 3 +-
.../client/TestHoodieJavaWriteClientInsert.java | 21 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 619 ++--
.../client/common/TestHoodieJavaEngineContext.java | 8 +-
.../common/TestMultipleHoodieJavaWriteClient.java | 227 ++
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 1412 +-------
.../TestHoodieJavaClientOnMergeOnReadStorage.java | 10 +-
.../hadoop/TestHoodieFileGroupReaderOnHive.java | 351 ++
.../commit/TestJavaCopyOnWriteActionExecutor.java | 36 +-
.../hudi/testutils/ArrayWritableTestUtil.java | 194 ++
.../testutils/HoodieJavaClientTestHarness.java | 196 +-
.../hudi/testutils/TestHoodieMetadataBase.java | 23 +-
hudi-client/hudi-spark-client/pom.xml | 28 +
.../org/apache/hudi/DefaultSparkRecordMerger.java | 121 +
.../apache/hudi/HoodieSparkFunctionalIndex.java | 236 ++
.../org/apache/hudi/HoodieSparkRecordMerger.java | 107 +-
.../hudi/OverwriteWithLatestSparkRecordMerger.java | 51 +
.../org/apache/hudi/client/SparkRDDReadClient.java | 13 +-
.../hudi/client/SparkRDDTableServiceClient.java | 13 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 61 +-
.../hudi/client/SparkTaskContextSupplier.java | 5 -
.../HoodieSparkBootstrapSchemaProvider.java | 7 +-
.../run/strategy/ExecutionStrategyUtil.java | 24 +-
.../MultipleSparkJobExecutionStrategy.java | 78 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 13 +-
.../client/common/HoodieSparkEngineContext.java | 13 +-
.../hudi/client/model/HoodieInternalRow.java | 1 +
.../client/utils/SparkMetadataWriterUtils.java | 26 +-
.../hudi/client/utils/SparkPartitionUtils.java | 4 +-
.../hudi/client/utils/SparkReleaseResources.java | 64 +
.../hudi/client/utils/SparkValidatorUtils.java | 5 +-
.../client/validator/SparkPreCommitValidator.java | 2 +-
.../hudi/common/model/HoodieSparkRecord.java | 17 +-
.../org/apache/hudi/data/HoodieJavaPairRDD.java | 23 +
.../java/org/apache/hudi/data/HoodieJavaRDD.java | 23 +
.../hudi/execution/SparkLazyInsertIterable.java | 5 +-
.../bulkinsert/RDDBucketIndexPartitioner.java | 7 +-
.../RDDCustomColumnsSortPartitioner.java | 23 +-
.../RDDSimpleBucketBulkInsertPartitioner.java | 1 +
.../bulkinsert/RDDSpatialCurveSortPartitioner.java | 7 +-
.../RowCustomColumnsSortPartitioner.java | 45 +-
.../bulkinsert/RowSpatialCurveSortPartitioner.java | 1 +
.../apache/hudi/index/SparkHoodieIndexFactory.java | 19 +-
.../index/bloom/HoodieFileProbingFunction.java | 12 +-
.../index/bloom/SparkHoodieBloomIndexHelper.java | 25 +-
.../bucket/HoodieSparkConsistentBucketIndex.java | 5 +-
.../functional/HoodieSparkFunctionalIndex.java | 235 --
.../io/storage/HoodieSparkFileReaderFactory.java | 27 +-
.../hudi/io/storage/HoodieSparkFileWriter.java | 3 +-
.../io/storage/HoodieSparkFileWriterFactory.java | 35 +-
.../hudi/io/storage/HoodieSparkIOFactory.java | 54 +
.../hudi/io/storage/HoodieSparkParquetReader.java | 43 +-
.../io/storage/HoodieSparkParquetStreamWriter.java | 10 +-
.../hudi/io/storage/HoodieSparkParquetWriter.java | 6 +-
.../row/HoodieInternalRowFileWriterFactory.java | 16 +-
.../row/HoodieInternalRowParquetWriter.java | 8 +-
.../hudi/io/storage/row/HoodieRowCreateHandle.java | 38 +-
.../io/storage/row/HoodieRowParquetConfig.java | 8 +-
.../storage/row/HoodieRowParquetWriteSupport.java | 4 +-
.../apache/hudi/keygen/ComplexKeyGenerator.java | 3 +-
.../org/apache/hudi/keygen/CustomKeyGenerator.java | 2 +-
.../org/apache/hudi/keygen/SimpleKeyGenerator.java | 3 +-
.../hudi/keygen/SparkKeyGeneratorInterface.java | 1 +
.../hudi/keygen/TimestampBasedKeyGenerator.java | 3 +-
.../keygen/UTF8StringPartitionPathFormatter.java | 1 +
.../factory/HoodieSparkKeyGeneratorFactory.java | 52 +-
.../SparkHoodieBackedTableMetadataWriter.java | 59 +-
.../SparkHoodieMetadataBulkInsertPartitioner.java | 12 +-
.../apache/hudi/metrics/DistributedRegistry.java | 9 +-
.../apache/hudi/sort/SpaceCurveSortingHelper.java | 10 +-
.../hudi/table/HoodieSparkCopyOnWriteTable.java | 39 +-
.../org/apache/hudi/table/HoodieSparkTable.java | 35 +-
.../bootstrap/BaseBootstrapMetadataHandler.java | 13 +-
.../bootstrap/MetadataBootstrapHandlerFactory.java | 9 +-
.../bootstrap/OrcBootstrapMetadataHandler.java | 17 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 15 +-
.../SparkBootstrapCommitActionExecutor.java | 19 +-
...SparkExecuteClusteringCommitActionExecutor.java | 4 +-
.../action/commit/AverageRecordSizeUtils.java | 90 +
.../commit/BaseSparkCommitActionExecutor.java | 42 +-
.../BucketBulkInsertDataInternalWriterHelper.java | 1 +
.../commit/BulkInsertDataInternalWriterHelper.java | 13 +-
.../action/commit/SparkBucketIndexPartitioner.java | 26 +-
.../SparkDeletePartitionCommitActionExecutor.java | 27 +-
.../action/commit/SparkHoodiePartitioner.java | 1 +
.../SparkInsertOverwriteCommitActionExecutor.java | 2 +-
.../commit/SparkInsertOverwritePartitioner.java | 5 +-
...rkInsertOverwriteTableCommitActionExecutor.java | 3 +-
.../commit/SparkPartitionTTLActionExecutor.java | 15 +-
.../table/action/commit/UpsertPartitioner.java | 69 +-
.../BaseSparkDeltaCommitActionExecutor.java | 2 +-
.../SparkUpsertDeltaCommitActionExecutor.java | 2 +-
.../SparkUpsertDeltaCommitPartitioner.java | 5 +-
.../org/apache/hudi/AvroConversionUtils.scala | 37 +-
.../hudi/BaseSparkInternalRowReaderContext.java | 54 +-
.../org/apache/hudi/HoodieConversionUtils.scala | 5 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 104 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 50 +-
.../scala/org/apache/hudi/HoodieUnsafeRDD.scala | 1 -
.../org/apache/hudi/SparkAdapterSupport.scala | 6 -
.../org/apache/hudi/SparkConversionUtils.scala | 1 +
.../SparkFileFormatInternalRowReaderContext.scala | 276 +-
.../hudi/util/CloseableInternalRowIterator.scala | 1 +
.../hudi/util/ExceptionWrappingIterator.scala | 44 +
.../org/apache/hudi/util/JavaScalaConverters.scala | 72 +
.../scala/org/apache/hudi/util/PathUtils.scala | 30 +-
.../org/apache/hudi/util/SparkKeyGenUtils.scala | 42 +-
.../apache/spark/HoodieSparkKryoRegistrar.scala | 18 +-
.../apache/spark/sql/BucketPartitionUtils.scala | 13 +-
.../scala/org/apache/spark/sql/DataFrameUtil.scala | 1 +
.../spark/sql/HoodieCatalystExpressionUtils.scala | 1 +
.../spark/sql/HoodieCatalystPlansUtils.scala | 12 +-
.../org/apache/spark/sql/HoodieDataTypeUtils.scala | 85 +-
.../apache/spark/sql/HoodieInternalRowUtils.scala | 17 +-
.../org/apache/spark/sql/HoodieUnsafeUtils.scala | 1 +
.../spark/sql/execution/SQLConfInjectingRDD.scala | 2 +-
.../HoodieSparkPartitionedFileUtils.scala | 6 +-
.../datasources/SparkParsePartitionUtil.scala | 127 +-
.../datasources/parquet/SparkParquetReader.scala | 50 +
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 33 +-
.../spark/sql/hudi/execution/RangeSample.scala | 1 +
.../callback/TestHoodieClientInitCallback.java | 8 +-
.../org/apache/hudi/client/TestClientRollback.java | 31 +-
.../hudi/client/TestCompactionAdminClient.java | 10 +-
.../hudi/client/TestFileBasedLockProvider.java | 16 +-
.../hudi/client/TestHoodieClientMultiWriter.java | 23 +-
.../java/org/apache/hudi/client/TestMultiFS.java | 23 +-
.../TestMultiWriterWithPreferWriterIngestion.java | 9 +-
.../hudi/client/TestPartitionTTLManagement.java | 10 +-
...erBasedDetectionStrategyWithZKLockProvider.java | 4 +-
...stSparkHoodieMetadataBulkInsertPartitioner.java | 11 +-
.../hudi/client/TestSparkRDDWriteClient.java | 8 +-
.../hudi/client/TestTableSchemaEvolution.java | 28 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 18 +-
.../TestUniformBootstrapModeSelector.java | 1 +
...parkConsistentBucketClusteringPlanStrategy.java | 6 +-
.../TestSparkSizeBasedClusteringPlanStrategy.java | 1 -
.../functional/TestConsistentBucketIndex.java | 15 +-
...DataValidationCheckForLogCompactionActions.java | 6 +-
.../functional/TestExternalPathHandling.java | 7 +-
.../functional/TestHoodieBackedMetadata.java | 858 +++--
.../functional/TestHoodieBackedTableMetadata.java | 193 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 1888 ++---------
.../TestHoodieClientOnMergeOnReadStorage.java | 14 +-
.../hudi/client/functional/TestHoodieIndex.java | 30 +-
.../client/functional/TestHoodieMetadataBase.java | 8 +-
.../functional/TestHoodieMetadataBootstrap.java | 20 +-
.../TestRemoteFileSystemViewWithMetadataTable.java | 22 +-
.../TestSavepointRestoreMergeOnRead.java | 23 +-
.../hudi/common/fs/NonSerializableFileSystem.java | 3 +-
.../org/apache/hudi/data/TestHoodieJavaRDD.java | 28 +
.../TestBoundedInMemoryExecutorInSpark.java | 2 +-
.../hudi/execution/TestBoundedInMemoryQueue.java | 2 +-
.../execution/TestDisruptorExecutionInSpark.java | 1 +
.../hudi/execution/TestDisruptorMessageQueue.java | 58 +-
.../hudi/execution/TestSimpleExecutionInSpark.java | 2 +-
.../TestBulkInsertInternalPartitioner.java | 7 +-
.../TestRDDSimpleBucketBulkInsertPartitioner.java | 5 +-
.../TestRowCustomColumnsSortPartitioner.java | 52 +
.../index/bloom/TestBloomIndexTagWithColStats.java | 9 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 84 +-
.../index/bloom/TestHoodieGlobalBloomIndex.java | 35 +-
.../index/bucket/TestHoodieSimpleBucketIndex.java | 2 +-
.../functional/TestHoodieSparkFunctionalIndex.java | 1 +
.../index/hbase/TestHBaseQPSResourceAllocator.java | 2 +-
.../index/hbase/TestSparkHoodieHBaseIndex.java | 17 +-
.../hudi/io/TestHoodieKeyLocationFetchHandle.java | 6 +-
.../org/apache/hudi/io/TestHoodieMergeHandle.java | 15 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 595 ++--
.../storage/TestHoodieAvroFileWriterFactory.java | 27 +-
.../java/org/apache/hudi/table/TestCleaner.java | 412 +--
.../apache/hudi/table/TestConsistencyGuard.java | 53 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 71 +-
.../table/action/bootstrap/TestBootstrapUtils.java | 24 +-
.../clean/TestCleanerInsertAndCleanByCommits.java | 4 +-
.../clean/TestCleanerInsertAndCleanByVersions.java | 4 +-
.../table/action/cluster/ClusteringTestUtils.java | 2 +-
...TestSparkBuildClusteringGroupsForPartition.java | 8 +-
.../action/commit/TestAverageRecordSizeUtils.java | 214 ++
.../commit/TestCopyOnWriteActionExecutor.java | 93 +-
.../table/action/commit/TestSparkWriteHelper.java | 23 +
.../table/action/commit/TestUpsertPartitioner.java | 23 +-
.../table/action/compact/CompactionTestBase.java | 44 +-
.../table/action/compact/TestAsyncCompaction.java | 71 +-
.../table/action/compact/TestHoodieCompactor.java | 81 +-
.../table/action/compact/TestInlineCompaction.java | 54 +-
.../strategy/TestHoodieCompactionStrategy.java | 53 +-
.../rollback/HoodieClientRollbackTestBase.java | 14 +-
.../TestCopyOnWriteRollbackActionExecutor.java | 49 +-
.../TestMergeOnReadRollbackActionExecutor.java | 109 +-
.../table/action/rollback/TestRollbackUtils.java | 19 +-
.../table/functional/TestCleanActionExecutor.java | 188 ++
.../table/functional/TestCleanPlanExecutor.java | 329 +-
.../TestHoodieSparkMergeOnReadTableCompaction.java | 82 +-
...HoodieSparkMergeOnReadTableIncrementalRead.java | 8 +-
...dieSparkMergeOnReadTableInsertUpdateDelete.java | 46 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 98 +-
.../TestMarkerBasedRollbackStrategy.java | 2 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 15 +-
.../TestTimelineServerBasedWriteMarkers.java | 24 +-
.../hudi/table/marker/TestWriteMarkersBase.java | 20 +-
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 137 +-
.../hudi/testutils/FunctionalTestHarness.java | 46 +-
.../hudi/testutils/HoodieCleanerTestBase.java | 45 +-
.../hudi/testutils/HoodieClientTestBase.java | 82 +-
.../hudi/testutils/HoodieClientTestUtils.java | 60 +-
.../testutils/HoodieSparkClientTestHarness.java | 151 +-
.../testutils/HoodieSparkWriteableTestTable.java | 32 +-
.../SparkClientFunctionalTestHarness.java | 67 +-
.../apache/spark/TestHoodieSparkKryoRegistrar.java | 86 +
.../apache/spark/sql/TestHoodieDataTypeUtils.java | 115 +
.../sql/hudi/execution/TestRangeSampleSort.java | 7 +-
hudi-common/pom.xml | 77 +-
hudi-common/src/main/avro/HoodieMetadata.avsc | 37 +
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 132 +-
.../org/apache/hudi/avro/AvroLogicalTypeEnum.java | 47 +
.../java/org/apache/hudi/avro/AvroSchemaUtils.java | 55 +-
.../apache/hudi/avro/ConvertingGenericData.java | 10 +
.../apache/hudi/avro/GenericAvroSerializer.java | 1 -
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 272 +-
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 85 -
.../hudi/avro/HoodieBloomFilterWriteSupport.java | 5 +-
.../java/org/apache/hudi/avro/JsonEncoder.java | 8 +-
.../apache/hudi/avro/MercifulJsonConverter.java | 474 ++-
.../avro/processors/DateLogicalTypeProcessor.java | 43 +
.../processors/DecimalLogicalTypeProcessor.java | 69 +
.../processors/DurationLogicalTypeProcessor.java | 66 +
.../hudi/avro/processors/EnumTypeProcessor.java | 33 +
.../hudi/avro/processors/FixedTypeProcessor.java | 38 +
.../hudi/avro/processors/JsonFieldProcessor.java | 39 +
.../LocalTimestampMicroLogicalTypeProcessor.java | 60 +
.../LocalTimestampMilliLogicalTypeProcessor.java | 57 +
.../org/apache/hudi/avro/processors/Parser.java | 83 +
.../avro/processors/TimeLogicalTypeProcessor.java | 266 ++
.../processors/TimeMicroLogicalTypeProcessor.java | 49 +
.../processors/TimeMilliLogicalTypeProcessor.java | 49 +
.../TimestampMicroLogicalTypeProcessor.java | 52 +
.../TimestampMilliLogicalTypeProcessor.java | 52 +
.../transaction/lock/InProcessLockProvider.java | 4 +-
.../org/apache/hudi/common/HoodieRollbackStat.java | 14 +-
.../hudi/common/bloom/InternalBloomFilter.java | 2 +-
.../hudi/common/bootstrap/FileStatusUtils.java | 98 +-
.../common/bootstrap/index/BootstrapIndex.java | 13 +-
.../bootstrap/index/HFileBootstrapIndex.java | 778 -----
.../common/bootstrap/index/NoOpBootstrapIndex.java | 13 +-
.../bootstrap/index/hfile/HFileBootstrapIndex.java | 171 +
.../index/hfile/HFileBootstrapIndexReader.java | 242 ++
.../apache/hudi/common/config/ConfigGroups.java | 14 +-
.../common/config/DFSPropertiesConfiguration.java | 250 --
.../hudi/common/config/HoodieCommonConfig.java | 3 +
.../apache/hudi/common/config/HoodieConfig.java | 11 +-
.../common/config/HoodieFunctionalIndexConfig.java | 302 --
.../hudi/common/config/HoodieIndexingConfig.java | 310 ++
.../hudi/common/config/HoodieMetadataConfig.java | 94 +-
.../hudi/common/config/HoodieReaderConfig.java | 14 +-
.../hudi/common/config/HoodieStorageConfig.java | 18 +-
.../config/HoodieTableServiceManagerConfig.java | 8 +-
.../hudi/common/config/LockConfiguration.java | 3 +
.../hudi/common/config/PropertiesConfig.java | 32 +
.../apache/hudi/common/config/RecordMergeMode.java | 36 +
.../common/config/TimestampKeyGeneratorConfig.java | 2 +-
.../DirectMarkerBasedDetectionStrategy.java | 32 +-
.../TimelineServerBasedDetectionStrategy.java | 9 +-
.../org/apache/hudi/common/data/HoodieData.java | 5 +
.../apache/hudi/common/data/HoodieListData.java | 5 +
.../hudi/common/data/HoodieListPairData.java | 5 +
.../apache/hudi/common/data/HoodiePairData.java | 5 +
.../hudi/common/engine/HoodieEngineContext.java | 26 +-
.../common/engine/HoodieLocalEngineContext.java | 16 +-
.../hudi/common/engine/HoodieReaderContext.java | 196 +-
.../common/engine/LocalTaskContextSupplier.java | 5 -
.../hudi/common/engine/TaskContextSupplier.java | 5 -
.../java/org/apache/hudi/common/fs/FSUtils.java | 575 ++--
.../hudi/common/fs/FailSafeConsistencyGuard.java | 52 +-
.../hudi/common/fs/NoOpConsistencyGuard.java | 49 +
.../hudi/common/fs/OptimisticConsistencyGuard.java | 20 +-
.../common/heartbeat/HoodieHeartbeatUtils.java | 25 +-
.../org/apache/hudi/common/model/ActionType.java | 2 +-
.../org/apache/hudi/common/model/BaseFile.java | 39 +-
.../common/model/BootstrapBaseFileMapping.java | 2 +-
.../hudi/common/model/BootstrapIndexType.java | 2 +-
.../hudi/common/model/ClusteringGroupInfo.java | 3 +-
.../hudi/common/model/ClusteringOperation.java | 3 +-
.../hudi/common/model/CompactionOperation.java | 14 +-
.../common/model/DefaultHoodieRecordPayload.java | 4 +-
.../hudi/common/model/FirstValueAvroPayload.java | 125 +
.../hudi/common/model/HoodieArchivedLogFile.java | 10 +-
.../hudi/common/model/HoodieAvroIndexedRecord.java | 8 +
.../hudi/common/model/HoodieAvroRecordMerger.java | 6 +-
.../apache/hudi/common/model/HoodieBaseFile.java | 50 +-
.../hudi/common/model/HoodieCleaningPolicy.java | 6 +-
.../common/model/HoodieColumnRangeMetadata.java | 18 +-
.../hudi/common/model/HoodieCommitMetadata.java | 91 +-
.../hudi/common/model/HoodieDeltaWriteStat.java | 14 +
.../model/HoodieFunctionalIndexDefinition.java | 90 -
.../model/HoodieFunctionalIndexMetadata.java | 95 -
.../hudi/common/model/HoodieIndexDefinition.java | 94 +
.../hudi/common/model/HoodieIndexMetadata.java | 96 +
.../org/apache/hudi/common/model/HoodieKey.java | 3 +-
.../apache/hudi/common/model/HoodieLogFile.java | 48 +-
.../common/model/HoodieMetadataRecordMerger.java | 51 +
.../apache/hudi/common/model/HoodieOperation.java | 4 +
.../hudi/common/model/HoodiePartitionMetadata.java | 145 +-
.../org/apache/hudi/common/model/HoodieRecord.java | 14 +-
.../hudi/common/model/HoodieRecordMerger.java | 25 +
.../hudi/common/model/HoodieRecordPayload.java | 5 +-
.../apache/hudi/common/model/HoodieWriteStat.java | 5 +-
.../apache/hudi/common/model/MetadataValues.java | 3 +-
.../OverwriteNonDefaultsWithLatestAvroPayload.java | 4 +-
.../model/OverwriteWithLatestAvroPayload.java | 2 -
.../common/model/OverwriteWithLatestMerger.java | 49 +
.../common/model/PartialUpdateAvroPayload.java | 12 +-
.../hudi/common/model/RecordPayloadType.java | 2 +-
.../apache/hudi/common/model/TableServiceType.java | 2 +-
.../hudi/common/model/WriteOperationType.java | 6 +-
.../hudi/common/table/HoodieTableConfig.java | 220 +-
.../hudi/common/table/HoodieTableMetaClient.java | 470 ++-
.../hudi/common/table/HoodieTableVersion.java | 14 +-
.../hudi/common/table/TableSchemaResolver.java | 147 +-
.../hudi/common/table/cdc/HoodieCDCExtractor.java | 62 +-
.../common/table/cdc/HoodieCDCInferenceCase.java | 47 +-
.../table/log/AbstractHoodieLogRecordReader.java | 285 +-
.../table/log/BaseHoodieLogRecordReader.java | 112 +-
.../log/BaseHoodieMergedLogRecordScanner.java | 257 ++
.../table/log/HoodieCDCLogRecordIterator.java | 11 +-
.../common/table/log/HoodieFileSliceReader.java | 98 +
.../hudi/common/table/log/HoodieLogFileReader.java | 175 +-
.../hudi/common/table/log/HoodieLogFormat.java | 52 +-
.../common/table/log/HoodieLogFormatReader.java | 23 +-
.../table/log/HoodieLogFormatReverseReader.java | 26 +-
.../common/table/log/HoodieLogFormatWriter.java | 291 --
.../table/log/HoodieMergedLogRecordReader.java | 88 +-
.../table/log/HoodieMergedLogRecordScanner.java | 295 +-
.../log/HoodieMetadataMergedLogRecordScanner.java | 276 ++
.../table/log/HoodieUnMergedLogRecordScanner.java | 33 +-
.../hudi/common/table/log/LogFileIterator.java | 0
.../hudi/common/table/log/LogReaderUtils.java | 56 -
.../table/log/block/HoodieAvroDataBlock.java | 140 +-
.../common/table/log/block/HoodieCommandBlock.java | 3 +-
.../common/table/log/block/HoodieCorruptBlock.java | 3 +-
.../common/table/log/block/HoodieDataBlock.java | 7 +-
.../common/table/log/block/HoodieDeleteBlock.java | 3 +-
.../table/log/block/HoodieHFileDataBlock.java | 160 +-
.../common/table/log/block/HoodieLogBlock.java | 21 +-
.../table/log/block/HoodieParquetDataBlock.java | 98 +-
.../read/HoodieBaseFileGroupRecordBuffer.java | 408 ++-
.../common/table/read/HoodieFileGroupReader.java | 239 +-
.../read/HoodieFileGroupReaderSchemaHandler.java | 200 ++
.../table/read/HoodieFileGroupReaderState.java | 36 -
.../read/HoodieKeyBasedFileGroupRecordBuffer.java | 53 +-
.../HoodiePositionBasedFileGroupRecordBuffer.java | 243 +-
.../read/HoodiePositionBasedSchemaHandler.java | 100 +
.../read/HoodieUnmergedFileGroupRecordBuffer.java | 16 +-
.../table/read/IncrementalQueryAnalyzer.java | 54 +-
.../table/timeline/HoodieActiveTimeline.java | 168 +-
.../table/timeline/HoodieArchivedTimeline.java | 16 +-
.../table/timeline/HoodieDefaultTimeline.java | 119 +-
.../hudi/common/table/timeline/HoodieInstant.java | 33 +-
.../table/timeline/HoodieInstantTimeGenerator.java | 28 +-
.../hudi/common/table/timeline/HoodieTimeline.java | 86 +-
.../hudi/common/table/timeline/LSMTimeline.java | 50 +-
.../table/timeline/MetadataConversionUtils.java | 17 +-
.../common/table/timeline/TimeGeneratorBase.java | 11 +-
.../hudi/common/table/timeline/TimeGenerators.java | 9 +-
.../common/table/timeline/TimelineDiffHelper.java | 66 +-
.../table/timeline/TimelineMetadataUtils.java | 4 +-
.../hudi/common/table/timeline/TimelineUtils.java | 22 +-
.../table/timeline/WaitBasedTimeGenerator.java | 6 +-
.../common/table/timeline/dto/BaseFileDTO.java | 11 +-
.../common/table/timeline/dto/FSPermissionDTO.java | 64 -
.../common/table/timeline/dto/FilePathDTO.java | 9 +-
.../common/table/timeline/dto/FileStatusDTO.java | 60 +-
.../common/table/timeline/dto/InstantStateDTO.java | 8 +-
.../hudi/common/table/timeline/dto/LogFileDTO.java | 8 +-
.../clean/CleanMetadataV1MigrationHandler.java | 9 +-
.../clean/CleanMetadataV2MigrationHandler.java | 5 +-
.../clean/CleanPlanV1MigrationHandler.java | 5 +-
.../clean/CleanPlanV2MigrationHandler.java | 5 +-
.../compaction/CompactionV1MigrationHandler.java | 9 +-
.../compaction/CompactionV2MigrationHandler.java | 8 +-
.../table/view/AbstractTableFileSystemView.java | 254 +-
.../common/table/view/FileSystemViewManager.java | 82 +-
.../table/view/FileSystemViewStorageConfig.java | 11 +-
.../table/view/HoodieTableFileSystemView.java | 32 +-
.../view/HoodieTablePreCommitFileSystemView.java | 4 +-
.../IncrementalTimelineSyncFileSystemView.java | 52 +-
.../table/view/PriorityBasedFileSystemView.java | 114 +-
.../view/RemoteHoodieTableFileSystemView.java | 340 +-
.../table/view/RocksDbBasedFileSystemView.java | 16 +-
.../view/SpillableMapBasedFileSystemView.java | 11 +-
.../common/table/view/SyncableFileSystemView.java | 2 +-
.../common/table/view/TableFileSystemView.java | 16 +-
.../org/apache/hudi/common/util/BaseFileUtils.java | 236 --
.../org/apache/hudi/common/util/CleanerUtils.java | 2 +-
.../apache/hudi/common/util/ClusteringUtils.java | 223 +-
.../org/apache/hudi/common/util/CommitUtils.java | 2 +-
.../apache/hudi/common/util/CompactionUtils.java | 18 +-
.../org/apache/hudi/common/util/ConfigUtils.java | 151 +-
.../apache/hudi/common/util/FileFormatUtils.java | 280 ++
.../common/util/HoodieCommonKryoRegistrar.java | 5 +-
.../hudi/common/util/InternalSchemaCache.java | 57 +-
.../org/apache/hudi/common/util/JsonUtils.java | 4 +-
.../org/apache/hudi/common/util/MarkerUtils.java | 121 +-
.../apache/hudi/common/util/OrcReaderIterator.java | 123 -
.../java/org/apache/hudi/common/util/OrcUtils.java | 293 --
.../org/apache/hudi/common/util/ParquetUtils.java | 530 ---
.../hudi/common/util/RocksDBSchemaHelper.java | 15 +-
.../hudi/common/util/SerializationUtils.java | 9 +-
.../org/apache/hudi/common/util/SortUtils.java | 77 +
.../apache/hudi/common/util/TablePathUtils.java | 45 +-
.../org/apache/hudi/common/util/TypeUtils.java | 3 +-
.../hudi/common/util/collection/FlatLists.java | 10 +-
.../hudi/common/util/collection/RocksDBDAO.java | 2 +-
.../hudi/common/util/hash/BucketIndexUtil.java | 45 +
.../org/apache/hudi/common/util/hash/HashID.java | 22 +-
.../util/queue/BaseHoodieQueueBasedExecutor.java | 3 +-
.../hudi/common/util/queue/DisruptorExecutor.java | 3 +-
.../common/util/queue/DisruptorMessageQueue.java | 5 +-
.../util/queue/FunctionBasedQueueProducer.java | 4 +-
.../common/util/queue/WaitStrategyFactory.java | 3 +-
.../metrics/HoodieMetricsCloudWatchConfig.java | 0
.../hudi/config/metrics/HoodieMetricsConfig.java | 430 +++
.../config/metrics/HoodieMetricsDatadogConfig.java | 0
.../metrics/HoodieMetricsGraphiteConfig.java | 0
.../config/metrics/HoodieMetricsJmxConfig.java | 0
.../hudi/config/metrics/HoodieMetricsM3Config.java | 132 +
.../metrics/HoodieMetricsPrometheusConfig.java | 0
.../org/apache/hudi/exception/ExceptionUtil.java | 3 +-
.../hudi/exception/HoodieAvroSchemaException.java | 31 +
.../org/apache/hudi/exception/HoodieException.java | 0
.../apache/hudi/exception/HoodieIOException.java | 0
.../HoodieIncompatibleSchemaException.java | 33 +
.../exception/HoodieJsonConversionException.java | 30 +
.../HoodieJsonToAvroConversionException.java | 33 +
.../exception/HoodieRecordCreationException.java | 32 +
.../exception/InvalidHoodieFileNameException.java | 27 +
.../hudi/exception/InvalidHoodiePathException.java | 8 +-
.../hudi/exception/TableNotFoundException.java | 12 +-
.../java/org/apache/hudi/expression/Literal.java | 1 +
.../index/functional/HoodieFunctionalIndex.java | 25 +
.../index/secondary/SecondaryIndexManager.java | 7 +-
.../hudi/internal/schema/InternalSchema.java | 47 +-
.../internal/schema/InternalSchemaBuilder.java | 5 +
.../schema/action/InternalSchemaMerger.java | 17 +
.../convert/AvroInternalSchemaConverter.java | 129 +-
.../io/FileBasedInternalSchemaStorageManager.java | 63 +-
.../schema/utils/AvroSchemaEvolutionUtils.java | 44 +-
.../internal/schema/visitor/NameToIDVisitor.java | 6 +-
.../schema/visitor/NameToPositionVisitor.java | 115 +
.../hudi/io/storage/HoodieAvroFileReader.java | 28 +-
.../hudi/io/storage/HoodieAvroFileReaderBase.java | 49 -
.../io/storage/HoodieAvroFileReaderFactory.java | 78 -
.../io/storage/HoodieAvroFileWriterFactory.java | 119 -
.../io/storage/HoodieAvroHFileReaderImplBase.java | 39 +-
.../hudi/io/storage/HoodieAvroHFileWriter.java | 205 --
.../hudi/io/storage/HoodieAvroOrcReader.java | 123 -
.../hudi/io/storage/HoodieAvroOrcWriter.java | 167 -
.../hudi/io/storage/HoodieAvroParquetReader.java | 196 --
.../hudi/io/storage/HoodieAvroParquetWriter.java | 86 -
.../hudi/io/storage/HoodieBaseParquetWriter.java | 162 -
.../hudi/io/storage/HoodieFileReaderFactory.java | 69 +-
.../apache/hudi/io/storage/HoodieFileWriter.java | 4 +-
.../hudi/io/storage/HoodieFileWriterFactory.java | 63 +-
.../io/storage/HoodieHBaseAvroHFileReader.java | 725 ----
.../apache/hudi/io/storage/HoodieHFileConfig.java | 108 -
.../apache/hudi/io/storage/HoodieHFileUtils.java | 100 -
.../apache/hudi/io/storage/HoodieIOFactory.java | 118 +
.../io/storage/HoodieNativeAvroHFileReader.java | 108 +-
.../apache/hudi/io/storage/HoodieOrcConfig.java | 15 +-
.../hudi/io/storage/HoodieParquetConfig.java | 81 -
.../hudi/io/storage/HoodieParquetStreamWriter.java | 106 -
.../hudi/io/storage/HoodieSeekingFileReader.java | 3 +-
.../org/apache/hudi/keygen/BaseKeyGenerator.java | 2 +
.../hudi/keygen/constant/KeyGeneratorOptions.java | 8 +
.../hudi/metadata/AbstractHoodieTableMetadata.java | 17 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 178 +-
.../metadata/FileSystemBackedTableMetadata.java | 134 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 388 ++-
.../metadata/HoodieMetadataFileSystemView.java | 17 +-
.../metadata/HoodieMetadataLogRecordReader.java | 54 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 35 +-
.../hudi/metadata/HoodieMetadataPayload.java | 458 +--
.../apache/hudi/metadata/HoodieTableMetadata.java | 55 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 1185 ++++---
.../hudi/metadata/MetadataPartitionType.java | 389 ++-
.../metadata/MetadataRecordsGenerationParams.java | 89 -
.../hudi/metrics/ConsoleMetricsReporter.java | 0
.../java/org/apache/hudi/metrics/HoodieGauge.java | 0
.../hudi/metrics/InMemoryMetricsReporter.java | 37 +
.../apache/hudi/metrics/JmxMetricsReporter.java | 141 +
.../org/apache/hudi/metrics/JmxReporterServer.java | 0
.../java/org/apache/hudi/metrics/MetricUtils.java | 82 +
.../main/java/org/apache/hudi/metrics/Metrics.java | 201 ++
.../hudi/metrics/MetricsGraphiteReporter.java | 93 +
.../org/apache/hudi/metrics/MetricsReporter.java | 0
.../hudi/metrics/MetricsReporterFactory.java | 101 +
.../apache/hudi/metrics/MetricsReporterType.java | 0
.../custom/CustomizableMetricsReporter.java | 0
.../hudi/metrics/datadog/DatadogHttpClient.java | 0
.../metrics/datadog/DatadogMetricsReporter.java | 87 +
.../hudi/metrics/datadog/DatadogReporter.java | 0
.../apache/hudi/metrics/m3/M3MetricsReporter.java | 124 +
.../hudi/metrics/m3/M3ScopeReporterAdaptor.java | 147 +
.../metrics/prometheus/PrometheusReporter.java | 132 +
.../prometheus/PushGatewayMetricsReporter.java | 99 +
.../metrics/prometheus/PushGatewayReporter.java | 0
.../AbstractUserDefinedMetricsReporter.java | 0
.../apache/hudi/storage/HoodieStorageUtils.java | 47 +
.../main/java/org/apache/hudi/util/Transient.java | 1 +
.../avro/HoodieAvroParquetReaderBuilder.java | 79 -
.../apache/parquet/avro/HoodieAvroReadSupport.java | 133 -
.../org/apache/hudi/avro/AvroSchemaTestUtils.java | 71 +
.../org/apache/hudi/avro/TestAvroSchemaUtils.java | 54 +-
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 132 +-
.../hudi/avro/TestMercifulJsonConverter.java | 675 ++++
.../lock/TestInProcessLockProvider.java | 509 ---
.../apache/hudi/common/bloom/TestBloomFilter.java | 172 -
.../hudi/common/bootstrap/TestBootstrapIndex.java | 210 --
.../org/apache/hudi/common/fs/TestFSUtils.java | 603 ----
.../fs/TestFSUtilsWithRetryWrapperEnable.java | 249 --
.../common/fs/TestHoodieWrapperFileSystem.java | 83 -
.../hudi/common/fs/inline/InLineFSUtilsTest.java | 62 -
.../common/fs/inline/TestInLineFileSystem.java | 388 ---
.../TestInLineFileSystemHFileInLiningBase.java | 180 -
.../TestInLineFileSystemWithHBaseHFileReader.java | 126 -
.../hudi/common/fs/inline/TestParquetInLining.java | 155 -
.../common/functional/TestHoodieLogFormat.java | 3040 -----------------
.../TestHoodieLogFormatAppendFailure.java | 156 -
.../apache/hudi/common/model/TestFileSlice.java | 7 +-
.../common/model/TestFirstValueAvroPayload.java | 81 +
.../hudi/common/model/TestHoodieBaseFile.java | 52 +-
.../common/model/TestHoodieCommitMetadata.java | 44 +
.../common/model/TestHoodieDeltaWriteStat.java | 25 +
.../hudi/common/model/TestHoodieFileGroup.java | 159 -
.../model/TestHoodieLSMTimelineManifest.java | 2 +-
.../hudi/common/model/TestHoodieLogFile.java | 36 +-
.../model/TestHoodieMetadataRecordMerger.java | 65 +
.../common/model/TestHoodiePartitionMetadata.java | 103 -
.../hudi/common/model/TestHoodieWriteStat.java | 14 +-
.../common/model/TestPartialUpdateAvroPayload.java | 1 -
.../hudi/common/table/TestHoodieTableConfig.java | 201 --
.../common/table/TestHoodieTableMetaClient.java | 128 -
.../hudi/common/table/TestTableSchemaResolver.java | 68 -
.../hudi/common/table/TestTimelineUtils.java | 654 ----
.../hudi/common/table/log/TestLogReaderUtils.java | 92 -
.../table/log/block/TestHoodieDeleteBlock.java | 151 -
.../common/table/read/CustomPayloadForTesting.java | 35 +
.../hudi/common/table/read/TestCustomMerger.java | 236 --
.../common/table/read/TestEventTimeMerging.java | 177 -
.../table/read/TestHoodieFileGroupReaderBase.java | 253 +-
.../table/timeline/TestHoodieActiveTimeline.java | 723 ----
.../common/table/timeline/TestHoodieInstant.java | 101 -
.../common/table/timeline/TestLSMTimeline.java | 2 +-
.../table/timeline/TestWaitBasedTimeGenerator.java | 147 -
.../view/TestHoodieTableFSViewWithClustering.java | 193 --
.../table/view/TestHoodieTableFileSystemView.java | 2295 -------------
.../table/view/TestIncrementalFSViewSync.java | 1020 ------
.../view/TestPriorityBasedFileSystemView.java | 135 +-
.../apache/hudi/common/testutils/Assertions.java | 10 +
.../hudi/common/testutils/CompactionTestUtils.java | 18 +-
.../hudi/common/testutils/FileCreateUtils.java | 205 +-
.../hudi/common/testutils/FileSystemTestUtils.java | 105 -
.../common/testutils/HoodieCommonTestHarness.java | 153 -
.../common/testutils/HoodieTestDataGenerator.java | 106 +-
.../hudi/common/testutils/HoodieTestTable.java | 1360 --------
.../hudi/common/testutils/HoodieTestUtils.java | 158 +-
.../common/testutils/InProcessTimeGenerator.java | 5 +-
.../hudi/common/testutils/RawTripTestPayload.java | 7 +-
.../hudi/common/testutils/SchemaTestUtil.java | 48 +-
.../testutils/reader/DataGenerationPlan.java | 18 +-
.../reader/HoodieFileGroupReaderTestHarness.java | 128 -
.../reader/HoodieFileGroupReaderTestUtils.java | 48 +-
.../testutils/reader/HoodieFileSliceTestUtils.java | 138 +-
.../testutils/reader/HoodieTestReaderContext.java | 36 +-
.../apache/hudi/common/util/TestAvroOrcUtils.java | 80 -
.../apache/hudi/common/util/TestBaseFileUtils.java | 90 +
.../hudi/common/util/TestClusteringUtils.java | 272 --
.../apache/hudi/common/util/TestCommitUtils.java | 217 --
.../hudi/common/util/TestCompactionUtils.java | 375 ---
.../apache/hudi/common/util/TestConfigUtils.java | 94 +-
.../util/TestDFSPropertiesConfiguration.java | 209 --
.../hudi/common/util/TestHoodieRecordUtils.java | 3 +-
.../apache/hudi/common/util/TestMarkerUtils.java | 91 -
.../hudi/common/util/TestOrcReaderIterator.java | 95 -
.../apache/hudi/common/util/TestParquetUtils.java | 273 --
.../apache/hudi/common/util/TestRetryHelper.java | 2 +-
.../org/apache/hudi/common/util/TestSortUtils.java | 110 +
.../apache/hudi/common/util/TestStringUtils.java | 124 -
.../hudi/common/util/TestTablePathUtils.java | 151 -
.../common/util/collection/TestBitCaskDiskMap.java | 260 --
.../util/collection/TestExternalSpillableMap.java | 472 ---
.../common/util/collection/TestRocksDBDAO.java | 16 +-
.../common/util/collection/TestRocksDbDiskMap.java | 208 --
.../hudi/common/util/hash/TestBucketIndexUtil.java | 152 +
.../apache/hudi/common/util/hash/TestHashID.java | 40 +-
.../hudi/expression/TestPartialBindVisitor.java | 1 +
.../internal/schema/action/TestMergeSchema.java | 13 +-
.../convert/TestAvroInternalSchemaConverter.java | 112 +
.../schema/utils/TestAvroSchemaEvolutionUtils.java | 17 +-
.../schema/utils/TestInternalSchemaUtils.java | 44 +-
.../storage/TestHoodieAvroFileReaderFactory.java | 64 -
.../io/storage/TestHoodieBaseParquetWriter.java | 123 -
.../storage/TestHoodieHBaseHFileReaderWriter.java | 205 --
.../io/storage/TestHoodieHFileReaderWriter.java | 87 -
.../storage/TestHoodieHFileReaderWriterBase.java | 486 ---
.../hudi/io/storage/TestHoodieOrcReaderWriter.java | 101 -
.../io/storage/TestHoodieReaderWriterBase.java | 283 --
.../io/storage/TestHoodieReaderWriterUtils.java | 2 +-
.../HoodieBackedTestDelayedTableMetadata.java | 4 +-
.../TestFileSystemBackedTableMetadata.java | 188 --
.../hudi/metadata/TestHoodieMetadataPayload.java | 155 -
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 149 -
.../hudi/metadata/TestMetadataPartitionType.java | 224 ++
.../java/org/apache/hudi/metrics/TestMetrics.java | 62 +
.../src/test/resources/complex-schema-evolved.avsc | 30 +
.../src/test/resources/complex-schema-evolved.data | 41 +
.../src/test/resources/date-type-invalid.avsc | 25 +
hudi-common/src/test/resources/date-type.avsc | 25 +
.../resources/decimal-logical-type-fixed-type.avsc | 35 +
.../resources/decimal-logical-type-invalid.avsc | 25 +
.../src/test/resources/decimal-logical-type.avsc | 25 +
.../resources/duration-logical-type-invalid.avsc | 33 +
.../src/test/resources/duration-logical-type.avsc | 33 +
.../src/test/resources/hash/magic_input.txt | 10 +
.../hash/xxhash_BITS_128_for_magic_input.txt | 10 +
.../hash/xxhash_BITS_32_for_magic_input.txt | 10 +
.../hash/xxhash_BITS_64_for_magic_input.txt | 10 +
.../resources/local-timestamp-logical-type.avsc | 26 +
.../local-timestamp-micros-logical-type.avsc | 25 +
.../local-timestamp-millis-logical-type.avsc | 25 +
.../src/test/resources/time-logical-type.avsc | 26 +
.../test/resources/timestamp-logical-type2.avsc | 26 +
.../src/test/resources/uuid-logical-type.avsc | 25 +
hudi-examples/hudi-examples-common/pom.xml | 19 +-
hudi-examples/hudi-examples-flink/pom.xml | 23 +-
.../examples/quickstart/HoodieFlinkQuickstart.java | 25 +-
.../factory/ContinuousFileSourceFactory.java | 5 +-
.../quickstart/utils/QuickstartConfigurations.java | 22 +-
.../quickstart/TestHoodieFlinkQuickstart.java | 5 +-
.../examples/quickstart/TestQuickstartData.java | 24 +-
hudi-examples/hudi-examples-java/pom.xml | 14 -
.../java/HoodieJavaWriteClientExample.java | 92 +-
hudi-examples/hudi-examples-k8s/Dockerfile | 31 +
hudi-examples/hudi-examples-k8s/README.md | 113 +
.../hudi-examples-k8s/config/hadoop/core-site.xml | 62 +
.../config/k8s/flink-deployment.yaml | 59 +
.../config/k8s/minio-standalone.yaml | 83 +
hudi-examples/hudi-examples-k8s/pom.xml | 200 ++
.../k8s/quickstart/HudiDataStreamWriter.java | 170 +
.../k8s/quickstart/utils/DataGenerator.java | 79 +
.../examples/common/ExampleDataSchemaProvider.java | 3 +-
.../hudi/examples/common/IdentityTransformer.java | 1 +
.../hudi/examples/common/RandomJsonSource.java | 7 +-
.../examples/spark/HoodieWriteClientExample.java | 98 +-
.../examples/spark/HoodieDataSourceExample.scala | 7 +-
.../examples/spark/HoodieMorCompactionJob.scala | 9 +-
hudi-examples/pom.xml | 1 +
hudi-flink-datasource/hudi-flink/pom.xml | 13 +
.../apache/hudi/configuration/FlinkOptions.java | 56 +-
.../apache/hudi/configuration/OptionsResolver.java | 59 +-
.../org/apache/hudi/sink/StreamWriteFunction.java | 2 +-
.../hudi/sink/StreamWriteOperatorCoordinator.java | 24 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 22 +-
.../sink/bucket/BucketStreamWriteFunction.java | 18 +-
.../bucket/ConsistentBucketAssignFunction.java | 3 +-
.../org/apache/hudi/sink/bulk/RowDataKeyGen.java | 45 +-
.../apache/hudi/sink/bulk/sort/SortOperator.java | 5 +-
.../hudi/sink/clustering/ClusteringCommitSink.java | 15 +-
.../hudi/sink/clustering/ClusteringOperator.java | 18 +-
.../sink/clustering/ClusteringPlanOperator.java | 5 +-
.../clustering/ClusteringPlanSourceFunction.java | 2 +-
.../sink/clustering/FlinkClusteringConfig.java | 10 +-
.../sink/clustering/HoodieFlinkClusteringJob.java | 12 +-
.../hudi/sink/compact/CompactionPlanEvent.java | 12 +
.../hudi/sink/compact/CompactionPlanOperator.java | 14 +-
.../hudi/sink/compact/FlinkCompactionConfig.java | 10 +-
.../sink/partitioner/BucketAssignFunction.java | 4 +-
.../hudi/sink/partitioner/BucketAssigner.java | 6 +-
.../sink/partitioner/BucketIndexPartitioner.java | 11 +-
.../partitioner/profile/DeltaWriteProfile.java | 2 +-
.../sink/partitioner/profile/WriteProfile.java | 4 +-
.../sink/partitioner/profile/WriteProfiles.java | 42 +-
.../apache/hudi/sink/utils/HiveSyncContext.java | 6 +-
.../java/org/apache/hudi/sink/utils/Pipelines.java | 11 +-
.../apache/hudi/source/ExpressionPredicates.java | 3 +-
.../java/org/apache/hudi/source/FileIndex.java | 74 +-
.../apache/hudi/source/IncrementalInputSplits.java | 70 +-
.../hudi/source/StreamReadMonitoringFunction.java | 1 +
.../partitioner/StreamReadAppendPartitioner.java | 38 +
.../StreamReadBucketIndexPartitioner.java | 40 +
.../selector/StreamReadAppendKeySelector.java | 31 +
.../selector/StreamReadBucketIndexKeySelector.java | 31 +
.../hudi/source/stats/ColumnStatsIndices.java | 6 +-
.../org/apache/hudi/table/HoodieTableFactory.java | 14 +-
.../org/apache/hudi/table/HoodieTableSource.java | 148 +-
.../apache/hudi/table/catalog/HiveSchemaUtils.java | 6 +-
.../apache/hudi/table/catalog/HoodieCatalog.java | 4 +
.../hudi/table/catalog/HoodieHiveCatalog.java | 30 +-
.../hudi/table/catalog/TableOptionProperties.java | 4 +-
.../apache/hudi/table/format/FilePathUtils.java | 4 +
.../org/apache/hudi/table/format/FormatUtils.java | 23 +-
.../hudi/table/format/InternalSchemaManager.java | 7 +-
.../apache/hudi/table/format/RecordIterators.java | 2 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 204 +-
.../table/format/mor/MergeOnReadInputFormat.java | 33 +-
.../hudi/table/lookup/HoodieLookupFunction.java | 206 ++
.../hudi/table/lookup/HoodieLookupTableReader.java | 87 +
.../apache/hudi/util/AvroToRowDataConverters.java | 42 +-
.../java/org/apache/hudi/util/ClusteringUtil.java | 9 +-
.../java/org/apache/hudi/util/CompactionUtil.java | 2 +-
.../java/org/apache/hudi/util/FlinkTables.java | 6 +-
.../org/apache/hudi/util/FlinkWriteClients.java | 4 +-
.../apache/hudi/util/RowDataCastProjection.java | 1 +
.../org/apache/hudi/util/RowDataProjection.java | 1 +
.../apache/hudi/util/RowDataToAvroConverters.java | 28 +-
.../org/apache/hudi/util/SerializableSchema.java | 108 +
.../java/org/apache/hudi/util/StreamerUtil.java | 95 +-
.../hudi/configuration/TestOptionsResolver.java | 28 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 85 +-
.../org/apache/hudi/sink/TestWriteCopyOnWrite.java | 63 +-
.../hudi/sink/TestWriteMergeOnReadWithCompact.java | 7 +-
.../TestWriteWithTimelineBasedCkpMetadata.java | 4 +-
.../hudi/sink/TestWriterWithPartitionTTl.java | 90 +
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 23 +-
.../bucket/ITTestConsistentBucketStreamWrite.java | 8 +-
.../ITTestFlinkConsistentHashingClustering.java | 4 +-
.../sink/cluster/ITTestHoodieFlinkClustering.java | 30 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 12 +-
.../apache/hudi/sink/compact/TestCustomSerDe.java | 17 +-
.../hudi/sink/partitioner/TestBucketAssigner.java | 4 +-
.../utils/BucketStreamWriteFunctionWrapper.java | 15 +-
.../hudi/sink/utils/BulkInsertFunctionWrapper.java | 5 +-
.../hudi/sink/utils/ClusteringFunctionWrapper.java | 11 +-
.../apache/hudi/sink/utils/CollectorOutput.java | 75 -
.../hudi/sink/utils/CompactFunctionWrapper.java | 9 +-
...ConsistentBucketStreamWriteFunctionWrapper.java | 2 +-
.../hudi/sink/utils/InsertFunctionWrapper.java | 4 +-
.../hudi/sink/utils/MockKeyedStateStore.java | 153 +
.../hudi/sink/utils/MockOperatorStateStore.java | 67 +-
.../sink/utils/MockStateInitializationContext.java | 41 +-
.../sink/utils/MockStreamingRuntimeContext.java | 14 +-
.../sink/utils/StreamWriteFunctionWrapper.java | 20 +-
.../hudi/sink/utils/TestFunctionWrapper.java | 8 +
.../org/apache/hudi/sink/utils/TestWriteBase.java | 80 +-
.../hudi/sink/utils/keyedstate/MockKeyContext.java | 39 +
.../sink/utils/keyedstate/MockKeyedListState.java | 98 +
.../sink/utils/keyedstate/MockKeyedMapState.java | 141 +
.../sink/utils/keyedstate/MockKeyedValueState.java | 85 +
.../hudi/source/TestExpressionEvaluators.java | 3 +-
.../hudi/source/TestExpressionPredicates.java | 4 +-
.../java/org/apache/hudi/source/TestFileIndex.java | 50 +-
.../hudi/source/TestIncrementalInputSplits.java | 173 +-
.../apache/hudi/source/TestStreamReadOperator.java | 5 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 186 +-
.../apache/hudi/table/ITTestSchemaEvolution.java | 6 +-
.../apache/hudi/table/TestHoodieTableSource.java | 73 +-
.../hudi/table/catalog/HoodieCatalogTestUtils.java | 7 +
.../hudi/table/catalog/TestHoodieCatalog.java | 73 +-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 118 +-
.../apache/hudi/table/format/TestInputFormat.java | 97 +-
.../table/format/cow/TestBlockLocationSort.java | 9 +-
.../org/apache/hudi/utils/FlinkMiniCluster.java | 2 -
.../org/apache/hudi/utils/TestClusteringUtil.java | 9 +-
.../org/apache/hudi/utils/TestCompactionUtil.java | 3 +-
.../test/java/org/apache/hudi/utils/TestData.java | 29 +-
.../hudi/utils/TestRowDataToAvroConverters.java | 8 +-
.../test/java/org/apache/hudi/utils/TestSQL.java | 12 +
.../org/apache/hudi/utils/TestStreamerUtil.java | 11 +-
.../test/java/org/apache/hudi/utils/TestUtils.java | 44 +-
.../apache/hudi/adapter/CollectOutputAdapter.java | 75 +
.../cow/vector/reader/EmptyColumnReader.java | 3 +-
.../apache/hudi/adapter/CollectOutputAdapter.java | 75 +
.../cow/vector/reader/EmptyColumnReader.java | 3 +-
.../apache/hudi/adapter/CollectOutputAdapter.java | 75 +
.../cow/vector/reader/EmptyColumnReader.java | 3 +-
.../apache/hudi/adapter/CollectOutputAdapter.java | 75 +
.../cow/vector/reader/EmptyColumnReader.java | 3 +-
.../apache/hudi/adapter/CollectOutputAdapter.java | 75 +
.../cow/vector/reader/AbstractColumnReader.java | 2 +-
.../cow/vector/reader/EmptyColumnReader.java | 3 +-
hudi-flink-datasource/hudi-flink1.19.x/pom.xml | 168 +
.../apache/hudi/adapter/CollectOutputAdapter.java | 81 +
.../adapter/DataStreamScanProviderAdapter.java | 0
.../adapter/DataStreamSinkProviderAdapter.java | 0
.../apache/hudi/adapter/HiveCatalogConstants.java | 0
.../apache/hudi/adapter/MaskingOutputAdapter.java | 73 +
.../hudi/adapter/OperatorCoordinatorAdapter.java | 0
.../hudi/adapter/SortCodeGeneratorAdapter.java | 0
.../adapter/SupportsRowLevelDeleteAdapter.java | 0
.../adapter/SupportsRowLevelUpdateAdapter.java | 0
.../main/java/org/apache/hudi/adapter/Utils.java | 0
.../table/format/cow/ParquetSplitReaderUtil.java | 0
.../table/format/cow/vector/HeapArrayVector.java | 0
.../table/format/cow/vector/HeapDecimalVector.java | 0
.../format/cow/vector/HeapMapColumnVector.java | 0
.../format/cow/vector/HeapRowColumnVector.java | 0
.../format/cow/vector/ParquetDecimalVector.java | 0
.../cow/vector/reader/AbstractColumnReader.java | 325 ++
.../cow/vector/reader/ArrayColumnReader.java | 0
.../vector/reader/BaseVectorizedColumnReader.java | 0
.../cow/vector/reader/EmptyColumnReader.java | 43 +
.../vector/reader/FixedLenBytesColumnReader.java | 0
.../vector/reader/Int64TimestampColumnReader.java | 0
.../format/cow/vector/reader/MapColumnReader.java | 0
.../reader/ParquetColumnarRowSplitReader.java | 0
.../cow/vector/reader/ParquetDataColumnReader.java | 0
.../reader/ParquetDataColumnReaderFactory.java | 0
.../format/cow/vector/reader/RowColumnReader.java | 0
.../format/cow/vector/reader/RunLengthDecoder.java | 0
.../apache/hudi/adapter/TestHoodieCatalogs.java | 0
.../org/apache/hudi/adapter/TestStreamConfigs.java | 0
hudi-flink-datasource/pom.xml | 1 +
hudi-gcp/pom.xml | 27 +
.../hudi/gcp/bigquery/BigQuerySchemaResolver.java | 4 +-
.../hudi/gcp/bigquery/BigQuerySyncConfig.java | 10 +-
.../apache/hudi/gcp/bigquery/BigQuerySyncTool.java | 23 +-
.../gcp/bigquery/HoodieBigQuerySyncClient.java | 64 +-
.../hudi/gcp/bigquery/TestBigQuerySyncConfig.java | 3 -
.../hudi/gcp/bigquery/TestBigQuerySyncTool.java | 72 +-
.../gcp/bigquery/TestBigQuerySyncToolArgs.java | 4 +-
.../gcp/bigquery/TestHoodieBigQuerySyncClient.java | 59 +-
hudi-hadoop-common/pom.xml | 30 +
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 83 +
.../bootstrap/index/HFileBootstrapIndex.java | 37 +
.../hfile/HBaseHFileBootstrapIndexReader.java | 283 ++
.../hfile/HBaseHFileBootstrapIndexWriter.java | 228 ++
.../common/config/DFSPropertiesConfiguration.java | 260 ++
.../common/table/ParquetTableSchemaResolver.java | 66 +
.../common/table/log/HoodieLogFormatWriter.java | 290 ++
.../org/apache/hudi/common/util/AvroOrcUtils.java | 0
.../org/apache/hudi/common/util/HFileUtils.java | 247 ++
.../apache/hudi/common/util/HadoopConfigUtils.java | 91 +
.../java/org/apache/hudi/common/util/OrcUtils.java | 320 ++
.../hudi/common/util/ParquetReaderIterator.java | 0
.../org/apache/hudi/common/util/ParquetUtils.java | 540 +++
.../apache/hudi/hadoop/fs/ConsistencyGuard.java | 92 -
.../org/apache/hudi/hadoop/fs/HadoopFSUtils.java | 461 ++-
.../hadoop/fs/HoodieRetryWrapperFileSystem.java | 13 +-
.../hadoop/fs/HoodieSerializableFileStatus.java | 2 +-
.../hudi/hadoop/fs/HoodieWrapperFileSystem.java | 119 +-
.../hudi/hadoop/fs/NoOpConsistencyGuard.java | 47 -
.../apache/hudi/hadoop/fs/SerializablePath.java | 71 -
.../hadoop/fs/SizeAwareFSDataOutputStream.java | 5 +-
.../hudi/hadoop/fs/inline/HadoopInLineFSUtils.java | 66 +
.../hudi/hadoop/fs/inline/InLineFSUtils.java | 125 -
.../hudi/hadoop/fs/inline/InLineFileSystem.java | 16 +-
.../io/hadoop/HoodieAvroFileReaderFactory.java | 80 +
.../io/hadoop/HoodieAvroFileWriterFactory.java | 131 +
.../hudi/io/hadoop/HoodieAvroHFileWriter.java | 213 ++
.../apache/hudi/io/hadoop/HoodieAvroOrcReader.java | 129 +
.../apache/hudi/io/hadoop/HoodieAvroOrcWriter.java | 175 +
.../hudi/io/hadoop/HoodieAvroParquetReader.java | 218 ++
.../io/hadoop/HoodieAvroParquetReaderIterator.java | 40 +
.../hudi/io/hadoop/HoodieAvroParquetWriter.java | 90 +
.../hudi/io/hadoop/HoodieBaseParquetWriter.java | 166 +
.../hudi/io/hadoop/HoodieHBaseAvroHFileReader.java | 718 ++++
.../apache/hudi/io/hadoop/HoodieHFileConfig.java | 110 +
.../apache/hudi/io/hadoop/HoodieHFileUtils.java | 156 +
.../hudi/io/hadoop/HoodieHadoopIOFactory.java | 117 +
.../hudi/io/hadoop/HoodieParquetStreamWriter.java | 109 +
.../apache/hudi/io/hadoop/OrcReaderIterator.java | 126 +
.../hudi/io/storage/HoodieHBaseKVComparator.java | 0
.../hudi/io/storage/HoodieParquetConfig.java | 82 +
.../parquet/io/OutputStreamBackedOutputFile.java | 0
.../storage/hadoop/HadoopStorageConfiguration.java | 60 +-
.../hudi/storage/hadoop/HoodieHadoopStorage.java | 120 +-
.../avro/HoodieAvroParquetReaderBuilder.java | 81 +
.../apache/parquet/avro/HoodieAvroReadSupport.java | 146 +
.../lock/TestInProcessLockProvider.java | 526 +++
.../apache/hudi/common/bloom/TestBloomFilter.java | 172 +
.../common/bootstrap/index/TestBootstrapIndex.java | 209 ++
.../org/apache/hudi/common/fs/TestFSUtils.java | 647 ++++
.../fs/TestFSUtilsWithRetryWrapperEnable.java | 271 ++
.../common/fs/TestHoodieWrapperFileSystem.java | 88 +
.../apache/hudi/common/fs/TestStorageSchemes.java | 0
.../hudi/common/fs/inline/InLineFSUtilsTest.java | 63 +
.../common/fs/inline/TestInLineFileSystem.java | 393 +++
.../TestInLineFileSystemHFileInLiningBase.java | 181 +
.../TestInLineFileSystemWithHBaseHFileReader.java | 126 +
.../TestInLineFileSystemWithHFileReader.java | 0
.../common/fs/inline/TestInMemoryFileSystem.java | 0
.../hudi/common/fs/inline/TestParquetInLining.java | 156 +
.../common/functional/TestHoodieLogFormat.java | 2954 +++++++++++++++++
.../TestHoodieLogFormatAppendFailure.java | 163 +
.../hudi/common/model/TestHoodieFileGroup.java | 159 +
.../common/model/TestHoodiePartitionMetadata.java | 109 +
.../apache/hudi/common/model/TestHoodieRecord.java | 0
.../common/model/TestHoodieRecordDelegate.java | 0
.../hudi/common/table/TestHoodieTableConfig.java | 239 ++
.../common/table/TestHoodieTableMetaClient.java | 273 ++
.../hudi/common/table/TestTableSchemaResolver.java | 122 +
.../hudi/common/table/TestTimelineUtils.java | 671 ++++
.../hudi/common/table/log/TestLogReaderUtils.java | 92 +
.../table/log/block/TestHoodieDeleteBlock.java | 152 +
.../hudi/common/table/read/TestCustomMerger.java | 278 ++
.../common/table/read/TestEventTimeMerging.java | 222 ++
.../table/read/TestOverwriteWithLatestMerger.java | 218 ++
.../table/timeline/TestHoodieActiveTimeline.java | 786 +++++
.../common/table/timeline/TestHoodieInstant.java | 102 +
.../table/timeline/TestWaitBasedTimeGenerator.java | 148 +
.../view/TestHoodieTableFSViewWithClustering.java | 197 ++
.../table/view/TestHoodieTableFileSystemView.java | 2509 ++++++++++++++
.../table/view/TestIncrementalFSViewSync.java | 1037 ++++++
.../TestRocksDBBasedIncrementalFSViewSync.java | 0
.../table/view/TestRocksDbBasedFileSystemView.java | 0
.../view/TestSpillableMapBasedFileSystemView.java | 0
...TestSpillableMapBasedIncrementalFSViewSync.java | 0
.../hudi/common/testutils/FileSystemTestUtils.java | 71 +
.../hudi/common/testutils/HadoopFSTestUtils.java | 43 +
.../common/testutils/HoodieCommonTestHarness.java | 162 +
.../hudi/common/testutils/HoodieTestTable.java | 1496 +++++++++
.../testutils/minicluster/HdfsTestService.java | 0
.../reader/HoodieFileGroupReaderTestHarness.java | 148 +
.../apache/hudi/common/util/TestAvroOrcUtils.java | 84 +
.../hudi/common/util/TestClusteringUtils.java | 274 ++
.../apache/hudi/common/util/TestCommitUtils.java | 219 ++
.../hudi/common/util/TestCompactionUtils.java | 445 +++
.../util/TestDFSPropertiesConfiguration.java | 211 ++
.../apache/hudi/common/util/TestFileIOUtils.java | 0
.../apache/hudi/common/util/TestHFileUtils.java | 59 +
.../hudi/common/util/TestHadoopConfigUtils.java | 63 +
.../apache/hudi/common/util/TestMarkerUtils.java | 90 +
.../common/util/TestParquetReaderIterator.java | 0
.../apache/hudi/common/util/TestParquetUtils.java | 389 +++
.../hudi/common/util/TestTablePathUtils.java | 166 +
.../common/util/collection/TestBitCaskDiskMap.java | 260 ++
.../util/collection/TestExternalSpillableMap.java | 472 +++
.../util/collection/TestRocksDbBasedMap.java | 0
.../common/util/collection/TestRocksDbDiskMap.java | 208 ++
.../TestFileBasedInternalSchemaStorageManager.java | 0
.../io/hadoop/TestHoodieAvroFileReaderFactory.java | 71 +
.../io/hadoop/TestHoodieBaseParquetWriter.java | 130 +
.../hadoop/TestHoodieHBaseHFileReaderWriter.java | 205 ++
.../io/hadoop/TestHoodieHFileReaderWriter.java | 92 +
.../io/hadoop/TestHoodieHFileReaderWriterBase.java | 490 +++
.../hudi/io/hadoop/TestHoodieHadoopIOFactory.java | 65 +
.../hudi/io/hadoop/TestHoodieOrcReaderWriter.java | 111 +
.../hudi/io/hadoop/TestHoodieReaderWriterBase.java | 289 ++
.../hudi/io/hadoop/TestOrcReaderIterator.java | 98 +
.../TestFileSystemBackedTableMetadata.java | 211 ++
.../hudi/metadata/TestHoodieMetadataPayload.java | 267 ++
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 233 ++
.../storage/hadoop/TestHoodieHadoopStorage.java | 2 +-
.../resources/external-config/hudi-defaults.conf | 0
.../src/test/resources/props/testdfs.properties | 17 +
hudi-hadoop-mr/pom.xml | 22 +
.../apache/hudi/hadoop/BootstrapBaseFileSplit.java | 3 +-
.../hudi/hadoop/DefaultHiveRecordMerger.java | 69 +
.../hudi/hadoop/HiveHoodieReaderContext.java | 253 ++
.../hudi/hadoop/HiveHoodieTableFileIndex.java | 11 +-
.../hadoop/HoodieCopyOnWriteTableInputFormat.java | 10 +-
.../HoodieFileGroupReaderBasedRecordReader.java | 310 ++
.../apache/hudi/hadoop/HoodieHFileInputFormat.java | 5 +-
.../hudi/hadoop/HoodieHFileRecordReader.java | 17 +-
.../org/apache/hudi/hadoop/HoodieHiveRecord.java | 221 ++
.../apache/hudi/hadoop/HoodieHiveRecordMerger.java | 42 +
.../hudi/hadoop/HoodieParquetInputFormat.java | 48 +-
.../hudi/hadoop/HoodieParquetInputFormatBase.java | 3 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 38 +-
.../org/apache/hudi/hadoop/InputPathHandler.java | 2 +-
.../OverwriteWithLatestHiveRecordMerger.java | 44 +
.../org/apache/hudi/hadoop/RealtimeFileStatus.java | 5 +-
.../hudi/hadoop/RecordReaderValueIterator.java | 13 +-
.../apache/hudi/hadoop/SchemaEvolutionContext.java | 36 +-
.../hadoop/hive/HoodieCombineHiveInputFormat.java | 1 +
.../hive/HoodieCombineRealtimeHiveSplit.java | 8 +-
.../realtime/AbstractRealtimeRecordReader.java | 12 +-
.../HoodieCombineRealtimeRecordReader.java | 51 +-
.../realtime/HoodieMergeOnReadSnapshotReader.java | 8 +-
.../HoodieMergeOnReadTableInputFormat.java | 43 +-
.../realtime/HoodieParquetRealtimeInputFormat.java | 14 +-
.../HoodieRealtimeBootstrapBaseFileSplit.java | 6 +-
.../hudi/hadoop/realtime/HoodieRealtimePath.java | 3 +-
.../realtime/RealtimeCompactedRecordReader.java | 8 +-
.../apache/hudi/hadoop/realtime/RealtimeSplit.java | 4 +-
.../realtime/RealtimeUnmergedRecordReader.java | 9 +-
.../hudi/hadoop/utils/HiveAvroSerializer.java | 5 +-
.../hadoop/utils/HoodieArrayWritableAvroUtils.java | 98 +
.../apache/hudi/hadoop/utils/HoodieHiveUtils.java | 5 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 101 +-
.../utils/HoodieRealtimeRecordReaderUtils.java | 19 +-
.../hudi/hadoop/utils/ObjectInspectorCache.java | 95 +
.../apache/hudi/hadoop/utils/shims/Hive2Shim.java | 4 +-
.../apache/hudi/hadoop/utils/shims/Hive3Shim.java | 32 +-
.../apache/hudi/hadoop/utils/shims/HiveShim.java | 2 +-
.../org/apache/hudi/hadoop/TestAnnotation.java | 4 +-
.../hudi/hadoop/TestHoodieHFileInputFormat.java | 26 +-
.../hudi/hadoop/TestHoodieParquetInputFormat.java | 148 +-
.../hudi/hadoop/TestHoodieROTablePathFilter.java | 7 +-
.../apache/hudi/hadoop/TestInputPathHandler.java | 4 +-
.../hive/TestHoodieCombineHiveInputFormat.java | 56 +-
.../TestHoodieMergeOnReadSnapshotReader.java | 28 +-
.../TestHoodieMergeOnReadTableInputFormat.java | 7 +-
.../realtime/TestHoodieRealtimeFileSplit.java | 3 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 88 +-
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 112 +-
.../hudi/hadoop/utils/TestHiveAvroSerializer.java | 2 +-
.../utils/TestHoodieArrayWritableAvroUtils.java | 88 +
.../utils/TestHoodieRealtimeInputFormatUtils.java | 7 +-
hudi-integ-test/pom.xml | 65 +-
.../testsuite/HoodieDeltaStreamerWrapper.java | 5 +-
.../hudi/integ/testsuite/HoodieTestSuiteJob.java | 15 +-
.../SparkDataSourceContinuousIngestTool.java | 4 +-
.../testsuite/configuration/DFSDeltaConfig.java | 8 +-
.../integ/testsuite/configuration/DeltaConfig.java | 10 +-
.../hudi/integ/testsuite/dag/WriterContext.java | 4 +-
.../dag/nodes/BaseValidateDatasetNode.java | 6 +-
.../integ/testsuite/dag/nodes/CompactNode.java | 8 +-
.../integ/testsuite/dag/nodes/RollbackNode.java | 23 +-
.../testsuite/dag/nodes/ScheduleCompactNode.java | 8 +-
.../dag/nodes/ValidateAsyncOperations.java | 4 +-
.../reader/DFSHoodieDatasetInputReader.java | 23 +-
.../streaming/StructuredStreamingSinkUtil.java | 168 +
.../testsuite/writer/AvroFileDeltaInputWriter.java | 7 +-
.../StructuredStreamingSinkTestWriter.scala | 104 +
.../TestDFSHoodieTestSuiteWriterAdapter.java | 3 +-
.../testsuite/job/TestHoodieTestSuiteJob.java | 21 +-
.../reader/TestDFSHoodieDatasetInputReader.java | 3 +-
hudi-io/pom.xml | 211 +-
.../apache/hudi/common/fs/ConsistencyGuard.java | 92 +
.../apache/hudi/common/util/ComparableVersion.java | 24 +-
.../org/apache/hudi/common/util/FileIOUtils.java | 95 +-
.../org/apache/hudi/common/util/StringUtils.java | 79 +
.../org/apache/hudi/storage/HoodieStorage.java | 124 +-
.../apache/hudi/storage/StorageConfiguration.java | 83 +-
.../java/org/apache/hudi/storage/StoragePath.java | 21 +-
.../org/apache/hudi/storage/StoragePathInfo.java | 30 +-
.../apache/hudi/storage/inline/InLineFSUtils.java | 108 +
.../hudi/common/testutils/NetworkTestUtils.java | 0
.../apache/hudi/common/util/TestStringUtils.java | 102 +-
.../io/storage/BaseTestStorageConfiguration.java | 58 +-
.../hudi/io/storage/TestHoodieStorageBase.java | 111 +-
.../apache/hudi/io/storage/TestStoragePath.java | 28 +-
.../hudi/io/storage/TestStoragePathInfo.java | 13 +
hudi-kafka-connect/pom.xml | 8 +
.../hudi/connect/utils/KafkaConnectUtils.java | 20 +-
.../writers/KafkaConnectTransactionServices.java | 13 +-
.../writers/KafkaConnectWriterProvider.java | 12 +-
.../connect/TestConnectTransactionParticipant.java | 2 +-
.../apache/hudi/connect/TestHdfsConfiguration.java | 7 +-
.../hudi/writers/TestBufferedConnectWriter.java | 7 +-
.../common/table/HoodieTableMetaserverClient.java | 6 +-
.../timeline/HoodieMetaserverBasedTimeline.java | 16 +-
.../hudi-metaserver/hudi-metaserver-server/pom.xml | 4 +-
.../hudi/metaserver/store/jdbc/BasicDao.java | 4 +-
hudi-platform-service/hudi-metaserver/pom.xml | 2 +-
hudi-spark-datasource/README.md | 10 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 26 +
.../main/java/org/apache/hudi/DataSourceUtils.java | 55 +-
.../hudi/HoodieSparkFunctionalIndexClient.java | 29 +-
.../BaseDatasetBulkInsertCommitActionExecutor.java | 5 +
.../DatasetBulkInsertCommitActionExecutor.java | 1 +
...setBulkInsertOverwriteCommitActionExecutor.java | 1 +
...lkInsertOverwriteTableCommitActionExecutor.java | 6 +-
.../apache/hudi/internal/BaseDefaultSource.java | 10 +-
.../hudi/internal/BaseWriterCommitMessage.java | 3 +-
.../internal/DataSourceInternalWriterHelper.java | 11 +-
.../scala/org/apache/hudi/AvroProjection.scala | 3 +-
.../org/apache/hudi/BaseFileOnlyRelation.scala | 15 +-
.../org/apache/hudi/BloomFiltersIndexSupport.scala | 90 +
.../scala/org/apache/hudi/BucketIndexSupport.scala | 224 ++
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 79 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 37 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 76 +-
.../scala/org/apache/hudi/FileSystemRelation.scala | 14 +-
.../org/apache/hudi/FunctionalIndexSupport.scala | 178 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 66 +-
.../org/apache/hudi/HoodieBootstrapMORRDD.scala | 9 +-
.../apache/hudi/HoodieBootstrapMORRelation.scala | 4 +-
.../scala/org/apache/hudi/HoodieBootstrapRDD.scala | 1 +
.../org/apache/hudi/HoodieBootstrapRelation.scala | 30 +-
.../scala/org/apache/hudi/HoodieCDCFileIndex.scala | 6 +-
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 33 +-
.../org/apache/hudi/HoodieCatalystUtils.scala | 1 +
.../org/apache/hudi/HoodieCreateRecordUtils.scala | 16 +-
.../org/apache/hudi/HoodieDataSourceHelper.scala | 16 +-
.../scala/org/apache/hudi/HoodieFileIndex.scala | 360 +-
.../hudi/HoodieHadoopFsRelationFactory.scala | 83 +-
.../apache/hudi/HoodieIncrementalFileIndex.scala | 27 +-
.../org/apache/hudi/HoodieMergeOnReadRDD.scala | 5 +-
.../hudi/HoodiePartitionCDCFileGroupMapping.scala | 1 -
.../hudi/HoodiePartitionFileSliceMapping.scala | 3 -
.../scala/org/apache/hudi/HoodieSchemaUtils.scala | 60 +-
.../org/apache/hudi/HoodieSparkConfUtils.scala | 1 +
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 239 +-
.../org/apache/hudi/HoodieStreamingSink.scala | 35 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 34 +-
.../org/apache/hudi/IncrementalRelation.scala | 37 +-
.../src/main/scala/org/apache/hudi/Iterators.scala | 75 +-
.../hudi/MergeOnReadIncrementalRelation.scala | 23 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 10 +-
.../apache/hudi/PartitionStatsIndexSupport.scala | 119 +
.../org/apache/hudi/RecordLevelIndexSupport.scala | 212 +-
.../org/apache/hudi/SecondaryIndexSupport.scala | 131 +
.../org/apache/hudi/SparkBaseIndexSupport.scala | 175 +
.../scala/org/apache/hudi/SparkFilterHelper.scala | 6 +-
.../apache/hudi/SparkHoodieTableFileIndex.scala | 121 +-
.../scala/org/apache/hudi/TimelineRelation.scala | 8 +-
.../org/apache/hudi/cdc/CDCFileGroupIterator.scala | 65 +-
.../scala/org/apache/hudi/cdc/CDCRelation.scala | 13 +-
.../scala/org/apache/hudi/cdc/HoodieCDCRDD.scala | 49 +-
.../cdc/InternalRowToJsonStringConverter.scala | 3 +-
.../org/apache/hudi/util/SparkConfigUtils.scala | 50 +
.../org/apache/spark/HoodieHadoopFSUtils.scala | 3 +-
.../datasources/HoodieInMemoryFileIndex.scala | 9 +-
.../sql/FileFormatUtilsForFileGroupReader.scala | 6 +-
.../apache/spark/sql/avro/SchemaConverters.scala | 4 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 34 +-
.../datasources/HoodieMultipleBaseFileFormat.scala | 13 +-
...odieFileGroupReaderBasedParquetFileFormat.scala | 467 +--
.../parquet/LegacyHoodieParquetFileFormat.scala | 4 +-
.../parquet/SparkParquetReaderBase.scala | 104 +
.../apache/spark/sql/hudi/DataSkippingUtils.scala | 4 +-
.../apache/spark/sql/hudi/HoodieOptionConfig.scala | 2 +-
.../spark/sql/hudi/HoodieSqlCommonUtils.scala | 41 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 85 +-
.../AlterHoodieTableAddColumnsCommand.scala | 6 +-
.../AlterHoodieTableAddPartitionCommand.scala | 16 +-
.../AlterHoodieTableDropPartitionCommand.scala | 2 +-
.../command/AlterHoodieTableRenameCommand.scala | 6 +-
.../hudi/command/CreateHoodieTableCommand.scala | 19 +-
.../sql/hudi/command/DropHoodieTableCommand.scala | 14 +-
...odieSparkValidateDuplicateKeyRecordMerger.scala | 9 +-
.../hudi/command/RepairHoodieTableCommand.scala | 23 +-
.../command/ShowHoodieCreateTableCommand.scala | 94 +
.../command/ShowHoodieTablePartitionsCommand.scala | 1 -
.../spark/sql/hudi/command/SqlKeyGenerator.scala | 2 +-
.../hudi/command/TruncateHoodieTableCommand.scala | 19 +-
.../hudi/command/ValidateDuplicateKeyPayload.scala | 1 -
.../sql/hudi/streaming/HoodieMetadataLog.scala | 6 +-
.../sql/hudi/streaming/HoodieSourceOffset.scala | 1 -
.../sql/hudi/streaming/HoodieStreamSource.scala | 23 +-
.../spark/sql/vectorized/ColumnarBatchUtils.java | 54 +
.../org/apache/hudi/TestHoodieSchemaUtils.java | 118 +-
.../apache/hudi/TestRecordLevelIndexSupport.scala | 104 +
.../apache/hudi/TestSecondaryIndexSupport.scala | 105 +
.../datasources/TestHoodieInMemoryFileIndex.scala | 7 +-
...odieFileGroupReaderBasedParquetFileFormat.scala | 96 -
...stSparkFileFormatInternalRowReaderContext.scala | 72 +
.../parquet/TestSparkParquetReaderFormat.scala | 60 +
.../spark/sql/hive/TestHiveClientUtils.scala | 2 +-
.../spark/sql/hudi/TestProvidesHoodieConfig.scala | 156 +
hudi-spark-datasource/hudi-spark/pom.xml | 26 +
.../org/apache/hudi/HoodieDataSourceHelpers.java | 56 +-
.../main/java/org/apache/hudi/QuickstartUtils.java | 7 +-
.../SparkFullBootstrapDataProviderBase.java | 74 +-
.../org/apache/hudi/cli/ArchiveExecutorUtils.java | 2 +-
.../apache/hudi/cli/BootstrapExecutorUtils.java | 23 +-
.../apache/hudi/cli/HDFSParquetImporterUtils.java | 8 +-
.../org/apache/hudi/payload/AWSDmsAvroPayload.java | 3 +-
.../org/apache/spark/sql/hudi/DedupeSparkJob.scala | 61 +-
.../sql/hudi/HoodieSparkSessionExtension.scala | 3 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 61 +-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 220 +-
.../analysis/HoodiePruneFileSourcePartitions.scala | 5 +-
.../hudi/command/CallProcedureHoodieCommand.scala | 1 +
.../hudi/command/CompactionHoodiePathCommand.scala | 4 +-
.../command/CompactionHoodieTableCommand.scala | 1 +
.../command/CompactionShowHoodiePathCommand.scala | 4 +-
.../command/CompactionShowHoodieTableCommand.scala | 1 +
.../command/CreateHoodieTableAsSelectCommand.scala | 5 +-
.../hudi/command/DeleteHoodieTableCommand.scala | 3 +-
.../spark/sql/hudi/command/IndexCommands.scala | 11 +-
.../command/InsertIntoHoodieTableCommand.scala | 3 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 9 +-
.../hudi/command/UpdateHoodieTableCommand.scala | 3 +-
.../spark/sql/hudi/command/UuidKeyGenerator.scala | 6 +-
.../hudi/command/payload/ExpressionPayload.scala | 8 +-
.../procedures/ArchiveCommitsProcedure.scala | 1 +
.../hudi/command/procedures/BaseProcedure.scala | 14 +
.../procedures/CommitsCompareProcedure.scala | 9 +-
.../command/procedures/CopyToTableProcedure.scala | 1 +
.../procedures/CopyToTempViewProcedure.scala | 1 +
.../procedures/CreateMetadataTableProcedure.scala | 18 +-
.../procedures/CreateSavepointProcedure.scala | 4 +-
.../command/procedures/DeleteMarkerProcedure.scala | 2 +
.../procedures/DeleteMetadataTableProcedure.scala | 4 +-
.../procedures/DeleteSavepointProcedure.scala | 4 +-
.../procedures/ExportInstantsProcedure.scala | 110 +-
.../procedures/HdfsParquetImportProcedure.scala | 2 +
.../hudi/command/procedures/HelpProcedure.scala | 1 +
.../command/procedures/HiveSyncProcedure.scala | 3 +-
.../command/procedures/HoodieProcedureUtils.scala | 3 +-
.../hudi/command/procedures/HoodieProcedures.scala | 2 +
.../procedures/InitMetadataTableProcedure.scala | 12 +-
.../sql/hudi/command/procedures/Procedure.scala | 4 -
.../RepairAddpartitionmetaProcedure.scala | 26 +-
.../RepairCorruptedCleanFilesProcedure.scala | 11 +-
.../procedures/RepairDeduplicateProcedure.scala | 6 +-
.../RepairMigratePartitionMetaProcedure.scala | 41 +-
.../RepairOverwriteHoodiePropsProcedure.scala | 25 +-
.../RollbackToInstantTimeProcedure.scala | 5 +-
.../procedures/RollbackToSavepointProcedure.scala | 4 +-
.../command/procedures/RunBootstrapProcedure.scala | 9 +-
.../command/procedures/RunCleanProcedure.scala | 2 +-
.../procedures/RunClusteringProcedure.scala | 21 +-
.../procedures/RunCompactionProcedure.scala | 23 +-
.../RunRollbackInflightTableServiceProcedure.scala | 143 +
.../hudi/command/procedures/RunTTLProcedure.scala | 2 +
.../procedures/ShowArchivedCommitsProcedure.scala | 13 +-
.../procedures/ShowBootstrapMappingProcedure.scala | 11 +-
.../ShowBootstrapPartitionsProcedure.scala | 3 +-
.../procedures/ShowClusteringProcedure.scala | 10 +-
.../ShowColumnStatsOverlapProcedure.scala | 29 +-
.../ShowCommitExtraMetadataProcedure.scala | 15 +-
.../procedures/ShowCommitFilesProcedure.scala | 18 +-
.../procedures/ShowCommitPartitionsProcedure.scala | 18 +-
.../procedures/ShowCommitWriteStatsProcedure.scala | 14 +-
.../command/procedures/ShowCommitsProcedure.scala | 11 +-
.../procedures/ShowCompactionProcedure.scala | 3 +-
.../procedures/ShowFileStatusProcedure.scala | 244 ++
.../procedures/ShowFileSystemViewProcedure.scala | 117 +-
.../procedures/ShowFsPathDetailProcedure.scala | 1 -
.../ShowHoodieLogFileMetadataProcedure.scala | 24 +-
.../ShowHoodieLogFileRecordsProcedure.scala | 29 +-
.../procedures/ShowInvalidParquetProcedure.scala | 29 +-
.../ShowMetadataTableColumnStatsProcedure.scala | 22 +-
.../ShowMetadataTableFilesProcedure.scala | 19 +-
.../ShowMetadataTablePartitionsProcedure.scala | 6 +-
.../ShowMetadataTableStatsProcedure.scala | 12 +-
.../procedures/ShowRollbacksProcedure.scala | 17 +-
.../procedures/ShowSavepointsProcedure.scala | 4 +-
.../procedures/ShowTablePropertiesProcedure.scala | 8 +-
.../procedures/StatsFileSizeProcedure.scala | 15 +-
.../StatsWriteAmplificationProcedure.scala | 7 +-
.../procedures/UpgradeOrDowngradeProcedure.scala | 9 +-
.../procedures/ValidateHoodieSyncProcedure.scala | 7 +-
.../ValidateMetadataTableFilesProcedure.scala | 72 +-
.../spark/sql/parser/HoodieCommonSqlParser.scala | 11 +-
.../sql/parser/HoodieSqlCommonAstBuilder.scala | 14 +-
.../src/test/java/HoodieJavaStreamingApp.java | 11 +-
.../org/apache/hudi/ColumnStatsIndexHelper.java | 35 +-
.../apache/hudi/TestDataSourceReadWithDeletes.java | 3 +-
.../java/org/apache/hudi/TestDataSourceUtils.java | 60 +-
.../hudi/TestHoodieMergeHandleWithSparkMerger.java | 17 +-
...stHoodiePositionBasedFileGroupRecordBuffer.java | 72 +-
.../java/org/apache/hudi/TestQuickstartUtils.java | 1 +
.../TestBulkInsertInternalPartitionerForRows.java | 15 +-
.../org/apache/hudi/functional/TestBootstrap.java | 68 +-
.../hudi/functional/TestBootstrapReadBase.java | 4 +-
.../TestDataSkippingWithMORColstats.java | 7 +-
.../functional/TestFiltersInFileGroupReader.java | 109 +
.../TestGlobalIndexEnableUpdatePartitions.java | 64 +-
.../functional/TestHiveTableSchemaEvolution.java | 56 +-
.../TestHoodieSparkMergeOnReadTableClustering.java | 19 +-
.../functional/TestNewHoodieParquetFileFormat.java | 12 +-
.../apache/hudi/functional/TestOrcBootstrap.java | 45 +-
.../TestSparkConsistentBucketClustering.java | 18 +-
.../hudi/functional/TestSparkParquetReader.java | 48 +
.../functional/TestSparkSortAndSizeClustering.java | 34 +-
.../apache/hudi/functional/TestWriteClient.java | 2 +-
.../row/TestHoodieInternalRowParquetWriter.java | 19 +-
.../io/storage/row/TestHoodieRowCreateHandle.java | 3 +-
.../hudi/keygen/TestComplexKeyGenerator.java | 3 +-
.../apache/hudi/keygen/TestCustomKeyGenerator.java | 38 +-
.../keygen/TestGlobalDeleteRecordGenerator.java | 3 +-
.../keygen/TestNonpartitionedKeyGenerator.java | 3 +-
.../apache/hudi/keygen/TestSimpleKeyGenerator.java | 3 +-
.../TestCreateKeyGeneratorByTypeWithFactory.java | 1 +
.../TestHoodieSparkKeyGeneratorFactory.java | 7 +-
.../apache/hudi/testutils/DataSourceTestUtils.java | 16 +-
.../hudi/testutils/HoodieSparkClientTestBase.java | 1 +
.../resources/external-config/hudi-defaults.conf | 2 +
.../apache/hudi/HoodieSparkWriterTestBase.scala | 3 +-
.../org/apache/hudi/ScalaAssertionSupport.scala | 2 +-
.../scala/org/apache/hudi/SparkDatasetMixin.scala | 7 +-
.../org/apache/hudi/TestAvroConversionUtils.scala | 15 +-
.../hudi/TestAvroSchemaResolutionSupport.scala | 339 +-
.../TestConvertFilterToCatalystExpression.scala | 6 +-
.../org/apache/hudi/TestDataSkippingUtils.scala | 4 +-
.../org/apache/hudi/TestDataSourceDefaults.scala | 7 +-
.../org/apache/hudi/TestDataSourceOptions.scala | 1 +
.../hudi/TestGenericRecordAndRowConsistency.scala | 3 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 103 +-
.../org/apache/hudi/TestHoodieParquetBloom.scala | 12 +-
.../org/apache/hudi/TestHoodieRelations.scala | 3 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 90 +-
.../TestHoodieSparkSqlWriterPartitionTTL.scala | 93 +
.../apache/hudi/TestHoodieSparkSqlWriterUtc.scala | 3 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 89 +-
.../hudi/TestParquetReaderCompatibility.scala | 327 ++
.../org/apache/hudi/TestSparkFilterHelper.scala | 3 +-
.../hudi/TestTableSchemaResolverWithSparkSQL.scala | 18 +-
.../model/TestHoodieRecordSerialization.scala | 3 +-
.../read/TestHoodieFileGroupReaderOnSpark.scala | 87 +-
.../TestSpark35RecordPositionMetadataColumn.scala | 56 +-
.../hudi/functional/ColumnStatIndexTestBase.scala | 15 +-
.../apache/hudi/functional/CommonOptionUtils.scala | 4 +-
.../functional/PartitionStatsIndexTestBase.scala | 279 ++
.../hudi/functional/RecordLevelIndexTestBase.scala | 90 +-
.../TestAutoGenerationOfRecordKeys.scala | 48 +-
.../hudi/functional/TestBasicSchemaEvolution.scala | 96 +-
.../functional/TestBloomFiltersIndexSupport.scala | 261 ++
.../hudi/functional/TestBucketIndexSupport.scala | 400 +++
.../apache/hudi/functional/TestCOWDataSource.scala | 372 ++-
.../hudi/functional/TestCOWDataSourceStorage.scala | 38 +-
.../hudi/functional/TestColumnStatsIndex.scala | 17 +-
.../functional/TestColumnStatsIndexWithSQL.scala | 10 +-
.../functional/TestDataSourceForBootstrap.scala | 11 +-
.../apache/hudi/functional/TestEmptyCommit.scala | 5 +-
.../hudi/functional/TestHoodieActiveTimeline.scala | 64 +-
.../TestHoodieMultipleBaseFileFormat.scala | 13 +-
.../TestIncrementalReadByStateTransitionTime.scala | 14 +-
.../TestIncrementalReadWithFullTableScan.scala | 19 +-
.../hudi/functional/TestLayoutOptimization.scala | 14 +-
.../apache/hudi/functional/TestMORDataSource.scala | 151 +-
.../hudi/functional/TestMORDataSourceStorage.scala | 26 +-
.../TestMORDataSourceWithBucketIndex.scala | 37 +-
.../hudi/functional/TestMetadataRecordIndex.scala | 16 +-
.../TestMetadataTableWithSparkDataSource.scala | 220 +-
.../hudi/functional/TestMetricsReporter.scala | 10 +-
.../functional/TestParquetColumnProjection.scala | 15 +-
.../functional/TestPartialUpdateAvroPayload.scala | 14 +-
.../hudi/functional/TestPartitionStatsIndex.scala | 271 ++
.../TestPartitionStatsIndexWithSql.scala | 405 +++
.../hudi/functional/TestRecordLevelIndex.scala | 80 +-
.../functional/TestRecordLevelIndexWithSQL.scala | 155 +-
.../functional/TestSecondaryIndexPruning.scala | 506 +++
.../hudi/functional/TestSevenToEightUpgrade.scala | 70 +
.../functional/TestSixToFiveDowngradeHandler.scala | 11 +-
.../hudi/functional/TestSparkDataSource.scala | 25 +-
.../TestSparkDataSourceDAGExecution.scala | 32 +-
.../hudi/functional/TestSparkSqlCoreFlow.scala | 17 +-
.../TestSparkSqlWithCustomKeyGenerator.scala | 768 +++++
.../TestSparkSqlWithTimestampKeyGenerator.scala | 168 +
.../apache/hudi/functional/TestSqlStatement.scala | 1 +
...TestStreamSourceReadByStateTransitionTime.scala | 11 +-
.../hudi/functional/TestStreamingSource.scala | 56 +-
.../hudi/functional/TestStructuredStreaming.scala | 89 +-
.../hudi/functional/TestTimeTravelQuery.scala | 19 +-
.../hudi/functional/cdc/HoodieCDCTestBase.scala | 47 +-
.../functional/cdc/TestCDCDataFrameSuite.scala | 125 +-
.../functional/cdc/TestCDCStreamingSuite.scala | 9 +-
.../org/apache/hudi/util/JavaConversions.scala | 23 +-
.../scala/org/apache/hudi/util/TestPathUtils.scala | 45 +-
.../util/TestSparkInternalSchemaConverter.scala | 94 +
.../spark/hudi/benchmark/HoodieBenchmark.scala | 9 +-
.../spark/hudi/benchmark/HoodieBenchmarkBase.scala | 1 +
.../spark/hudi/benchmark/HoodieBenchmarks.scala | 4 +-
.../org/apache/spark/sql/avro/TestAvroSerDe.scala | 4 +-
.../spark/sql/avro/TestSchemaConverters.scala | 1 +
.../execution/benchmark/AvroSerDerBenchmark.scala | 5 +-
.../benchmark/BoundInMemoryExecutorBenchmark.scala | 5 +-
.../benchmark/CowTableReadBenchmark.scala | 11 +-
.../benchmark/LSMTimelineReadBenchmark.scala | 9 +-
.../ReadAndWriteWithoutAvroBenchmark.scala | 19 +-
.../benchmark/SpaceCurveOptimizeBenchmark.scala | 15 +-
.../TestHoodiePruneFileSourcePartitions.scala | 15 +-
.../hudi/command/index/TestFunctionalIndex.scala | 163 +-
.../sql/hudi/command/index/TestIndexSyntax.scala | 3 +-
.../hudi/command/index/TestSecondaryIndex.scala | 104 +-
.../sql/hudi/common/HoodieSparkSqlTestBase.scala | 49 +-
.../hudi/common/TestHoodieInternalRowUtils.scala | 5 +-
.../sql/hudi/common/TestHoodieOptionConfig.scala | 5 +-
.../TestNestedSchemaPruningOptimization.scala | 9 +-
.../apache/spark/sql/hudi/common/TestSqlConf.scala | 18 +-
.../apache/spark/sql/hudi/ddl/TestAlterTable.scala | 34 +-
.../sql/hudi/ddl/TestAlterTableDropPartition.scala | 201 +-
.../spark/sql/hudi/ddl/TestCreateTable.scala | 108 +-
.../apache/spark/sql/hudi/ddl/TestSpark3DDL.scala | 356 +-
.../spark/sql/hudi/dml/TestCDCForSparkSQL.scala | 96 +-
.../spark/sql/hudi/dml/TestCompactionTable.scala | 70 +
.../spark/sql/hudi/dml/TestDataSkippingQuery.scala | 245 +-
.../spark/sql/hudi/dml/TestDeleteTable.scala | 17 +-
.../apache/spark/sql/hudi/dml/TestDropTable.scala | 3 +-
.../hudi/dml/TestHoodieTableValuedFunction.scala | 97 +-
.../spark/sql/hudi/dml/TestInsertTable.scala | 812 +++--
.../sql/hudi/dml/TestMergeIntoLogOnlyTable.scala | 1 +
.../spark/sql/hudi/dml/TestMergeIntoTable.scala | 185 +-
.../spark/sql/hudi/dml/TestMergeIntoTable2.scala | 8 +-
.../TestMergeIntoTableWithNonRecordKeyField.scala | 3 +-
.../hudi/dml/TestPartialUpdateForMergeInto.scala | 65 +-
.../spark/sql/hudi/dml/TestRepairTable.scala | 3 +-
.../spark/sql/hudi/dml/TestShowPartitions.scala | 1 +
.../spark/sql/hudi/dml/TestTimeTravelTable.scala | 282 +-
.../spark/sql/hudi/dml/TestTruncateTable.scala | 1 +
.../spark/sql/hudi/dml/TestUpdateTable.scala | 157 +-
.../hudi/procedure/TestBootstrapProcedure.scala | 5 +-
.../sql/hudi/procedure/TestCallCommandParser.scala | 2 +
.../sql/hudi/procedure/TestCleanProcedure.scala | 8 +-
.../hudi/procedure/TestClusteringProcedure.scala | 44 +-
.../sql/hudi/procedure/TestCommitsProcedure.scala | 4 +-
.../hudi/procedure/TestCompactionProcedure.scala | 10 +-
.../procedure/TestCopyToTempViewProcedure.scala | 1 +
.../sql/hudi/procedure/TestFsViewProcedure.scala | 86 +-
.../procedure/TestHdfsParquetImportProcedure.scala | 53 +-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 92 +-
...tRunRollbackInflightTableServiceProcedure.scala | 122 +
.../procedure/TestShowFileStatusProcedure.scala | 330 ++
.../TestShowInvalidParquetProcedure.scala | 45 +-
.../sql/hudi/procedure/TestTTLProcedure.scala | 4 +-
.../TestUpgradeOrDowngradeProcedure.scala | 88 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 13 +
.../org/apache/hudi/internal/DefaultSource.java | 5 +-
.../internal/HoodieDataSourceInternalWriter.java | 4 +-
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 6 +-
.../apache/spark/sql/adapter/Spark2Adapter.scala | 34 +-
.../HoodieSpark2PartitionedFileUtils.scala | 13 +-
.../datasources/parquet/Spark24ParquetReader.scala | 271 ++
.../HoodieBulkInsertInternalWriterTestBase.java | 2 +-
.../TestHoodieDataSourceInternalWriter.java | 27 +-
hudi-spark-datasource/hudi-spark3-common/pom.xml | 8 +
.../apache/hudi/spark3/internal/DefaultSource.java | 5 +-
.../HoodieDataSourceInternalBatchWrite.java | 6 +-
.../HoodieDataSourceInternalBatchWriteBuilder.java | 10 +-
.../internal/HoodieDataSourceInternalTable.java | 10 +-
.../spark3/internal/HoodieWriterCommitMessage.java | 1 +
.../apache/hudi/spark3/internal/ReflectUtil.java | 44 +-
.../Spark3HoodieVectorizedParquetRecordReader.java | 192 ++
.../org/apache/hudi/Spark3DefaultSource.scala | 47 +
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 16 +-
.../apache/spark/sql/ResolveInsertionBase.scala | 96 +
.../spark/sql/adapter/BaseSpark3Adapter.scala | 30 +-
.../HoodieFileSystemViewTableValuedFunction.scala | 0
.../logcal/HoodieMetadataTableValuedFunction.scala | 0
.../sql/catalyst/plans/logcal/HoodieQuery.scala | 0
.../catalyst/plans/logcal/HoodieTableChanges.scala | 0
.../logcal/HoodieTimelineTableValuedFunction.scala | 0
.../spark/sql/catalyst/plans/logical/Index.scala | 0
.../datasources/Spark3ParsePartitionUtil.scala | 15 +-
.../Spark3ParquetSchemaEvolutionUtils.scala | 221 ++
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 358 ++
.../sql/hudi/analysis/TableValuedFunctions.scala | 0
.../spark/sql/hudi/catalog/BasicStagedTable.scala | 0
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 423 +++
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 126 +
.../spark/sql/hudi/catalog/HoodieStagedTable.scala | 101 +
.../spark/sql/hudi/catalog/TableCreationMode.java | 0
.../spark/sql/hudi/command/AlterTableCommand.scala | 342 ++
.../sql/hudi/logical/TimeTravelRelation.scala | 0
hudi-spark-datasource/hudi-spark3.0.x/pom.xml | 283 --
...Spark30HoodieVectorizedParquetRecordReader.java | 187 --
...org.apache.spark.sql.sources.DataSourceRegister | 19 -
.../org/apache/hudi/Spark30DefaultSource.scala | 24 -
.../org/apache/hudi/Spark30HoodieFileScanRDD.scala | 35 -
.../spark/sql/HoodieSpark30CatalogUtils.scala | 29 -
.../sql/HoodieSpark30CatalystExpressionUtils.scala | 157 -
.../spark/sql/HoodieSpark30CatalystPlanUtils.scala | 85 -
.../spark/sql/HoodieSpark30SchemaUtils.scala | 39 -
.../apache/spark/sql/adapter/Spark3_0Adapter.scala | 121 -
.../apache/spark/sql/avro/AvroDeserializer.scala | 493 ---
.../org/apache/spark/sql/avro/AvroSerializer.scala | 390 ---
.../org/apache/spark/sql/avro/AvroUtils.scala | 115 -
.../sql/avro/HoodieSpark3_0AvroDeserializer.scala | 34 -
.../sql/avro/HoodieSpark3_0AvroSerializer.scala | 29 -
.../catalyst/analysis/HoodieSpark30Analysis.scala | 189 --
.../sql/catalyst/plans/logical/HStatments.scala | 66 -
.../HoodieSpark30PartitionedFileUtils.scala | 51 -
.../datasources/Spark30NestedSchemaPruning.scala | 272 --
.../Spark30LegacyHoodieParquetFileFormat.scala | 414 ---
.../hudi/Spark30ResolveHudiAlterTableCommand.scala | 297 --
.../hudi/command/Spark30AlterTableCommand.scala | 318 --
.../sql/parser/HoodieSpark30SqlAstBuilder.scala | 125 -
.../parser/HoodieSpark3_0ExtendedSqlParser.scala | 34 -
.../spark/sql/vectorized/ColumnarUtils.scala | 32 -
.../HoodieBulkInsertInternalWriterTestBase.java | 174 -
.../TestHoodieBulkInsertDataInternalWriter.java | 175 -
.../TestHoodieDataSourceInternalBatchWrite.java | 331 --
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 283 --
...Spark31HoodieVectorizedParquetRecordReader.java | 188 --
...org.apache.spark.sql.sources.DataSourceRegister | 19 -
.../org/apache/hudi/Spark31DefaultSource.scala | 24 -
.../org/apache/hudi/Spark31HoodieFileScanRDD.scala | 35 -
.../spark/sql/HoodieSpark31CatalogUtils.scala | 30 -
.../sql/HoodieSpark31CatalystExpressionUtils.scala | 114 -
.../spark/sql/HoodieSpark31CatalystPlanUtils.scala | 86 -
.../spark/sql/HoodieSpark31SchemaUtils.scala | 39 -
.../apache/spark/sql/adapter/Spark3_1Adapter.scala | 124 -
.../apache/spark/sql/avro/AvroDeserializer.scala | 497 ---
.../org/apache/spark/sql/avro/AvroSerializer.scala | 389 ---
.../org/apache/spark/sql/avro/AvroUtils.scala | 103 -
.../sql/avro/HoodieSpark3_1AvroDeserializer.scala | 37 -
.../sql/avro/HoodieSpark3_1AvroSerializer.scala | 29 -
.../catalyst/analysis/HoodieSpark31Analysis.scala | 215 --
.../sql/catalyst/plans/logical/HStatments.scala | 66 -
.../HoodieSpark31PartitionedFileUtils.scala | 51 -
.../datasources/Spark31NestedSchemaPruning.scala | 198 --
.../Spark31LegacyHoodieParquetFileFormat.scala | 421 ---
.../hudi/Spark31ResolveHudiAlterTableCommand.scala | 283 --
.../hudi/command/Spark31AlterTableCommand.scala | 318 --
.../sql/parser/HoodieSpark31SqlAstBuilder.scala | 125 -
.../parser/HoodieSpark3_1ExtendedSqlParser.scala | 35 -
.../spark/sql/vectorized/ColumnarUtils.scala | 32 -
.../HoodieBulkInsertInternalWriterTestBase.java | 174 -
.../TestHoodieBulkInsertDataInternalWriter.java | 175 -
.../TestHoodieDataSourceInternalBatchWrite.java | 331 --
hudi-spark-datasource/hudi-spark3.2.x/pom.xml | 321 --
.../src/main/antlr4/imports/SqlBase.g4 | 1940 -----------
.../apache/hudi/spark/sql/parser/HoodieSqlBase.g4 | 40 -
...org.apache.spark.sql.sources.DataSourceRegister | 19 -
.../org/apache/hudi/Spark32HoodieFileScanRDD.scala | 32 -
.../spark/sql/HoodieSpark32CatalogUtils.scala | 29 -
.../sql/HoodieSpark32CatalystExpressionUtils.scala | 117 -
.../spark/sql/HoodieSpark32CatalystPlanUtils.scala | 130 -
.../spark/sql/HoodieSpark32SchemaUtils.scala | 39 -
.../apache/spark/sql/adapter/Spark3_2Adapter.scala | 126 -
.../apache/spark/sql/avro/AvroDeserializer.scala | 513 ---
.../org/apache/spark/sql/avro/AvroSerializer.scala | 450 ---
.../org/apache/spark/sql/avro/AvroUtils.scala | 117 -
.../sql/avro/HoodieSpark3_2AvroDeserializer.scala | 32 -
.../sql/avro/HoodieSpark3_2AvroSerializer.scala | 29 -
.../HoodieSpark32PartitionedFileUtils.scala | 51 -
.../datasources/Spark32NestedSchemaPruning.scala | 199 --
.../parquet/Spark32DataSourceUtils.scala | 77 -
.../Spark32LegacyHoodieParquetFileFormat.scala | 516 ---
.../hudi/Spark32ResolveHudiAlterTableCommand.scala | 70 -
.../sql/hudi/analysis/HoodieSpark32Analysis.scala | 66 -
.../HoodieSpark3_2ExtendedSqlAstBuilder.scala | 3480 --------------------
.../parser/HoodieSpark3_2ExtendedSqlParser.scala | 199 --
.../spark/sql/vectorized/ColumnarUtils.scala | 32 -
.../HoodieBulkInsertInternalWriterTestBase.java | 174 -
.../TestHoodieBulkInsertDataInternalWriter.java | 175 -
.../TestHoodieDataSourceInternalBatchWrite.java | 331 --
.../hudi-spark3.2plus-common/pom.xml | 235 --
...k32PlusHoodieVectorizedParquetRecordReader.java | 192 --
.../org/apache/hudi/Spark32PlusDefaultSource.scala | 47 -
.../hudi/analysis/HoodieSpark32PlusAnalysis.scala | 358 --
.../spark/sql/hudi/catalog/HoodieCatalog.scala | 421 ---
.../sql/hudi/catalog/HoodieInternalV2Table.scala | 124 -
.../spark/sql/hudi/catalog/HoodieStagedTable.scala | 99 -
.../spark/sql/hudi/command/AlterTableCommand.scala | 342 --
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 23 +-
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../spark/sql/HoodieSpark33CatalystPlanUtils.scala | 21 +-
.../apache/spark/sql/adapter/Spark3_3Adapter.scala | 25 +-
.../apache/spark/sql/avro/AvroDeserializer.scala | 1 +
.../org/apache/spark/sql/avro/AvroSerializer.scala | 11 +-
.../org/apache/spark/sql/avro/AvroUtils.scala | 11 +-
.../HoodieSpark33PartitionedFileUtils.scala | 13 +-
.../datasources/Spark33NestedSchemaPruning.scala | 1 +
.../Spark33LegacyHoodieParquetFileFormat.scala | 146 +-
.../datasources/parquet/Spark33ParquetReader.scala | 273 ++
.../hudi/Spark33ResolveHudiAlterTableCommand.scala | 7 +-
.../sql/hudi/analysis/HoodieSpark33Analysis.scala | 2 +-
.../HoodieSpark3_3ExtendedSqlAstBuilder.scala | 6 +-
.../parser/HoodieSpark3_3ExtendedSqlParser.scala | 5 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 2 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 38 +-
.../hudi/spark3/internal/TestReflectUtil.java | 54 -
hudi-spark-datasource/hudi-spark3.4.x/pom.xml | 16 +-
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../spark/sql/HoodieSpark34CatalystPlanUtils.scala | 32 +
.../apache/spark/sql/adapter/Spark3_4Adapter.scala | 20 +-
.../HoodieSpark34PartitionedFileUtils.scala | 12 +-
.../Spark34LegacyHoodieParquetFileFormat.scala | 139 +-
.../datasources/parquet/Spark34ParquetReader.scala | 283 ++
.../hudi/Spark34ResolveHudiAlterTableCommand.scala | 7 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 2 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 26 +-
.../hudi/spark3/internal/TestReflectUtil.java | 54 -
hudi-spark-datasource/hudi-spark3.5.x/pom.xml | 20 +-
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../spark/sql/HoodieSpark35CatalystPlanUtils.scala | 18 +
.../apache/spark/sql/adapter/Spark3_5Adapter.scala | 26 +-
.../HoodieSpark35PartitionedFileUtils.scala | 11 +-
.../Spark35LegacyHoodieParquetFileFormat.scala | 144 +-
.../datasources/parquet/Spark35ParquetReader.scala | 290 ++
.../hudi/Spark35ResolveHudiAlterTableCommand.scala | 7 +-
.../HoodieSpark3_5ExtendedSqlAstBuilder.scala | 2 +
.../HoodieBulkInsertInternalWriterTestBase.java | 2 +-
.../TestHoodieDataSourceInternalBatchWrite.java | 26 +-
.../hudi/spark3/internal/TestReflectUtil.java | 54 -
.../java/org/apache/hudi/sync/adb/AdbSyncTool.java | 11 +-
.../apache/hudi/sync/adb/HoodieAdbJdbcClient.java | 22 +-
hudi-sync/hudi-datahub-sync/pom.xml | 23 +-
.../hudi/sync/datahub/DataHubSyncClient.java | 7 +-
.../apache/hudi/sync/datahub/DataHubSyncTool.java | 16 +-
.../hudi/sync/datahub/TestDataHubSyncClient.java | 12 +-
.../hudi/sync/datahub/TestDataHubSyncTool.java | 66 +
hudi-sync/hudi-hive-sync/pom.xml | 27 +-
hudi-sync/hudi-hive-sync/run_sync_tool.sh | 4 +-
.../java/org/apache/hudi/hive/HiveSyncConfig.java | 8 +
.../org/apache/hudi/hive/HiveSyncConfigHolder.java | 3 +-
.../java/org/apache/hudi/hive/HiveSyncTool.java | 133 +-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 75 +-
.../hudi/hive/MultiPartKeysValueExtractor.java | 2 +-
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 8 +-
.../org/apache/hudi/hive/ddl/JDBCExecutor.java | 2 +-
.../hudi/hive/ddl/QueryBasedDDLExecutor.java | 8 +-
.../hudi/hive/transaction/lock/Heartbeat.java | 43 +
.../lock/HiveMetastoreBasedLockProvider.java | 34 +-
.../apache/hudi/hive/util/FilterGenVisitor.java | 2 +-
.../org/apache/hudi/hive/util/HiveSchemaUtil.java | 23 -
.../hudi/hive/util/PartitionFilterGenerator.java | 11 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 343 +-
.../hudi/hive/TestMultiPartKeysValueExtractor.java | 3 +-
.../hudi/hive/TestParquet2SparkSchemaUtils.java | 3 +-
.../TestHiveMetastoreBasedLockProvider.java | 20 +-
.../testutils/HiveSyncFunctionalTestHarness.java | 12 +-
.../hudi/hive/testutils/HiveTestCluster.java | 6 +-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 182 +-
.../hive/util/TestPartitionFilterGenerator.java | 1 +
hudi-sync/hudi-sync-common/pom.xml | 14 +-
.../org/apache/hudi/hive/SchemaDifference.java | 0
.../hudi/sync/common/HoodieMetaSyncOperations.java | 38 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 48 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 20 +-
.../apache/hudi/sync/common/HoodieSyncTool.java | 19 +-
.../sync/common/metrics/HoodieMetaSyncMetrics.java | 118 +
.../hudi/sync/common/util/ManifestFileWriter.java | 70 +-
.../sync/common/util/Parquet2SparkSchemaUtils.java | 1 +
.../hudi/sync/common/util/SyncUtilHelpers.java | 48 +-
.../hudi/sync/common/TestHoodieSyncClient.java | 72 +
.../hudi/sync/common/TestHoodieSyncTool.java | 40 +
.../common/metrics/TestHoodieMetaSyncMetrics.java | 106 +
.../sync/common/util/TestManifestFileWriter.java | 18 +-
.../hudi/sync/common/util/TestSyncUtilHelpers.java | 45 +-
hudi-tests-common/pom.xml | 5 +-
hudi-timeline-service/pom.xml | 13 +
.../hudi/timeline/service/RequestHandler.java | 530 +--
.../hudi/timeline/service/TimelineService.java | 44 +-
.../timeline/service/handlers/BaseFileHandler.java | 11 +-
.../service/handlers/FileSliceHandler.java | 21 +-
.../hudi/timeline/service/handlers/Handler.java | 15 +-
.../service/handlers/InstantStateHandler.java | 23 +-
.../timeline/service/handlers/MarkerHandler.java | 19 +-
.../timeline/service/handlers/TimelineHandler.java | 11 +-
.../AsyncTimelineServerBasedDetectionStrategy.java | 12 +-
.../MarkerBasedEarlyConflictDetectionRunnable.java | 28 +-
.../service/handlers/marker/MarkerDirState.java | 33 +-
.../TestRemoteHoodieTableFileSystemView.java | 20 +-
...tMarkerBasedEarlyConflictDetectionRunnable.java | 21 +-
hudi-utilities/pom.xml | 43 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 3 +-
.../hudi/utilities/HiveIncrementalPuller.java | 10 +-
.../org/apache/hudi/utilities/HoodieCleaner.java | 2 +-
.../apache/hudi/utilities/HoodieClusteringJob.java | 19 +-
.../hudi/utilities/HoodieCompactionAdminTool.java | 13 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 10 +-
.../hudi/utilities/HoodieDataTableUtils.java | 13 +-
.../hudi/utilities/HoodieDataTableValidator.java | 24 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 8 +-
.../utilities/HoodieMetadataTableValidator.java | 430 ++-
.../apache/hudi/utilities/HoodieRepairTool.java | 51 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 27 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 111 +-
.../org/apache/hudi/utilities/HoodieTTLJob.java | 2 +-
.../hudi/utilities/HoodieWithTimelineServer.java | 3 +-
.../org/apache/hudi/utilities/TableSizeStats.java | 17 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 131 +-
...heckpointFromAnotherHoodieTimelineProvider.java | 5 +-
.../hudi/utilities/config/CloudSourceConfig.java | 7 +
.../utilities/config/HoodieIncrSourceConfig.java | 8 +
.../utilities/config/HoodieStreamerConfig.java | 7 +
.../hudi/utilities/config/KafkaSourceConfig.java | 53 +-
.../utilities/config/ParquetDFSSourceConfig.java | 2 +-
.../config/S3EventsHoodieIncrSourceConfig.java | 3 +
.../utilities/config/SqlFileBasedSourceConfig.java | 3 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 3 +-
.../deltastreamer/HoodieDeltaStreamerMetrics.java | 11 +-
.../deser/KafkaAvroSchemaDeserializer.java | 7 +-
.../ingestion/HoodieIngestionMetrics.java | 21 +-
.../hudi/utilities/multitable/CleanTask.java | 1 +
.../hudi/utilities/multitable/ClusteringTask.java | 1 +
.../hudi/utilities/multitable/CompactionTask.java | 1 +
.../multitable/MultiTableServiceUtils.java | 17 +-
.../hudi/utilities/perf/TimelineServerPerf.java | 49 +-
.../schema/ProtoClassBasedSchemaProvider.java | 10 +-
.../utilities/schema/SchemaRegistryProvider.java | 186 +-
.../apache/hudi/utilities/schema/SchemaSet.java | 4 +-
.../utilities/schema/SimpleSchemaProvider.java | 3 +-
.../converter/JsonToAvroSchemaConverter.java | 12 +-
.../ProtoSchemaToAvroSchemaConverter.java | 43 +
.../postprocessor/ChainedSchemaPostProcessor.java | 2 +-
.../hudi/utilities/sources/AvroKafkaSource.java | 4 +-
.../sources/GcsEventsHoodieIncrSource.java | 73 +-
.../hudi/utilities/sources/GcsEventsSource.java | 9 +-
.../hudi/utilities/sources/HoodieIncrSource.java | 69 +-
.../sources/HoodieRetryingKafkaConsumer.java | 99 +
.../hudi/utilities/sources/JsonKafkaSource.java | 22 +-
.../apache/hudi/utilities/sources/KafkaSource.java | 58 +-
.../hudi/utilities/sources/ORCDFSSource.java | 1 +
.../hudi/utilities/sources/ProtoKafkaSource.java | 44 +-
.../apache/hudi/utilities/sources/RowSource.java | 17 +-
.../sources/S3EventsHoodieIncrSource.java | 90 +-
.../hudi/utilities/sources/S3EventsSource.java | 4 +-
.../sources/SnapshotLoadQuerySplitter.java | 17 +-
.../utilities/sources/helpers/AvroConvertor.java | 7 +-
.../sources/helpers/CloudDataFetcher.java | 85 +-
.../helpers/CloudObjectsSelectorCommon.java | 300 +-
.../utilities/sources/helpers/DFSPathSelector.java | 4 +-
.../sources/helpers/DatePartitionPathSelector.java | 11 +-
.../sources/helpers/IncrSourceHelper.java | 34 +-
.../utilities/sources/helpers/KafkaOffsetGen.java | 201 +-
.../sources/helpers/ProtoConversionUtil.java | 81 +-
.../utilities/sources/helpers/QueryRunner.java | 4 +-
.../sources/helpers/S3EventsMetaSelector.java | 4 +-
.../helpers/gcs/GcsObjectMetadataFetcher.java | 84 -
.../processor/JsonKafkaSourcePostProcessor.java | 2 +-
.../utilities/streamer/BaseErrorTableWriter.java | 19 +-
.../hudi/utilities/streamer/BootstrapExecutor.java | 9 +-
.../apache/hudi/utilities/streamer/ErrorEvent.java | 19 +
.../hudi/utilities/streamer/ErrorTableUtils.java | 33 +-
.../hudi/utilities/streamer/HoodieStreamer.java | 52 +-
.../utilities/streamer/HoodieStreamerMetrics.java | 58 +-
.../utilities/streamer/HoodieStreamerUtils.java | 45 +-
.../utilities/streamer/SchedulerConfGenerator.java | 8 +-
.../utilities/streamer/SourceFormatAdapter.java | 11 +-
.../utilities/streamer/SparkSampleWritesUtils.java | 20 +-
.../apache/hudi/utilities/streamer/StreamSync.java | 297 +-
.../transform/SqlFileBasedTransformer.java | 9 +-
.../transform/SqlQueryBasedTransformer.java | 8 +-
.../hudi/utilities/TestHiveIncrementalPuller.java | 6 +-
.../apache/hudi/utilities/TestHoodieIndexer.java | 14 +-
.../TestHoodieMetadataTableValidator.java | 428 +++
.../hudi/utilities/TestHoodieRepairTool.java | 60 +-
.../utilities/TestManifestFileWriterSpark.java | 117 +
.../org/apache/hudi/utilities/TestUtilHelpers.java | 4 +-
.../callback/TestKafkaCallbackProvider.java | 1 +
.../TestKafkaConnectHdfsProvider.java | 8 +-
.../hudi/utilities/config/SourceTestConfig.java | 15 +-
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 331 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 584 ++--
...TestHoodieDeltaStreamerSchemaEvolutionBase.java | 30 +-
...oodieDeltaStreamerSchemaEvolutionExtensive.java | 8 +-
...estHoodieDeltaStreamerSchemaEvolutionQuick.java | 41 +-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 146 +-
.../TestHoodieMultiTableDeltaStreamer.java | 17 +-
.../deltastreamer/TestSchedulerConfGenerator.java | 2 +-
.../deltastreamer/TestSparkSampleWritesUtils.java | 2 +
.../deser/TestKafkaAvroSchemaDeserializer.java | 8 +-
.../TestErrorTableAwareChainedTransformer.java | 6 +-
.../functional/TestHDFSParquetImporter.java | 68 +-
.../functional/TestHiveSchemaProvider.java | 10 +-
.../functional/TestHoodieSnapshotCopier.java | 30 +-
.../functional/TestHoodieSnapshotExporter.java | 222 +-
.../functional/TestJdbcbasedSchemaProvider.java | 20 +-
.../TestHoodieMultiTableServicesMain.java | 15 +-
.../offlinejob/HoodieOfflineJobTestBase.java | 14 +-
.../offlinejob/TestHoodieClusteringJob.java | 55 +-
.../offlinejob/TestHoodieCompactorJob.java | 38 +-
.../schema/TestSchemaRegistryProvider.java | 142 +-
.../converter/TestJsonToAvroSchemaConverter.java | 3 +-
.../TestProtoSchemaToAvroSchemaConverter.java | 52 +
.../utilities/sources/BaseTestKafkaSource.java | 33 +-
.../hudi/utilities/sources/TestAvroDFSSource.java | 7 +-
.../utilities/sources/TestAvroKafkaSource.java | 59 +-
.../hudi/utilities/sources/TestCsvDFSSource.java | 9 +-
.../sources/TestGcsEventsHoodieIncrSource.java | 110 +-
.../utilities/sources/TestHoodieIncrSource.java | 140 +-
.../hudi/utilities/sources/TestJdbcSource.java | 85 +-
.../hudi/utilities/sources/TestJsonDFSSource.java | 53 +-
.../utilities/sources/TestJsonKafkaSource.java | 15 +-
.../sources/TestJsonKafkaSourcePostProcessor.java | 2 +-
.../utilities/sources/TestParquetDFSSource.java | 5 +-
.../utilities/sources/TestProtoKafkaSource.java | 75 +-
.../sources/TestS3EventsHoodieIncrSource.java | 156 +-
.../hudi/utilities/sources/TestS3EventsSource.java | 2 +-
.../utilities/sources/TestSqlFileBasedSource.java | 14 +-
.../hudi/utilities/sources/TestSqlSource.java | 2 +-
.../debezium/TestAbstractDebeziumSource.java | 9 +-
.../sources/helpers/TestCheckpointUtils.java | 167 +-
.../sources/helpers/TestCloudObjectsSelector.java | 2 +-
.../helpers/TestCloudObjectsSelectorCommon.java | 120 +-
.../helpers/TestDFSPathSelectorCommonMethods.java | 22 +-
.../helpers/TestDatePartitionPathSelector.java | 54 +-
.../sources/helpers/TestIncrSourceHelper.java | 16 +-
.../sources/helpers/TestKafkaOffsetGen.java | 110 +-
.../sources/helpers/TestProtoConversionUtil.java | 30 +-
.../sources/helpers/TestS3EventsMetaSelector.java | 2 +-
.../helpers/TestSnapshotQuerySplitterImpl.java | 5 +-
.../utilities/streamer/TestErrorTableUtils.java | 77 +
.../streamer/TestHoodieStreamerMetrics.java | 71 +
.../streamer/TestHoodieStreamerUtils.java | 35 +-
.../streamer/TestStreamSyncUnitTests.java | 239 ++
.../hudi/utilities/testutils/JdbcTestUtils.java | 5 +
...ClientFunctionalTestHarnessWithHiveSupport.java | 1 +
.../utilities/testutils/UtilitiesTestBase.java | 134 +-
.../testutils/sources/AbstractBaseTestSource.java | 24 +-
.../sources/AbstractDFSSourceTestBase.java | 7 +-
.../sources/DistributedTestDataSource.java | 11 +-
.../transform/TestSqlFileBasedTransformer.java | 14 +-
.../transform/TestSqlQueryBasedTransformer.java | 7 +-
.../src/test/resources/data/nested_data_1.json | 1 +
.../src/test/resources/data/nested_data_2.json | 1 +
.../src/test/resources/data/nested_data_3.json | 1 +
.../partitioned/country=US/state=TX/old_data.json | 1 +
.../resources/exporter/sql-file-transformer.sql | 19 +
.../resources/schema-provider/proto/evolved.proto | 51 +
.../proto/sample_schema_defaults.avsc | 2 +-
...ple_schema_wrapped_and_timestamp_as_record.avsc | 4 +-
.../test/resources/schema/nested_data_schema.avsc | 81 +
.../test/resources/schema/sample_data_schema.avsc | 3 +-
.../streamer-config/dfs-source.properties | 6 +-
.../invalid_hive_sync_uber_config.properties | 6 +-
.../streamer-config/kafka-source.properties | 6 +-
.../short_trip_uber_config.properties | 12 +-
.../streamer-config/sql-transformer.properties | 2 +-
.../streamer-config/uber_config.properties | 10 +-
packaging/bundle-validation/README.md | 10 +-
packaging/bundle-validation/base/Dockerfile | 18 +-
.../base/build_flink1146hive313spark302.sh | 26 -
.../base/build_flink1146hive313spark313.sh | 26 -
.../base/build_flink1146hive313spark323.sh | 26 -
.../base/build_flink1146hive313spark331.sh | 27 +
.../base/build_flink1153hive313spark323.sh | 26 -
.../base/build_flink1153hive313spark334.sh | 27 +
.../base/build_flink1162hive313spark343.sh | 27 +
.../base/build_flink1170hive313spark351.sh | 27 +
.../base/build_flink1180hive313spark350scala213.sh | 28 +
.../base/build_flink1180hive313spark351scala213.sh | 28 +
.../base/build_flink1190hive313spark350.sh | 27 +
.../base/build_flink1190hive313spark350scala213.sh | 28 +
.../base/build_flink1190hive313spark351scala213.sh | 28 +
packaging/bundle-validation/ci_run.sh | 121 +-
packaging/bundle-validation/run_docker_java17.sh | 43 +-
.../spark_hadoop_mr/validate.scala | 22 +
.../bundle-validation/spark_hadoop_mr/write.scala | 4 +-
packaging/bundle-validation/validate.sh | 38 +-
packaging/hudi-aws-bundle/pom.xml | 1 -
packaging/hudi-cli-bundle/hudi-cli-with-bundle.sh | 6 +-
packaging/hudi-flink-bundle/pom.xml | 2 +-
packaging/hudi-integ-test-bundle/pom.xml | 12 +-
packaging/hudi-metaserver-server-bundle/pom.xml | 2 +-
packaging/hudi-spark-bundle/pom.xml | 8 +-
packaging/hudi-utilities-bundle/pom.xml | 9 +-
packaging/hudi-utilities-slim-bundle/pom.xml | 1 +
pom.xml | 342 +-
release/release_guide.md | 8 +-
rfc/README.md | 7 +-
rfc/rfc-76/rfc-76.md | 2 +-
rfc/rfc-80/append-handle.png | Bin 0 -> 7938 bytes
rfc/rfc-80/bucket.png | Bin 0 -> 6246 bytes
rfc/rfc-80/family-compaction.png | Bin 0 -> 46659 bytes
rfc/rfc-80/file-group.png | Bin 0 -> 22283 bytes
rfc/rfc-80/grand-compaction.png | Bin 0 -> 92566 bytes
rfc/rfc-80/process-read.png | Bin 0 -> 29983 bytes
rfc/rfc-80/process-write.png | Bin 0 -> 69560 bytes
rfc/rfc-80/rfc-80.md | 187 ++
rfc/rfc-80/row-reader.png | Bin 0 -> 49108 bytes
rfc/rfc-80/table.png | Bin 0 -> 54443 bytes
scripts/ci/move_surefire_reports.sh | 58 -
scripts/pr_compliance.py | 4 +-
scripts/release/deploy_staging_jars.sh | 32 +-
scripts/release/validate_source_copyright.sh | 4 +-
scripts/release/validate_staged_bundles.sh | 42 +-
style/checkstyle-suppressions.xml | 2 +
style/checkstyle.xml | 13 +-
style/scalastyle.xml | 8 +-
2208 files changed, 94089 insertions(+), 74133 deletions(-)
create mode 100644 .idea/icon.png
create mode 100644 hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedImplicitPartitionKeyLockProvider.java
create mode 100644 hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProviderBase.java
create mode 100644 hudi-aws/src/test/java/org/apache/hudi/aws/sync/MockAwsGlueCatalogSyncTool.java
create mode 100644 hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java
create mode 100644 hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAwsGlueSyncTool.java
create mode 100644 hudi-aws/src/test/java/org/apache/hudi/aws/testutils/GlueTestUtil.java
create mode 100644 hudi-aws/src/test/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProviderBaseTest.java
create mode 100644 hudi-aws/src/test/java/org/apache/hudi/aws/utils/TestS3Utils.java
create mode 100644 hudi-cli/src/main/resources/version.txt
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/BaseZookeeperBasedLockProvider.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/lock/ZookeeperBasedImplicitBasePathLockProvider.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/common/table/log/HoodieFileSliceReader.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsM3Config.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/exception/HoodieDeletePartitionPendingTableServiceException.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/InMemoryMetricsReporter.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricUtils.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/Metrics.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/datadog/DatadogMetricsReporter.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/m3/M3MetricsReporter.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/m3/M3ScopeReporterAdaptor.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/prometheus/PrometheusReporter.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/prometheus/PushGatewayMetricsReporter.java
delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/SerializablePathFilter.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToSevenDowngradeHandler.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToSixDowngradeHandler.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SixToSevenUpgradeHandler.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngradeUtils.java
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/client/http/TestCallbackHttpClient.java
delete mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/http/TestCallbackHttpClient.java
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategyWithMORTable.java
delete mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/index/TestHoodieIndexUtils.java
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestEightToSevenDowngradeHandler.java
create mode 100644 hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java
create mode 100644 hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkPartitionTTLActionExecutor.java
create mode 100644 hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/common/TestMultipleHoodieJavaWriteClient.java
create mode 100644 hudi-client/hudi-java-client/src/test/java/org/apache/hudi/hadoop/TestHoodieFileGroupReaderOnHive.java
create mode 100644 hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/ArrayWritableTestUtil.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/DefaultSparkRecordMerger.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/HoodieSparkFunctionalIndex.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/OverwriteWithLatestSparkRecordMerger.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkReleaseResources.java
delete mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/functional/HoodieSparkFunctionalIndex.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java
create mode 100644 hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/ExceptionWrappingIterator.scala
create mode 100644 hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/JavaScalaConverters.scala
create mode 100644 hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkParquetReader.scala
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestRowCustomColumnsSortPartitioner.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanActionExecutor.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/spark/TestHoodieSparkKryoRegistrar.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/spark/sql/TestHoodieDataTypeUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/AvroLogicalTypeEnum.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/DateLogicalTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/DecimalLogicalTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/DurationLogicalTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/EnumTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/FixedTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/JsonFieldProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/LocalTimestampMicroLogicalTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/LocalTimestampMilliLogicalTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/Parser.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/TimeLogicalTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/TimeMicroLogicalTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/TimeMilliLogicalTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/TimestampMicroLogicalTypeProcessor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/processors/TimestampMilliLogicalTypeProcessor.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndex.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HFileBootstrapIndexReader.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/config/HoodieFunctionalIndexConfig.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/config/HoodieIndexingConfig.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/config/PropertiesConfig.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/config/RecordMergeMode.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/fs/NoOpConsistencyGuard.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/FirstValueAvroPayload.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFunctionalIndexDefinition.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFunctionalIndexMetadata.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/HoodieIndexDefinition.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/HoodieIndexMetadata.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/HoodieMetadataRecordMerger.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/OverwriteWithLatestMerger.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/table/log/BaseHoodieMergedLogRecordScanner.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieFileSliceReader.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMetadataMergedLogRecordScanner.java
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/common/table/log/LogFileIterator.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReaderSchemaHandler.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReaderState.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodiePositionBasedSchemaHandler.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FSPermissionDTO.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/BaseFileUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/OrcReaderIterator.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/SortUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/hash/BucketIndexUtil.java
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsCloudWatchConfig.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsDatadogConfig.java (100%)
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsGraphiteConfig.java (100%)
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsJmxConfig.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsM3Config.java
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsPrometheusConfig.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/exception/HoodieAvroSchemaException.java
copy {hudi-io => hudi-common}/src/main/java/org/apache/hudi/exception/HoodieException.java (100%)
copy {hudi-io => hudi-common}/src/main/java/org/apache/hudi/exception/HoodieIOException.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/exception/HoodieIncompatibleSchemaException.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/exception/HoodieJsonConversionException.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/exception/HoodieJsonToAvroConversionException.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/exception/HoodieRecordCreationException.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/exception/InvalidHoodieFileNameException.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/internal/schema/visitor/NameToPositionVisitor.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroFileReaderBase.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroFileReaderFactory.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroFileWriterFactory.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroHFileWriter.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroOrcReader.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroOrcWriter.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroParquetReader.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroParquetWriter.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieBaseParquetWriter.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieHBaseAvroHFileReader.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileConfig.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetStreamWriter.java
delete mode 100644 hudi-common/src/main/java/org/apache/hudi/metadata/MetadataRecordsGenerationParams.java
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/metrics/ConsoleMetricsReporter.java (100%)
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/metrics/HoodieGauge.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/InMemoryMetricsReporter.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/metrics/JmxReporterServer.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/MetricUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/Metrics.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/metrics/MetricsReporter.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java (100%)
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/metrics/custom/CustomizableMetricsReporter.java (100%)
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/metrics/datadog/DatadogHttpClient.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/datadog/DatadogMetricsReporter.java
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/metrics/datadog/DatadogReporter.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/m3/M3MetricsReporter.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/m3/M3ScopeReporterAdaptor.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/prometheus/PrometheusReporter.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/metrics/prometheus/PushGatewayMetricsReporter.java
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/metrics/prometheus/PushGatewayReporter.java (100%)
rename {hudi-client/hudi-client-common => hudi-common}/src/main/java/org/apache/hudi/metrics/userdefined/AbstractUserDefinedMetricsReporter.java (100%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/storage/HoodieStorageUtils.java
delete mode 100644 hudi-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetReaderBuilder.java
delete mode 100644 hudi-common/src/main/java/org/apache/parquet/avro/HoodieAvroReadSupport.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/avro/AvroSchemaTestUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/client/transaction/lock/TestInProcessLockProvider.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/bloom/TestBloomFilter.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/bootstrap/TestBootstrapIndex.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtilsWithRetryWrapperEnable.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/TestHoodieWrapperFileSystem.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/inline/InLineFSUtilsTest.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemHFileInLiningBase.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHBaseHFileReader.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestParquetInLining.java
delete mode 100755 hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/model/TestFirstValueAvroPayload.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieMetadataRecordMerger.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/log/TestLogReaderUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieDeleteBlock.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/read/CustomPayloadForTesting.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/read/TestCustomMerger.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/read/TestEventTimeMerging.java
delete mode 100755 hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieInstant.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestWaitBasedTimeGenerator.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFSViewWithClustering.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestHarness.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestAvroOrcUtils.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestBaseFileUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestMarkerUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestOrcReaderIterator.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestSortUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestStringUtils.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
delete mode 100755 hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbDiskMap.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/hash/TestBucketIndexUtil.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/internal/schema/convert/TestAvroInternalSchemaConverter.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileReaderFactory.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieBaseParquetWriter.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHBaseHFileReaderWriter.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriterBase.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/metadata/TestMetadataPartitionType.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/metrics/TestMetrics.java
create mode 100644 hudi-common/src/test/resources/complex-schema-evolved.avsc
create mode 100644 hudi-common/src/test/resources/complex-schema-evolved.data
create mode 100644 hudi-common/src/test/resources/date-type-invalid.avsc
create mode 100644 hudi-common/src/test/resources/date-type.avsc
create mode 100644 hudi-common/src/test/resources/decimal-logical-type-fixed-type.avsc
create mode 100644 hudi-common/src/test/resources/decimal-logical-type-invalid.avsc
create mode 100644 hudi-common/src/test/resources/decimal-logical-type.avsc
create mode 100644 hudi-common/src/test/resources/duration-logical-type-invalid.avsc
create mode 100644 hudi-common/src/test/resources/duration-logical-type.avsc
create mode 100644 hudi-common/src/test/resources/hash/magic_input.txt
create mode 100644 hudi-common/src/test/resources/hash/xxhash_BITS_128_for_magic_input.txt
create mode 100644 hudi-common/src/test/resources/hash/xxhash_BITS_32_for_magic_input.txt
create mode 100644 hudi-common/src/test/resources/hash/xxhash_BITS_64_for_magic_input.txt
create mode 100644 hudi-common/src/test/resources/local-timestamp-logical-type.avsc
create mode 100644 hudi-common/src/test/resources/local-timestamp-micros-logical-type.avsc
create mode 100644 hudi-common/src/test/resources/local-timestamp-millis-logical-type.avsc
create mode 100644 hudi-common/src/test/resources/time-logical-type.avsc
create mode 100644 hudi-common/src/test/resources/timestamp-logical-type2.avsc
create mode 100644 hudi-common/src/test/resources/uuid-logical-type.avsc
create mode 100644 hudi-examples/hudi-examples-k8s/Dockerfile
create mode 100644 hudi-examples/hudi-examples-k8s/README.md
create mode 100644 hudi-examples/hudi-examples-k8s/config/hadoop/core-site.xml
create mode 100644 hudi-examples/hudi-examples-k8s/config/k8s/flink-deployment.yaml
create mode 100644 hudi-examples/hudi-examples-k8s/config/k8s/minio-standalone.yaml
create mode 100644 hudi-examples/hudi-examples-k8s/pom.xml
create mode 100644 hudi-examples/hudi-examples-k8s/src/main/java/org/apache/hudi/examples/k8s/quickstart/HudiDataStreamWriter.java
create mode 100644 hudi-examples/hudi-examples-k8s/src/main/java/org/apache/hudi/examples/k8s/quickstart/utils/DataGenerator.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/rebalance/partitioner/StreamReadAppendPartitioner.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/rebalance/partitioner/StreamReadBucketIndexPartitioner.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/rebalance/selector/StreamReadAppendKeySelector.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/rebalance/selector/StreamReadBucketIndexKeySelector.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupFunction.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/lookup/HoodieLookupTableReader.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/SerializableSchema.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriterWithPartitionTTl.java
delete mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/CollectorOutput.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/MockKeyedStateStore.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/keyedstate/MockKeyContext.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/keyedstate/MockKeyedListState.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/keyedstate/MockKeyedMapState.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/utils/keyedstate/MockKeyedValueState.java
create mode 100644 hudi-flink-datasource/hudi-flink1.14.x/src/main/java/org/apache/hudi/adapter/CollectOutputAdapter.java
create mode 100644 hudi-flink-datasource/hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/CollectOutputAdapter.java
create mode 100644 hudi-flink-datasource/hudi-flink1.16.x/src/main/java/org/apache/hudi/adapter/CollectOutputAdapter.java
create mode 100644 hudi-flink-datasource/hudi-flink1.17.x/src/main/java/org/apache/hudi/adapter/CollectOutputAdapter.java
create mode 100644 hudi-flink-datasource/hudi-flink1.18.x/src/main/java/org/apache/hudi/adapter/CollectOutputAdapter.java
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/pom.xml
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/CollectOutputAdapter.java
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/adapter/DataStreamScanProviderAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/adapter/HiveCatalogConstants.java (100%)
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/adapter/MaskingOutputAdapter.java
copy hudi-flink-datasource/{hudi-flink1.16.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/adapter/OperatorCoordinatorAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.16.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/adapter/SortCodeGeneratorAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.17.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/adapter/SupportsRowLevelDeleteAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.17.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/adapter/SupportsRowLevelUpdateAdapter.java (100%)
copy hudi-flink-datasource/{hudi-flink1.18.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/adapter/Utils.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapArrayVector.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapDecimalVector.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapMapColumnVector.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/HeapRowColumnVector.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/ParquetDecimalVector.java (100%)
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/AbstractColumnReader.java
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ArrayColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/BaseVectorizedColumnReader.java (100%)
create mode 100644 hudi-flink-datasource/hudi-flink1.19.x/src/main/java/org/apache/hudi/table/format/cow/vector/reader/EmptyColumnReader.java
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/FixedLenBytesColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/Int64TimestampColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/MapColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetColumnarRowSplitReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.14.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.14.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/ParquetDataColumnReaderFactory.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RowColumnReader.java (100%)
copy hudi-flink-datasource/{hudi-flink1.15.x => hudi-flink1.19.x}/src/main/java/org/apache/hudi/table/format/cow/vector/reader/RunLengthDecoder.java (100%)
copy hudi-flink-datasource/{hudi-flink1.17.x => hudi-flink1.19.x}/src/test/java/org/apache/hudi/adapter/TestHoodieCatalogs.java (100%)
copy hudi-flink-datasource/{hudi-flink1.16.x => hudi-flink1.19.x}/src/test/java/org/apache/hudi/adapter/TestStreamConfigs.java (100%)
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HBaseHFileBootstrapIndexReader.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HBaseHFileBootstrapIndexWriter.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/ParquetTableSchemaResolver.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
rename {hudi-common => hudi-hadoop-common}/src/main/java/org/apache/hudi/common/util/AvroOrcUtils.java (100%)
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HadoopConfigUtils.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
rename {hudi-common => hudi-hadoop-common}/src/main/java/org/apache/hudi/common/util/ParquetReaderIterator.java (100%)
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
delete mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/ConsistencyGuard.java
delete mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/NoOpConsistencyGuard.java
delete mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/SerializablePath.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/HadoopInLineFSUtils.java
delete mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/inline/InLineFSUtils.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroHFileWriter.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcWriter.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReaderIterator.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetWriter.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieBaseParquetWriter.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHBaseAvroHFileReader.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHFileConfig.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHFileUtils.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHadoopIOFactory.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieParquetStreamWriter.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/OrcReaderIterator.java
rename {hudi-common => hudi-hadoop-common}/src/main/java/org/apache/hudi/io/storage/HoodieHBaseKVComparator.java (100%)
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java
rename {hudi-common => hudi-hadoop-common}/src/main/java/org/apache/hudi/parquet/io/OutputStreamBackedOutputFile.java (100%)
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetReaderBuilder.java
create mode 100644 hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroReadSupport.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/client/transaction/lock/TestInProcessLockProvider.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/bloom/TestBloomFilter.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/bootstrap/index/TestBootstrapIndex.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestFSUtilsWithRetryWrapperEnable.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/TestHoodieWrapperFileSystem.java
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java (100%)
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/InLineFSUtilsTest.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemHFileInLiningBase.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHBaseHFileReader.java
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHFileReader.java (100%)
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/fs/inline/TestInMemoryFileSystem.java (100%)
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/TestParquetInLining.java
create mode 100755 hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormatAppendFailure.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/model/TestHoodieFileGroup.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/model/TestHoodiePartitionMetadata.java
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/model/TestHoodieRecord.java (100%)
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/model/TestHoodieRecordDelegate.java (100%)
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTableSchemaResolver.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/TestLogReaderUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/log/block/TestHoodieDeleteBlock.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/read/TestCustomMerger.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/read/TestEventTimeMerging.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/read/TestOverwriteWithLatestMerger.java
create mode 100755 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieInstant.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/timeline/TestWaitBasedTimeGenerator.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFSViewWithClustering.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/table/view/TestRocksDBBasedIncrementalFSViewSync.java (100%)
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java (100%)
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedFileSystemView.java (100%)
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/table/view/TestSpillableMapBasedIncrementalFSViewSync.java (100%)
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/FileSystemTestUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HadoopFSTestUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/testutils/minicluster/HdfsTestService.java (100%)
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieFileGroupReaderTestHarness.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestAvroOrcUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestClusteringUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java (100%)
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestHFileUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestHadoopConfigUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestMarkerUtils.java
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/util/TestParquetReaderIterator.java (100%)
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
create mode 100755 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java (100%)
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbDiskMap.java
rename {hudi-common => hudi-hadoop-common}/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java (100%)
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieBaseParquetWriter.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHBaseHFileReaderWriter.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriterBase.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieReaderWriterBase.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestOrcReaderIterator.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
rename {hudi-common => hudi-hadoop-common}/src/test/resources/external-config/hudi-defaults.conf (100%)
create mode 100644 hudi-hadoop-common/src/test/resources/props/testdfs.properties
create mode 100644 hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/DefaultHiveRecordMerger.java
create mode 100644 hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveHoodieReaderContext.java
create mode 100644 hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieFileGroupReaderBasedRecordReader.java
create mode 100644 hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHiveRecord.java
create mode 100644 hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHiveRecordMerger.java
create mode 100644 hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/OverwriteWithLatestHiveRecordMerger.java
create mode 100644 hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieArrayWritableAvroUtils.java
create mode 100644 hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/ObjectInspectorCache.java
create mode 100644 hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHoodieArrayWritableAvroUtils.java
create mode 100644 hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/streaming/StructuredStreamingSinkUtil.java
create mode 100644 hudi-integ-test/src/main/scala/org/apache/hudi/integ/testsuite/streaming/StructuredStreamingSinkTestWriter.scala
create mode 100644 hudi-io/src/main/java/org/apache/hudi/common/fs/ConsistencyGuard.java
create mode 100644 hudi-io/src/main/java/org/apache/hudi/storage/inline/InLineFSUtils.java
rename {hudi-common => hudi-io}/src/test/java/org/apache/hudi/common/testutils/NetworkTestUtils.java (100%)
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BloomFiltersIndexSupport.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BucketIndexSupport.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/PartitionStatsIndexSupport.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SecondaryIndexSupport.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkBaseIndexSupport.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/util/SparkConfigUtils.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkParquetReaderBase.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ShowHoodieCreateTableCommand.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/vectorized/ColumnarBatchUtils.java
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestRecordLevelIndexSupport.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestSecondaryIndexSupport.scala
delete mode 100644 hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/execution/datasources/parquet/TestHoodieFileGroupReaderBasedParquetFileFormat.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/execution/datasources/parquet/TestSparkFileFormatInternalRowReaderContext.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/TestSparkParquetReaderFormat.scala
create mode 100644 hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/spark/sql/hudi/TestProvidesHoodieConfig.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunRollbackInflightTableServiceProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFileStatusProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestFiltersInFileGroupReader.java
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkParquetReader.java
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterPartitionTTL.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestParquetReaderCompatibility.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/PartitionStatsIndexTestBase.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBloomFiltersIndexSupport.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBucketIndexSupport.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSevenToEightUpgrade.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithTimestampKeyGenerator.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestSparkInternalSchemaConverter.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRunRollbackInflightTableServiceProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestShowFileStatusProcedure.scala
create mode 100644 hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark24ParquetReader.scala
create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark3HoodieVectorizedParquetRecordReader.java
create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala
create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/ResolveInsertionBase.scala
rename hudi-spark-datasource/{hudi-spark3.2plus-common => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieFileSystemViewTableValuedFunction.scala (100%)
rename hudi-spark-datasource/{hudi-spark3.2plus-common => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieMetadataTableValuedFunction.scala (100%)
rename hudi-spark-datasource/{hudi-spark3.2plus-common => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieQuery.scala (100%)
rename hudi-spark-datasource/{hudi-spark3.2plus-common => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieTableChanges.scala (100%)
rename hudi-spark-datasource/{hudi-spark3.2plus-common => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/catalyst/plans/logcal/HoodieTimelineTableValuedFunction.scala (100%)
rename hudi-spark-datasource/{hudi-spark3.2plus-common => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Index.scala (100%)
create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark3ParquetSchemaEvolutionUtils.scala
create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala
rename hudi-spark-datasource/{hudi-spark3.2plus-common => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/hudi/analysis/TableValuedFunctions.scala (100%)
rename hudi-spark-datasource/{hudi-spark3.2plus-common => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/hudi/catalog/BasicStagedTable.scala (100%)
create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
rename hudi-spark-datasource/{hudi-spark3.2plus-common => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/hudi/catalog/TableCreationMode.java (100%)
create mode 100644 hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
rename hudi-spark-datasource/{hudi-spark3.2plus-common => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/hudi/logical/TimeTravelRelation.scala (100%)
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/pom.xml
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark30HoodieVectorizedParquetRecordReader.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/hudi/Spark30DefaultSource.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/hudi/Spark30HoodieFileScanRDD.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalogUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalystExpressionUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30CatalystPlanUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark30SchemaUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_0Adapter.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_0AvroDeserializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_0AvroSerializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/catalyst/analysis/HoodieSpark30Analysis.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HStatments.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark30PartitionedFileUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark30NestedSchemaPruning.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark30LegacyHoodieParquetFileFormat.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/hudi/Spark30ResolveHudiAlterTableCommand.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/hudi/command/Spark30AlterTableCommand.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark30SqlAstBuilder.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_0ExtendedSqlParser.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/main/scala/org/apache/spark/sql/vectorized/ColumnarUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.0.x/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/pom.xml
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark31HoodieVectorizedParquetRecordReader.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31DefaultSource.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark31HoodieFileScanRDD.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalogUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystExpressionUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31CatalystPlanUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/HoodieSpark31SchemaUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_1Adapter.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_1AvroDeserializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_1AvroSerializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/catalyst/analysis/HoodieSpark31Analysis.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/HStatments.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark31PartitionedFileUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark31NestedSchemaPruning.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark31LegacyHoodieParquetFileFormat.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/hudi/Spark31ResolveHudiAlterTableCommand.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/hudi/command/Spark31AlterTableCommand.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark31SqlAstBuilder.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_1ExtendedSqlParser.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/main/scala/org/apache/spark/sql/vectorized/ColumnarUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/test/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/test/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.1.x/src/test/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/pom.xml
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/antlr4/imports/SqlBase.g4
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlBase.g4
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/hudi/Spark32HoodieFileScanRDD.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalogUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystExpressionUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark32CatalystPlanUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/HoodieSpark32SchemaUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/adapter/Spark3_2Adapter.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroDeserializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/avro/HoodieSpark3_2AvroSerializer.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/HoodieSpark32PartitionedFileUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/Spark32NestedSchemaPruning.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32DataSourceUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark32LegacyHoodieParquetFileFormat.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/hudi/Spark32ResolveHudiAlterTableCommand.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark32Analysis.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlAstBuilder.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/main/scala/org/apache/spark/sql/vectorized/ColumnarUtils.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/test/java/org/apache/hudi/internal/HoodieBulkInsertInternalWriterTestBase.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/test/java/org/apache/hudi/spark3/internal/TestHoodieBulkInsertDataInternalWriter.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.2.x/src/test/java/org/apache/hudi/spark3/internal/TestHoodieDataSourceInternalBatchWrite.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.2plus-common/pom.xml
delete mode 100644 hudi-spark-datasource/hudi-spark3.2plus-common/src/main/java/org/apache/spark/sql/execution/datasources/parquet/Spark32PlusHoodieVectorizedParquetRecordReader.java
delete mode 100644 hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/hudi/Spark32PlusDefaultSource.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark32PlusAnalysis.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieInternalV2Table.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieStagedTable.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.2plus-common/src/main/scala/org/apache/spark/sql/hudi/command/AlterTableCommand.scala
create mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark33ParquetReader.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.3.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
create mode 100644 hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark34ParquetReader.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.4.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
create mode 100644 hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/Spark35ParquetReader.scala
delete mode 100644 hudi-spark-datasource/hudi-spark3.5.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
create mode 100644 hudi-sync/hudi-datahub-sync/src/test/java/org/apache/hudi/sync/datahub/TestDataHubSyncTool.java
create mode 100644 hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/transaction/lock/Heartbeat.java
rename hudi-sync/{hudi-hive-sync => hudi-sync-common}/src/main/java/org/apache/hudi/hive/SchemaDifference.java (100%)
create mode 100644 hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/metrics/HoodieMetaSyncMetrics.java
create mode 100644 hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/TestHoodieSyncClient.java
create mode 100644 hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/TestHoodieSyncTool.java
create mode 100644 hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/metrics/TestHoodieMetaSyncMetrics.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/converter/ProtoSchemaToAvroSchemaConverter.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieRetryingKafkaConsumer.java
delete mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/gcs/GcsObjectMetadataFetcher.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/TestManifestFileWriterSpark.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/converter/TestProtoSchemaToAvroSchemaConverter.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestErrorTableUtils.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieStreamerMetrics.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestStreamSyncUnitTests.java
create mode 100644 hudi-utilities/src/test/resources/data/nested_data_1.json
create mode 100644 hudi-utilities/src/test/resources/data/nested_data_2.json
create mode 100644 hudi-utilities/src/test/resources/data/nested_data_3.json
create mode 100644 hudi-utilities/src/test/resources/data/partitioned/country=US/state=TX/old_data.json
create mode 100644 hudi-utilities/src/test/resources/exporter/sql-file-transformer.sql
create mode 100644 hudi-utilities/src/test/resources/schema-provider/proto/evolved.proto
create mode 100644 hudi-utilities/src/test/resources/schema/nested_data_schema.avsc
delete mode 100755 packaging/bundle-validation/base/build_flink1146hive313spark302.sh
delete mode 100755 packaging/bundle-validation/base/build_flink1146hive313spark313.sh
delete mode 100755 packaging/bundle-validation/base/build_flink1146hive313spark323.sh
create mode 100755 packaging/bundle-validation/base/build_flink1146hive313spark331.sh
delete mode 100755 packaging/bundle-validation/base/build_flink1153hive313spark323.sh
create mode 100755 packaging/bundle-validation/base/build_flink1153hive313spark334.sh
create mode 100755 packaging/bundle-validation/base/build_flink1162hive313spark343.sh
create mode 100755 packaging/bundle-validation/base/build_flink1170hive313spark351.sh
create mode 100755 packaging/bundle-validation/base/build_flink1180hive313spark350scala213.sh
create mode 100755 packaging/bundle-validation/base/build_flink1180hive313spark351scala213.sh
create mode 100755 packaging/bundle-validation/base/build_flink1190hive313spark350.sh
create mode 100755 packaging/bundle-validation/base/build_flink1190hive313spark350scala213.sh
create mode 100755 packaging/bundle-validation/base/build_flink1190hive313spark351scala213.sh
create mode 100644 packaging/bundle-validation/spark_hadoop_mr/validate.scala
create mode 100644 rfc/rfc-80/append-handle.png
create mode 100644 rfc/rfc-80/bucket.png
create mode 100644 rfc/rfc-80/family-compaction.png
create mode 100644 rfc/rfc-80/file-group.png
create mode 100644 rfc/rfc-80/grand-compaction.png
create mode 100644 rfc/rfc-80/process-read.png
create mode 100644 rfc/rfc-80/process-write.png
create mode 100644 rfc/rfc-80/rfc-80.md
create mode 100644 rfc/rfc-80/row-reader.png
create mode 100644 rfc/rfc-80/table.png
delete mode 100755 scripts/ci/move_surefire_reports.sh