You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2023/04/01 07:49:20 UTC
[hudi] branch dependabot/maven/packaging/hudi-cli-bundle/com.google.code.gson-gson-2.8.9 updated (b5389217bd1 -> c1b20713d7a)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a change to branch dependabot/maven/packaging/hudi-cli-bundle/com.google.code.gson-gson-2.8.9
in repository https://gitbox.apache.org/repos/asf/hudi.git
discard b5389217bd1 Bump gson from 2.6.2 to 2.8.9 in /packaging/hudi-cli-bundle
add 828a85e8c2e [MINOR] Refactor tests - spark read only once to df, then assert (#8045)
add 5af78c665bd [HUDI-5845] Remove usage of deprecated getTableAvroSchemaWithoutMetadataFields. (#8032)
add e131505a4f0 [HUDI-5838] Mask sensitive info while printing hudi properties in DeltaStreamer (#8027)
add f22d1c9f26a [HUDI-5794] Failing new commits on any pending restore commits (#7948)
add ba3db4e1339 [MINOR] Reduce verbosity of the most noisy log messages (#6269)
add 58e9ce710b4 [HUDI-5850] Fix timestamp(6) field long overflow (#8052)
add dd083c41942 [HUDI-5855] Release resource actively for Flink hive meta sync (#8050)
add 3e12b46d717 [MINOR] fix the compilation failure for master branch. (#8059)
add 31e94abf902 [HUDI-5835] After performing the update operation, the hoodie table cannot be read normally by spark (#8026)
add 30175e992ef [MINOR] schema evolution relax decimal type conversion conditions (#8063)
add b6fdcb7ada2 [HUDI-5853] Add infer functions to BQ sync configs (#8053)
add dc8b078af68 [HUDI-5080] Unpersist only relevant RDDs instead of all (#7914)
add c7cebf445f4 [HUDI-5843] multiwriter deltastreamer checkpoints (#8043)
add b6490c160bf [HUDI-5813] Error table support for error events (#7982)
add bb240f7beaf [HUDI-5777] Support Metrics for Multiple Tables Simultaneously (#7934)
add 04c2ddfd8f9 [HUDI-5852] Release the HoodieSyncTool actively in time where feasible (#8064)
add a72ca04eeea [HUDI-5808] Add Support for kaffka ofsets in jsonkafkasource and avrokafkasource (#8011)
add 47356a57930 [HUDI-5862] Strength MARKER based rollback for log files with a single commit (#8077)
add f6518e87b55 [HUDI-5869] Fix Some Typos in Hudi-Common (#8083)
add eca57b51866 [HUDI-5867] Upgrade commons-io to v2.11.0 (#8081)
add 6b178ce978c [HUDI-4442] [HUDI-5001] Sanitize JsonConversion and RowSource (#8010)
add 8e7524574ce [HUDI-5728] HoodieTimelineArchiver archives the latest instant before inflight replacecommit (#7891)
add cfe490fcb23 [HUDI-5665] Adding support to re-use table configs (#7901)
add 81e6e854883 [HUDI-5847] Add support for multiple metric reporters and metric labels (#8041)
add 2a52bc03d90 [HUDI-5812] Optimize the data size check in HoodieBaseParquetWriter (#7978)
add 18d528f33d8 [HUDI-5736] Common de-coupling column drop flag and schema validation flag (#7895)
add d40a6211f64 [HUDI-5796] Adding auto inferring partition from incoming df (#7951)
add 2ddcf96cddb [HUDI-5863] Fix HoodieMetadataFileSystemView serving stale view at the timeline server (#8079)
add 9bb6b55440c [HUDI-4372] Enable matadata table by default for flink (#8070)
add bce038967e3 [HUDI-5866] Fix unnecessary log messages during bulk insert in Spark (#8095)
add 1db22d84493 [HUDI-5876] Remove usage of deprecated TableConfig (#8094)
add 0fd3c2c6ca3 [HUDI-5878] Optimize the txn metadata for BaseCommitActionExecutor (#8099)
add dcedf139d32 Fix some comments on schema change (#8084)
add 168005f020d [HUDI-5872] Abstraction for DeltaSyncService and DeltaStreamerMetrics (#8086)
add 46a2ccf56ee [HUDI-5881] Handle pending clean instants while running savepoint (#8105)
add 2ba1423ff5e [HUDI-5883] Avoid throwing error if data table does not exist in Metadata Table Validator (#8108)
add c3bfd129f8e [HUDI-5882] Close table metadata and file system view instances in Metadata Table Validator (#8106)
add c6441134ba6 [MINOR] Fix testHoodieFlinkClusteringScheduleAfterArchive (#8110)
add a8312a9b8c3 [HUDI-5887] Distinguish the single writer enabling metadata table and multi-writer use cases for lock guard (#8111)
add bab75b6c60c [HUDI-4911] Following the first patch, fix the inefficient code (#8127)
add 2675118d95c [HUDI-5941] Fix for RowData class cast exception (#8145)
add 79428391bac [HUDI-5851] Improvement of data skipping, only converts expressions to evaluators once (#8051)
add 8129a85eebb [HUDI-5333] Ignore file system type of basePath when using RocksDbBasedFileSystemView (#7387)
add 51d03515728 [HUDI-5857] Insert overwrite into bucket table would generate new file group id (#8072)
add 6589ebe8486 [HUDI-5904] support more than one update actions in merge into table (#8133)
add 84fcc3851cc [HUDI-5913] Table can not read correctly when computed column is in the midst (#8154)
add 5416f001dc7 [HUDI-5917] Fix HoodieRetryWrapperFileSystem getDefaultReplication (#8150)
add 9c44339e4bb [HUDI-5909] Reuse hive client if possible (#8139)
add b162330a1f3 [HUDI-5919] Fix the validation of partition listing in metadata table validator (#8156)
add 25c2f594938 [MINOR] Remove redundant variable in MetadataBootstrapHandlerFactory (#8158)
add ca76bc5788b [HUDI-5713] Add advanced property for configs (#7869)
add 07c6d8b1845 [MINOR] Use ExecutorFactory in BootstrapHandler (#7808)
add 1b756d62e19 [HUDI-5915] Fixed load ckpMeatadata error when using minio (#8149)
add c48db6d8ebe [HUDI-4675] add unittest for RebalancedSparkHoodieHBaseIndex (#6458)
add c392fb7ff8c [HUDI-4630] Add transformer capability to individual feeds in MultiTableDeltaStreamer (#6726)
add 38e4078d23b [MINOR] Ignoring warn msg for timeline server for metadata table (#8168)
add bfb01bdfc1e [HUDI-5930] Fix build failure due to ClosableIterator (#8177)
add bd4b13926ed [HUDI-5926] Improve cleaner parallelism (#8171)
add 17fdd10fdc3 [HUDI-5925] Improve bootstrap parallelism (#8170)
add 0d6f656e5bb [HUDI-5927] Improve parallelism of deleting invalid files (#8172)
add d760ed99734 [HUDI-5933] Revert "[HUDI-4630] Add transformer capability to individual feeds in MultiTableDeltaStreamer (#6726)" (#8183)
add ca552c6a8ae [MINOR] bot.yml ignore more filetype. (#7890)
add eb921591562 [HUDI-5938] No need to init the properties for each data flushing (#8192)
add d889cc77347 [HUDI-5939] Revert "[HUDI-4675] add unittest for RebalancedSparkHoodieHBaseIndex (#6458)" (#8197)
add b79ce80f709 [HUDI-5931] Improve the description of operation in HoodieDeltaStreamer (#8175)
add b466e66bad2 [HUDI-5922] Reuse IMetaStoreClient between HoodieHiveSyncClient and DDLExecutor (#8165)
add fbb51a6f154 [HUDI-5947] Update the README for flink jar building (#8212)
add fc3e81f24e7 [HUDI-5407] Following the first patch, add changes for Flink (#8204)
add e51b4575cb7 [HUDI-5920] Improve documentation of parallelism configs (#8157)
add 344df73c6b5 [HUDI-5865] Fix table service client to instantiate with timeline server (#8080)
add 102b535c970 [HUDI-5785] Enhance Spark Datasource tests (#7938)
add 42185a6118f [HUDI-5879] Extends evaluators to support evaluate based on column values (#8218)
add ce21873f332 [MINOR] Fix typo in ValidateMetadataTableFilesProcedure (#8202)
add f7c0d31fa3f [HUDI-5950] Fixing pending instant deduction to trigger compaction in MDT (#8223)
add 607ca5a664d [HUDI-5965] Add fallback keys for Flink options (#8252)
add 749a93ba269 [HUDI-5781] Refactor SQL transformer configs to use HoodieConfig and ConfigProperty (#8155)
add ff921568226 fix error info not show in CreateHoodieTableCommand (#8251)
add a75da8cd6aa [HUDI-5822] Fix bucket stream writer fileId not found exception (#8263)
add e849ad828e5 [MINOR] Fix typos in hudi-client and hudi-spark-datasource (#8230)
add 22ff13b9a14 [MINOR] Fix job name in prometheus.properties (#8239)
add 91fafcf3bc3 [HUDI-5960] Allow bootstrap procedure to throw an exception when execution fails (#8241)
add 14719735c4e [HUDI-5961] SparkBulkInsertDeltaCommitActionExecutor lacks bulkInsertPartitioner construction parameters (#8242)
add 23be4531e53 [HUDI-5962] Adding timeline server support to integ test suite (#8248)
add bf2e64caa6b [HUDI-5957] Fix table props not being properly propagated for HoodieCLIUtils (#8243)
add dbadacc9768 [HUDI-5958] Replace deprecated TableSchema with ResolvedSchema (#8237)
add a36c8e0f973 [HUDI-5967] Add partition ordering for full table scans (#8269)
add 6916803f7a4 [HUDI-5941] Support savepoint call procedure with base path in Spark SQL (#8271)
add 41026ef1fea [HUDI-5289] Avoiding repeated trigger of clustering dag (#8275)
add 7da1127c14f [HUDI-5978] Update timeline timezone when write in spark (#8284)
add 1a526eea748 [HUDI-5979] Add dependencies to hudi-trino-bundle needed for Trino connector (#8285)
add 099c05a0aee [HUDI-5985] Fix orc version for spark 3.3 (#8294)
add a1509e25f5c [HUDI-5891] Fix clustering on bootstrapped tables (#8206)
add cd4f7f551b8 [HUDI-5984] Enable FT for spark3.x versions in CI (#8293)
add 192ee432ee2 Remove duplicated WriteOperationType.INSERT.value (#7999)
add 04ec593413e [HUDI-5977] Fix Date to String column schema evolution (#8280)
add e1741ddc7d9 [MINOR] Fix typo for method in AvroSchemaConverter (#8306)
add 2023302ebee [HUDI-5986] Empty preCombineKey should never be stored in hoodie.properties (#8296)
add 7243393c688 [HUDI-5952] Fix NPE when use kafka callback (#8227)
add 21f83594a9c [HUDI-5976] Add fs in the constructor of HoodieAvroHFileReader (#8277)
add 16848bc46cb [HUDI-5928] Fixing shutting down deltastreamer properly when post write termination strategy is enabled (#8173)
add f4f329e1d19 Fix typo in use of ShellOption (#7963)
add f076c0a8a4a [HUDI-5993] Connection leak for lock provider (#8304)
add 8310e250ff7 [HUDI-5992] Fix (de)serialization for avro versions > 1.10.0 (#8307)
add 2e7a93cabb9 [HUDI-6003] Add schema evolution nullability test case (#8320)
add cb1395a820f [HUDI-5893] Mark advanced configs (#8295)
add 09f5d4f583b [HUDI-5907] Allow skip saving checkpoint in deltastreamer (#8137)
add 4b995a8c5d3 [HUDI-6005] Auto generate client id for Flink multi writer (#8323)
add f7a3c26960e [HUDI-5999] Fix bootstrap relation for nonpartitioned bootstrap table (#8317)
add 7c664fd1f82 [HUDI-6010] Always write parquets for insert overwrite operation (#8339)
add c53d9fbe019 [HUDI-5900] Clean up unused metadata configs (#8125)
add f051c8a0ad2 [HUDI-5929] Automatically infer key generator type (#8176)
add e9c5b58838d [MINOR] Remove unnecessary KryoSerializable interface in HoodieSparkReocrd class signature (#8333)
add b752df2a2f9 [HUDI-5921] Partition path should be considered in BucketIndexConcurrentFileWritesConflictResolutionStrategy (#8163)
add 6fd885fb3dc [HUDI-5740] Refactor Deltastreamer and schema providers to use HoodieConfig/ConfigProperty (#8152)
add c7397013f64 [HUDI-5954] Infer cleaning policy based on clean configs (#8238)
add c1b20713d7a Bump gson from 2.6.2 to 2.8.9 in /packaging/hudi-cli-bundle
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b5389217bd1)
\
N -- N -- N refs/heads/dependabot/maven/packaging/hudi-cli-bundle/com.google.code.gson-gson-2.8.9 (c1b20713d7a)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/bot.yml | 20 +-
README.md | 34 +-
.../demo/config/test-suite/compact-test.properties | 2 +-
.../config/test-suite/multi-writer-1.properties | 2 +-
.../config/test-suite/multi-writer-2.properties | 2 +-
.../test-suite/multi-writer-local-1.properties | 2 +-
.../test-suite/multi-writer-local-2.properties | 2 +-
.../test-suite/multi-writer-local-3.properties | 2 +-
.../test-suite/multi-writer-local-4.properties | 2 +-
.../test-suite/templates/test.properties.template | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
.../test-aggressive-clean-archival.properties | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
...clustering-aggressive-clean-archival.properties | 2 +-
.../test-clustering-inline-compact.properties | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
...g-metadata-aggressive-clean-archival.properties | 2 +-
.../config/test-suite/test-clustering.properties | 2 +-
.../test-suite/test-inline-compact.properties | 2 +-
...essive-clean-archival-inline-compact.properties | 2 +-
...t-metadata-aggressive-clean-archival.properties | 2 +-
.../test-metadata-inline-compact.properties | 2 +-
.../config/test-suite/test-metadata.properties | 2 +-
.../test-nonpartitioned-inline-compact.properties | 3 +-
...npartitioned-metadata-inline-compact.properties | 3 +-
.../test-nonpartitioned-metadata.properties | 3 +-
.../test-suite/test-nonpartitioned.properties | 3 +-
docker/demo/config/test-suite/test.properties | 2 +-
.../hudi/aws/sync/AwsGlueCatalogSyncTool.java | 11 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 6 +-
.../org/apache/hudi/client/BaseHoodieClient.java | 3 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 16 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 70 +--
.../apache/hudi/client/HoodieTimelineArchiver.java | 4 +-
...urrentFileWritesConflictResolutionStrategy.java | 22 +-
.../client/transaction/ConcurrentOperation.java | 37 +-
.../DirectMarkerTransactionManager.java | 7 +-
...urrentFileWritesConflictResolutionStrategy.java | 9 +-
.../client/transaction/TransactionManager.java | 19 +-
.../hudi/client/transaction/lock/LockManager.java | 1 +
.../lock/metrics/HoodieLockMetrics.java | 4 +-
.../apache/hudi/config/HoodieArchivalConfig.java | 11 +-
.../apache/hudi/config/HoodieBootstrapConfig.java | 10 +-
.../org/apache/hudi/config/HoodieCleanConfig.java | 88 ++-
.../apache/hudi/config/HoodieErrorTableConfig.java | 100 ++++
.../org/apache/hudi/config/HoodieIndexConfig.java | 18 +-
.../org/apache/hudi/config/HoodieLockConfig.java | 8 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 87 ++-
.../hudi/config/metrics/HoodieMetricsConfig.java | 6 +
.../hudi/index/bloom/ListBasedIndexFileFilter.java | 2 +-
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 31 +-
.../keygen/NonpartitionedAvroKeyGenerator.java | 5 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 31 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 90 +--
.../java/org/apache/hudi/metrics/MetricUtils.java | 81 +++
.../main/java/org/apache/hudi/metrics/Metrics.java | 136 +++--
.../hudi/metrics/MetricsReporterFactory.java | 14 +-
.../hudi/metrics/datadog/DatadogHttpClient.java | 20 +-
.../metrics/datadog/DatadogMetricsReporter.java | 2 +-
.../hudi/metrics/datadog/DatadogReporter.java | 27 +-
.../prometheus/PushGatewayMetricsReporter.java | 26 +-
.../metrics/prometheus/PushGatewayReporter.java | 42 +-
.../apache/hudi/table/BulkInsertPartitioner.java | 2 +-
.../java/org/apache/hudi/table/HoodieTable.java | 72 +--
.../table/action/bootstrap/BootstrapUtils.java | 15 +-
.../table/action/clean/CleanActionExecutor.java | 4 +-
.../hudi/table/action/clean/CleanPlanner.java | 43 +-
.../action/commit/BaseCommitActionExecutor.java | 38 +-
.../hudi/table/action/compact/CompactHelpers.java | 2 +
.../hudi/table/action/compact/HoodieCompactor.java | 2 +-
.../compact/RunCompactionActionExecutor.java | 4 +-
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../table/action/rollback/BaseRollbackHelper.java | 7 +-
.../rollback/BaseRollbackPlanActionExecutor.java | 2 +-
.../rollback/ListingBasedRollbackStrategy.java | 30 +-
.../rollback/MarkerBasedRollbackStrategy.java | 12 +-
.../SerializableHoodieRollbackRequest.java | 2 +-
.../action/savepoint/SavepointActionExecutor.java | 31 +-
.../marker/TimelineServerBasedWriteMarkers.java | 2 +-
.../hudi/table/upgrade/DowngradeHandler.java | 2 +-
...urrentFileWritesConflictResolutionStrategy.java | 48 +-
.../hudi/client/transaction/TestLockManager.java | 111 ++++
...urrentFileWritesConflictResolutionStrategy.java | 20 +-
.../apache/hudi/config/TestHoodieWriteConfig.java | 106 +++-
.../org/apache/hudi/keygen/TestKeyGenUtils.java | 26 +
.../factory/TestHoodieAvroKeyGeneratorFactory.java | 3 +-
.../hudi/metrics/TestHoodieConsoleMetrics.java | 15 +-
.../hudi/metrics/TestHoodieGraphiteMetrics.java | 15 +-
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 40 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 129 ++---
.../hudi/metrics/TestMetricsReporterFactory.java | 4 +-
.../datadog/TestDatadogMetricsReporter.java | 33 +-
.../metrics/prometheus/TestPrometheusReporter.java | 11 +-
.../prometheus/TestPushGateWayReporter.java | 82 ++-
.../src/test/resources/datadog.properties | 25 +
.../src/test/resources/prometheus.properties | 24 +
.../hudi/client/HoodieFlinkTableServiceClient.java | 7 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 28 +-
.../client/common/HoodieFlinkEngineContext.java | 16 +
.../apache/hudi/io/FlinkWriteHandleFactory.java | 6 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 2 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 6 +
.../HoodieFlinkMergeOnReadTableCompactor.java | 3 +-
.../apache/hudi/client/TestFlinkWriteClient.java | 64 +++
.../hudi/client/HoodieJavaTableServiceClient.java | 7 +-
.../apache/hudi/client/HoodieJavaWriteClient.java | 4 +-
.../client/common/HoodieJavaEngineContext.java | 21 +-
.../HoodieJavaMergeOnReadTableCompactor.java | 3 +-
.../client/TestHoodieJavaWriteClientInsert.java | 46 ++
.../hudi/client/SparkRDDTableServiceClient.java | 7 +-
.../apache/hudi/client/SparkRDDWriteClient.java | 15 +-
.../MultipleSparkJobExecutionStrategy.java | 7 +-
.../client/common/HoodieSparkEngineContext.java | 36 +-
.../client/utils/SparkInternalSchemaConverter.java | 4 +-
.../hudi/common/model/HoodieSparkRecord.java | 3 +-
.../java/org/apache/hudi/data/HoodieJavaRDD.java | 16 +-
.../hudi/io/storage/HoodieSparkFileReader.java | 2 +-
.../factory/HoodieSparkKeyGeneratorFactory.java | 78 ++-
.../bootstrap/MetadataBootstrapHandlerFactory.java | 1 -
.../bootstrap/OrcBootstrapMetadataHandler.java | 25 +-
.../bootstrap/ParquetBootstrapMetadataHandler.java | 15 +-
.../SparkBootstrapCommitActionExecutor.java | 5 +-
.../SparkBootstrapDeltaCommitActionExecutor.java | 1 +
.../commit/BaseSparkCommitActionExecutor.java | 9 +-
.../commit/BulkInsertDataInternalWriterHelper.java | 5 +-
.../action/commit/SparkBucketIndexPartitioner.java | 11 +
.../SparkInsertOverwriteCommitActionExecutor.java | 14 +
.../commit/SparkInsertOverwritePartitioner.java | 15 +
.../table/action/commit/UpsertPartitioner.java | 19 +-
.../HoodieSparkMergeOnReadTableCompactor.java | 6 +-
.../org/apache/hudi/AvroConversionUtils.scala | 1 +
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 79 ++-
.../scala/org/apache/spark/sql/DataFrameUtil.scala | 2 +-
.../spark/sql/hudi/execution/RangeSample.scala | 2 +-
.../org/apache/hudi/client/TestClientRollback.java | 4 +-
.../hudi/client/TestHoodieClientMultiWriter.java | 2 +-
.../hudi/client/TestSparkRDDWriteClient.java | 168 ++++++
.../common/TestHoodieSparkEngineContext.java | 51 ++
.../functional/TestHoodieBackedMetadata.java | 164 +++---
.../functional/TestHoodieBackedTableMetadata.java | 1 -
.../TestHoodieClientOnCopyOnWriteStorage.java | 59 +-
.../client/functional/TestHoodieMetadataBase.java | 31 +-
.../functional/TestHoodieMetadataBootstrap.java | 6 +-
.../TestRemoteFileSystemViewWithMetadataTable.java | 275 ++++++++++
.../hudi/execution/TestSimpleExecutionInSpark.java | 5 +-
.../index/hbase/TestSparkHoodieHBaseIndex.java | 22 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 24 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 53 +-
.../table/action/compact/TestInlineCompaction.java | 4 +-
.../strategy/TestHoodieCompactionStrategy.java | 2 +-
.../TestHoodieSparkMergeOnReadTableCompaction.java | 2 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 4 +-
.../TestMarkerBasedRollbackStrategy.java | 54 +-
.../hudi/testutils/HoodieClientTestHarness.java | 2 -
.../SparkClientFunctionalTestHarness.java | 16 +-
hudi-common/pom.xml | 9 +-
.../org/apache/hudi/BaseHoodieTableFileIndex.java | 3 +-
.../java/org/apache/hudi/avro/AvroSchemaUtils.java | 65 +++
.../apache/hudi/avro/GenericAvroSerializer.java | 146 +++++
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 86 ++-
.../apache/hudi/avro/MercifulJsonConverter.java | 120 +++-
.../apache/hudi/common/config/ConfigGroups.java | 10 +-
.../apache/hudi/common/config/ConfigProperty.java | 38 +-
.../hudi/common/config/HoodieMetadataConfig.java | 87 +--
.../org/apache/hudi/common/data/HoodieData.java | 56 +-
.../apache/hudi/common/data/HoodieListData.java | 11 +
.../hudi/common/engine/HoodieEngineContext.java | 6 +
.../common/engine/HoodieLocalEngineContext.java | 16 +
.../apache/hudi/common/fs/ConsistencyGuard.java | 2 +-
.../hudi/common/fs/ConsistencyGuardConfig.java | 4 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 10 +-
.../hudi/common/fs/FailSafeConsistencyGuard.java | 2 +-
.../common/fs/HoodieRetryWrapperFileSystem.java | 11 +
.../hudi/common/fs/OptimisticConsistencyGuard.java | 2 +-
.../hudi/common/fs/SizeAwareDataOutputStream.java | 4 +-
.../debezium/AbstractDebeziumAvroPayload.java | 2 +-
.../hudi/common/table/HoodieTableMetaClient.java | 2 +-
.../table/log/block/HoodieHFileDataBlock.java | 10 +-
.../table/log/block/HoodieParquetDataBlock.java | 6 +-
.../table/view/AbstractTableFileSystemView.java | 36 +-
.../IncrementalTimelineSyncFileSystemView.java | 20 +-
.../apache/hudi/common/util/ClusteringUtils.java | 35 +-
.../apache/hudi/common/util/CollectionUtils.java | 5 +-
.../org/apache/hudi/common/util/CommitUtils.java | 23 +-
.../hudi/common/util/SerializationUtils.java | 3 +
.../hudi/common/util/collection/RocksDBDAO.java | 3 +-
.../internal/schema/utils/SchemaChangeUtils.java | 9 +-
.../hudi/io/storage/HoodieBaseParquetWriter.java | 38 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 11 +-
.../metadata/HoodieMetadataFileSystemView.java | 20 +-
.../org/apache/hudi/avro/TestAvroSchemaUtils.java | 57 ++
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 11 +
.../hudi/avro/TestMercifulJsonConverter.java | 102 ++++
.../hudi/common/config/TestConfigProperty.java | 10 +
.../fs/TestFSUtilsWithRetryWrapperEnable.java | 22 +
.../hudi/common/testutils/FileCreateUtils.java | 16 +-
.../common/testutils/HoodieTestDataGenerator.java | 20 +
.../io/storage/TestHoodieBaseParquetWriter.java | 122 +++++
.../HoodieBackedTestDelayedTableMetadata.java | 54 ++
.../examples/quickstart/HoodieFlinkQuickstart.java | 9 +-
.../factory/CollectSinkTableFactory.java | 8 +-
.../quickstart/utils/QuickstartConfigurations.java | 16 +-
.../quickstart/TestHoodieSparkQuickstart.java | 6 +
.../apache/hudi/configuration/FlinkOptions.java | 14 +-
.../hudi/configuration/OptionsInference.java | 28 +
.../apache/hudi/configuration/OptionsResolver.java | 8 +
.../org/apache/hudi/sink/StreamWriteFunction.java | 10 +-
.../hudi/sink/StreamWriteOperatorCoordinator.java | 23 +-
.../sink/bucket/BucketStreamWriteFunction.java | 17 +-
.../hudi/sink/bulk/sort/SortOperatorGen.java | 2 +-
.../sink/clustering/HoodieFlinkClusteringJob.java | 3 +
.../hudi/sink/compact/HoodieFlinkCompactor.java | 3 +
.../org/apache/hudi/sink/meta/CkpMetadata.java | 11 +-
.../java/org/apache/hudi/source/DataPruner.java | 140 +++++
.../apache/hudi/source/ExpressionEvaluators.java | 586 ++++++++++++++++++++
.../java/org/apache/hudi/source/FileIndex.java | 46 +-
.../apache/hudi/source/IncrementalInputSplits.java | 8 +-
.../org/apache/hudi/source/stats/ColumnStats.java | 72 +++
.../hudi/source/stats/ExpressionEvaluator.java | 605 ---------------------
.../apache/hudi/streamer/HoodieFlinkStreamer.java | 1 +
.../org/apache/hudi/table/HoodieTableSink.java | 2 +
.../org/apache/hudi/table/HoodieTableSource.java | 5 +-
.../hudi/table/catalog/HoodieHiveCatalog.java | 2 +-
.../org/apache/hudi/util/AvroSchemaConverter.java | 4 +-
.../main/java/org/apache/hudi/util/ClientIds.java | 276 ++++++++++
.../java/org/apache/hudi/util/CompactionUtil.java | 19 +-
.../apache/hudi/util/RowDataToAvroConverters.java | 4 +-
.../apache/hudi/util/ViewStorageProperties.java | 2 -
.../hudi/configuration/TestOptionsInference.java | 78 +++
.../apache/hudi/sink/ITTestDataStreamWrite.java | 52 ++
.../sink/TestStreamWriteOperatorCoordinator.java | 10 +-
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 184 +++++++
.../sink/cluster/ITTestHoodieFlinkClustering.java | 133 +++++
.../apache/hudi/sink/compact/TestCustomSerDe.java | 81 +++
.../hudi/source/TestExpressionEvaluators.java | 412 ++++++++++++++
.../hudi/source/TestIncrementalInputSplits.java | 37 ++
.../hudi/source/stats/TestExpressionEvaluator.java | 403 --------------
.../apache/hudi/table/ITTestHoodieDataSource.java | 61 ++-
.../org/apache/hudi/utils/TestClusteringUtil.java | 4 +
.../org/apache/hudi/utils/TestCompactionUtil.java | 13 +
.../resources/test_read_schema_dropped_age.avsc | 36 +-
.../reader/ParquetColumnarRowSplitReader.java | 3 +-
.../hudi/gcp/bigquery/BigQuerySyncConfig.java | 24 +-
.../hudi/gcp/bigquery/TestBigQuerySyncConfig.java | 86 ++-
hudi-integ-test/pom.xml | 10 -
.../testsuite/HoodieContinousTestSuiteWriter.java | 10 +-
.../testsuite/HoodieDeltaStreamerWrapper.java | 8 +-
.../testsuite/HoodieInlineTestSuiteWriter.java | 14 +-
.../integ/testsuite/dag/nodes/HiveQueryNode.java | 6 +-
.../testsuite/dag/nodes/SparkDeleteNode.scala | 2 +-
.../dag/nodes/SparkDeletePartitionNode.scala | 2 +-
.../testsuite/dag/nodes/SparkInsertNode.scala | 2 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 6 +
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 2 +-
.../scala/org/apache/hudi/DataSourceOptions.scala | 98 ++--
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 35 +-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 45 +-
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 10 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 62 ++-
.../org/apache/hudi/HoodieStreamingSink.scala | 10 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 22 +-
.../org/apache/hudi/IncrementalRelation.scala | 4 +-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 2 +-
.../datasources/HoodieInMemoryFileIndex.scala | 4 +-
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 2 +-
.../apache/spark/sql/hive/HiveClientUtils.scala | 17 +-
.../AlterHoodieTableChangeColumnCommand.scala | 4 +-
.../hudi/command/CreateHoodieTableCommand.scala | 2 +-
.../HoodieBulkInsertInternalWriterTestBase.java | 3 +-
.../spark/sql/hive/TestHiveClientUtils.scala} | 30 +-
.../apache/hudi/cli/BootstrapExecutorUtils.java | 4 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 38 +-
.../hudi/command/payload/ExpressionPayload.scala | 7 +-
.../procedures/CreateSavepointProcedure.scala | 8 +-
.../procedures/DeleteSavepointProcedure.scala | 8 +-
.../procedures/RollbackToSavepointProcedure.scala | 8 +-
.../command/procedures/RunBootstrapProcedure.scala | 8 +-
.../procedures/ShowSavepointsProcedure.scala | 6 +-
.../ValidateMetadataTableFilesProcedure.scala | 2 +-
.../hudi-spark/src/test/java/HoodieJavaApp.java | 12 -
.../src/test/java/HoodieJavaGenerateApp.java | 6 -
.../functional/TestHiveTableSchemaEvolution.java | 37 +-
.../TestHoodieSparkKeyGeneratorFactory.java | 56 +-
.../org/apache/hudi/TestHoodieFileIndex.scala | 11 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 68 +--
.../apache/hudi/functional/TestCOWDataSource.scala | 340 +++++++++++-
.../hudi/functional/TestCOWDataSourceStorage.scala | 4 +
.../hudi/functional/TestColumnStatsIndex.scala | 15 +-
.../functional/TestDataSourceForBootstrap.scala | 66 ++-
.../apache/hudi/functional/TestMORDataSource.scala | 5 +-
.../hudi/functional/TestMORDataSourceStorage.scala | 10 +-
.../TestMORDataSourceWithBucketIndex.scala | 29 +
.../TestMetadataTableWithSparkDataSource.scala | 17 +-
.../functional/TestParquetColumnProjection.scala | 8 +-
.../hudi/functional/TestStreamingSource.scala | 4 +
.../hudi/functional/TestTimeTravelQuery.scala | 11 +-
.../functional/cdc/TestCDCDataFrameSuite.scala | 4 +-
.../spark/sql/avro/TestSchemaConverters.scala | 2 +-
.../sql/hudi/TestAlterTableDropPartition.scala | 10 +-
.../apache/spark/sql/hudi/TestCreateTable.scala | 5 +-
.../apache/spark/sql/hudi/TestDeleteTable.scala | 2 -
.../apache/spark/sql/hudi/TestInsertTable.scala | 154 +++++-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 55 +-
.../org/apache/spark/sql/hudi/TestSpark3DDL.scala | 98 +++-
.../apache/spark/sql/hudi/TestTruncateTable.scala | 3 -
.../apache/spark/sql/hudi/TestUpdateTable.scala | 34 ++
.../hudi/procedure/TestBootstrapProcedure.scala | 40 ++
.../hudi/procedure/TestClusteringProcedure.scala | 31 ++
.../hudi/procedure/TestSavepointsProcedure.scala | 70 ++-
.../parquet/Spark24HoodieParquetFileFormat.scala | 6 +-
.../parquet/Spark31HoodieParquetFileFormat.scala | 6 +-
.../Spark32PlusHoodieParquetFileFormat.scala | 6 +-
.../java/org/apache/hudi/hive/HiveSyncConfig.java | 2 -
.../java/org/apache/hudi/hive/HiveSyncTool.java | 12 +-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 9 +-
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 13 +-
.../apache/hudi/hive/ddl/HiveQueryDDLExecutor.java | 15 +-
.../lock/HiveMetastoreBasedLockProvider.java | 3 +-
.../hudi/hive/util/IMetaStoreClientUtil.java | 47 ++
.../org/apache/hudi/hive/TestHiveSyncTool.java | 92 ++--
.../testutils/HiveSyncFunctionalTestHarness.java | 5 +-
.../apache/hudi/hive/testutils/HiveTestUtil.java | 3 +-
.../hudi/sync/common/util/SyncUtilHelpers.java | 4 +-
.../hudi/timeline/service/RequestHandler.java | 27 +-
.../timeline/service/handlers/MarkerHandler.java | 2 +-
.../hudi/utilities/HoodieDropPartitionsTool.java | 5 +-
.../utilities/HoodieMetadataTableValidator.java | 94 ++--
.../org/apache/hudi/utilities/UtilHelpers.java | 40 +-
.../kafka/HoodieWriteCommitKafkaCallback.java | 4 +-
.../checkpointing/InitialCheckPointProvider.java | 18 +-
.../config/FilebasedSchemaProviderConfig.java | 49 ++
.../utilities/config/HiveSchemaProviderConfig.java | 59 ++
.../config/HoodieDeltaStreamerConfig.java | 89 +++
.../config/HoodieSchemaProviderConfig.java | 82 +++
.../config/JdbcbasedSchemaProviderConfig.java | 77 +++
.../ProtoClassBasedSchemaProviderConfig.java | 69 +++
.../config/SchemaProviderPostProcessorConfig.java | 76 +++
.../utilities/config/SqlTransformerConfig.java | 46 ++
.../deltastreamer/BaseErrorTableWriter.java | 70 +++
.../utilities/deltastreamer/BootstrapExecutor.java | 4 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 205 +++++--
.../DeltastreamerMultiWriterCkptUpdateFunc.java | 98 ++++
.../hudi/utilities/deltastreamer/ErrorEvent.java | 58 ++
.../utilities/deltastreamer/ErrorTableUtils.java | 66 +++
.../deltastreamer/HoodieDeltaStreamer.java | 167 +++---
.../deltastreamer/HoodieDeltaStreamerMetrics.java | 123 +++--
.../HoodieMultiTableDeltaStreamer.java | 57 +-
.../deltastreamer/SourceFormatAdapter.java | 195 ++++++-
.../ingestion/HoodieIngestionException.java | 25 +-
.../ingestion/HoodieIngestionMetrics.java | 57 ++
.../ingestion/HoodieIngestionService.java | 185 +++++++
.../utilities/schema/FilebasedSchemaProvider.java | 39 +-
.../hudi/utilities/schema/HiveSchemaProvider.java | 23 +-
.../utilities/schema/JdbcbasedSchemaProvider.java | 36 +-
.../utilities/schema/KafkaOffsetPostProcessor.java | 71 +++
.../schema/ProtoClassBasedSchemaProvider.java | 52 +-
.../hudi/utilities/schema/SchemaPostProcessor.java | 9 +-
.../utilities/schema/SchemaRegistryProvider.java | 25 +-
.../utilities/schema/SparkAvroPostProcessor.java | 5 +-
.../DropColumnSchemaPostProcessor.java | 12 +-
.../add/AddPrimitiveColumnSchemaPostProcessor.java | 15 +-
.../add/BaseSchemaPostProcessorConfig.java | 53 --
.../hudi/utilities/sources/AvroKafkaSource.java | 41 +-
.../sources/GcsEventsHoodieIncrSource.java | 2 +-
.../hudi/utilities/sources/GcsEventsSource.java | 7 +-
.../hudi/utilities/sources/JsonKafkaSource.java | 47 +-
.../apache/hudi/utilities/sources/KafkaSource.java | 16 +-
.../hudi/utilities/sources/ProtoKafkaSource.java | 13 +-
.../utilities/sources/debezium/DebeziumSource.java | 28 +-
.../sources/debezium/MysqlDebeziumSource.java | 4 +-
.../sources/debezium/PostgresDebeziumSource.java | 4 +-
.../utilities/sources/helpers/AvroConvertor.java | 56 +-
.../utilities/sources/helpers/KafkaOffsetGen.java | 7 +-
.../sources/helpers/SanitizationUtils.java | 198 +++++++
.../transform/SqlFileBasedTransformer.java | 16 +-
.../transform/SqlQueryBasedTransformer.java | 13 +-
.../hudi/utilities/TestHiveIncrementalPuller.java | 10 +-
.../hudi/utilities/TestSchemaPostProcessor.java | 27 +-
.../callback/TestKafkaCallbackProvider.java | 90 +++
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 33 +-
.../deltastreamer/TestHoodieDeltaStreamer.java | 208 ++++++-
.../TestHoodieMultiTableDeltaStreamer.java | 8 +-
.../deltastreamer/TestSourceFormatAdapter.java | 189 +++++++
.../schema/TestFilebasedSchemaProvider.java | 99 ++++
.../schema/TestProtoClassBasedSchemaProvider.java | 17 +-
.../utilities/sources/BaseTestKafkaSource.java | 4 +-
.../utilities/sources/TestAvroKafkaSource.java | 188 +++++++
.../utilities/sources/TestGenericRddTransform.java | 57 ++
.../utilities/sources/TestJsonKafkaSource.java | 173 +++++-
.../sources/TestJsonKafkaSourcePostProcessor.java | 14 +-
.../utilities/sources/TestProtoKafkaSource.java | 5 +-
.../debezium/TestAbstractDebeziumSource.java | 4 +-
.../sources/helpers/TestKafkaOffsetGen.java | 4 +-
.../sources/helpers/TestSanitizationUtils.java | 141 +++++
.../utilities/testutils/SanitizationTestUtils.java | 173 ++++++
.../src/test/resources/data/avro_sanitization.json | 2 +
.../data/avro_sanitization_bad_naming_in.json | 2 +
...ro_sanitization_bad_naming_nested_array_in.json | 2 +
...o_sanitization_bad_naming_nested_array_out.json | 2 +
...avro_sanitization_bad_naming_nested_map_in.json | 2 +
...vro_sanitization_bad_naming_nested_map_out.json | 2 +
.../data/avro_sanitization_bad_naming_out.json | 2 +
.../file_schema_provider_invalid.avsc | 43 +-
.../file_schema_provider_valid.avsc | 42 +-
packaging/bundle-validation/flink/insert.sql | 1 +
packaging/hudi-flink-bundle/pom.xml | 3 +
packaging/hudi-integ-test-bundle/pom.xml | 17 +-
packaging/hudi-trino-bundle/pom.xml | 18 +
pom.xml | 12 +-
409 files changed, 11541 insertions(+), 3353 deletions(-)
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieErrorTableConfig.java
create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricUtils.java
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestLockManager.java
create mode 100644 hudi-client/hudi-client-common/src/test/resources/datadog.properties
create mode 100644 hudi-client/hudi-client-common/src/test/resources/prometheus.properties
create mode 100644 hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/client/TestFlinkWriteClient.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkRDDWriteClient.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java
create mode 100644 hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/GenericAvroSerializer.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/avro/TestMercifulJsonConverter.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieBaseParquetWriter.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/metadata/HoodieBackedTestDelayedTableMetadata.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/DataPruner.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/ExpressionEvaluators.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ColumnStats.java
delete mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/source/stats/ExpressionEvaluator.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/ClientIds.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/configuration/TestOptionsInference.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bucket/ITTestBucketStreamWrite.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/compact/TestCustomSerDe.java
create mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestExpressionEvaluators.java
delete mode 100644 hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/stats/TestExpressionEvaluator.java
copy hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReader.java => hudi-flink-datasource/hudi-flink/src/test/resources/test_read_schema_dropped_age.avsc (59%)
copy hudi-spark-datasource/hudi-spark-common/src/{main/scala/org/apache/spark/sql/hive/HiveClientUtils.scala => test/scala/org/apache/spark/sql/hive/TestHiveClientUtils.scala} (58%)
create mode 100644 hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/IMetaStoreClientUtil.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/config/FilebasedSchemaProviderConfig.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/config/HiveSchemaProviderConfig.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/config/HoodieDeltaStreamerConfig.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/config/HoodieSchemaProviderConfig.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/config/JdbcbasedSchemaProviderConfig.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/config/ProtoClassBasedSchemaProviderConfig.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/config/SchemaProviderPostProcessorConfig.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/config/SqlTransformerConfig.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BaseErrorTableWriter.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltastreamerMultiWriterCkptUpdateFunc.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/ErrorEvent.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/ErrorTableUtils.java
copy hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReader.java => hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionException.java (60%)
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionMetrics.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/ingestion/HoodieIngestionService.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/KafkaOffsetPostProcessor.java
delete mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/add/BaseSchemaPostProcessorConfig.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/SanitizationUtils.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/callback/TestKafkaCallbackProvider.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSourceFormatAdapter.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/schema/TestFilebasedSchemaProvider.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestAvroKafkaSource.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGenericRddTransform.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/helpers/TestSanitizationUtils.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/SanitizationTestUtils.java
create mode 100644 hudi-utilities/src/test/resources/data/avro_sanitization.json
create mode 100644 hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_in.json
create mode 100644 hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_nested_array_in.json
create mode 100644 hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_nested_array_out.json
create mode 100644 hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_nested_map_in.json
create mode 100644 hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_nested_map_out.json
create mode 100644 hudi-utilities/src/test/resources/data/avro_sanitization_bad_naming_out.json
copy hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReader.java => hudi-utilities/src/test/resources/delta-streamer-config/file_schema_provider_invalid.avsc (55%)
copy hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReader.java => hudi-utilities/src/test/resources/delta-streamer-config/file_schema_provider_valid.avsc (57%)