You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by na...@apache.org on 2020/04/07 05:18:16 UTC
[incubator-hudi] branch hudi_test_suite_refactor updated (f380005
-> 29b4fdf)
This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a change to branch hudi_test_suite_refactor
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git.
discard f380005 trigger rebuild
discard 162eb50 [MINOR] Fix compile error after rebasing the branch
discard c1035d0 [HUDI-592] Remove duplicated dependencies in the pom file of test suite module
discard 5b2ffff [HUDI-503] Add hudi test suite documentation into the README file of the test suite module (#1191)
discard 64f9924 [MINOR] Fix TestHoodieTestSuiteJob#testComplexDag failure
discard a823b93 [HUDI-441] Rename WorkflowDagGenerator and some class names in test package
discard ea6389d Fixed resource leak in HiveTestService about hive meta store
discard 188199b [HUDI-591] Support Spark version upgrade
discard c9c3319 [HUDI-442] Fix TestComplexKeyGenerator#testSingleValueKeyGenerator and testMultipleValueKeyGenerator NPE
discard 89788a5 [MINOR] Fix compile error about the deletion of HoodieActiveTimeline#createNewCommitTime
discard c3caeb9 [HUDI-391] Rename module name from hudi-bench to hudi-test-suite and fix some checkstyle issues (#1102)
discard 1f5051d [HUDI-394] Provide a basic implementation of test suite
add 20ed251 [HUDI-571] Add show archived compaction(s) to CLI
add aaa6cf9 [MINOR] Fix some typos
add 24e7381 [MINOR] Code Cleanup, remove redundant code (#1337)
add b8f9d0e [HUDI-615]: Add some methods and test cases for StringUtils. (#1338)
add 0049323 [HUDI-108] Removing 2GB spark partition limitations in HoodieBloomIndex with spark 2.4.4 (#1315)
add 8c6138c [MINOR] Add javadoc to SchedulerConfGenerator and code clean (#1340)
add c2b08cd [HUDI-617] Add support for types implementing CharSequence (#1339)
add f9d2f66 [HUDI-622]: Remove VisibleForTesting annotation and import from code (#1343)
add 185ff64 Refactoring getter to avoid double extrametadata in json representation
add 078d482 [HUDI-624]: Split some of the code from PR for HUDI-479 (#1344)
add 4e7fcde [HUDI-597] Enable incremental pulling from defined partitions (#1348)
add 83c8ad5 [HUDI-625] Fixing performance issues around DiskBasedMap & kryo (#1352)
add 11fb2c2 [HUDI-580] Fix incorrect license header in files
add 323fffa [HUDI-606] Improve execute build_local_docker_images.sh script
add b7f35be [HUDI-618] Adding unit tests for PriorityBasedFileSystemView (#1345)
add cacd9a3 [HUDI-636] Fix could not get sources warnings while compiling (#1357)
add 71170fa [HUDI-554] Cleanup package structure in hudi-client (#1346)
add acf359c [HUDI-627] Aggregate code coverage and publish to codecov.io during CI (#1347)
add 0cde27e [HUDI-599] Update release guide & release scripts due to the change of scala 2.12 build (#1364)
add 9160084 [MINOR] Fix cut_release_branch script missed a double quotation marks (#1365)
add 0dc8e49 Moving to 0.6.0-SNAPSHOT on master branch.
add 2d04014 [HUDI-607] Fix to allow creation/syncing of Hive tables partitioned by Date type columns (#1330)
add 8306205 [HUDI-332]Add operation type (insert/upsert/bulkinsert/delete) to HoodieCommitMetadata (#1157)
add 9d46ce3 [HUDI -409] Match header and footer block length to improve corrupted block detection (#1332)
add 5f85c26 [HUDI-584] Relocate spark-avro dependency by maven-shade-plugin (#1290)
add ccbf543 [HUDI-654] Rename hudi-hive to hudi-hive-sync
add 3d37818 [CLI] Add export to table
add ee5b32f [HUDI-652] Decouple HoodieReadClient and AbstractHoodieClient to break the inheritance chain (#1372)
add 170ee88 [HUDI-553] Building/Running Hudi on higher java versions (#1369)
add 90227ee [HUDI-673] Rename hudi-hive-bundle to hudi-hive-sync-bundle
add fdcd3b1 [HUDI-680] Update Jackson databind to 2.6.7.3 (#1385)
add 5f8bf97 [HUDI-671] Added unit-test for HBaseIndex (#1381)
add 415882f [HUDI-581] NOTICE need more work as it missing content form included 3rd party ALv2 licensed NOTICE files (#1354)
add 2137ecf [HUDI-676] Address issues towards removing use of WIP Disclaimer (#1386)
add f93e64f [HUDI-681]Remove embeddedTimelineService from HoodieReadClient (#1388)
add 44700d5 [HUDI-344] Hudi Dataset Snapshot Exporter (#1360)
add 77d5b92 [HUDI-668] Added additional unit-tests for HUDI metrics. (#1380)
add cf0a4c1 [HUDI-670] Added test cases for TestDiskBasedMap. (#1379)
add dd7cf38 [HUDI-688] Paring down the NOTICE file to minimum required notices (#1391)
add 1ca912a [HUDI-667] Fixing delete tests for DeltaStreamer (#1395)
add 7194514 [HUDI-689] Change CLI command names to not have overlap (#1392)
add 7d66831 [MINOR] Removing code which is duplicated from the base class HoodieWriteHandle. (#1399)
add 0f892ef [HUDI-692] Add delete savepoint for cli (#1397)
add c40a0d4 [HUDI-656][Performance] Return a dummy Spark relation after writing the DataFrame (#1394)
add fb7fba3 [HUDI-646] fix failing test due to improper filesytem cleanup (#1373)
add 99b7e9e [HUDI-629]: Replace Guava's Hashing with an equivalent in NumericUtils.java (#1350)
add 14323cb [HUDI-344] Improve exporter tests (#1404)
add 23afe7a [HUDI-710] Fixing failure in Staging Validation Script (#1403)
add 55e6d34 [HUDI-694]Add unit test for SparkEnvCommand (#1401)
add 3ef9e88 [HUDI-715] Fix duplicate name in TableCommand (#1410)
add 418f9bb Add constructor to HoodieROTablePathFilter (#1413)
add f1d7bb3 [HUDI-695]Add unit test for TableCommand (#1411)
add e301903 [HUDI-539] Make ROPathFilter conf member serializable (#1415)
add 0a4902e [HUDI-437] Support user-defined index (#1408)
add 779edc0 [HUDI-344] Add partitioner param to Exporter (#1405)
add 1e321c2 [HUDI-209] Implement JMX metrics reporter (#1106)
add cf765df [HUDI-76] Add CSV Source support for Hudi Delta Streamer
add a752b7b Merge pull request #1165 from yihua/HUDI-76-deltastreamer-csv-source
add 14e0c95 [HUDI-400] Check upgrade from old plan to new plan for compaction (#1422)
add 21c45e1 [HUDI-726]Delete unused method in HoodieDeltaStreamer (#1426)
add eeab532 [HUDI-725] Remove init log in the constructor of DeltaSync (#1425)
add 83fb965 [HUDI-650] Modify handleUpdate path to validate partitionPath (#1368)
add c5030f7 [HUDI-720] NOTICE file needs to add more content based on the NOTICE files of the ASF projects that hudi bundles (#1417)
add 8b00791 [MINOR] cleanup redundant comment and unused variable and fix typo (#1435)
add 06652aa [MINOR] Add omissive param desc on method doc and cleanup redundant code (#1437)
add 1e1d9e1 [HUDI-616] Fixed parquet files getting created on local FS (#1434)
add 38c3ccc [HUDI-663] Fix HoodieDeltaStreamer offset not handled correctly (#1377)
add 0241b21 [HUDI-65] commitTime rename to instantTime (#1431)
add cafc870 [HUDI-697]Add unit test for ArchivedCommitsCommand (#1424)
add bc82e2b [HUDI-711] Refactor exporter main logic (#1436)
add 5eed6c9 [MINOR] Fix javadoc of InsertBucket (#1445)
add e101ea9 [MINOR] Update DOAP with 0.5.2 Release (#1448)
add 8b0a400 [HUDI-678] Make config package spark free (#1418)
add 1713f68 [MINOR] Add error message when check arguments (#1451)
add 8c30013 HUDI-479: Eliminate or Minimize use of Guava if possible (#1159)
add 04449f3 [HUDI-743]: Remove FileIOUtils.close() (#1461)
add ac73bdc [HUDI-430] Adding InlineFileSystem to support embedding any file format as an InlineFile (#1176)
add 07c3c5d [HUDI-679] Make io package Spark free (#1460)
add e057c27 [HUDI-744] Restructure hudi-common and clean up files under util packages (#1462)
add fad4bd3 [HUDI-745] CI should fail PRs with unapproved license files (#1464)
add fa36082 [HUDI-746] Reduce build warnings < 10 (#1465)
add 1f5b0c7 [HUDI-724] Parallelize getSmallFiles for partitions (#1421)
add 9f51b99 [MINOR] Updated HoodieMergeOnReadTestUtils for future testing requirements (#1456)
add dbc9acd [HUDI-716] Exception: Not an Avro data file when running HoodieCleanClient.runClean (#1432)
add ce0a4c6 [HUDI-713] Fix conversion of Spark array of struct type to Avro schema (#1406)
add 9ecf0cc [HUDI-742] Fix Java Math Exception (#1466)
add 78b3194 [HUDI-751] Fix some coding issues reported by FindBugs (#1470)
add c146ca9 [HUDI-754] Configure .asf.yaml for Hudi Github repository (#1472)
add 2a611f4 [HUDI-749] Fix hudi-timeline-server-bundle run_server.sh start error (#1477)
add 5b53b0d [HUDI-731] Add ChainedTransformer (#1440)
add 194e20e [MINOR] Fix label issue in .asf.yaml (#1478)
add bd716ec [MINIOR] Add license header for .asf.yaml and adjust labels
add 639ec20 [HUDI-562] Enable testing at debug log level
add 29b4fdf [HUDI-394] Provide a basic implementation of test suite
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f380005)
\
N -- N -- N refs/heads/hudi_test_suite_refactor (29b4fdf)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
...ark.sql.sources.DataSourceRegister => .asf.yaml | 18 +-
.travis.yml | 2 +
DISCLAIMER | 10 +
DISCLAIMER-WIP | 26 -
LICENSE | 17 +-
NOTICE | 146 ++++-
README.md | 21 +
doap_HUDI.rdf | 5 +
docker/build_local_docker_images.sh | 7 +-
docker/compose/hadoop.env | 3 +-
docker/demo/compaction.commands | 3 +-
docker/demo/config/base.properties | 3 +-
docker/demo/config/dfs-source.properties | 3 +-
docker/demo/config/kafka-source.properties | 3 +-
docker/demo/get_min_commit_time.sh | 3 +-
docker/demo/hive-batch1.commands | 3 +-
docker/demo/hive-batch2-after-compaction.commands | 3 +-
docker/demo/hive-incremental.commands | 3 +-
docker/demo/hive-table-check.commands | 3 +-
docker/demo/setup_demo_container.sh | 5 +-
docker/hoodie/hadoop/base/Dockerfile | 3 +-
docker/hoodie/hadoop/base/entrypoint.sh | 2 -
docker/hoodie/hadoop/base/export_container_ip.sh | 3 +-
docker/hoodie/hadoop/base/pom.xml | 2 +-
docker/hoodie/hadoop/datanode/Dockerfile | 3 +-
docker/hoodie/hadoop/datanode/pom.xml | 2 +-
docker/hoodie/hadoop/datanode/run_dn.sh | 2 -
docker/hoodie/hadoop/historyserver/Dockerfile | 3 +-
docker/hoodie/hadoop/historyserver/pom.xml | 2 +-
docker/hoodie/hadoop/historyserver/run_history.sh | 2 -
docker/hoodie/hadoop/hive_base/Dockerfile | 5 +-
.../hive_base/conf/beeline-log4j2.properties | 3 +-
docker/hoodie/hadoop/hive_base/conf/hive-env.sh | 3 +-
.../hive_base/conf/hive-exec-log4j2.properties | 3 +-
.../hadoop/hive_base/conf/hive-log4j2.properties | 3 +-
.../hive_base/conf/llap-daemon-log4j2.properties | 3 +-
docker/hoodie/hadoop/hive_base/entrypoint.sh | 2 -
docker/hoodie/hadoop/hive_base/pom.xml | 4 +-
docker/hoodie/hadoop/hive_base/startup.sh | 2 -
docker/hoodie/hadoop/namenode/Dockerfile | 3 +-
docker/hoodie/hadoop/namenode/pom.xml | 2 +-
docker/hoodie/hadoop/namenode/run_nn.sh | 2 -
docker/hoodie/hadoop/pom.xml | 2 +-
docker/hoodie/hadoop/prestobase/Dockerfile | 3 +-
docker/hoodie/hadoop/prestobase/bin/entrypoint.sh | 2 -
docker/hoodie/hadoop/prestobase/bin/mustache.sh | 2 -
.../hadoop/prestobase/etc/catalog/hive.properties | 4 +-
.../hadoop/prestobase/etc/catalog/jmx.properties | 4 +-
.../prestobase/etc/catalog/localfile.properties | 4 +-
.../prestobase/etc/coordinator.properties.mustache | 4 +-
.../hadoop/prestobase/etc/jvm.config.mustache | 4 +-
docker/hoodie/hadoop/prestobase/etc/log.properties | 4 +-
.../hadoop/prestobase/etc/node.properties.mustache | 4 +-
.../prestobase/etc/worker.properties.mustache | 4 +-
docker/hoodie/hadoop/prestobase/lib/mustache.sh | 3 +-
docker/hoodie/hadoop/prestobase/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/Dockerfile | 3 +-
docker/hoodie/hadoop/spark_base/execute-step.sh | 2 -
docker/hoodie/hadoop/spark_base/finish-step.sh | 2 -
docker/hoodie/hadoop/spark_base/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/wait-for-step.sh | 2 -
docker/hoodie/hadoop/sparkadhoc/Dockerfile | 3 +-
docker/hoodie/hadoop/sparkadhoc/adhoc.sh | 2 -
docker/hoodie/hadoop/sparkadhoc/pom.xml | 2 +-
docker/hoodie/hadoop/sparkmaster/Dockerfile | 3 +-
docker/hoodie/hadoop/sparkmaster/master.sh | 2 -
docker/hoodie/hadoop/sparkmaster/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/Dockerfile | 3 +-
docker/hoodie/hadoop/sparkworker/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/worker.sh | 2 -
docker/setup_demo.sh | 2 -
docker/stop_demo.sh | 2 -
hudi-cli/conf/hudi-env.sh | 2 -
hudi-cli/hudi-cli.sh | 2 -
hudi-cli/pom.xml | 23 +-
.../main/java/org/apache/hudi/cli/HoodieCLI.java | 18 +-
.../org/apache/hudi/cli/HoodiePrintHelper.java | 30 +
.../hudi/cli/commands/ArchivedCommitsCommand.java | 4 +-
.../apache/hudi/cli/commands/CleansCommand.java | 18 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 80 +--
.../hudi/cli/commands/CompactionCommand.java | 259 ++++++---
.../hudi/cli/commands/FileSystemViewCommand.java | 2 +-
.../hudi/cli/commands/HoodieLogFileCommand.java | 12 +-
.../hudi/cli/commands/HoodieSyncCommand.java | 2 +-
.../apache/hudi/cli/commands/RepairsCommand.java | 23 +-
.../apache/hudi/cli/commands/RollbacksCommand.java | 16 +-
.../hudi/cli/commands/SavepointsCommand.java | 55 +-
.../apache/hudi/cli/commands/SparkEnvCommand.java | 6 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 24 +-
.../org/apache/hudi/cli/commands/StatsCommand.java | 28 +-
.../org/apache/hudi/cli/commands/TableCommand.java | 4 +-
.../{UtilsCommand.java => TempViewCommand.java} | 29 +-
.../java/org/apache/hudi/cli/utils/CommitUtil.java | 25 +-
.../java/org/apache/hudi/cli/utils/HiveUtil.java | 16 +-
.../hudi/cli/utils/SparkTempViewProvider.java | 135 +++++
.../java/org/apache/hudi/cli/utils/SparkUtil.java | 9 +-
.../apache/hudi/cli/utils/TempViewProvider.java | 16 +-
.../scala/org/apache/hudi/cli/DedupeSparkJob.scala | 6 +-
.../scala/org/apache/hudi/cli/SparkHelpers.scala | 7 +-
.../hudi/cli/AbstractShellIntegrationTest.java} | 41 +-
.../cli/commands/TestArchivedCommitsCommand.java | 194 +++++++
.../hudi/cli/commands/TestSparkEnvCommand.java | 54 ++
.../apache/hudi/cli/commands/TestTableCommand.java | 143 +++++
.../common/HoodieTestCommitMetadataGenerator.java | 125 ++++
.../hudi/cli/common/HoodieTestCommitUtilities.java | 54 ++
.../test/resources/log4j-surefire-quiet.properties | 22 +-
.../src/test/resources/log4j-surefire.properties | 0
hudi-client/pom.xml | 8 +-
.../hudi/{ => client}/AbstractHoodieClient.java | 4 +-
.../{ => client}/AbstractHoodieWriteClient.java | 89 ++-
.../hudi/{ => client}/CompactionAdminClient.java | 38 +-
.../hudi/{ => client}/HoodieCleanClient.java | 30 +-
.../apache/hudi/{ => client}/HoodieReadClient.java | 33 +-
.../hudi/{ => client}/HoodieWriteClient.java | 237 ++++----
.../hudi/client/SparkTaskContextSupplier.java | 25 +-
.../org/apache/hudi/{ => client}/WriteStatus.java | 3 +-
.../client/embedded/EmbeddedTimelineService.java | 2 +-
.../org/apache/hudi/client/utils/ClientUtils.java | 2 +-
.../utils}/LazyIterableIterator.java | 4 +-
.../utils}/ParquetReaderIterator.java | 2 +-
.../apache/hudi/client/utils/SparkConfigUtils.java | 94 ++++
.../apache/hudi/config/HoodieCompactionConfig.java | 21 +-
.../apache/hudi/config/HoodieHBaseIndexConfig.java | 3 +-
.../org/apache/hudi/config/HoodieIndexConfig.java | 12 +-
.../org/apache/hudi/config/HoodieMemoryConfig.java | 47 +-
.../apache/hudi/config/HoodieMetricsConfig.java | 17 +-
.../apache/hudi/config/HoodieStorageConfig.java | 2 +
.../org/apache/hudi/config/HoodieWriteConfig.java | 51 +-
.../{func => execution}/BulkInsertMapFunction.java | 16 +-
.../CopyOnWriteLazyInsertIterable.java | 23 +-
.../MergeOnReadLazyInsertIterable.java | 15 +-
.../SparkBoundedInMemoryExecutor.java | 2 +-
.../java/org/apache/hudi/index/HoodieIndex.java | 16 +-
.../org/apache/hudi/index/InMemoryHashIndex.java | 4 +-
.../hudi/index/bloom/BloomIndexFileInfo.java | 14 +-
.../bloom/BucketizedBloomCheckPartitioner.java | 9 +-
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 83 +--
.../index/bloom/HoodieBloomIndexCheckFunction.java | 2 +-
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 5 +-
.../org/apache/hudi/index/hbase/HBaseIndex.java | 13 +-
.../org/apache/hudi/io/HoodieAppendHandle.java | 40 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 24 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 2 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 98 +---
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 34 +-
.../hudi/io/storage/HoodieParquetWriter.java | 22 +-
.../io/storage/HoodieStorageWriterFactory.java | 19 +-
.../org/apache/hudi/metrics/HoodieMetrics.java | 4 +-
.../hudi/metrics/InMemoryMetricsReporter.java | 5 +
.../apache/hudi/metrics/JmxMetricsReporter.java | 94 +++-
.../org/apache/hudi/metrics/JmxReporterServer.java | 164 ++++++
.../main/java/org/apache/hudi/metrics/Metrics.java | 6 +-
.../hudi/metrics/MetricsGraphiteReporter.java | 7 +
.../org/apache/hudi/metrics/MetricsReporter.java | 5 +
.../hudi/metrics/MetricsReporterFactory.java | 2 +-
.../CleanHelper.java} | 31 +-
.../hudi/{io => table}/HoodieCommitArchiveLog.java | 25 +-
.../apache/hudi/table/HoodieCopyOnWriteTable.java | 135 +++--
.../apache/hudi/table/HoodieMergeOnReadTable.java | 56 +-
.../java/org/apache/hudi/table/HoodieTable.java | 41 +-
.../org/apache/hudi/table/WorkloadProfile.java | 4 +
.../{io => table}/compact/HoodieCompactor.java | 4 +-
.../compact/HoodieMergeOnReadTableCompactor.java | 43 +-
.../{func => table/compact}/OperationResult.java | 2 +-
.../strategy/BoundedIOCompactionStrategy.java | 7 +-
.../BoundedPartitionAwareCompactionStrategy.java | 7 +-
.../compact/strategy/CompactionStrategy.java | 11 +-
.../strategy/DayBasedCompactionStrategy.java | 11 +-
.../LogFileSizeBasedCompactionStrategy.java | 2 +-
.../strategy/UnBoundedCompactionStrategy.java | 2 +-
.../UnBoundedPartitionAwareCompactionStrategy.java | 2 +-
.../RollbackHelper.java} | 23 +-
.../hudi/table/{ => rollback}/RollbackRequest.java | 6 +-
hudi-client/src/test/java/HoodieClientExample.java | 6 +-
.../hudi/{ => client}/TestClientRollback.java | 8 +-
.../{ => client}/TestCompactionAdminClient.java | 10 +-
.../hudi/{ => client}/TestHoodieClientBase.java | 53 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 132 ++---
.../hudi/{ => client}/TestHoodieReadClient.java | 73 ++-
.../org/apache/hudi/{ => client}/TestMultiFS.java | 7 +-
.../TestUpdateSchemaEvolution.java} | 30 +-
.../apache/hudi/{ => client}/TestWriteStatus.java | 2 +-
.../utils}/TestParquetReaderIterator.java | 2 +-
.../hudi/{ => common}/HoodieClientTestHarness.java | 11 +-
.../apache/hudi/common/HoodieClientTestUtils.java | 66 +--
.../hudi/common/HoodieMergeOnReadTestUtils.java | 16 +-
.../hudi/common/HoodieTestDataGenerator.java | 237 +++++---
.../org/apache/hudi/common/TestRawTripPayload.java | 2 +-
.../{ => common}/config/TestHoodieWriteConfig.java | 8 +-
.../TestBoundedInMemoryExecutor.java | 12 +-
.../TestBoundedInMemoryQueue.java | 18 +-
.../hudi/index/TestHBaseQPSResourceAllocator.java | 2 +-
.../java/org/apache/hudi/index/TestHbaseIndex.java | 139 ++++-
.../org/apache/hudi/index/TestHoodieIndex.java | 86 ++-
.../bloom/TestBucketizedBloomCheckPartitioner.java | 17 +
.../hudi/index/bloom/TestHoodieBloomIndex.java | 47 +-
.../index/bloom/TestHoodieGlobalBloomIndex.java | 29 +-
.../apache/hudi/io/TestHoodieCommitArchiveLog.java | 31 +-
.../org/apache/hudi/io/TestHoodieMergeHandle.java | 10 +-
.../io/storage/TestHoodieStorageWriterFactory.java | 14 +-
.../hudi/io/strategy/TestHoodieBaseFile.java | 57 --
.../apache/hudi/io/strategy/TestHoodieLogFile.java | 47 --
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 28 +-
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 116 +++-
.../org/apache/hudi/{ => table}/TestCleaner.java | 117 ++--
.../hudi/{ => table}/TestConsistencyGuard.java | 9 +-
.../apache/hudi/table/TestCopyOnWriteTable.java | 52 +-
.../apache/hudi/table/TestHoodieRecordSizing.java | 2 +-
.../apache/hudi/table/TestMergeOnReadTable.java | 249 +++++---
.../{ => table/compact}/TestAsyncCompaction.java | 33 +-
.../{io => table/compact}/TestHoodieCompactor.java | 10 +-
.../strategy/TestHoodieCompactionStrategy.java | 182 ++++--
.../test/resources/log4j-surefire-quiet.properties | 18 +-
.../src/test/resources/log4j-surefire.properties | 21 +-
hudi-common/pom.xml | 17 +-
.../src/main/avro/HoodieCommitMetadata.avsc | 5 +
.../{common/util => avro}/HoodieAvroUtils.java | 6 +-
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 4 +-
.../apache/hudi/avro/MercifulJsonConverter.java | 25 +-
.../common/bloom/{filter => }/BloomFilter.java | 2 +-
.../bloom/{filter => }/BloomFilterFactory.java | 2 +-
.../bloom/{filter => }/BloomFilterTypeCode.java | 2 +-
.../bloom/{filter => }/BloomFilterUtils.java | 2 +-
.../HoodieDynamicBoundedBloomFilter.java | 9 +-
.../{filter => }/InternalDynamicBloomFilter.java | 2 +-
.../common/bloom/{filter => }/InternalFilter.java | 6 +-
.../bloom/{filter => }/SimpleBloomFilter.java | 9 +-
.../DFSPropertiesConfiguration.java | 2 +-
.../{ => common}/config/DefaultHoodieConfig.java | 2 +-
.../{ => config}/SerializableConfiguration.java | 3 +-
.../common/{util => config}/TypedProperties.java | 2 +-
.../hudi/common/{util => fs}/ConsistencyGuard.java | 2 +-
.../{util => fs}/ConsistencyGuardConfig.java | 4 +-
.../apache/hudi/common/{util => fs}/FSUtils.java | 41 +-
.../{util => fs}/FailSafeConsistencyGuard.java | 11 +-
.../storage => fs}/HoodieWrapperFileSystem.java | 6 +-
.../common/{util => fs}/NoOpConsistencyGuard.java | 2 +-
.../{storage => fs}/SizeAwareDataInputStream.java | 2 +-
.../storage => fs}/SizeAwareDataOutputStream.java | 2 +-
.../SizeAwareFSDataOutputStream.java | 3 +-
.../common/{storage => fs}/StorageSchemes.java | 2 +-
.../hudi/common/fs/inline/InLineFSUtils.java | 98 ++++
.../hudi/common/fs/inline/InLineFileSystem.java | 133 +++++
.../common/fs/inline/InLineFsDataInputStream.java | 114 ++++
.../hudi/common/fs/inline/InMemoryFileSystem.java | 138 +++++
.../apache/hudi/common/model/BaseAvroPayload.java | 2 +-
.../hudi/common/model/CompactionOperation.java | 2 +-
.../hudi/common/model/HoodieAvroPayload.java | 2 +-
.../apache/hudi/common/model/HoodieBaseFile.java | 3 +-
.../hudi/common/model/HoodieCommitMetadata.java | 24 +-
.../apache/hudi/common/model/HoodieFileGroup.java | 10 +-
.../org/apache/hudi/common/model/HoodieKey.java | 7 +-
.../apache/hudi/common/model/HoodieLogFile.java | 4 +-
.../hudi/common/model/HoodiePartitionMetadata.java | 4 +-
.../org/apache/hudi/common/model/HoodieRecord.java | 29 +-
.../hudi/common/model/HoodieRecordLocation.java | 7 +-
.../common/model/HoodieRollingStatMetadata.java | 2 +-
.../model/OverwriteWithLatestAvroPayload.java | 2 +-
.../hudi/common/model/WriteOperationType.java | 72 +++
.../hudi/common/table/HoodieTableConfig.java | 2 +-
.../hudi/common/table/HoodieTableMetaClient.java | 33 +-
.../table/log/AbstractHoodieLogRecordScanner.java | 2 +-
.../hudi/common/table/log/HoodieLogFileReader.java | 25 +-
.../hudi/common/table/log/HoodieLogFormat.java | 12 +-
.../common/table/log/HoodieLogFormatWriter.java | 7 +-
.../common/{util => table/log}/LogReaderUtils.java | 6 +-
.../table/log/block/HoodieAvroDataBlock.java | 7 +-
.../common/table/log/block/HoodieDeleteBlock.java | 2 +-
.../common/table/log/block/HoodieLogBlock.java | 21 +-
.../table/timeline/HoodieActiveTimeline.java | 58 +-
.../table/timeline/HoodieArchivedTimeline.java | 16 +-
.../table/timeline/HoodieDefaultTimeline.java | 32 +-
.../hudi/common/table/timeline/HoodieInstant.java | 9 +-
.../table/{ => timeline}/HoodieTimeline.java | 47 +-
.../timeline}/TimelineDiffHelper.java | 5 +-
.../table/{ => timeline}/TimelineLayout.java | 5 +-
.../timeline/TimelineMetadataUtils.java} | 30 +-
.../common/table/timeline/dto/TimelineDTO.java | 2 +-
.../timeline}/versioning/AbstractMigratorBase.java | 2 +-
.../timeline}/versioning/MetadataMigrator.java | 9 +-
.../versioning}/TimelineLayoutVersion.java | 8 +-
.../timeline}/versioning/VersionMigrator.java | 2 +-
.../versioning/clean/CleanMetadataMigrator.java | 4 +-
.../versioning/clean/CleanV1MigrationHandler.java | 15 +-
.../versioning/clean/CleanV2MigrationHandler.java | 8 +-
.../compaction/CompactionPlanMigrator.java | 4 +-
.../compaction/CompactionV1MigrationHandler.java | 17 +-
.../compaction/CompactionV2MigrationHandler.java | 17 +-
.../table/view/AbstractTableFileSystemView.java | 28 +-
.../common/table/view/FileSystemViewManager.java | 5 +-
.../table/view/FileSystemViewStorageConfig.java | 7 +-
.../table/view/HoodieTableFileSystemView.java | 22 +-
.../IncrementalTimelineSyncFileSystemView.java | 16 +-
.../table/view/PriorityBasedFileSystemView.java | 3 +-
.../view/RemoteHoodieTableFileSystemView.java | 55 +-
.../table/view/RocksDbBasedFileSystemView.java | 12 +-
.../view/SpillableMapBasedFileSystemView.java | 2 +-
.../table/{ => view}/SyncableFileSystemView.java | 6 +-
.../table/{ => view}/TableFileSystemView.java | 3 +-
.../util/{Functions.java => Base64CodecUtil.java} | 39 +-
.../hudi/common/util/BufferedRandomAccessFile.java | 8 +-
.../org/apache/hudi/common/util/CleanerUtils.java | 22 +-
.../apache/hudi/common/util/CollectionUtils.java | 111 ++++
.../apache/hudi/common/util/CompactionUtils.java | 11 +-
.../org/apache/hudi/common/util/HoodieTimer.java | 2 +-
.../org/apache/hudi/common/util/NumericUtils.java | 30 +
.../hudi/common/util/ObjectSizeCalculator.java | 39 +-
.../org/apache/hudi/common/util/ParquetUtils.java | 8 +-
.../apache/hudi/common/util/ReflectionUtils.java | 69 ++-
.../hudi/common/util/SerializationUtils.java | 45 +-
.../apache/hudi/common/util/SpillableMapUtils.java | 2 +-
.../org/apache/hudi/common/util/StringUtils.java | 27 +
.../apache/hudi/common/util/ValidationUtils.java | 70 +++
.../hudi/common/util/collection/DiskBasedMap.java | 4 +-
.../common/util/collection/RocksDBBasedMap.java | 1 -
.../common/util/{ => collection}/RocksDBDAO.java | 26 +-
.../common/util/queue/BoundedInMemoryQueue.java | 9 +-
.../{common/util => avro}/TestHoodieAvroUtils.java | 2 +-
.../hudi/common/HoodieCommonTestHarness.java | 4 +-
.../common/bloom/{filter => }/TestBloomFilter.java | 2 +-
.../TestInternalDynamicBloomFilter.java | 2 +-
.../hudi/common/{util => fs}/TestFSUtils.java | 60 +-
.../hudi/common/fs/inline/FileSystemTestUtils.java | 66 +++
.../hudi/common/fs/inline/TestHFileInLining.java | 241 ++++++++
.../common/fs/inline/TestInLineFileSystem.java | 356 ++++++++++++
.../common/fs/inline/TestInMemoryFileSystem.java | 145 +++++
.../hudi/common/minicluster/HdfsTestService.java | 32 +-
.../common/minicluster/ZookeeperTestService.java | 15 +-
.../hudi/common/model/AvroBinaryTestPayload.java | 2 +-
.../apache/hudi/common/model/HoodieTestUtils.java | 148 ++---
.../common/model/TestHoodieCommitMetadata.java | 23 +
.../hudi/common/model/TestHoodieWriteStat.java | 6 +-
.../hudi/common/storage/TestStorageSchemes.java | 2 +
.../common/table/TestHoodieTableMetaClient.java | 1 +
.../hudi/common/table/TestTimelineLayout.java | 4 +-
.../hudi/common/table/log/TestHoodieLogFormat.java | 73 ++-
.../log/TestHoodieLogFormatAppendFailure.java | 4 +-
.../{string => timeline}/MockHoodieTimeline.java | 6 +-
.../TestHoodieActiveTimeline.java | 74 ++-
.../table/view/TestHoodieTableFileSystemView.java | 41 +-
.../table/view/TestIncrementalFSViewSync.java | 57 +-
.../view/TestPriorityBasedFileSystemView.java | 626 +++++++++++++++++++++
.../TestRocksDBBasedIncrementalFSViewSync.java | 3 +-
.../table/view/TestRocksDbBasedFileSystemView.java | 3 +-
.../view/TestSpillableMapBasedFileSystemView.java | 3 +-
...TestSpillableMapBasedIncrementalFSViewSync.java | 3 +-
.../hudi/common/util/CompactionTestUtils.java | 23 +-
.../apache/hudi/common/util/SchemaTestUtil.java | 13 +-
.../hudi/common/util/TestBase64CodecUtil.java | 30 +-
.../hudi/common/util/TestCompactionUtils.java | 36 +-
.../util/TestDFSPropertiesConfiguration.java | 2 +
.../apache/hudi/common/util/TestNumericUtils.java | 26 +
.../apache/hudi/common/util/TestParquetUtils.java | 7 +-
.../org/apache/hudi/common/util/TestRecord.java | 12 +-
.../apache/hudi/common/util/TestStringUtils.java | 64 +++
.../common/util/collection/TestDiskBasedMap.java | 27 +-
.../util/collection/TestExternalSpillableMap.java | 6 +-
.../util/{ => collection}/TestRocksDBManager.java | 3 +-
.../test/resources/log4j-surefire-quiet.properties | 18 +-
.../src/test/resources/log4j-surefire.properties | 19 +-
hudi-common/src/test/resources/old-version.commit | 20 +
hudi-hadoop-mr/pom.xml | 6 +-
.../org/apache/hudi/hadoop/HoodieHiveUtil.java | 9 +-
.../hudi/hadoop/HoodieParquetInputFormat.java | 42 +-
.../hudi/hadoop/HoodieROTablePathFilter.java | 19 +-
.../org/apache/hudi/hadoop/InputPathHandler.java | 18 +-
.../hadoop/hive/HoodieCombineHiveInputFormat.java | 8 +-
.../realtime/AbstractRealtimeRecordReader.java | 4 +-
.../realtime/HoodieParquetRealtimeInputFormat.java | 12 +-
.../realtime/RealtimeCompactedRecordReader.java | 4 +-
.../realtime/RealtimeUnmergedRecordReader.java | 5 +-
.../apache/hudi/hadoop/InputFormatTestUtil.java | 8 +-
.../apache/hudi/hadoop/InputPathHandlerTest.java | 24 +-
.../hudi/hadoop/TestHoodieParquetInputFormat.java | 24 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 48 +-
.../test/resources/log4j-surefire-quiet.properties | 18 +-
.../src/test/resources/log4j-surefire.properties | 17 +-
{hudi-hive => hudi-hive-sync}/pom.xml | 13 +-
{hudi-hive => hudi-hive-sync}/run_sync_tool.sh | 4 +-
{hudi-hive => hudi-hive-sync}/src/assembly/src.xml | 0
.../java/org/apache/hudi/hive/HiveSyncConfig.java | 0
.../java/org/apache/hudi/hive/HiveSyncTool.java | 4 +-
.../org/apache/hudi/hive/HoodieHiveClient.java | 24 +-
.../apache/hudi/hive/HoodieHiveSyncException.java | 0
.../hudi/hive/MultiPartKeysValueExtractor.java | 4 +-
.../apache/hudi/hive/NonPartitionedExtractor.java | 0
.../apache/hudi/hive/PartitionValueExtractor.java | 0
.../org/apache/hudi/hive/SchemaDifference.java | 38 +-
.../SlashEncodedDayPartitionValueExtractor.java | 1 +
.../apache/hudi/hive/util/ColumnNameXLator.java | 13 +-
.../java/org/apache/hudi/hive/util/SchemaUtil.java | 13 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 23 +-
.../test/java/org/apache/hudi/hive/TestUtil.java | 67 +--
.../org/apache/hudi/hive/util/HiveTestService.java | 17 +-
.../test/resources/log4j-surefire-quiet.properties | 22 +-
.../src/test/resources/log4j-surefire.properties | 18 +-
.../test/resources/log4j-surefire-quiet.properties | 23 -
hudi-integ-test/pom.xml | 9 +-
.../java/org/apache/hudi/integ/ITTestBase.java | 4 +-
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 40 +-
.../test/resources/log4j-surefire-quiet.properties | 18 +-
.../src/test/resources/log4j-surefire.properties | 17 +-
hudi-spark/pom.xml | 13 +-
hudi-spark/run_hoodie_app.sh | 2 -
.../main/java/org/apache/hudi/DataSourceUtils.java | 72 ++-
.../org/apache/hudi/HoodieDataSourceHelpers.java | 7 +-
.../main/java/org/apache/hudi/QuickstartUtils.java | 2 +-
.../apache/hudi/keygen/ComplexKeyGenerator.java | 2 +-
.../hudi/keygen/GlobalDeleteKeyGenerator.java | 10 +-
.../java/org/apache/hudi/keygen/KeyGenerator.java | 2 +-
.../hudi/keygen/NonpartitionedKeyGenerator.java | 2 +-
.../org/apache/hudi/keygen/SimpleKeyGenerator.java | 2 +-
...org.apache.spark.sql.sources.DataSourceRegister | 3 +-
.../org/apache/hudi/AvroConversionHelper.scala | 14 +-
.../org/apache/hudi/AvroConversionUtils.scala | 10 -
.../scala/org/apache/hudi/DataSourceOptions.scala | 9 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 20 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 33 +-
.../scala/org/apache/hudi/HudiEmptyRelation.scala | 26 +-
.../org/apache/hudi/IncrementalRelation.scala | 18 +-
hudi-spark/src/test/java/DataSourceUtilsTest.java | 62 ++
hudi-spark/src/test/java/HoodieJavaApp.java | 4 +-
.../test/resources/log4j-surefire-quiet.properties | 18 +-
.../src/test/resources/log4j-surefire.properties | 21 +-
hudi-spark/src/test/scala/TestDataSource.scala | 11 +-
.../src/test/scala/TestDataSourceDefaults.scala | 3 +-
hudi-timeline-service/pom.xml | 6 +-
.../timeline/service/FileSystemViewHandler.java | 10 +-
.../hudi/timeline/service/TimelineService.java | 4 +-
.../view/TestRemoteHoodieTableFileSystemView.java | 6 +-
.../test/resources/log4j-surefire-quiet.properties | 18 +-
.../src/test/resources/log4j-surefire.properties | 19 +-
hudi-utilities/pom.xml | 11 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 11 +-
.../org/apache/hudi/utilities/HoodieCleaner.java | 6 +-
.../hudi/utilities/HoodieCompactionAdminTool.java | 8 +-
.../org/apache/hudi/utilities/HoodieCompactor.java | 8 +-
.../hudi/utilities/HoodieSnapshotCopier.java | 12 +-
.../hudi/utilities/HoodieSnapshotExporter.java | 272 +++++++++
.../hudi/utilities/HoodieWithTimelineServer.java | 5 +-
.../org/apache/hudi/utilities/UtilHelpers.java | 76 +--
.../adhoc/UpgradePayloadFromUberToApache.java | 2 +-
.../hudi/utilities/deltastreamer/Compactor.java | 5 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 66 +--
.../deltastreamer/HoodieDeltaStreamer.java | 92 +--
.../deltastreamer/SchedulerConfGenerator.java | 36 +-
.../exception/HoodieSnapshotExporterException.java | 10 +
.../keygen/TimestampBasedKeyGenerator.java | 6 +-
.../hudi/utilities/perf/TimelineServerPerf.java | 5 +-
.../utilities/schema/FilebasedSchemaProvider.java | 4 +-
.../utilities/schema/JdbcbasedSchemaProvider.java | 5 +-
.../schema/NullTargetSchemaRegistryProvider.java | 2 +-
.../utilities/schema/RowBasedSchemaProvider.java | 2 +-
.../hudi/utilities/schema/SchemaProvider.java | 2 +-
.../utilities/schema/SchemaRegistryProvider.java | 2 +-
.../hudi/utilities/sources/AvroDFSSource.java | 2 +-
.../hudi/utilities/sources/AvroKafkaSource.java | 2 +-
.../apache/hudi/utilities/sources/AvroSource.java | 2 +-
.../hudi/utilities/sources/CsvDFSSource.java | 128 +++++
.../hudi/utilities/sources/HiveIncrPullSource.java | 12 +-
.../hudi/utilities/sources/HoodieIncrSource.java | 4 +-
.../hudi/utilities/sources/JsonDFSSource.java | 2 +-
.../hudi/utilities/sources/JsonKafkaSource.java | 2 +-
.../apache/hudi/utilities/sources/JsonSource.java | 2 +-
.../hudi/utilities/sources/ParquetDFSSource.java | 2 +-
.../apache/hudi/utilities/sources/RowSource.java | 2 +-
.../org/apache/hudi/utilities/sources/Source.java | 5 +-
.../utilities/sources/helpers/AvroConvertor.java | 1 +
.../utilities/sources/helpers/DFSPathSelector.java | 6 +-
.../sources/helpers/IncrSourceHelper.java | 16 +-
.../utilities/sources/helpers/KafkaOffsetGen.java | 6 +-
.../utilities/transform/AWSDmsTransformer.java | 2 +-
...DmsTransformer.java => ChainedTransformer.java} | 37 +-
.../utilities/transform/FlatteningTransformer.java | 2 +-
.../transform/SqlQueryBasedTransformer.java | 2 +-
.../hudi/utilities/transform/Transformer.java | 2 +-
.../hudi/utilities/TestHDFSParquetImporter.java | 6 +-
.../hudi/utilities/TestHoodieDeltaStreamer.java | 276 ++++++---
.../hudi/utilities/TestHoodieSnapshotCopier.java | 4 +-
.../hudi/utilities/TestHoodieSnapshotExporter.java | 325 +++++++++++
.../utilities/TestJdbcbasedSchemaProvider.java | 2 +-
.../utilities/TestTimestampBasedKeyGenerator.java | 7 +-
.../org/apache/hudi/utilities/TestUtilHelpers.java | 101 ++++
.../apache/hudi/utilities/UtilitiesTestBase.java | 99 +++-
.../utilities/inline/fs/TestParquetInLining.java | 153 +++++
.../utilities/sources/AbstractBaseTestSource.java | 12 +-
.../sources/AbstractDFSSourceTestBase.java | 7 +-
.../sources/DistributedTestDataSource.java | 10 +-
...estJsonDFSSource.java => TestCsvDFSSource.java} | 18 +-
.../hudi/utilities/sources/TestDataSource.java | 8 +-
.../hudi/utilities/sources/TestJsonDFSSource.java | 2 +-
.../hudi/utilities/sources/TestKafkaSource.java | 2 +-
.../utilities/sources/TestParquetDFSSource.java | 2 +-
.../transform/TestChainedTransformer.java | 92 +++
.../{ => transform}/TestFlatteningTransformer.java | 4 +-
.../{source.avsc => source-flattened.avsc} | 25 +-
.../resources/delta-streamer-config/source.avsc | 27 +
.../sql-transformer.properties | 2 +-
.../{source.avsc => target-flattened.avsc} | 30 +-
.../resources/delta-streamer-config/target.avsc | 26 +
.../test/resources/log4j-surefire-quiet.properties | 18 +-
.../src/test/resources/log4j-surefire.properties | 17 +-
packaging/hudi-hadoop-mr-bundle/pom.xml | 2 +-
.../pom.xml | 8 +-
.../java/org/apache/hudi/hive/bundle/Main.java | 0
packaging/hudi-presto-bundle/pom.xml | 2 +-
packaging/hudi-spark-bundle/pom.xml | 51 +-
packaging/hudi-timeline-server-bundle/pom.xml | 2 +-
.../hudi-timeline-server-bundle/run_server.sh | 5 +-
packaging/hudi-utilities-bundle/pom.xml | 6 +-
pom.xml | 58 +-
scripts/release/cut_release_branch.sh | 2 +-
scripts/release/deploy_staging_jars.sh | 2 +-
scripts/release/validate_staged_release.sh | 10 +-
scripts/run_travis_tests.sh | 2 -
style/checkstyle.xml | 4 +-
516 files changed, 9663 insertions(+), 3732 deletions(-)
copy hudi-spark/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister => .asf.yaml (73%)
create mode 100644 DISCLAIMER
delete mode 100644 DISCLAIMER-WIP
copy hudi-cli/src/main/java/org/apache/hudi/cli/commands/{UtilsCommand.java => TempViewCommand.java} (56%)
create mode 100644 hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java
copy hudi-client/src/main/java/org/apache/hudi/exception/HoodieCompactionException.java => hudi-cli/src/main/java/org/apache/hudi/cli/utils/TempViewProvider.java (75%)
copy hudi-cli/src/{main/java/org/apache/hudi/cli/Main.java => test/java/org/apache/hudi/cli/AbstractShellIntegrationTest.java} (51%)
create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java
create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java
create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitMetadataGenerator.java
create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/common/HoodieTestCommitUtilities.java
copy hudi-common/src/test/resources/log4j-surefire.properties => hudi-cli/src/test/resources/log4j-surefire-quiet.properties (59%)
copy {hudi-utilities => hudi-cli}/src/test/resources/log4j-surefire.properties (100%)
rename hudi-client/src/main/java/org/apache/hudi/{ => client}/AbstractHoodieClient.java (98%)
rename hudi-client/src/main/java/org/apache/hudi/{ => client}/AbstractHoodieWriteClient.java (87%)
rename hudi-client/src/main/java/org/apache/hudi/{ => client}/CompactionAdminClient.java (94%)
rename hudi-client/src/main/java/org/apache/hudi/{ => client}/HoodieCleanClient.java (90%)
rename hudi-client/src/main/java/org/apache/hudi/{ => client}/HoodieReadClient.java (89%)
rename hudi-client/src/main/java/org/apache/hudi/{ => client}/HoodieWriteClient.java (87%)
copy hudi-hive/src/main/java/org/apache/hudi/hive/PartitionValueExtractor.java => hudi-client/src/main/java/org/apache/hudi/client/SparkTaskContextSupplier.java (62%)
rename hudi-client/src/main/java/org/apache/hudi/{ => client}/WriteStatus.java (98%)
rename hudi-client/src/main/java/org/apache/hudi/{func => client/utils}/LazyIterableIterator.java (98%)
rename hudi-client/src/main/java/org/apache/hudi/{func => client/utils}/ParquetReaderIterator.java (98%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/client/utils/SparkConfigUtils.java
rename hudi-client/src/main/java/org/apache/hudi/{func => execution}/BulkInsertMapFunction.java (80%)
rename hudi-client/src/main/java/org/apache/hudi/{func => execution}/CopyOnWriteLazyInsertIterable.java (86%)
rename hudi-client/src/main/java/org/apache/hudi/{func => execution}/MergeOnReadLazyInsertIterable.java (77%)
rename hudi-client/src/main/java/org/apache/hudi/{func => execution}/SparkBoundedInMemoryExecutor.java (98%)
create mode 100644 hudi-client/src/main/java/org/apache/hudi/metrics/JmxReporterServer.java
rename hudi-client/src/main/java/org/apache/hudi/{io/HoodieCleanHelper.java => table/CleanHelper.java} (93%)
rename hudi-client/src/main/java/org/apache/hudi/{io => table}/HoodieCommitArchiveLog.java (94%)
rename hudi-client/src/main/java/org/apache/hudi/{io => table}/compact/HoodieCompactor.java (96%)
rename hudi-client/src/main/java/org/apache/hudi/{io => table}/compact/HoodieMergeOnReadTableCompactor.java (88%)
rename hudi-client/src/main/java/org/apache/hudi/{func => table/compact}/OperationResult.java (98%)
rename hudi-client/src/main/java/org/apache/hudi/{io => table}/compact/strategy/BoundedIOCompactionStrategy.java (92%)
rename hudi-client/src/main/java/org/apache/hudi/{io => table}/compact/strategy/BoundedPartitionAwareCompactionStrategy.java (94%)
rename hudi-client/src/main/java/org/apache/hudi/{io => table}/compact/strategy/CompactionStrategy.java (95%)
rename hudi-client/src/main/java/org/apache/hudi/{io => table}/compact/strategy/DayBasedCompactionStrategy.java (90%)
rename hudi-client/src/main/java/org/apache/hudi/{io => table}/compact/strategy/LogFileSizeBasedCompactionStrategy.java (98%)
rename hudi-client/src/main/java/org/apache/hudi/{io => table}/compact/strategy/UnBoundedCompactionStrategy.java (97%)
rename hudi-client/src/main/java/org/apache/hudi/{io => table}/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java (98%)
rename hudi-client/src/main/java/org/apache/hudi/table/{RollbackExecutor.java => rollback/RollbackHelper.java} (93%)
rename hudi-client/src/main/java/org/apache/hudi/table/{ => rollback}/RollbackRequest.java (96%)
rename hudi-client/src/test/java/org/apache/hudi/{ => client}/TestClientRollback.java (99%)
rename hudi-client/src/test/java/org/apache/hudi/{ => client}/TestCompactionAdminClient.java (98%)
rename hudi-client/src/test/java/org/apache/hudi/{ => client}/TestHoodieClientBase.java (92%)
rename hudi-client/src/test/java/org/apache/hudi/{ => client}/TestHoodieClientOnCopyOnWriteStorage.java (92%)
rename hudi-client/src/test/java/org/apache/hudi/{ => client}/TestHoodieReadClient.java (82%)
rename hudi-client/src/test/java/org/apache/hudi/{ => client}/TestMultiFS.java (96%)
rename hudi-client/src/test/java/org/apache/hudi/{func/TestUpdateMapFunction.java => client/TestUpdateSchemaEvolution.java} (87%)
rename hudi-client/src/test/java/org/apache/hudi/{ => client}/TestWriteStatus.java (98%)
rename hudi-client/src/test/java/org/apache/hudi/{func => client/utils}/TestParquetReaderIterator.java (98%)
rename hudi-client/src/test/java/org/apache/hudi/{ => common}/HoodieClientTestHarness.java (96%)
rename hudi-client/src/test/java/org/apache/hudi/{ => common}/config/TestHoodieWriteConfig.java (91%)
rename hudi-client/src/test/java/org/apache/hudi/{func => execution}/TestBoundedInMemoryExecutor.java (89%)
rename hudi-client/src/test/java/org/apache/hudi/{func => execution}/TestBoundedInMemoryQueue.java (96%)
delete mode 100644 hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieBaseFile.java
delete mode 100644 hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieLogFile.java
mode change 100644 => 100755 hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
rename hudi-client/src/test/java/org/apache/hudi/{ => table}/TestCleaner.java (93%)
rename hudi-client/src/test/java/org/apache/hudi/{ => table}/TestConsistencyGuard.java (95%)
rename hudi-client/src/test/java/org/apache/hudi/{ => table/compact}/TestAsyncCompaction.java (97%)
rename hudi-client/src/test/java/org/apache/hudi/{io => table/compact}/TestHoodieCompactor.java (97%)
rename hudi-client/src/test/java/org/apache/hudi/{io => table/compact}/strategy/TestHoodieCompactionStrategy.java (71%)
rename hudi-common/src/main/java/org/apache/hudi/{common/util => avro}/HoodieAvroUtils.java (98%)
rename hudi-common/src/main/java/org/apache/hudi/common/bloom/{filter => }/BloomFilter.java (96%)
rename hudi-common/src/main/java/org/apache/hudi/common/bloom/{filter => }/BloomFilterFactory.java (98%)
rename hudi-common/src/main/java/org/apache/hudi/common/bloom/{filter => }/BloomFilterTypeCode.java (95%)
rename hudi-common/src/main/java/org/apache/hudi/common/bloom/{filter => }/BloomFilterUtils.java (97%)
rename hudi-common/src/main/java/org/apache/hudi/common/bloom/{filter => }/HoodieDynamicBoundedBloomFilter.java (94%)
rename hudi-common/src/main/java/org/apache/hudi/common/bloom/{filter => }/InternalDynamicBloomFilter.java (99%)
rename hudi-common/src/main/java/org/apache/hudi/common/bloom/{filter => }/InternalFilter.java (97%)
rename hudi-common/src/main/java/org/apache/hudi/common/bloom/{filter => }/SimpleBloomFilter.java (95%)
rename hudi-common/src/main/java/org/apache/hudi/common/{util => config}/DFSPropertiesConfiguration.java (99%)
rename hudi-common/src/main/java/org/apache/hudi/{ => common}/config/DefaultHoodieConfig.java (97%)
rename hudi-common/src/main/java/org/apache/hudi/common/{ => config}/SerializableConfiguration.java (95%)
rename hudi-common/src/main/java/org/apache/hudi/common/{util => config}/TypedProperties.java (98%)
rename hudi-common/src/main/java/org/apache/hudi/common/{util => fs}/ConsistencyGuard.java (98%)
rename hudi-common/src/main/java/org/apache/hudi/common/{util => fs}/ConsistencyGuardConfig.java (98%)
rename hudi-common/src/main/java/org/apache/hudi/common/{util => fs}/FSUtils.java (93%)
rename hudi-common/src/main/java/org/apache/hudi/common/{util => fs}/FailSafeConsistencyGuard.java (96%)
rename hudi-common/src/main/java/org/apache/hudi/common/{io/storage => fs}/HoodieWrapperFileSystem.java (99%)
rename hudi-common/src/main/java/org/apache/hudi/common/{util => fs}/NoOpConsistencyGuard.java (97%)
rename hudi-common/src/main/java/org/apache/hudi/common/{storage => fs}/SizeAwareDataInputStream.java (97%)
rename hudi-common/src/main/java/org/apache/hudi/common/{util/collection/io/storage => fs}/SizeAwareDataOutputStream.java (97%)
rename hudi-common/src/main/java/org/apache/hudi/common/{io/storage => fs}/SizeAwareFSDataOutputStream.java (96%)
rename hudi-common/src/main/java/org/apache/hudi/common/{storage => fs}/StorageSchemes.java (98%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFSUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFileSystem.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InLineFsDataInputStream.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/fs/inline/InMemoryFileSystem.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/WriteOperationType.java
rename hudi-common/src/main/java/org/apache/hudi/common/{util => table/log}/LogReaderUtils.java (95%)
rename hudi-common/src/main/java/org/apache/hudi/common/table/{ => timeline}/HoodieTimeline.java (87%)
rename hudi-common/src/main/java/org/apache/hudi/common/{util => table/timeline}/TimelineDiffHelper.java (97%)
rename hudi-common/src/main/java/org/apache/hudi/common/table/{ => timeline}/TimelineLayout.java (94%)
rename hudi-common/src/main/java/org/apache/hudi/common/{util/AvroUtils.java => table/timeline/TimelineMetadataUtils.java} (87%)
rename hudi-common/src/main/java/org/apache/hudi/common/{ => table/timeline}/versioning/AbstractMigratorBase.java (95%)
rename hudi-common/src/main/java/org/apache/hudi/common/{ => table/timeline}/versioning/MetadataMigrator.java (93%)
rename hudi-common/src/main/java/org/apache/hudi/common/{model => table/timeline/versioning}/TimelineLayoutVersion.java (91%)
rename hudi-common/src/main/java/org/apache/hudi/common/{ => table/timeline}/versioning/VersionMigrator.java (96%)
rename hudi-common/src/main/java/org/apache/hudi/common/{ => table/timeline}/versioning/clean/CleanMetadataMigrator.java (90%)
rename hudi-common/src/main/java/org/apache/hudi/common/{ => table/timeline}/versioning/clean/CleanV1MigrationHandler.java (91%)
rename hudi-common/src/main/java/org/apache/hudi/common/{ => table/timeline}/versioning/clean/CleanV2MigrationHandler.java (94%)
rename hudi-common/src/main/java/org/apache/hudi/common/{ => table/timeline}/versioning/compaction/CompactionPlanMigrator.java (90%)
rename hudi-common/src/main/java/org/apache/hudi/common/{ => table/timeline}/versioning/compaction/CompactionV1MigrationHandler.java (85%)
rename hudi-common/src/main/java/org/apache/hudi/common/{ => table/timeline}/versioning/compaction/CompactionV2MigrationHandler.java (80%)
rename hudi-common/src/main/java/org/apache/hudi/common/table/{ => view}/SyncableFileSystemView.java (88%)
rename hudi-common/src/main/java/org/apache/hudi/common/table/{ => view}/TableFileSystemView.java (98%)
copy hudi-common/src/main/java/org/apache/hudi/common/util/{Functions.java => Base64CodecUtil.java} (55%)
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/util/ValidationUtils.java
rename hudi-common/src/main/java/org/apache/hudi/common/util/{ => collection}/RocksDBDAO.java (95%)
rename hudi-common/src/test/java/org/apache/hudi/{common/util => avro}/TestHoodieAvroUtils.java (98%)
rename hudi-common/src/test/java/org/apache/hudi/common/bloom/{filter => }/TestBloomFilter.java (98%)
rename hudi-common/src/test/java/org/apache/hudi/common/bloom/{filter => }/TestInternalDynamicBloomFilter.java (97%)
rename hudi-common/src/test/java/org/apache/hudi/common/{util => fs}/TestFSUtils.java (86%)
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/inline/FileSystemTestUtils.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestHFileInLining.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystem.java
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestInMemoryFileSystem.java
rename hudi-common/src/test/java/org/apache/hudi/common/table/{string => timeline}/MockHoodieTimeline.java (86%)
rename hudi-common/src/test/java/org/apache/hudi/common/table/{string => timeline}/TestHoodieActiveTimeline.java (87%)
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java
copy hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHiveIncrementalPuller.java => hudi-common/src/test/java/org/apache/hudi/common/util/TestBase64CodecUtil.java (62%)
create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/util/TestStringUtils.java
mode change 100644 => 100755 hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java
rename hudi-common/src/test/java/org/apache/hudi/common/util/{ => collection}/TestRocksDBManager.java (98%)
create mode 100644 hudi-common/src/test/resources/old-version.commit
rename {hudi-hive => hudi-hive-sync}/pom.xml (96%)
rename {hudi-hive => hudi-hive-sync}/run_sync_tool.sh (90%)
rename {hudi-hive => hudi-hive-sync}/src/assembly/src.xml (100%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java (100%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/HiveSyncTool.java (98%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java (97%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/HoodieHiveSyncException.java (100%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java (89%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java (100%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/PartitionValueExtractor.java (100%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/SchemaDifference.java (79%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/SlashEncodedDayPartitionValueExtractor.java (98%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/util/ColumnNameXLator.java (80%)
rename {hudi-hive => hudi-hive-sync}/src/main/java/org/apache/hudi/hive/util/SchemaUtil.java (98%)
rename {hudi-hive => hudi-hive-sync}/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java (97%)
rename {hudi-hive => hudi-hive-sync}/src/test/java/org/apache/hudi/hive/TestUtil.java (89%)
rename {hudi-hive => hudi-hive-sync}/src/test/java/org/apache/hudi/hive/util/HiveTestService.java (95%)
copy hudi-common/src/test/resources/log4j-surefire.properties => hudi-hive-sync/src/test/resources/log4j-surefire-quiet.properties (59%)
rename {hudi-hive => hudi-hive-sync}/src/test/resources/log4j-surefire.properties (62%)
delete mode 100644 hudi-hive/src/test/resources/log4j-surefire-quiet.properties
copy hudi-client/src/main/java/org/apache/hudi/exception/HoodieAppendException.java => hudi-spark/src/main/scala/org/apache/hudi/HudiEmptyRelation.scala (60%)
create mode 100644 hudi-spark/src/test/java/DataSourceUtilsTest.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieSnapshotExporterException.java
create mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/CsvDFSSource.java
copy hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/{AWSDmsTransformer.java => ChainedTransformer.java} (58%)
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotExporter.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/TestUtilHelpers.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java
copy hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/{TestJsonDFSSource.java => TestCsvDFSSource.java} (67%)
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestChainedTransformer.java
rename hudi-utilities/src/test/java/org/apache/hudi/utilities/{ => transform}/TestFlatteningTransformer.java (95%)
copy hudi-utilities/src/test/resources/delta-streamer-config/{source.avsc => source-flattened.avsc} (83%)
copy hudi-utilities/src/test/resources/delta-streamer-config/{source.avsc => target-flattened.avsc} (83%)
rename packaging/{hudi-hive-bundle => hudi-hive-sync-bundle}/pom.xml (95%)
rename packaging/{hudi-hive-bundle => hudi-hive-sync-bundle}/src/main/java/org/apache/hudi/hive/bundle/Main.java (100%)