You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by na...@apache.org on 2020/07/17 23:56:07 UTC
[hudi] branch hudi_test_suite_refactor updated (0702a48 -> 786b36e)
This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a change to branch hudi_test_suite_refactor
in repository https://gitbox.apache.org/repos/asf/hudi.git.
discard 0702a48 [HUDI-394] Provide a basic implementation of test suite
add 2be924f [HUDI-760]Remove Rolling Stat management from Hudi Writer (#1739)
add 37ea795 [HUDI-539] Make HoodieROTablePathFilter implement Configurable (#1784)
add 574dcf9 [MINOR] Relocate jetty during shading/packaging for Databricks runtime (#1781)
add 3b9a305 [HUDI-996] Add functional test suite for hudi-utilities (#1746)
add be85a6c [HUDI-1004] Support update metrics in HoodieDeltaStreamerMetrics (#1732)
add 8c4ff18 [HUDI-1064]Trim hoodie table name (#1805)
add 7b2a947 [HUDI-1069] Remove duplicate assertNoWriteErrors() (#1797)
add 086853c [HUDI-1080] Fix backward compatibility for com.uber inputformats
add d58644b [HUDI-1062]Remove unnecessary maxEvent check and add some log in KafkaOffsetGen (#1779)
add c7f1a78 [HUDI-728]: Implemented custom key generator (#1433)
add 9627a38 [HUDI-916]: Added support for multiple input formats in TimestampBasedKeyGenerator (#1648)
add abfebd3 [MINOR] Update parameter description (#1821)
add 20ac7c3 [HUDI-994] Make TestHBaseQPSResourceAllocator a unit test (#1820)
add 10e4572 [HUDI-1078]Fix IllegalArgumentException in Delete data demo of Quick-Start Guide (#1808)
add 21bb1b5 [HUDI-1068] Fixing deletes in global bloom when update partition path is set (#1793)
add f5dc8ca [HUDI-994] Split TestHBaseIndex to unit tests (#1818)
add b399b4a [HUDI-996] Add functional test in hudi-client (#1824)
add bf1d36f [HUDI-1087] Handle decimal type for realtime record reader with SparkSQL (#1831)
add 786b36e [HUDI-394] Provide a basic implementation of test suite
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (0702a48)
\
N -- N -- N refs/heads/hudi_test_suite_refactor (786b36e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
hudi-client/pom.xml | 18 ++
.../hudi/client/AbstractHoodieWriteClient.java | 47 +--
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 8 +-
.../org/apache/hudi/index/hbase/HBaseIndex.java | 8 +-
.../hudi/index/simple/HoodieGlobalSimpleIndex.java | 8 +-
.../{MetricsReporter.java => HoodieGauge.java} | 30 +-
.../main/java/org/apache/hudi/metrics/Metrics.java | 4 +-
.../action/commit/BaseCommitActionExecutor.java | 15 +-
.../org/apache/hudi/ClientFunctionalTestSuite.java | 32 ++
.../org/apache/hudi/client/TestClientRollback.java | 1 +
.../TestHoodieClientOnCopyOnWriteStorage.java | 241 +++++++++++----
.../apache/hudi/client/TestHoodieReadClient.java | 1 +
.../org/apache/hudi/index/TestHoodieIndex.java | 8 +-
.../apache/hudi/index/hbase/TestHBaseIndex.java | 248 ++++++---------
.../hbase/TestHBaseIndexUsage.java} | 27 +-
.../hbase/TestHBasePutBatchSizeCalculator.java | 66 ++++
.../index/hbase/TestHBaseQPSResourceAllocator.java | 42 +--
.../org/apache/hudi/io/TestHoodieMergeHandle.java | 14 +-
.../java/org/apache/hudi/table/TestCleaner.java | 1 +
.../hudi/table/TestHoodieMergeOnReadTable.java | 132 +++-----
.../table/action/compact/TestAsyncCompaction.java | 1 +
.../java/org/apache/hudi/testutils/Assertions.java | 41 +++
.../hudi/testutils/FunctionalTestHarness.java | 160 ++++++++++
.../hudi/testutils/HoodieClientTestBase.java | 13 +-
.../hudi/testutils/HoodieClientTestUtils.java | 21 ++
.../hudi/testutils/providers/DFSProvider.java | 34 +++
.../providers/HoodieMetaClientProvider.java | 40 +++
.../providers/HoodieWriteClientProvider.java} | 28 +-
.../hudi/testutils/providers/SparkProvider.java | 55 ++++
.../testutils/minicluster/HdfsTestService.java | 4 +-
.../hadoop/hive/HoodieCombineHiveInputFormat.java | 60 ++++
.../hudi/hadoop/HoodieROTablePathFilter.java | 13 +-
.../hadoop/hive/HoodieCombineHiveInputFormat.java | 44 ++-
.../utils/HoodieRealtimeRecordReaderUtils.java | 7 +-
.../test/resources/log4j-surefire-quiet.properties | 1 +
.../main/java/org/apache/hudi/DataSourceUtils.java | 16 +
.../main/java/org/apache/hudi/QuickstartUtils.java | 2 +-
.../exception/HoodieDeltaStreamerException.java | 4 +-
.../apache/hudi/keygen/ComplexKeyGenerator.java | 46 +--
.../org/apache/hudi/keygen/CustomKeyGenerator.java | 131 ++++++++
.../hudi/keygen/GlobalDeleteKeyGenerator.java | 13 +-
.../hudi/keygen/NonpartitionedKeyGenerator.java | 6 +-
.../org/apache/hudi/keygen/SimpleKeyGenerator.java | 23 +-
.../hudi}/keygen/TimestampBasedKeyGenerator.java | 107 ++++---
.../hudi/keygen/parser/HoodieDateTimeParser.java} | 36 ++-
.../keygen/parser/HoodieDateTimeParserImpl.java | 122 ++++++++
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 21 +-
.../hudi/keygen/TestComplexKeyGenerator.java | 88 ++++++
.../apache/hudi/keygen/TestCustomKeyGenerator.java | 169 ++++++++++
.../hudi/keygen/TestGlobalDeleteKeyGenerator.java | 78 +++++
.../hudi/keygen/TestKeyGeneratorUtilities.java} | 33 +-
.../apache/hudi/keygen/TestSimpleKeyGenerator.java | 97 ++++++
.../keygen/TestTimestampBasedKeyGenerator.java | 255 ++++++++++++++++
hudi-utilities/pom.xml | 26 ++
.../org/apache/hudi/utilities/UtilHelpers.java | 7 -
.../hudi/utilities/deltastreamer/DeltaSync.java | 2 +-
.../deltastreamer/HoodieDeltaStreamer.java | 2 +-
.../HoodieMultiTableDeltaStreamer.java | 2 +-
.../utilities/sources/helpers/KafkaOffsetGen.java | 11 +-
.../hudi/utilities/TestHoodieSnapshotExporter.java | 57 ++++
.../TestKafkaConnectHdfsProvider.java | 30 +-
.../TestAWSDatabaseMigrationServiceSource.java | 31 +-
.../TestChainedTransformer.java | 41 +--
.../functional/TestHDFSParquetImporter.java | 340 +++++++++------------
.../functional/TestHoodieSnapshotCopier.java | 36 +--
.../functional/TestHoodieSnapshotExporter.java | 125 +++-----
.../functional/TestJdbcbasedSchemaProvider.java | 24 +-
.../functional/UtilitiesFunctionalTestSuite.java | 32 ++
.../keygen/TestTimestampBasedKeyGenerator.java | 98 ------
.../hudi/utilities/sources/TestKafkaSource.java | 36 ++-
.../transform/TestChainedTransformer.java | 52 ----
packaging/hudi-spark-bundle/pom.xml | 27 +-
pom.xml | 26 +-
style/checkstyle.xml | 2 +-
74 files changed, 2509 insertions(+), 1218 deletions(-)
copy hudi-client/src/main/java/org/apache/hudi/metrics/{MetricsReporter.java => HoodieGauge.java} (61%)
create mode 100644 hudi-client/src/test/java/org/apache/hudi/ClientFunctionalTestSuite.java
copy hudi-client/src/test/java/org/apache/hudi/{metrics/TestMetricsReporterFactory.java => index/hbase/TestHBaseIndexUsage.java} (66%)
create mode 100644 hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBasePutBatchSizeCalculator.java
create mode 100644 hudi-client/src/test/java/org/apache/hudi/testutils/Assertions.java
create mode 100644 hudi-client/src/test/java/org/apache/hudi/testutils/FunctionalTestHarness.java
create mode 100644 hudi-client/src/test/java/org/apache/hudi/testutils/providers/DFSProvider.java
create mode 100644 hudi-client/src/test/java/org/apache/hudi/testutils/providers/HoodieMetaClientProvider.java
copy hudi-client/src/test/java/org/apache/hudi/{metrics/TestMetricsReporterFactory.java => testutils/providers/HoodieWriteClientProvider.java} (52%)
create mode 100644 hudi-client/src/test/java/org/apache/hudi/testutils/providers/SparkProvider.java
create mode 100644 hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/hive/HoodieCombineHiveInputFormat.java
rename {hudi-utilities/src/main/java/org/apache/hudi/utilities => hudi-spark/src/main/java/org/apache/hudi}/exception/HoodieDeltaStreamerException.java (91%)
create mode 100644 hudi-spark/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java
rename {hudi-utilities/src/main/java/org/apache/hudi/utilities => hudi-spark/src/main/java/org/apache/hudi}/keygen/TimestampBasedKeyGenerator.java (53%)
copy hudi-spark/src/main/{scala/org/apache/hudi/package.scala => java/org/apache/hudi/keygen/parser/HoodieDateTimeParser.java} (51%)
create mode 100644 hudi-spark/src/main/java/org/apache/hudi/keygen/parser/HoodieDateTimeParserImpl.java
create mode 100644 hudi-spark/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java
create mode 100644 hudi-spark/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java
create mode 100644 hudi-spark/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteKeyGenerator.java
copy hudi-spark/src/{main/java/org/apache/hudi/keygen/KeyGenerator.java => test/java/org/apache/hudi/keygen/TestKeyGeneratorUtilities.java} (55%)
create mode 100644 hudi-spark/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java
create mode 100644 hudi-spark/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotExporter.java
copy hudi-utilities/src/test/java/org/apache/hudi/utilities/{transform => functional}/TestChainedTransformer.java (62%)
create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/UtilitiesFunctionalTestSuite.java
delete mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/keygen/TestTimestampBasedKeyGenerator.java