You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2019/12/12 12:46:19 UTC
[incubator-hudi] branch hudi_test_suite created (now 4ac72f4)
This is an automated email from the ASF dual-hosted git repository.
vinoyang pushed a change to branch hudi_test_suite
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git.
at 4ac72f4 Rename module name from hudi-bench to hudi-end-to-end-tests
This branch includes the following new commits:
new 4ac72f4 Rename module name from hudi-bench to hudi-end-to-end-tests
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-hudi] 01/01: Rename module name from hudi-bench to
hudi-end-to-end-tests
Posted by vi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vinoyang pushed a commit to branch hudi_test_suite
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
commit 4ac72f4f62800725a469b9a8bc8110acb0b18fb1
Author: yanghua <ya...@gmail.com>
AuthorDate: Tue Dec 3 19:44:15 2019 +0800
Rename module name from hudi-bench to hudi-end-to-end-tests
---
docker/hoodie/hadoop/hive_base/Dockerfile | 2 +-
docker/hoodie/hadoop/hive_base/pom.xml | 4 ++--
{hudi-bench => hudi-end-to-end-tests}/pom.xml | 2 +-
.../prepare_integration_suite.sh | 4 ++--
.../apache/hudi/e2e}/DFSDeltaWriterAdapter.java | 6 +++---
.../apache/hudi/e2e}/DFSSparkAvroDeltaWriter.java | 8 ++++----
.../org/apache/hudi/e2e}/DeltaInputFormat.java | 2 +-
.../java/org/apache/hudi/e2e}/DeltaOutputType.java | 2 +-
.../org/apache/hudi/e2e}/DeltaWriterAdapter.java | 4 ++--
.../org/apache/hudi/e2e}/DeltaWriterFactory.java | 10 ++++-----
.../hudi/e2e}/configuration/DFSDeltaConfig.java | 6 +++---
.../hudi/e2e}/configuration/DeltaConfig.java | 6 +++---
.../hudi/e2e}/converter/UpdateConverter.java | 6 +++---
.../java/org/apache/hudi/e2e}/dag/DagUtils.java | 9 ++++----
.../org/apache/hudi/e2e}/dag/ExecutionContext.java | 9 ++++----
.../java/org/apache/hudi/e2e}/dag/WorkflowDag.java | 4 ++--
.../apache/hudi/e2e}/dag/WorkflowDagGenerator.java | 12 +++++------
.../apache/hudi/e2e}/dag/nodes/BulkInsertNode.java | 6 +++---
.../org/apache/hudi/e2e}/dag/nodes/CleanNode.java | 4 ++--
.../apache/hudi/e2e}/dag/nodes/CompactNode.java | 6 +++---
.../org/apache/hudi/e2e}/dag/nodes/DagNode.java | 6 +++---
.../apache/hudi/e2e}/dag/nodes/HiveQueryNode.java | 8 ++++----
.../apache/hudi/e2e}/dag/nodes/HiveSyncNode.java | 8 ++++----
.../org/apache/hudi/e2e}/dag/nodes/InsertNode.java | 10 ++++-----
.../apache/hudi/e2e}/dag/nodes/RollbackNode.java | 6 +++---
.../hudi/e2e}/dag/nodes/ScheduleCompactNode.java | 6 +++---
.../hudi/e2e}/dag/nodes/SparkSQLQueryNode.java | 8 ++++----
.../org/apache/hudi/e2e}/dag/nodes/UpsertNode.java | 8 ++++----
.../apache/hudi/e2e}/dag/nodes/ValidateNode.java | 6 +++---
.../hudi/e2e}/dag/scheduler/DagScheduler.java | 12 +++++------
.../apache/hudi/e2e}/generator/DeltaGenerator.java | 24 +++++++++++-----------
.../FlexibleSchemaRecordGenerationIterator.java | 2 +-
.../GenericRecordFullPayloadGenerator.java | 2 +-
.../GenericRecordFullPayloadSizeEstimator.java | 2 +-
.../GenericRecordPartialPayloadGenerator.java | 2 +-
.../generator/LazyRecordGeneratorIterator.java | 2 +-
.../e2e}/generator/UpdateGeneratorIterator.java | 2 +-
.../e2e}/helpers/DFSTestSuitePathSelector.java | 2 +-
.../hudi/e2e}/helpers/HiveServiceProvider.java | 6 +++---
.../hudi/e2e}/job/HoodieDeltaStreamerWrapper.java | 2 +-
.../apache/hudi/e2e}/job/HoodieTestSuiteJob.java | 24 +++++++++++-----------
.../hudi/e2e}/reader/DFSAvroDeltaInputReader.java | 8 ++++----
.../hudi/e2e}/reader/DFSDeltaInputReader.java | 2 +-
.../e2e}/reader/DFSHoodieDatasetInputReader.java | 2 +-
.../e2e}/reader/DFSParquetDeltaInputReader.java | 6 +++---
.../apache/hudi/e2e}/reader/DeltaInputReader.java | 2 +-
.../apache/hudi/e2e}/reader/SparkBasedReader.java | 2 +-
.../hudi/e2e}/writer/AvroDeltaInputWriter.java | 2 +-
.../apache/hudi/e2e}/writer/DeltaInputWriter.java | 2 +-
.../org/apache/hudi/e2e}/writer/DeltaWriter.java | 6 +++---
.../hudi/e2e}/writer/FileDeltaInputWriter.java | 2 +-
.../e2e}/writer/SparkAvroDeltaInputWriter.java | 2 +-
.../org/apache/hudi/e2e}/writer/WriteStats.java | 2 +-
.../hudi/e2e}/TestDFSDeltaWriterAdapter.java | 16 +++++++--------
.../apache/hudi/e2e}/TestFileDeltaInputWriter.java | 14 ++++++-------
.../e2e}/configuration/TestWorkflowBuilder.java | 12 +++++------
.../hudi/e2e}/converter/TestUpdateConverter.java | 4 ++--
.../org/apache/hudi/e2e}/dag/TestComplexDag.java | 14 +++++++------
.../org/apache/hudi/e2e}/dag/TestDagUtils.java | 13 +++++++-----
.../org/apache/hudi/e2e}/dag/TestHiveSyncDag.java | 14 +++++++------
.../apache/hudi/e2e}/dag/TestInsertOnlyDag.java | 10 +++++----
.../apache/hudi/e2e}/dag/TestInsertUpsertDag.java | 12 ++++++-----
.../TestGenericRecordPayloadEstimator.java | 7 ++++---
.../TestGenericRecordPayloadGenerator.java | 16 ++++++++-------
.../hudi/e2e}/generator/TestWorkloadGenerator.java | 22 ++++++++++----------
.../hudi/e2e}/job/TestHoodieTestSuiteJob.java | 22 ++++++++++----------
.../e2e}/reader/TestDFSAvroDeltaInputReader.java | 4 ++--
.../reader/TestDFSHoodieDatasetInputReader.java | 5 +++--
.../java/org/apache/hudi/e2e}/utils/TestUtils.java | 2 +-
.../apache/hudi/e2e}/writer/TestDeltaWriter.java | 7 ++++---
.../resources/hudi-bench-config/base.properties | 0
.../hudi-bench-config/complex-source.avsc | 0
.../complex-workflow-dag-cow.yaml | 0
.../test/resources/hudi-bench-config/source.avsc | 0
.../test/resources/hudi-bench-config/target.avsc | 0
.../test/resources/log4j-surefire-quiet.properties | 0
.../src/test/resources/log4j-surefire.properties | 0
.../pom.xml | 6 +++---
.../java/org/apache/hudi/e2e}/bundle/Main.java | 2 +-
79 files changed, 258 insertions(+), 242 deletions(-)
diff --git a/docker/hoodie/hadoop/hive_base/Dockerfile b/docker/hoodie/hadoop/hive_base/Dockerfile
index 1273c7e..d2b5650 100644
--- a/docker/hoodie/hadoop/hive_base/Dockerfile
+++ b/docker/hoodie/hadoop/hive_base/Dockerfile
@@ -56,7 +56,7 @@ ENV HUDI_HADOOP_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hood
ENV HUDI_HIVE_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-hive-bundle.jar
ENV HUDI_SPARK_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-spark-bundle.jar
ENV HUDI_UTILITIES_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-utilities.jar
-ENV HUDI_BENCH_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-bench-bundle.jar
+ENV HUDI_END_TO_END_TESTS_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-end-to-end-tests-bundle.jar
COPY startup.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/startup.sh
diff --git a/docker/hoodie/hadoop/hive_base/pom.xml b/docker/hoodie/hadoop/hive_base/pom.xml
index 8a13fb4..12608e4 100644
--- a/docker/hoodie/hadoop/hive_base/pom.xml
+++ b/docker/hoodie/hadoop/hive_base/pom.xml
@@ -62,8 +62,8 @@
file="${project.basedir}/../../../../packaging/hudi-utilities-bundle/target/hudi-utilities-bundle-${project.version}.jar"
tofile="target/hoodie-utilities.jar"/>
<copy
- file="${project.basedir}/../../../../packaging/hudi-bench-bundle/target/hudi-bench-bundle-${project.version}.jar"
- tofile="target/hoodie-bench-bundle.jar"/>
+ file="${project.basedir}/../../../../packaging/hudi-end-to-end-tests-bundle/target/hudi-end-to-end-tests-bundle-${project.version}.jar"
+ tofile="target/hoodie-end-to-end-tests-bundle.jar"/>
</tasks>
</configuration>
<goals>
diff --git a/hudi-bench/pom.xml b/hudi-end-to-end-tests/pom.xml
similarity index 99%
rename from hudi-bench/pom.xml
rename to hudi-end-to-end-tests/pom.xml
index 69cc78b..7cb1bd2 100644
--- a/hudi-bench/pom.xml
+++ b/hudi-end-to-end-tests/pom.xml
@@ -24,7 +24,7 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>hudi-bench</artifactId>
+ <artifactId>hudi-end-to-end-tests</artifactId>
<packaging>jar</packaging>
<properties>
diff --git a/hudi-bench/prepare_integration_suite.sh b/hudi-end-to-end-tests/prepare_integration_suite.sh
similarity index 95%
rename from hudi-bench/prepare_integration_suite.sh
rename to hudi-end-to-end-tests/prepare_integration_suite.sh
index 69ecdd9..9e28988 100644
--- a/hudi-bench/prepare_integration_suite.sh
+++ b/hudi-end-to-end-tests/prepare_integration_suite.sh
@@ -24,8 +24,8 @@ echo "spark-submit --packages com.databricks:spark-avro_2.11:4.0.0 \
--master $0 \
--deploy-mode $1 \
--properties-file $2 \
---class org.apache.hudi.bench.job.HoodieTestSuiteJob \
-`ls target/hudi-bench-*-SNAPSHOT.jar` \
+--class org.apache.hudi.e2e.job.HoodieTestSuiteJob \
+`ls target/hudi-end-to-end-tests-*-SNAPSHOT.jar` \
--source-class $3 \
--source-ordering-field $4 \
--input-base-path $5 \
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DFSDeltaWriterAdapter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSDeltaWriterAdapter.java
similarity index 94%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/DFSDeltaWriterAdapter.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSDeltaWriterAdapter.java
index e3f1f7b..d743dc3 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/DFSDeltaWriterAdapter.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSDeltaWriterAdapter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench;
+package org.apache.hudi.e2e;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
@@ -24,8 +24,8 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hudi.bench.writer.FileDeltaInputWriter;
-import org.apache.hudi.bench.writer.WriteStats;
+import org.apache.hudi.e2e.writer.FileDeltaInputWriter;
+import org.apache.hudi.e2e.writer.WriteStats;
/**
* {@link org.apache.hadoop.hdfs.DistributedFileSystem} (or {@link org.apache.hadoop.fs.LocalFileSystem}) based delta
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DFSSparkAvroDeltaWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSSparkAvroDeltaWriter.java
similarity index 88%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/DFSSparkAvroDeltaWriter.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSSparkAvroDeltaWriter.java
index 8c7c8b5..515d0da 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/DFSSparkAvroDeltaWriter.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSSparkAvroDeltaWriter.java
@@ -16,13 +16,13 @@
* limitations under the License.
*/
-package org.apache.hudi.bench;
+package org.apache.hudi.e2e;
import java.io.IOException;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hudi.bench.DeltaWriterAdapter.SparkBasedDeltaWriter;
-import org.apache.hudi.bench.writer.DeltaInputWriter;
-import org.apache.hudi.bench.writer.WriteStats;
+import org.apache.hudi.e2e.DeltaWriterAdapter.SparkBasedDeltaWriter;
+import org.apache.hudi.e2e.writer.DeltaInputWriter;
+import org.apache.hudi.e2e.writer.WriteStats;
import org.apache.spark.api.java.JavaRDD;
/**
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaInputFormat.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaInputFormat.java
similarity index 96%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/DeltaInputFormat.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaInputFormat.java
index 46d1425..6175b7c 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaInputFormat.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaInputFormat.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench;
+package org.apache.hudi.e2e;
/**
* Supported delta input data formats
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaOutputType.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaOutputType.java
similarity index 96%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/DeltaOutputType.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaOutputType.java
index 77a1b0a..bdd9c29 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaOutputType.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaOutputType.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench;
+package org.apache.hudi.e2e;
/**
* Supported output destination types for the generated delta workload
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterAdapter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterAdapter.java
similarity index 93%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterAdapter.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterAdapter.java
index da001db..06955b9 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterAdapter.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterAdapter.java
@@ -16,12 +16,12 @@
* limitations under the License.
*/
-package org.apache.hudi.bench;
+package org.apache.hudi.e2e;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
-import org.apache.hudi.bench.writer.WriteStats;
+import org.apache.hudi.e2e.writer.WriteStats;
import org.apache.spark.api.java.JavaRDD;
public interface DeltaWriterAdapter<I> {
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterFactory.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterFactory.java
similarity index 89%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterFactory.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterFactory.java
index aeefa48..078d04d 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterFactory.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterFactory.java
@@ -16,14 +16,14 @@
* limitations under the License.
*/
-package org.apache.hudi.bench;
+package org.apache.hudi.e2e;
import java.io.IOException;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hudi.bench.configuration.DFSDeltaConfig;
-import org.apache.hudi.bench.configuration.DeltaConfig;
-import org.apache.hudi.bench.writer.AvroDeltaInputWriter;
-import org.apache.hudi.bench.writer.FileDeltaInputWriter;
+import org.apache.hudi.e2e.configuration.DFSDeltaConfig;
+import org.apache.hudi.e2e.configuration.DeltaConfig;
+import org.apache.hudi.e2e.writer.AvroDeltaInputWriter;
+import org.apache.hudi.e2e.writer.FileDeltaInputWriter;
import org.apache.hudi.common.util.StringUtils;
/**
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DFSDeltaConfig.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DFSDeltaConfig.java
similarity index 94%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DFSDeltaConfig.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DFSDeltaConfig.java
index 3781f3c..57d4a56 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DFSDeltaConfig.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DFSDeltaConfig.java
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.configuration;
+package org.apache.hudi.e2e.configuration;
-import org.apache.hudi.bench.DeltaInputFormat;
-import org.apache.hudi.bench.DeltaOutputType;
+import org.apache.hudi.e2e.DeltaInputFormat;
+import org.apache.hudi.e2e.DeltaOutputType;
import org.apache.hudi.common.SerializableConfiguration;
/**
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DeltaConfig.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DeltaConfig.java
similarity index 98%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DeltaConfig.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DeltaConfig.java
index 11aa072..6c0b351 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DeltaConfig.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DeltaConfig.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.configuration;
+package org.apache.hudi.e2e.configuration;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
@@ -26,8 +26,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hudi.bench.DeltaInputFormat;
-import org.apache.hudi.bench.DeltaOutputType;
+import org.apache.hudi.e2e.DeltaInputFormat;
+import org.apache.hudi.e2e.DeltaOutputType;
import org.apache.hudi.common.SerializableConfiguration;
import org.apache.hudi.common.util.collection.Pair;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/converter/UpdateConverter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/converter/UpdateConverter.java
similarity index 92%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/converter/UpdateConverter.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/converter/UpdateConverter.java
index 7cf48c3..5d58bd3 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/converter/UpdateConverter.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/converter/UpdateConverter.java
@@ -16,12 +16,12 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.converter;
+package org.apache.hudi.e2e.converter;
import java.util.List;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hudi.bench.generator.LazyRecordGeneratorIterator;
-import org.apache.hudi.bench.generator.UpdateGeneratorIterator;
+import org.apache.hudi.e2e.generator.LazyRecordGeneratorIterator;
+import org.apache.hudi.e2e.generator.UpdateGeneratorIterator;
import org.apache.hudi.utilities.converter.Converter;
import org.apache.spark.api.java.JavaRDD;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/DagUtils.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/DagUtils.java
similarity index 96%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/DagUtils.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/DagUtils.java
index 93a04b3..2d93e7a 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/DagUtils.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/DagUtils.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag;
+package org.apache.hudi.e2e.dag;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonNode;
@@ -37,9 +37,8 @@ import java.util.Map.Entry;
import java.util.stream.Collectors;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hudi.bench.configuration.DeltaConfig;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.nodes.DagNode;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.Pair;
@@ -119,7 +118,7 @@ public class DagUtils {
private static DagNode convertJsonToDagNode(JsonNode node, String type) {
try {
- DeltaConfig.Config config = Config.newBuilder().withConfigsMap(convertJsonNodeToMap(node)).build();
+ Config config = Config.newBuilder().withConfigsMap(convertJsonNodeToMap(node)).build();
return (DagNode) ReflectionUtils.loadClass(generateFQN(type), config);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/ExecutionContext.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/ExecutionContext.java
similarity index 87%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/ExecutionContext.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/ExecutionContext.java
index 03b585f..6b640f5 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/ExecutionContext.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/ExecutionContext.java
@@ -16,16 +16,17 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag;
+package org.apache.hudi.e2e.dag;
import java.io.Serializable;
-import org.apache.hudi.bench.generator.DeltaGenerator;
-import org.apache.hudi.bench.writer.DeltaWriter;
+import org.apache.hudi.e2e.generator.DeltaGenerator;
+import org.apache.hudi.e2e.writer.DeltaWriter;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
import org.apache.spark.api.java.JavaSparkContext;
/**
* This wraps the context needed for an execution of
- * a {@link org.apache.hudi.bench.dag.nodes.DagNode#execute(ExecutionContext)}
+ * a {@link DagNode#execute(ExecutionContext)}
*/
public class ExecutionContext implements Serializable {
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDag.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDag.java
similarity index 93%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDag.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDag.java
index b88bd74..bcd45c2 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDag.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDag.java
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag;
+package org.apache.hudi.e2e.dag;
import java.util.List;
-import org.apache.hudi.bench.dag.nodes.DagNode;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
/**
* Workflow dag that encapsulates all execute nodes
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDagGenerator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDagGenerator.java
similarity index 88%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDagGenerator.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDagGenerator.java
index a5602ce..171e27f 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDagGenerator.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDagGenerator.java
@@ -16,15 +16,15 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag;
+package org.apache.hudi.e2e.dag;
import java.util.ArrayList;
import java.util.List;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.nodes.DagNode;
-import org.apache.hudi.bench.dag.nodes.HiveQueryNode;
-import org.apache.hudi.bench.dag.nodes.InsertNode;
-import org.apache.hudi.bench.dag.nodes.UpsertNode;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
+import org.apache.hudi.e2e.dag.nodes.HiveQueryNode;
+import org.apache.hudi.e2e.dag.nodes.InsertNode;
+import org.apache.hudi.e2e.dag.nodes.UpsertNode;
import org.apache.hudi.common.util.collection.Pair;
/**
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/BulkInsertNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/BulkInsertNode.java
similarity index 89%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/BulkInsertNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/BulkInsertNode.java
index e855075..b4bf25d 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/BulkInsertNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/BulkInsertNode.java
@@ -16,11 +16,11 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
import org.apache.hudi.WriteStatus;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.writer.DeltaWriter;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.writer.DeltaWriter;
import org.apache.hudi.common.util.Option;
import org.apache.spark.api.java.JavaRDD;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CleanNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CleanNode.java
similarity index 92%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CleanNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CleanNode.java
index 6e43259..861c338 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CleanNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CleanNode.java
@@ -16,9 +16,9 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
-import org.apache.hudi.bench.dag.ExecutionContext;
+import org.apache.hudi.e2e.dag.ExecutionContext;
public class CleanNode extends DagNode<Boolean> {
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CompactNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CompactNode.java
similarity index 92%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CompactNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CompactNode.java
index 2c65252..bd62fb9 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CompactNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CompactNode.java
@@ -16,11 +16,11 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
import org.apache.hudi.WriteStatus;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.ExecutionContext;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.ExecutionContext;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/DagNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/DagNode.java
similarity index 95%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/DagNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/DagNode.java
index 3f4083a..321cee9 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/DagNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/DagNode.java
@@ -16,15 +16,15 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.ExecutionContext;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.ExecutionContext;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveQueryNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveQueryNode.java
similarity index 93%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveQueryNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveQueryNode.java
index 8c5cac5..c58f00a 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveQueryNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveQueryNode.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -24,9 +24,9 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import org.apache.hudi.DataSourceUtils;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.ExecutionContext;
-import org.apache.hudi.bench.helpers.HiveServiceProvider;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.ExecutionContext;
+import org.apache.hudi.e2e.helpers.HiveServiceProvider;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.hive.HiveSyncConfig;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveSyncNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveSyncNode.java
similarity index 88%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveSyncNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveSyncNode.java
index 7ad06c6..c91a77a 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveSyncNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveSyncNode.java
@@ -16,11 +16,11 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.ExecutionContext;
-import org.apache.hudi.bench.helpers.HiveServiceProvider;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.ExecutionContext;
+import org.apache.hudi.e2e.helpers.HiveServiceProvider;
public class HiveSyncNode extends DagNode<Boolean> {
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/InsertNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/InsertNode.java
similarity index 89%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/InsertNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/InsertNode.java
index 73a1e22..8573a28 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/InsertNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/InsertNode.java
@@ -16,13 +16,13 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
import org.apache.hudi.WriteStatus;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.ExecutionContext;
-import org.apache.hudi.bench.generator.DeltaGenerator;
-import org.apache.hudi.bench.writer.DeltaWriter;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.ExecutionContext;
+import org.apache.hudi.e2e.generator.DeltaGenerator;
+import org.apache.hudi.e2e.writer.DeltaWriter;
import org.apache.hudi.common.util.Option;
import org.apache.spark.api.java.JavaRDD;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/RollbackNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/RollbackNode.java
similarity index 92%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/RollbackNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/RollbackNode.java
index 1128368..16bcc1c 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/RollbackNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/RollbackNode.java
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.ExecutionContext;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.ExecutionContext;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ScheduleCompactNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ScheduleCompactNode.java
similarity index 94%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ScheduleCompactNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ScheduleCompactNode.java
index d983af2..1ffda7b 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ScheduleCompactNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ScheduleCompactNode.java
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.ExecutionContext;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.ExecutionContext;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/SparkSQLQueryNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/SparkSQLQueryNode.java
similarity index 91%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/SparkSQLQueryNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/SparkSQLQueryNode.java
index d6d931e..14cb7d2 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/SparkSQLQueryNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/SparkSQLQueryNode.java
@@ -16,11 +16,11 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.ExecutionContext;
-import org.apache.hudi.bench.helpers.HiveServiceProvider;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.ExecutionContext;
+import org.apache.hudi.e2e.helpers.HiveServiceProvider;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/UpsertNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/UpsertNode.java
similarity index 90%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/UpsertNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/UpsertNode.java
index 60b013f..b9345df 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/UpsertNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/UpsertNode.java
@@ -16,12 +16,12 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
import org.apache.hudi.WriteStatus;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.generator.DeltaGenerator;
-import org.apache.hudi.bench.writer.DeltaWriter;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.generator.DeltaGenerator;
+import org.apache.hudi.e2e.writer.DeltaWriter;
import org.apache.hudi.common.util.Option;
import org.apache.spark.api.java.JavaRDD;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ValidateNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ValidateNode.java
similarity index 91%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ValidateNode.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ValidateNode.java
index 3f28abc..ea51864 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ValidateNode.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ValidateNode.java
@@ -16,12 +16,12 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.nodes;
+package org.apache.hudi.e2e.dag.nodes;
import java.util.List;
import java.util.function.Function;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.ExecutionContext;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.ExecutionContext;
public class ValidateNode<R> extends DagNode {
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/scheduler/DagScheduler.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/scheduler/DagScheduler.java
similarity index 91%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/scheduler/DagScheduler.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/scheduler/DagScheduler.java
index 1dde398..db74f0c 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/scheduler/DagScheduler.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/scheduler/DagScheduler.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag.scheduler;
+package org.apache.hudi.e2e.dag.scheduler;
import java.util.ArrayList;
import java.util.HashSet;
@@ -28,11 +28,11 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import org.apache.hudi.bench.dag.ExecutionContext;
-import org.apache.hudi.bench.dag.WorkflowDag;
-import org.apache.hudi.bench.dag.nodes.DagNode;
-import org.apache.hudi.bench.generator.DeltaGenerator;
-import org.apache.hudi.bench.writer.DeltaWriter;
+import org.apache.hudi.e2e.dag.ExecutionContext;
+import org.apache.hudi.e2e.dag.WorkflowDag;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
+import org.apache.hudi.e2e.generator.DeltaGenerator;
+import org.apache.hudi.e2e.writer.DeltaWriter;
import org.apache.hudi.exception.HoodieException;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/DeltaGenerator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/DeltaGenerator.java
similarity index 94%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/DeltaGenerator.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/DeltaGenerator.java
index 335fecf..a7fe7bd 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/DeltaGenerator.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/DeltaGenerator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.generator;
+package org.apache.hudi.e2e.generator;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
@@ -32,17 +32,17 @@ import java.util.Map;
import java.util.stream.StreamSupport;
import org.apache.avro.generic.GenericRecord;
import org.apache.hudi.KeyGenerator;
-import org.apache.hudi.bench.DeltaOutputType;
-import org.apache.hudi.bench.DeltaWriterAdapter;
-import org.apache.hudi.bench.DeltaWriterFactory;
-import org.apache.hudi.bench.configuration.DFSDeltaConfig;
-import org.apache.hudi.bench.configuration.DeltaConfig;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.converter.UpdateConverter;
-import org.apache.hudi.bench.reader.DFSAvroDeltaInputReader;
-import org.apache.hudi.bench.reader.DFSHoodieDatasetInputReader;
-import org.apache.hudi.bench.reader.DeltaInputReader;
-import org.apache.hudi.bench.writer.WriteStats;
+import org.apache.hudi.e2e.DeltaOutputType;
+import org.apache.hudi.e2e.DeltaWriterAdapter;
+import org.apache.hudi.e2e.DeltaWriterFactory;
+import org.apache.hudi.e2e.configuration.DFSDeltaConfig;
+import org.apache.hudi.e2e.configuration.DeltaConfig;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.converter.UpdateConverter;
+import org.apache.hudi.e2e.reader.DFSAvroDeltaInputReader;
+import org.apache.hudi.e2e.reader.DFSHoodieDatasetInputReader;
+import org.apache.hudi.e2e.reader.DeltaInputReader;
+import org.apache.hudi.e2e.writer.WriteStats;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.utilities.converter.Converter;
import org.apache.log4j.LogManager;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/FlexibleSchemaRecordGenerationIterator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/FlexibleSchemaRecordGenerationIterator.java
similarity index 98%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/FlexibleSchemaRecordGenerationIterator.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/FlexibleSchemaRecordGenerationIterator.java
index a982792..cf23460 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/FlexibleSchemaRecordGenerationIterator.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/FlexibleSchemaRecordGenerationIterator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.generator;
+package org.apache.hudi.e2e.generator;
import java.util.Iterator;
import java.util.List;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadGenerator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadGenerator.java
similarity index 99%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadGenerator.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadGenerator.java
index 2232cc5..09001e0 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadGenerator.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadGenerator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.generator;
+package org.apache.hudi.e2e.generator;
import com.google.common.annotations.VisibleForTesting;
import java.io.Serializable;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadSizeEstimator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadSizeEstimator.java
similarity index 99%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadSizeEstimator.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadSizeEstimator.java
index 1d515e9..ca32cf8 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadSizeEstimator.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadSizeEstimator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.generator;
+package org.apache.hudi.e2e.generator;
import java.io.Serializable;
import java.util.List;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordPartialPayloadGenerator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordPartialPayloadGenerator.java
similarity index 98%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordPartialPayloadGenerator.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordPartialPayloadGenerator.java
index 04174cd..091724f 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordPartialPayloadGenerator.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordPartialPayloadGenerator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.generator;
+package org.apache.hudi.e2e.generator;
import com.google.common.annotations.VisibleForTesting;
import org.apache.avro.Schema;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/LazyRecordGeneratorIterator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/LazyRecordGeneratorIterator.java
similarity index 97%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/LazyRecordGeneratorIterator.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/LazyRecordGeneratorIterator.java
index 902aaee..0878736 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/LazyRecordGeneratorIterator.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/LazyRecordGeneratorIterator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.generator;
+package org.apache.hudi.e2e.generator;
import java.util.Iterator;
import org.apache.avro.generic.GenericRecord;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/UpdateGeneratorIterator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/UpdateGeneratorIterator.java
similarity index 97%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/UpdateGeneratorIterator.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/UpdateGeneratorIterator.java
index 235b090..bfe0526 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/UpdateGeneratorIterator.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/UpdateGeneratorIterator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.generator;
+package org.apache.hudi.e2e.generator;
import java.util.ArrayList;
import java.util.Iterator;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/helpers/DFSTestSuitePathSelector.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/DFSTestSuitePathSelector.java
similarity index 98%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/helpers/DFSTestSuitePathSelector.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/DFSTestSuitePathSelector.java
index 3167407..182d5aa 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/helpers/DFSTestSuitePathSelector.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/DFSTestSuitePathSelector.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.helpers;
+package org.apache.hudi.e2e.helpers;
import java.io.IOException;
import java.util.ArrayList;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/helpers/HiveServiceProvider.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/HiveServiceProvider.java
similarity index 92%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/helpers/HiveServiceProvider.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/HiveServiceProvider.java
index 5ae2ed1..6cee004 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/helpers/HiveServiceProvider.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/HiveServiceProvider.java
@@ -16,13 +16,13 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.helpers;
+package org.apache.hudi.e2e.helpers;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hive.service.server.HiveServer2;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.writer.DeltaWriter;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.writer.DeltaWriter;
import org.apache.hudi.hive.util.HiveTestService;
public class HiveServiceProvider {
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieDeltaStreamerWrapper.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieDeltaStreamerWrapper.java
similarity index 98%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieDeltaStreamerWrapper.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieDeltaStreamerWrapper.java
index e0f77df..96cf6e4 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieDeltaStreamerWrapper.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieDeltaStreamerWrapper.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.job;
+package org.apache.hudi.e2e.job;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.conf.HiveConf;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieTestSuiteJob.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieTestSuiteJob.java
similarity index 91%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieTestSuiteJob.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieTestSuiteJob.java
index b9e52d3..90bd9af 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieTestSuiteJob.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieTestSuiteJob.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.job;
+package org.apache.hudi.e2e.job;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
@@ -27,15 +27,15 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hudi.DataSourceUtils;
import org.apache.hudi.KeyGenerator;
-import org.apache.hudi.bench.DeltaInputFormat;
-import org.apache.hudi.bench.DeltaOutputType;
-import org.apache.hudi.bench.configuration.DFSDeltaConfig;
-import org.apache.hudi.bench.dag.DagUtils;
-import org.apache.hudi.bench.dag.WorkflowDag;
-import org.apache.hudi.bench.dag.WorkflowDagGenerator;
-import org.apache.hudi.bench.dag.scheduler.DagScheduler;
-import org.apache.hudi.bench.generator.DeltaGenerator;
-import org.apache.hudi.bench.writer.DeltaWriter;
+import org.apache.hudi.e2e.DeltaInputFormat;
+import org.apache.hudi.e2e.DeltaOutputType;
+import org.apache.hudi.e2e.configuration.DFSDeltaConfig;
+import org.apache.hudi.e2e.dag.DagUtils;
+import org.apache.hudi.e2e.dag.WorkflowDag;
+import org.apache.hudi.e2e.dag.WorkflowDagGenerator;
+import org.apache.hudi.e2e.dag.scheduler.DagScheduler;
+import org.apache.hudi.e2e.generator.DeltaGenerator;
+import org.apache.hudi.e2e.writer.DeltaWriter;
import org.apache.hudi.common.SerializableConfiguration;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.FSUtils;
@@ -164,11 +164,11 @@ public class HoodieTestSuiteJob {
public String workloadYamlPath;
@Parameter(names = {"--delta-output-type"}, description = "Subclass of "
- + "org.apache.hudi.bench.workload.DeltaOutputType to readAvro data.")
+ + "org.apache.hudi.e2e.workload.DeltaOutputType to readAvro data.")
public String outputTypeName = DeltaOutputType.DFS.name();
@Parameter(names = {"--delta-input-format"}, description = "Subclass of "
- + "org.apache.hudi.bench.workload.DeltaOutputType to read avro data.")
+ + "org.apache.hudi.e2e.workload.DeltaOutputType to read avro data.")
public String inputFormatName = DeltaInputFormat.AVRO.name();
@Parameter(names = {"--input-file-size"}, description = "The min/max size of the input files to generate",
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSAvroDeltaInputReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSAvroDeltaInputReader.java
similarity index 94%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSAvroDeltaInputReader.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSAvroDeltaInputReader.java
index 7d06c77..622687b 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSAvroDeltaInputReader.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSAvroDeltaInputReader.java
@@ -16,15 +16,15 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.reader;
+package org.apache.hudi.e2e.reader;
import java.io.IOException;
import java.util.Arrays;
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.hudi.bench.DeltaInputFormat;
-import org.apache.hudi.bench.DeltaOutputType;
-import org.apache.hudi.bench.writer.AvroDeltaInputWriter;
+import org.apache.hudi.e2e.DeltaInputFormat;
+import org.apache.hudi.e2e.DeltaOutputType;
+import org.apache.hudi.e2e.writer.AvroDeltaInputWriter;
import org.apache.hudi.common.util.Option;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.SparkSession;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSDeltaInputReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSDeltaInputReader.java
similarity index 99%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSDeltaInputReader.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSDeltaInputReader.java
index 9d3e5f2..bc2ffb3 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSDeltaInputReader.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSDeltaInputReader.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.reader;
+package org.apache.hudi.e2e.reader;
import java.io.IOException;
import java.util.ArrayList;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSHoodieDatasetInputReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSHoodieDatasetInputReader.java
similarity index 99%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSHoodieDatasetInputReader.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSHoodieDatasetInputReader.java
index 462acc7..f96cdef 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSHoodieDatasetInputReader.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSHoodieDatasetInputReader.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.reader;
+package org.apache.hudi.e2e.reader;
import static java.util.Map.Entry.comparingByValue;
import static java.util.stream.Collectors.toMap;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSParquetDeltaInputReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSParquetDeltaInputReader.java
similarity index 96%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSParquetDeltaInputReader.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSParquetDeltaInputReader.java
index c0016a2..776b16e 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSParquetDeltaInputReader.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSParquetDeltaInputReader.java
@@ -16,15 +16,15 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.reader;
+package org.apache.hudi.e2e.reader;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.fs.PathFilter;
-import org.apache.hudi.bench.DeltaInputFormat;
-import org.apache.hudi.bench.DeltaOutputType;
+import org.apache.hudi.e2e.DeltaInputFormat;
+import org.apache.hudi.e2e.DeltaOutputType;
import org.apache.hudi.common.util.Option;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.SparkSession;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DeltaInputReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DeltaInputReader.java
similarity index 98%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/DeltaInputReader.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DeltaInputReader.java
index c322c88..e72b2c5 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DeltaInputReader.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DeltaInputReader.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.reader;
+package org.apache.hudi.e2e.reader;
import java.io.IOException;
import java.io.Serializable;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/SparkBasedReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/SparkBasedReader.java
similarity index 98%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/SparkBasedReader.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/SparkBasedReader.java
index 6a28923..31bffc7 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/SparkBasedReader.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/SparkBasedReader.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.reader;
+package org.apache.hudi.e2e.reader;
import java.util.List;
import org.apache.avro.generic.GenericRecord;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/AvroDeltaInputWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/AvroDeltaInputWriter.java
similarity index 99%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/AvroDeltaInputWriter.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/AvroDeltaInputWriter.java
index d53c39c..38f42e8 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/AvroDeltaInputWriter.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/AvroDeltaInputWriter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.writer;
+package org.apache.hudi.e2e.writer;
import java.io.IOException;
import java.io.OutputStream;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaInputWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaInputWriter.java
similarity index 97%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaInputWriter.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaInputWriter.java
index e16e25d..41153b3 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaInputWriter.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaInputWriter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.writer;
+package org.apache.hudi.e2e.writer;
import java.io.IOException;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaWriter.java
similarity index 97%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaWriter.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaWriter.java
index 5c18ff2..13dedd9 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaWriter.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaWriter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.writer;
+package org.apache.hudi.e2e.writer;
import java.util.HashMap;
import java.util.Map;
@@ -26,8 +26,8 @@ import org.apache.hudi.HoodieReadClient;
import org.apache.hudi.HoodieWriteClient;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
-import org.apache.hudi.bench.job.HoodieDeltaStreamerWrapper;
-import org.apache.hudi.bench.job.HoodieTestSuiteJob.HoodieTestSuiteConfig;
+import org.apache.hudi.e2e.job.HoodieDeltaStreamerWrapper;
+import org.apache.hudi.e2e.job.HoodieTestSuiteJob.HoodieTestSuiteConfig;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.util.Option;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/FileDeltaInputWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/FileDeltaInputWriter.java
similarity index 97%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/FileDeltaInputWriter.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/FileDeltaInputWriter.java
index 4dc6bed..9210a27 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/FileDeltaInputWriter.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/FileDeltaInputWriter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.writer;
+package org.apache.hudi.e2e.writer;
import java.io.IOException;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/SparkAvroDeltaInputWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/SparkAvroDeltaInputWriter.java
similarity index 98%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/SparkAvroDeltaInputWriter.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/SparkAvroDeltaInputWriter.java
index dcf744a..a9aee9c 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/SparkAvroDeltaInputWriter.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/SparkAvroDeltaInputWriter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.writer;
+package org.apache.hudi.e2e.writer;
import java.io.IOException;
import javax.ws.rs.NotSupportedException;
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/WriteStats.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/WriteStats.java
similarity index 98%
rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/WriteStats.java
rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/WriteStats.java
index 1ba673e..6ded854 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/WriteStats.java
+++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/WriteStats.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.writer;
+package org.apache.hudi.e2e.writer;
import java.io.Serializable;
import java.util.ArrayList;
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/TestDFSDeltaWriterAdapter.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestDFSDeltaWriterAdapter.java
similarity index 91%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/TestDFSDeltaWriterAdapter.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestDFSDeltaWriterAdapter.java
index e9e596a..b4cb13c 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/TestDFSDeltaWriterAdapter.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestDFSDeltaWriterAdapter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench;
+package org.apache.hudi.e2e;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.when;
@@ -27,12 +27,12 @@ import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hudi.bench.configuration.DFSDeltaConfig;
-import org.apache.hudi.bench.configuration.DeltaConfig;
-import org.apache.hudi.bench.generator.FlexibleSchemaRecordGenerationIterator;
-import org.apache.hudi.bench.utils.TestUtils;
-import org.apache.hudi.bench.writer.FileDeltaInputWriter;
-import org.apache.hudi.bench.writer.WriteStats;
+import org.apache.hudi.e2e.configuration.DFSDeltaConfig;
+import org.apache.hudi.e2e.configuration.DeltaConfig;
+import org.apache.hudi.e2e.generator.FlexibleSchemaRecordGenerationIterator;
+import org.apache.hudi.e2e.utils.TestUtils;
+import org.apache.hudi.e2e.writer.FileDeltaInputWriter;
+import org.apache.hudi.e2e.writer.WriteStats;
import org.apache.hudi.common.SerializableConfiguration;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.utilities.UtilitiesTestBase;
@@ -63,7 +63,7 @@ public class TestDFSDeltaWriterAdapter extends UtilitiesTestBase {
@Before
public void setup() throws Exception {
super.setup();
- schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-bench-config/complex-source.avsc"),
+ schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-end-to-end-tests-config/complex-source.avsc"),
jsc);
}
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/TestFileDeltaInputWriter.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestFileDeltaInputWriter.java
similarity index 93%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/TestFileDeltaInputWriter.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestFileDeltaInputWriter.java
index 30e0190..78f8756 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/TestFileDeltaInputWriter.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestFileDeltaInputWriter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench;
+package org.apache.hudi.e2e;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -32,11 +32,11 @@ import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hudi.bench.generator.GenericRecordFullPayloadGenerator;
-import org.apache.hudi.bench.reader.SparkBasedReader;
-import org.apache.hudi.bench.writer.AvroDeltaInputWriter;
-import org.apache.hudi.bench.writer.FileDeltaInputWriter;
-import org.apache.hudi.bench.writer.WriteStats;
+import org.apache.hudi.e2e.generator.GenericRecordFullPayloadGenerator;
+import org.apache.hudi.e2e.reader.SparkBasedReader;
+import org.apache.hudi.e2e.writer.AvroDeltaInputWriter;
+import org.apache.hudi.e2e.writer.FileDeltaInputWriter;
+import org.apache.hudi.e2e.writer.WriteStats;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.utilities.UtilitiesTestBase;
@@ -65,7 +65,7 @@ public class TestFileDeltaInputWriter extends UtilitiesTestBase {
@Before
public void setup() throws Exception {
super.setup();
- schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-bench-config/complex-source.avsc"),
+ schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-end-to-end-tests-config/complex-source.avsc"),
jsc);
}
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/configuration/TestWorkflowBuilder.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/configuration/TestWorkflowBuilder.java
similarity index 89%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/configuration/TestWorkflowBuilder.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/configuration/TestWorkflowBuilder.java
index 77cc42c..8838518 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/configuration/TestWorkflowBuilder.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/configuration/TestWorkflowBuilder.java
@@ -16,18 +16,18 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.configuration;
+package org.apache.hudi.e2e.configuration;
import static junit.framework.Assert.assertTrue;
import static junit.framework.TestCase.assertEquals;
import java.util.ArrayList;
import java.util.List;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.WorkflowDag;
-import org.apache.hudi.bench.dag.nodes.DagNode;
-import org.apache.hudi.bench.dag.nodes.InsertNode;
-import org.apache.hudi.bench.dag.nodes.UpsertNode;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.WorkflowDag;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
+import org.apache.hudi.e2e.dag.nodes.InsertNode;
+import org.apache.hudi.e2e.dag.nodes.UpsertNode;
import org.junit.Test;
public class TestWorkflowBuilder {
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/converter/TestUpdateConverter.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/converter/TestUpdateConverter.java
similarity index 97%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/converter/TestUpdateConverter.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/converter/TestUpdateConverter.java
index cb476d7..d8440f0 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/converter/TestUpdateConverter.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/converter/TestUpdateConverter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.converter;
+package org.apache.hudi.e2e.converter;
import static junit.framework.TestCase.assertTrue;
@@ -25,7 +25,7 @@ import java.util.List;
import java.util.Map;
import org.apache.avro.Schema.Field;
import org.apache.avro.generic.GenericRecord;
-import org.apache.hudi.bench.utils.TestUtils;
+import org.apache.hudi.e2e.utils.TestUtils;
import org.apache.hudi.utilities.UtilHelpers;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestComplexDag.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestComplexDag.java
similarity index 88%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestComplexDag.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestComplexDag.java
index c9492b4..4bdb0dd 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestComplexDag.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestComplexDag.java
@@ -16,17 +16,19 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag;
+package org.apache.hudi.e2e.dag;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import org.apache.hudi.WriteStatus;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.nodes.DagNode;
-import org.apache.hudi.bench.dag.nodes.InsertNode;
-import org.apache.hudi.bench.dag.nodes.UpsertNode;
-import org.apache.hudi.bench.dag.nodes.ValidateNode;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.WorkflowDag;
+import org.apache.hudi.e2e.dag.WorkflowDagGenerator;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
+import org.apache.hudi.e2e.dag.nodes.InsertNode;
+import org.apache.hudi.e2e.dag.nodes.UpsertNode;
+import org.apache.hudi.e2e.dag.nodes.ValidateNode;
import org.apache.spark.api.java.JavaRDD;
public class TestComplexDag extends WorkflowDagGenerator {
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestDagUtils.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestDagUtils.java
similarity index 88%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestDagUtils.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestDagUtils.java
index afeaf6a..640c4fa 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestDagUtils.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestDagUtils.java
@@ -16,13 +16,16 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag;
+package org.apache.hudi.e2e.dag;
import java.util.ArrayList;
import java.util.List;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.nodes.DagNode;
-import org.apache.hudi.bench.dag.nodes.InsertNode;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.DagUtils;
+import org.apache.hudi.e2e.dag.WorkflowDag;
+import org.apache.hudi.e2e.dag.WorkflowDagGenerator;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
+import org.apache.hudi.e2e.dag.nodes.InsertNode;
import org.junit.Assert;
import org.junit.Test;
@@ -38,7 +41,7 @@ public class TestDagUtils {
@Test
public void testConvertYamlToDag() throws Exception {
WorkflowDag dag = DagUtils.convertYamlToDag(
- DagUtils.toString(getClass().getResourceAsStream("/hudi-bench-config/complex-workflow-dag-cow.yaml")));
+ DagUtils.toString(getClass().getResourceAsStream("/hudi-end-to-end-tests-config/complex-workflow-dag-cow.yaml")));
Assert.assertEquals(dag.getNodeList().size(), 1);
Assert.assertEquals(((DagNode) dag.getNodeList().get(0)).getParentNodes().size(), 0);
Assert.assertEquals(((DagNode) dag.getNodeList().get(0)).getChildNodes().size(), 1);
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestHiveSyncDag.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestHiveSyncDag.java
similarity index 81%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestHiveSyncDag.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestHiveSyncDag.java
index 17c9730..ee85e18 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestHiveSyncDag.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestHiveSyncDag.java
@@ -16,16 +16,18 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag;
+package org.apache.hudi.e2e.dag;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.nodes.DagNode;
-import org.apache.hudi.bench.dag.nodes.HiveQueryNode;
-import org.apache.hudi.bench.dag.nodes.HiveSyncNode;
-import org.apache.hudi.bench.dag.nodes.InsertNode;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.WorkflowDag;
+import org.apache.hudi.e2e.dag.WorkflowDagGenerator;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
+import org.apache.hudi.e2e.dag.nodes.HiveQueryNode;
+import org.apache.hudi.e2e.dag.nodes.HiveSyncNode;
+import org.apache.hudi.e2e.dag.nodes.InsertNode;
import org.apache.hudi.common.util.collection.Pair;
public class TestHiveSyncDag extends WorkflowDagGenerator {
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertOnlyDag.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertOnlyDag.java
similarity index 84%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertOnlyDag.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertOnlyDag.java
index f50fe97..23dc61e 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertOnlyDag.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertOnlyDag.java
@@ -16,13 +16,15 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag;
+package org.apache.hudi.e2e.dag;
import java.util.ArrayList;
import java.util.List;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.nodes.DagNode;
-import org.apache.hudi.bench.dag.nodes.InsertNode;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.WorkflowDag;
+import org.apache.hudi.e2e.dag.WorkflowDagGenerator;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
+import org.apache.hudi.e2e.dag.nodes.InsertNode;
public class TestInsertOnlyDag extends WorkflowDagGenerator {
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertUpsertDag.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertUpsertDag.java
similarity index 82%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertUpsertDag.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertUpsertDag.java
index 63d898d..1b0d5a9 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertUpsertDag.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertUpsertDag.java
@@ -16,14 +16,16 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.dag;
+package org.apache.hudi.e2e.dag;
import java.util.ArrayList;
import java.util.List;
-import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-import org.apache.hudi.bench.dag.nodes.DagNode;
-import org.apache.hudi.bench.dag.nodes.InsertNode;
-import org.apache.hudi.bench.dag.nodes.UpsertNode;
+import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+import org.apache.hudi.e2e.dag.WorkflowDag;
+import org.apache.hudi.e2e.dag.WorkflowDagGenerator;
+import org.apache.hudi.e2e.dag.nodes.DagNode;
+import org.apache.hudi.e2e.dag.nodes.InsertNode;
+import org.apache.hudi.e2e.dag.nodes.UpsertNode;
public class TestInsertUpsertDag extends WorkflowDagGenerator {
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadEstimator.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadEstimator.java
similarity index 88%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadEstimator.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadEstimator.java
index e616813..a3fe808 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadEstimator.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadEstimator.java
@@ -16,12 +16,13 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.generator;
+package org.apache.hudi.e2e.generator;
import static junit.framework.TestCase.assertEquals;
import org.apache.avro.Schema;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.e2e.generator.GenericRecordFullPayloadSizeEstimator;
import org.junit.Test;
public class TestGenericRecordPayloadEstimator {
@@ -29,7 +30,7 @@ public class TestGenericRecordPayloadEstimator {
@Test
public void testSimpleSchemaSize() throws Exception {
Schema schema = new Schema.Parser().parse(getClass().getClassLoader()
- .getResourceAsStream("hudi-bench-config/source.avsc"));
+ .getResourceAsStream("hudi-end-to-end-tests-config/source.avsc"));
GenericRecordFullPayloadSizeEstimator estimator =
new GenericRecordFullPayloadSizeEstimator(schema);
Pair<Integer, Integer> estimateAndNumComplexFields = estimator.typeEstimateAndNumComplexFields();
@@ -40,7 +41,7 @@ public class TestGenericRecordPayloadEstimator {
@Test
public void testComplexSchemaSize() throws Exception {
Schema schema = new Schema.Parser().parse(getClass().getClassLoader()
- .getResourceAsStream("hudi-bench-config/complex-source.avsc"));
+ .getResourceAsStream("hudi-end-to-end-tests-config/complex-source.avsc"));
GenericRecordFullPayloadSizeEstimator estimator =
new GenericRecordFullPayloadSizeEstimator(schema);
Pair<Integer, Integer> estimateAndNumComplexFields = estimator.typeEstimateAndNumComplexFields();
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadGenerator.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadGenerator.java
similarity index 88%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadGenerator.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadGenerator.java
index e2c3ae6..bb3e9b3 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadGenerator.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadGenerator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.generator;
+package org.apache.hudi.e2e.generator;
import static junit.framework.TestCase.assertTrue;
@@ -28,6 +28,8 @@ import java.util.stream.IntStream;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.hudi.common.util.HoodieAvroUtils;
+import org.apache.hudi.e2e.generator.GenericRecordFullPayloadGenerator;
+import org.apache.hudi.e2e.generator.GenericRecordPartialPayloadGenerator;
import org.junit.Assert;
import org.junit.Test;
@@ -36,7 +38,7 @@ public class TestGenericRecordPayloadGenerator {
@Test
public void testSimplePayload() throws Exception {
Schema schema = new Schema.Parser().parse(getClass().getClassLoader()
- .getResourceAsStream("hudi-bench-config/source.avsc"));
+ .getResourceAsStream("hudi-end-to-end-tests-config/source.avsc"));
GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(schema);
GenericRecord record = payloadGenerator.getNewPayload();
// The generated payload should validate with the provided schema
@@ -46,7 +48,7 @@ public class TestGenericRecordPayloadGenerator {
@Test
public void testComplexPayload() throws IOException {
Schema schema = new Schema.Parser().parse(getClass().getClassLoader()
- .getResourceAsStream("hudi-bench-config/complex-source.avsc"));
+ .getResourceAsStream("hudi-end-to-end-tests-config/complex-source.avsc"));
GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(schema);
GenericRecord record = payloadGenerator.getNewPayload();
// The generated payload should validate with the provided schema
@@ -56,7 +58,7 @@ public class TestGenericRecordPayloadGenerator {
@Test
public void testComplexPartialPayload() throws IOException {
Schema schema = new Schema.Parser().parse(getClass().getClassLoader()
- .getResourceAsStream("hudi-bench-config/complex-source.avsc"));
+ .getResourceAsStream("hudi-end-to-end-tests-config/complex-source.avsc"));
GenericRecordPartialPayloadGenerator payloadGenerator = new GenericRecordPartialPayloadGenerator(schema);
IntStream.range(0, 10).forEach(a -> {
GenericRecord record = payloadGenerator.getNewPayload();
@@ -68,7 +70,7 @@ public class TestGenericRecordPayloadGenerator {
@Test
public void testUpdatePayloadGenerator() throws IOException {
Schema schema = new Schema.Parser().parse(getClass().getClassLoader()
- .getResourceAsStream("hudi-bench-config/source.avsc"));
+ .getResourceAsStream("hudi-end-to-end-tests-config/source.avsc"));
GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(schema);
List<String> insertRowKeys = new ArrayList<>();
List<String> updateRowKeys = new ArrayList<>();
@@ -98,7 +100,7 @@ public class TestGenericRecordPayloadGenerator {
@Test
public void testSimplePayloadWithLargeMinSize() throws Exception {
Schema schema = new Schema.Parser().parse(getClass().getClassLoader()
- .getResourceAsStream("hudi-bench-config/source.avsc"));
+ .getResourceAsStream("hudi-end-to-end-tests-config/source.avsc"));
int minPayloadSize = 1000;
GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(schema,
minPayloadSize);
@@ -110,7 +112,7 @@ public class TestGenericRecordPayloadGenerator {
@Test
public void testComplexPayloadWithLargeMinSize() throws Exception {
Schema schema = new Schema.Parser().parse(getClass().getClassLoader()
- .getResourceAsStream("hudi-bench-config/complex-source.avsc"));
+ .getResourceAsStream("hudi-end-to-end-tests-config/complex-source.avsc"));
int minPayloadSize = 10000;
GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(
schema, minPayloadSize);
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestWorkloadGenerator.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestWorkloadGenerator.java
similarity index 96%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestWorkloadGenerator.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestWorkloadGenerator.java
index 5f10005..1c1d26b 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestWorkloadGenerator.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestWorkloadGenerator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.generator;///*
+package org.apache.hudi.e2e.generator;///*
// * Copyright (c) 2019 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
// *
// * Licensed under the Apache License, Version 2.0 (the "License");
@@ -34,22 +34,22 @@ package org.apache.hudi.bench.generator;///*
// *
// */
//
-//package org.apache.hudi.bench.generator;
+//package org.apache.hudi.e2e.generator;
//
//import static junit.framework.TestCase.assertEquals;
//import static junit.framework.TestCase.assertFalse;
//import static org.mockito.Mockito.when;
//
-//import org.apache.hudi.bench.configuration.DeltaConfig.Config;
-//import org.apache.hudi.bench.input.config.DFSDeltaConfig;
-//import org.apache.hudi.bench.input.config.DeltaConfig;
+//import org.apache.hudi.e2e.configuration.DeltaConfig.Config;
+//import org.apache.hudi.e2e.input.config.DFSDeltaConfig;
+//import org.apache.hudi.e2e.input.config.DeltaConfig;
//import org.apache.hudi.common.SerializableConfiguration;
//import org.apache.hudi.common.util.FSUtils;
-//import org.apache.hudi.bench.job.operation.Operation;
-//import org.apache.hudi.bench.input.DeltaInputFormat;
-//import org.apache.hudi.bench.input.DeltaOutputType;
-//import org.apache.hudi.bench.input.reader.DFSAvroDeltaInputReader;
-//import org.apache.hudi.bench.utils.TestUtils;
+//import org.apache.hudi.e2e.job.operation.Operation;
+//import org.apache.hudi.e2e.input.DeltaInputFormat;
+//import org.apache.hudi.e2e.input.DeltaOutputType;
+//import org.apache.hudi.e2e.input.reader.DFSAvroDeltaInputReader;
+//import org.apache.hudi.e2e.utils.TestUtils;
//import org.apache.hudi.utilities.UtilitiesTestBase;
//import org.apache.hudi.utilities.schema.FilebasedSchemaProvider;
//import java.io.IOException;
@@ -95,7 +95,7 @@ package org.apache.hudi.bench.generator;///*
// @Before
// public void setup() throws Exception {
// super.setup();
-// schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-bench-config/source.avsc"),
+// schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-end-to-end-tests-config/source.avsc"),
// jsc);
// }
//
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/job/TestHoodieTestSuiteJob.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/job/TestHoodieTestSuiteJob.java
similarity index 92%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/job/TestHoodieTestSuiteJob.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/job/TestHoodieTestSuiteJob.java
index d1acd5f..2918003 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/job/TestHoodieTestSuiteJob.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/job/TestHoodieTestSuiteJob.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.job;
+package org.apache.hudi.e2e.job;
import static org.junit.Assert.assertEquals;
@@ -26,13 +26,13 @@ import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.DataSourceWriteOptions;
-import org.apache.hudi.bench.DeltaInputFormat;
-import org.apache.hudi.bench.DeltaOutputType;
-import org.apache.hudi.bench.dag.TestHiveSyncDag;
-import org.apache.hudi.bench.dag.TestInsertOnlyDag;
-import org.apache.hudi.bench.dag.TestInsertUpsertDag;
-import org.apache.hudi.bench.dag.WorkflowDagGenerator;
-import org.apache.hudi.bench.job.HoodieTestSuiteJob.HoodieTestSuiteConfig;
+import org.apache.hudi.e2e.DeltaInputFormat;
+import org.apache.hudi.e2e.DeltaOutputType;
+import org.apache.hudi.e2e.dag.TestHiveSyncDag;
+import org.apache.hudi.e2e.dag.TestInsertOnlyDag;
+import org.apache.hudi.e2e.dag.TestInsertUpsertDag;
+import org.apache.hudi.e2e.dag.WorkflowDagGenerator;
+import org.apache.hudi.e2e.job.HoodieTestSuiteJob.HoodieTestSuiteConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.TypedProperties;
import org.apache.hudi.utilities.UtilitiesTestBase;
@@ -72,9 +72,9 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
public static void initClass() throws Exception {
UtilitiesTestBase.initClass();
// prepare the configs.
- UtilitiesTestBase.Helpers.copyToDFS("hudi-bench-config/base.properties", dfs, dfsBasePath + "/base.properties");
- UtilitiesTestBase.Helpers.copyToDFS("hudi-bench-config/source.avsc", dfs, dfsBasePath + "/source.avsc");
- UtilitiesTestBase.Helpers.copyToDFS("hudi-bench-config/target.avsc", dfs, dfsBasePath + "/target.avsc");
+ UtilitiesTestBase.Helpers.copyToDFS("hudi-end-to-end-tests-config/base.properties", dfs, dfsBasePath + "/base.properties");
+ UtilitiesTestBase.Helpers.copyToDFS("hudi-end-to-end-tests-config/source.avsc", dfs, dfsBasePath + "/source.avsc");
+ UtilitiesTestBase.Helpers.copyToDFS("hudi-end-to-end-tests-config/target.avsc", dfs, dfsBasePath + "/target.avsc");
TypedProperties props = new TypedProperties();
props.setProperty("hoodie.datasource.write.recordkey.field", "_row_key");
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSAvroDeltaInputReader.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSAvroDeltaInputReader.java
similarity index 96%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSAvroDeltaInputReader.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSAvroDeltaInputReader.java
index 9cfb70f..1529cbc 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSAvroDeltaInputReader.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSAvroDeltaInputReader.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.reader;
+package org.apache.hudi.e2e.reader;
import static junit.framework.Assert.assertTrue;
import static junit.framework.TestCase.assertEquals;
@@ -26,7 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hudi.bench.utils.TestUtils;
+import org.apache.hudi.e2e.utils.TestUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.utilities.UtilitiesTestBase;
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSHoodieDatasetInputReader.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSHoodieDatasetInputReader.java
similarity index 96%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSHoodieDatasetInputReader.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSHoodieDatasetInputReader.java
index dba57b0..db817c6 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSHoodieDatasetInputReader.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSHoodieDatasetInputReader.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.reader;
+package org.apache.hudi.e2e.reader;
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertTrue;
@@ -32,6 +32,7 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.util.HoodieAvroUtils;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.e2e.reader.DFSHoodieDatasetInputReader;
import org.apache.hudi.utilities.UtilitiesTestBase;
import org.apache.hudi.utilities.schema.FilebasedSchemaProvider;
import org.apache.spark.api.java.JavaRDD;
@@ -58,7 +59,7 @@ public class TestDFSHoodieDatasetInputReader extends UtilitiesTestBase {
@Before
public void setup() throws Exception {
super.setup();
- schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-bench-config/complex-source.avsc"),
+ schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-end-to-end-tests-config/complex-source.avsc"),
jsc);
HoodieTestUtils.init(jsc.hadoopConfiguration(), dfsBasePath);
}
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/utils/TestUtils.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/utils/TestUtils.java
similarity index 98%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/utils/TestUtils.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/utils/TestUtils.java
index af2b575..742e3ef 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/utils/TestUtils.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/utils/TestUtils.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.utils;
+package org.apache.hudi.e2e.utils;
import java.util.List;
import org.apache.avro.Schema;
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/writer/TestDeltaWriter.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/writer/TestDeltaWriter.java
similarity index 96%
rename from hudi-bench/src/test/java/org/apache/hudi/bench/writer/TestDeltaWriter.java
rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/writer/TestDeltaWriter.java
index 692f552..3316b4c 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/writer/TestDeltaWriter.java
+++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/writer/TestDeltaWriter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.writer;
+package org.apache.hudi.e2e.writer;
import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertEquals;
@@ -29,10 +29,11 @@ import static org.powermock.api.mockito.PowerMockito.when;
import java.util.Properties;
import org.apache.hudi.HoodieReadClient;
import org.apache.hudi.HoodieWriteClient;
-import org.apache.hudi.bench.job.HoodieDeltaStreamerWrapper;
-import org.apache.hudi.bench.job.HoodieTestSuiteJob.HoodieTestSuiteConfig;
+import org.apache.hudi.e2e.job.HoodieDeltaStreamerWrapper;
+import org.apache.hudi.e2e.job.HoodieTestSuiteJob.HoodieTestSuiteConfig;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.e2e.writer.DeltaWriter;
import org.apache.spark.api.java.JavaSparkContext;
import org.junit.Test;
import org.junit.runner.RunWith;
diff --git a/hudi-bench/src/test/resources/hudi-bench-config/base.properties b/hudi-end-to-end-tests/src/test/resources/hudi-bench-config/base.properties
similarity index 100%
rename from hudi-bench/src/test/resources/hudi-bench-config/base.properties
rename to hudi-end-to-end-tests/src/test/resources/hudi-bench-config/base.properties
diff --git a/hudi-bench/src/test/resources/hudi-bench-config/complex-source.avsc b/hudi-end-to-end-tests/src/test/resources/hudi-bench-config/complex-source.avsc
similarity index 100%
rename from hudi-bench/src/test/resources/hudi-bench-config/complex-source.avsc
rename to hudi-end-to-end-tests/src/test/resources/hudi-bench-config/complex-source.avsc
diff --git a/hudi-bench/src/test/resources/hudi-bench-config/complex-workflow-dag-cow.yaml b/hudi-end-to-end-tests/src/test/resources/hudi-bench-config/complex-workflow-dag-cow.yaml
similarity index 100%
rename from hudi-bench/src/test/resources/hudi-bench-config/complex-workflow-dag-cow.yaml
rename to hudi-end-to-end-tests/src/test/resources/hudi-bench-config/complex-workflow-dag-cow.yaml
diff --git a/hudi-bench/src/test/resources/hudi-bench-config/source.avsc b/hudi-end-to-end-tests/src/test/resources/hudi-bench-config/source.avsc
similarity index 100%
rename from hudi-bench/src/test/resources/hudi-bench-config/source.avsc
rename to hudi-end-to-end-tests/src/test/resources/hudi-bench-config/source.avsc
diff --git a/hudi-bench/src/test/resources/hudi-bench-config/target.avsc b/hudi-end-to-end-tests/src/test/resources/hudi-bench-config/target.avsc
similarity index 100%
rename from hudi-bench/src/test/resources/hudi-bench-config/target.avsc
rename to hudi-end-to-end-tests/src/test/resources/hudi-bench-config/target.avsc
diff --git a/hudi-bench/src/test/resources/log4j-surefire-quiet.properties b/hudi-end-to-end-tests/src/test/resources/log4j-surefire-quiet.properties
similarity index 100%
rename from hudi-bench/src/test/resources/log4j-surefire-quiet.properties
rename to hudi-end-to-end-tests/src/test/resources/log4j-surefire-quiet.properties
diff --git a/hudi-bench/src/test/resources/log4j-surefire.properties b/hudi-end-to-end-tests/src/test/resources/log4j-surefire.properties
similarity index 100%
rename from hudi-bench/src/test/resources/log4j-surefire.properties
rename to hudi-end-to-end-tests/src/test/resources/log4j-surefire.properties
diff --git a/packaging/hudi-bench-bundle/pom.xml b/packaging/hudi-end-to-end-tests-bundle/pom.xml
similarity index 99%
rename from packaging/hudi-bench-bundle/pom.xml
rename to packaging/hudi-end-to-end-tests-bundle/pom.xml
index 60c12bd..07dfb20 100644
--- a/packaging/hudi-bench-bundle/pom.xml
+++ b/packaging/hudi-end-to-end-tests-bundle/pom.xml
@@ -24,7 +24,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>hudi-bench-bundle</artifactId>
+ <artifactId>hudi-end-to-end-tests-bundle</artifactId>
<packaging>jar</packaging>
<properties>
@@ -72,7 +72,7 @@
<include>org.apache.hudi:hudi-hive</include>
<include>org.apache.hudi:hudi-hadoop-mr</include>
<include>org.apache.hudi:hudi-timeline-service</include>
- <include>org.apache.hudi:hudi-bench</include>
+ <include>org.apache.hudi:hudi-end-to-end-tests</include>
<include>com.beust:jcommander</include>
<include>com.twitter:bijection-avro_2.11</include>
<include>com.twitter:bijection-core_2.11</include>
@@ -396,7 +396,7 @@
<dependency>
<groupId>org.apache.hudi</groupId>
- <artifactId>hudi-bench</artifactId>
+ <artifactId>hudi-end-to-end-tests</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/packaging/hudi-bench-bundle/src/main/java/org/apache/hudi/bench/bundle/Main.java b/packaging/hudi-end-to-end-tests-bundle/src/main/java/org/apache/hudi/e2e/bundle/Main.java
similarity index 97%
rename from packaging/hudi-bench-bundle/src/main/java/org/apache/hudi/bench/bundle/Main.java
rename to packaging/hudi-end-to-end-tests-bundle/src/main/java/org/apache/hudi/e2e/bundle/Main.java
index 134141e..7e9e573 100644
--- a/packaging/hudi-bench-bundle/src/main/java/org/apache/hudi/bench/bundle/Main.java
+++ b/packaging/hudi-end-to-end-tests-bundle/src/main/java/org/apache/hudi/e2e/bundle/Main.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.bench.bundle;
+package org.apache.hudi.e2e.bundle;
import org.apache.hudi.common.util.ReflectionUtils;