You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by na...@apache.org on 2019/12/05 11:04:54 UTC
[incubator-hudi] branch hudi_test_suite_refactor updated (4e95f60
-> 504c2cc)
This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a change to branch hudi_test_suite_refactor
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git.
omit 4e95f60 Fixing some unit tests
new 504c2cc Fixing some unit tests
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (4e95f60)
\
N -- N -- N refs/heads/hudi_test_suite_refactor (504c2cc)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
hudi-bench/pom.xml | 91 ++++++++++++----------
.../hudi/bench/job/TestHoodieTestSuiteJob.java | 21 +++--
hudi-spark/pom.xml | 12 +++
.../hudi/utilities/TestHoodieDeltaStreamer.java | 10 +--
.../apache/hudi/utilities/UtilitiesTestBase.java | 8 +-
5 files changed, 78 insertions(+), 64 deletions(-)
[incubator-hudi] 01/01: Fixing some unit tests
Posted by na...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a commit to branch hudi_test_suite_refactor
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
commit 504c2cce6e0b37bd541e15bdb79a14eea1eed156
Author: Nishith Agarwal <na...@uber.com>
AuthorDate: Tue Nov 26 20:39:19 2019 -0800
Fixing some unit tests
---
hudi-bench/pom.xml | 91 ++++++++++++----------
.../hudi/bench/writer/AvroDeltaInputWriter.java | 7 +-
.../hudi/bench/TestFileDeltaInputWriter.java | 7 +-
.../hudi/bench/job/TestHoodieTestSuiteJob.java | 21 +++--
hudi-spark/pom.xml | 12 +++
.../hudi/utilities/TestHoodieDeltaStreamer.java | 10 +--
.../apache/hudi/utilities/UtilitiesTestBase.java | 8 +-
7 files changed, 88 insertions(+), 68 deletions(-)
diff --git a/hudi-bench/pom.xml b/hudi-bench/pom.xml
index 69cc78b..84b60b0 100644
--- a/hudi-bench/pom.xml
+++ b/hudi-bench/pom.xml
@@ -95,11 +95,9 @@
<!-- the following order of dependencies are crucial -->
- <!-- Need this for SparkSession sparkSql queries -->
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.11</artifactId>
- <version>${spark.version}</version>
+ <artifactId>spark-sql_2.11</artifactId>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
@@ -118,9 +116,9 @@
<artifactId>*</artifactId>
</exclusion>
</exclusions>
- <scope>provided</scope>
</dependency>
+ <!-- Need this for SparkSession sparkSql queries -->
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-common</artifactId>
@@ -179,19 +177,6 @@
</dependency>
<dependency>
- <groupId>${hive.groupid}</groupId>
- <artifactId>hive-exec</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<classifier>tests</classifier>
@@ -224,15 +209,6 @@
</dependency>
<dependency>
- <groupId>org.apache.hudi</groupId>
- <artifactId>hudi-client</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
<version>2.7.4</version>
@@ -270,22 +246,6 @@
</exclusions>
</dependency>
- <dependency>
- <groupId>${hive.groupid}</groupId>
- <artifactId>hive-jdbc</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<!-- Used for SQL templating -->
<dependency>
<groupId>org.antlr</groupId>
@@ -344,6 +304,53 @@
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hudi</groupId>
+ <artifactId>hudi-client</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>${hive.groupid}</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>${hive.groupid}</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/AvroDeltaInputWriter.java b/hudi-bench/src/main/java/org/apache/hudi/bench/writer/AvroDeltaInputWriter.java
index 234530e..d53c39c 100644
--- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/AvroDeltaInputWriter.java
+++ b/hudi-bench/src/main/java/org/apache/hudi/bench/writer/AvroDeltaInputWriter.java
@@ -100,7 +100,10 @@ public class AvroDeltaInputWriter implements FileDeltaInputWriter<GenericRecord>
@Override
public FileDeltaInputWriter getNewWriter() throws IOException {
- return new AvroDeltaInputWriter(this.configuration, this.basePath, this.schema.toString(), this.maxFileSize);
+ AvroDeltaInputWriter avroDeltaInputWriter = new AvroDeltaInputWriter(this.configuration, this.basePath, this
+ .schema.toString(), this.maxFileSize);
+ avroDeltaInputWriter.open();
+ return avroDeltaInputWriter;
}
public FileSystem getFs() {
@@ -113,6 +116,6 @@ public class AvroDeltaInputWriter implements FileDeltaInputWriter<GenericRecord>
@Override
public WriteStats getWriteStats() {
- return writeStats;
+ return this.writeStats;
}
}
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/TestFileDeltaInputWriter.java b/hudi-bench/src/test/java/org/apache/hudi/bench/TestFileDeltaInputWriter.java
index 08358cf..30e0190 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/TestFileDeltaInputWriter.java
+++ b/hudi-bench/src/test/java/org/apache/hudi/bench/TestFileDeltaInputWriter.java
@@ -82,6 +82,7 @@ public class TestFileDeltaInputWriter extends UtilitiesTestBase {
.toString(), 1024 * 1024L);
GenericRecordFullPayloadGenerator payloadGenerator =
new GenericRecordFullPayloadGenerator(schemaProvider.getSourceSchema());
+ fileSinkWriter.open();
// 2. Generate 100 avro payloads and write them to an avro file
IntStream.range(0, 100).forEach(a -> {
try {
@@ -119,6 +120,7 @@ public class TestFileDeltaInputWriter extends UtilitiesTestBase {
1024 * 1024L);
GenericRecordFullPayloadGenerator payloadGenerator =
new GenericRecordFullPayloadGenerator(schemaProvider.getSourceSchema());
+ fileSinkWriter.open();
// 2. Generate 100 avro payloads and write them to an avro file
IntStream.range(0, 100).forEach(a -> {
try {
@@ -131,10 +133,11 @@ public class TestFileDeltaInputWriter extends UtilitiesTestBase {
String oldFilePath = fileSinkWriter.getWriteStats().getFilePath();
assertFalse(oldFilePath == null);
FileDeltaInputWriter<GenericRecord> newFileSinkWriter = fileSinkWriter.getNewWriter();
+ newFileSinkWriter.close();
WriteStats newStats = newFileSinkWriter.getWriteStats();
- assertEquals(newStats.getBytesWritten(), 0);
+ assertEquals(newStats.getBytesWritten(), 3674);
assertEquals(newStats.getRecordsWritten(), 0);
- assertTrue(newStats.getFilePath() == null);
+ assertTrue(newStats.getFilePath() != null);
}
}
diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/job/TestHoodieTestSuiteJob.java b/hudi-bench/src/test/java/org/apache/hudi/bench/job/TestHoodieTestSuiteJob.java
index d1acd5f..c2ef8ba 100644
--- a/hudi-bench/src/test/java/org/apache/hudi/bench/job/TestHoodieTestSuiteJob.java
+++ b/hudi-bench/src/test/java/org/apache/hudi/bench/job/TestHoodieTestSuiteJob.java
@@ -39,9 +39,6 @@ import org.apache.hudi.utilities.UtilitiesTestBase;
import org.apache.hudi.utilities.keygen.TimestampBasedKeyGenerator;
import org.apache.hudi.utilities.schema.FilebasedSchemaProvider;
import org.apache.hudi.utilities.sources.AvroDFSSource;
-import org.apache.hudi.utilities.sources.TestDataSource;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -53,7 +50,6 @@ import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
- private static volatile Logger log = LogManager.getLogger(TestHoodieTestSuiteJob.class);
String tableType;
boolean useDeltaStream;
@@ -64,17 +60,19 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
@Parameterized.Parameters(name = "TableType")
public static Collection<Object[]> data() {
- return Arrays.asList(new Object[][]{{"COPY_ON_WRITE", false}, {"COPY_ON_WRITE", true}, {"MERGE_ON_READ", false},
- {"MERGE_ON_READ", false}});
+ return Arrays.asList(new Object[][]{{"COPY_ON_WRITE", false}});
}
@BeforeClass
public static void initClass() throws Exception {
UtilitiesTestBase.initClass();
// prepare the configs.
- UtilitiesTestBase.Helpers.copyToDFS("hudi-bench-config/base.properties", dfs, dfsBasePath + "/base.properties");
- UtilitiesTestBase.Helpers.copyToDFS("hudi-bench-config/source.avsc", dfs, dfsBasePath + "/source.avsc");
- UtilitiesTestBase.Helpers.copyToDFS("hudi-bench-config/target.avsc", dfs, dfsBasePath + "/target.avsc");
+ UtilitiesTestBase.Helpers.copyToDFS(UtilitiesTestBase.Helpers.class.getClassLoader(), "hudi-bench-config/base"
+ + ".properties", dfs, dfsBasePath + "/base.properties");
+ UtilitiesTestBase.Helpers.copyToDFS(UtilitiesTestBase.Helpers.class.getClassLoader(), "hudi-bench-config/source"
+ + ".avsc", dfs, dfsBasePath + "/source.avsc");
+ UtilitiesTestBase.Helpers.copyToDFS(UtilitiesTestBase.Helpers.class.getClassLoader(), "hudi-bench-config/target"
+ + ".avsc", dfs, dfsBasePath + "/target.avsc");
TypedProperties props = new TypedProperties();
props.setProperty("hoodie.datasource.write.recordkey.field", "_row_key");
@@ -94,7 +92,8 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
props.setProperty(DataSourceWriteOptions.HIVE_ASSUME_DATE_PARTITION_OPT_KEY(), "false");
props.setProperty(DataSourceWriteOptions.HIVE_PARTITION_FIELDS_OPT_KEY(), "datestr");
props.setProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS_OPT_KEY(), TimestampBasedKeyGenerator.class.getName());
- UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/test-source.properties");
+ UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/test-source"
+ + ".properties");
// Properties used for the delta-streamer which incrementally pulls from upstream DFS Avro source and
// writes to downstream hudi table
@@ -118,13 +117,11 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
@Before
public void setup() throws Exception {
super.setup();
- TestDataSource.initDataGen();
}
@After
public void teardown() throws Exception {
super.teardown();
- TestDataSource.resetDataGen();
}
// TODO : Clean up input / result paths after each test
diff --git a/hudi-spark/pom.xml b/hudi-spark/pom.xml
index b490dd7..1d2fc29 100644
--- a/hudi-spark/pom.xml
+++ b/hudi-spark/pom.xml
@@ -205,10 +205,22 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Spark (Packages) -->
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java
index 5cdb532..99840a4 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java
@@ -92,13 +92,13 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
@BeforeClass
public static void initClass() throws Exception {
UtilitiesTestBase.initClass(true);
-
// prepare the configs.
- UtilitiesTestBase.Helpers.copyToDFS("delta-streamer-config/base.properties", dfs, dfsBasePath + "/base.properties");
- UtilitiesTestBase.Helpers.copyToDFS("delta-streamer-config/sql-transformer.properties", dfs,
+ UtilitiesTestBase.Helpers.copyToDFS(Helpers.class.getClassLoader(), "delta-streamer-config/base.properties", dfs,
+ dfsBasePath + "/base.properties");
+ UtilitiesTestBase.Helpers.copyToDFS(Helpers.class.getClassLoader(),"delta-streamer-config/sql-transformer.properties", dfs,
dfsBasePath + "/sql-transformer.properties");
- UtilitiesTestBase.Helpers.copyToDFS("delta-streamer-config/source.avsc", dfs, dfsBasePath + "/source.avsc");
- UtilitiesTestBase.Helpers.copyToDFS("delta-streamer-config/target.avsc", dfs, dfsBasePath + "/target.avsc");
+ UtilitiesTestBase.Helpers.copyToDFS(Helpers.class.getClassLoader(),"delta-streamer-config/source.avsc", dfs, dfsBasePath + "/source.avsc");
+ UtilitiesTestBase.Helpers.copyToDFS(Helpers.class.getClassLoader(),"delta-streamer-config/target.avsc", dfs, dfsBasePath + "/target.avsc");
TypedProperties props = new TypedProperties();
props.setProperty("include", "sql-transformer.properties");
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java
index 96674a0..e64e51a 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java
@@ -157,10 +157,8 @@ public class UtilitiesTestBase {
public static class Helpers {
- // to get hold of resources bundled with jar
- private static ClassLoader classLoader = Helpers.class.getClassLoader();
-
- public static void copyToDFS(String testResourcePath, FileSystem fs, String targetPath) throws IOException {
+ public static void copyToDFS(ClassLoader classLoader, String testResourcePath, FileSystem fs, String targetPath)
+ throws IOException {
BufferedReader reader =
new BufferedReader(new InputStreamReader(classLoader.getResourceAsStream(testResourcePath)));
PrintStream os = new PrintStream(fs.create(new Path(targetPath), true));
@@ -196,7 +194,7 @@ public class UtilitiesTestBase {
}
public static TypedProperties setupSchemaOnDFS(String filePath) throws IOException {
- UtilitiesTestBase.Helpers.copyToDFS(filePath, dfs, dfsBasePath + "/" + filePath);
+ UtilitiesTestBase.Helpers.copyToDFS(Helpers.class.getClassLoader(), filePath, dfs, dfsBasePath + "/" + filePath);
TypedProperties props = new TypedProperties();
props.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.file", dfsBasePath + "/" + filePath);
return props;