You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by lh...@apache.org on 2020/03/16 23:57:54 UTC
[samza] branch master updated: SAMZA-2486: Update
TestFileReaderSystemAdmin to use test resource files (#1315)
This is an automated email from the ASF dual-hosted git repository.
lhaiesp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git
The following commit(s) were added to refs/heads/master by this push:
new 9bd3d61 SAMZA-2486: Update TestFileReaderSystemAdmin to use test resource files (#1315)
9bd3d61 is described below
commit 9bd3d616b3699f344e92ccaadb4194f3b4179132
Author: Ke Wu <ke...@icloud.com>
AuthorDate: Mon Mar 16 16:57:48 2020 -0700
SAMZA-2486: Update TestFileReaderSystemAdmin to use test resource files (#1315)
Symptom:
samza-core tests are randomly failing.
Cause:
TestFileReaderSystemAdmin and TestFileReaderSystemConsumer are creating and deleting the same set of files, when runnig in parallel, FileNotFoundException could happen depending on the timing.
Changes:
1. Update TestFileReaderSystemAdmin to use test resource files.
2. Update assert conditions as we removed tailing spaces and newlines in the files.
3. Keep TestFileReaderSystemConsumer as it is because it is also appending data to test files, thus cannot share to use the same test resource file.
Tests:
./gradlew test
API Changes:
None
Upgrade Instructions:
None
Usage Instructions:
None
Co-authored-by: Ke Wu <kw...@linkedin.com>
---
build.gradle | 1 +
samza-core/src/test/resources/empty.txt | 0
samza-core/src/test/resources/moreEnter.txt | 4 ++
samza-core/src/test/resources/noEnter.txt | 1 +
samza-core/src/test/resources/oneEnter.txt | 2 +
samza-core/src/test/resources/twoEnter.txt | 3 +
.../filereader/TestFileReaderSystemAdmin.scala | 71 ++++++++--------------
7 files changed, 38 insertions(+), 44 deletions(-)
diff --git a/build.gradle b/build.gradle
index 16402ec..038c606 100644
--- a/build.gradle
+++ b/build.gradle
@@ -90,6 +90,7 @@ rat {
'README.md',
'RELEASE.md',
'samza-core/src/test/resources/classloader/samza-framework-api-classes.txt',
+ 'samza-core/src/test/resources/*.txt',
'samza-test/src/main/resources/**',
'samza-hdfs/src/main/resources/**',
'samza-hdfs/src/test/resources/**',
diff --git a/samza-core/src/test/resources/empty.txt b/samza-core/src/test/resources/empty.txt
new file mode 100644
index 0000000..e69de29
diff --git a/samza-core/src/test/resources/moreEnter.txt b/samza-core/src/test/resources/moreEnter.txt
new file mode 100644
index 0000000..a275be3
--- /dev/null
+++ b/samza-core/src/test/resources/moreEnter.txt
@@ -0,0 +1,4 @@
+first line
+second line
+third line
+other lines
diff --git a/samza-core/src/test/resources/noEnter.txt b/samza-core/src/test/resources/noEnter.txt
new file mode 100644
index 0000000..6cc3bfa
--- /dev/null
+++ b/samza-core/src/test/resources/noEnter.txt
@@ -0,0 +1 @@
+first line
\ No newline at end of file
diff --git a/samza-core/src/test/resources/oneEnter.txt b/samza-core/src/test/resources/oneEnter.txt
new file mode 100644
index 0000000..9c40573
--- /dev/null
+++ b/samza-core/src/test/resources/oneEnter.txt
@@ -0,0 +1,2 @@
+first line
+second line
\ No newline at end of file
diff --git a/samza-core/src/test/resources/twoEnter.txt b/samza-core/src/test/resources/twoEnter.txt
new file mode 100644
index 0000000..c1d496e
--- /dev/null
+++ b/samza-core/src/test/resources/twoEnter.txt
@@ -0,0 +1,3 @@
+first line
+second line
+other lines
\ No newline at end of file
diff --git a/samza-core/src/test/scala/org/apache/samza/system/filereader/TestFileReaderSystemAdmin.scala b/samza-core/src/test/scala/org/apache/samza/system/filereader/TestFileReaderSystemAdmin.scala
index 8cc593a..96098c2 100644
--- a/samza-core/src/test/scala/org/apache/samza/system/filereader/TestFileReaderSystemAdmin.scala
+++ b/samza-core/src/test/scala/org/apache/samza/system/filereader/TestFileReaderSystemAdmin.scala
@@ -19,60 +19,38 @@
package org.apache.samza.system.filereader
-import java.io.PrintWriter
-import java.io.File
-
import org.apache.samza.system.SystemStreamPartition
import org.apache.samza.Partition
import org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata
import org.junit.Assert._
import org.junit.Test
-import org.junit.Before
-import org.junit.After
import org.scalatest.junit.AssertionsForJUnit
-import scala.collection.mutable.HashMap
import scala.collection.JavaConverters._
+import scala.collection.mutable
class TestFileReaderSystemAdmin extends AssertionsForJUnit {
- val files = List("empty.txt", "noEnter.txt", "oneEnter.txt", "twoEnter.txt", "moreEnter.txt")
-
- @Before
- def createFiles {
- files.foreach(file => {
- val writer = new PrintWriter(new File(file))
- file match {
- case "empty.txt" =>
- case "noEnter.txt" => writer.write("first line")
- case "oneEnter.txt" => writer.write("first line \nsecond line")
- case "twoEnter.txt" => writer.write("first line \nsecond line \nother lines")
- case "moreEnter.txt" => writer.write("first line \nsecond line \nthird line \nother lines \n")
- }
- writer.close
- })
- }
-
- @After
- def deleteFiles {
- files.foreach(file => (new File(file)).delete)
- }
+ val files = List(
+ getClass.getResource("/empty.txt").getPath,
+ getClass.getResource("/noEnter.txt").getPath,
+ getClass.getResource("/oneEnter.txt").getPath,
+ getClass.getResource("/twoEnter.txt").getPath,
+ getClass.getResource("/moreEnter.txt").getPath)
@Test
def testGetOffsetsAfter {
val fileReaderSystemAdmin = new FileReaderSystemAdmin
- val ssp1 = new SystemStreamPartition("file-reader", files(0), new Partition(0))
- val ssp2 = new SystemStreamPartition("file-reader", files(1), new Partition(0))
val ssp3 = new SystemStreamPartition("file-reader", files(2), new Partition(0))
val ssp4 = new SystemStreamPartition("file-reader", files(3), new Partition(0))
val ssp5 = new SystemStreamPartition("file-reader", files(4), new Partition(0))
val offsets: java.util.Map[SystemStreamPartition, String] =
- HashMap(ssp3 -> "0", ssp4 -> "12", ssp5 -> "25").asJava
+ mutable.HashMap(ssp3 -> "0", ssp4 -> "12", ssp5 -> "25").asJava
val afterOffsets = fileReaderSystemAdmin.getOffsetsAfter(offsets)
- assertEquals("12", afterOffsets.get(ssp3))
- assertEquals("25", afterOffsets.get(ssp4))
- assertEquals("37", afterOffsets.get(ssp5))
+ assertEquals("11", afterOffsets.get(ssp3))
+ assertEquals("23", afterOffsets.get(ssp4))
+ assertEquals("34", afterOffsets.get(ssp5))
}
@Test
@@ -81,20 +59,25 @@ class TestFileReaderSystemAdmin extends AssertionsForJUnit {
val allMetadata = fileReaderSystemAdmin.getSystemStreamMetadata(setAsJavaSetConverter(files.toSet).asJava)
val expectedEmpty = new SystemStreamPartitionMetadata(null, null, "0")
val expectedNoEntry = new SystemStreamPartitionMetadata("0", "0", "0")
- val expectedOneEntry = new SystemStreamPartitionMetadata("0", "0", "12")
- val expectedTwoEntry = new SystemStreamPartitionMetadata("0", "12", "25")
- val expectedMoreEntry = new SystemStreamPartitionMetadata("0", "37", "50")
+ val expectedOneEntry = new SystemStreamPartitionMetadata("0", "0", "11")
+ val expectedTwoEntry = new SystemStreamPartitionMetadata("0", "11", "23")
+ val expectedMoreEntry = new SystemStreamPartitionMetadata("0", "34", "46")
allMetadata.asScala.foreach { entry =>
{
- val result = (entry._2).getSystemStreamPartitionMetadata().get(new Partition(0))
- entry._1 match {
- case "empty.txt" => assertEquals(expectedEmpty, result)
- case "noEnter.txt" => assertEquals(expectedNoEntry, result)
- case "oneEnter.txt" => assertEquals(expectedOneEntry, result)
- case "twoEnter.txt" => assertEquals(expectedTwoEntry, result)
- case "moreEnter.txt" => assertEquals(expectedMoreEntry, result)
- }
+ val result = entry._2.getSystemStreamPartitionMetadata.get(new Partition(0))
+ if (entry._1.endsWith("empty.txt")) {
+ assertEquals(expectedEmpty, result)
+ } else if (entry._1.endsWith("noEnter.txt")) {
+ assertEquals(expectedNoEntry, result)
+ } else if (entry._1.endsWith("oneEnter.txt")) {
+ assertEquals(expectedOneEntry, result)
+ } else if (entry._1.endsWith("twoEnter.txt")) {
+ assertEquals(expectedTwoEntry, result)
+ } else if (entry._1.endsWith("moreEnter.txt")) {
+ assertEquals(expectedMoreEntry, result)
+ } else
+ fail()
}
}
}