You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2020/08/15 07:36:30 UTC
[beam] branch master updated: [BEAM-8025] Update tests to use
TemporaryFolder instead of rolling their own implementation.
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f45b39a [BEAM-8025] Update tests to use TemporaryFolder instead of rolling their own implementation.
new ee08991 Merge pull request #12592 from [BEAM-8025] Update tests to use TemporaryFolder instead of rolling their own implementation.
f45b39a is described below
commit f45b39a33a656a80af10b32f5023fc40c659cf0a
Author: Luke Cwik <lc...@google.com>
AuthorDate: Fri Aug 14 19:38:27 2020 -0700
[BEAM-8025] Update tests to use TemporaryFolder instead of rolling their own implementation.
This prevents cross-test contanimation if multiple instances of the same test are executing at the same time on different executors, simplifies the implementation and migrates it to a reliable mechanism.
Example failure:
https://ci-beam.apache.org/job/beam_PreCommit_Java_Phrase/2620/testReport/junit/org.apache.beam.runners.samza.runtime/SamzaTimerInternalsFactoryTest/testProcessingTimeTimers/
---
.../runtime/SamzaTimerInternalsFactoryTest.java | 21 ++++++++++++---------
.../beam/sdk/io/cassandra/CassandraIOTest.java | 22 ++++++++--------------
2 files changed, 20 insertions(+), 23 deletions(-)
diff --git a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaTimerInternalsFactoryTest.java b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaTimerInternalsFactoryTest.java
index d0d6126..02b1cad 100644
--- a/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaTimerInternalsFactoryTest.java
+++ b/runners/samza/src/test/java/org/apache/beam/runners/samza/runtime/SamzaTimerInternalsFactoryTest.java
@@ -24,7 +24,6 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@@ -57,7 +56,9 @@ import org.apache.samza.storage.kv.RocksDbKeyValueStore;
import org.apache.samza.storage.kv.SerializedKeyValueStore;
import org.apache.samza.storage.kv.SerializedKeyValueStoreMetrics;
import org.joda.time.Instant;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.rocksdb.FlushOptions;
import org.rocksdb.Options;
import org.rocksdb.WriteOptions;
@@ -67,13 +68,15 @@ import org.rocksdb.WriteOptions;
* timers.
*/
public class SamzaTimerInternalsFactoryTest {
- private static KeyValueStore<ByteArray, byte[]> createStore(String name) {
+ @Rule public transient TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private KeyValueStore<ByteArray, byte[]> createStore() {
final Options options = new Options();
options.setCreateIfMissing(true);
RocksDbKeyValueStore rocksStore =
new RocksDbKeyValueStore(
- new File(System.getProperty("java.io.tmpdir") + "/" + name),
+ temporaryFolder.getRoot(),
options,
new MapConfig(),
false,
@@ -137,7 +140,7 @@ public class SamzaTimerInternalsFactoryTest {
final SamzaPipelineOptions pipelineOptions =
PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
- final KeyValueStore<ByteArray, byte[]> store = createStore("store1");
+ final KeyValueStore<ByteArray, byte[]> store = createStore();
final SamzaTimerInternalsFactory<String> timerInternalsFactory =
createTimerInternalsFactory(null, "timer", pipelineOptions, store);
@@ -175,7 +178,7 @@ public class SamzaTimerInternalsFactoryTest {
final SamzaPipelineOptions pipelineOptions =
PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
- KeyValueStore<ByteArray, byte[]> store = createStore("store2");
+ KeyValueStore<ByteArray, byte[]> store = createStore();
final SamzaTimerInternalsFactory<String> timerInternalsFactory =
createTimerInternalsFactory(null, "timer", pipelineOptions, store);
@@ -195,7 +198,7 @@ public class SamzaTimerInternalsFactoryTest {
store.close();
// restore by creating a new instance
- store = createStore("store2");
+ store = createStore();
final SamzaTimerInternalsFactory<String> restoredFactory =
createTimerInternalsFactory(null, "timer", pipelineOptions, store);
@@ -220,7 +223,7 @@ public class SamzaTimerInternalsFactoryTest {
final SamzaPipelineOptions pipelineOptions =
PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
- KeyValueStore<ByteArray, byte[]> store = createStore("store3");
+ KeyValueStore<ByteArray, byte[]> store = createStore();
TestTimerRegistry timerRegistry = new TestTimerRegistry();
final SamzaTimerInternalsFactory<String> timerInternalsFactory =
@@ -243,7 +246,7 @@ public class SamzaTimerInternalsFactoryTest {
store.close();
// restore by creating a new instance
- store = createStore("store3");
+ store = createStore();
TestTimerRegistry restoredRegistry = new TestTimerRegistry();
final SamzaTimerInternalsFactory<String> restoredFactory =
createTimerInternalsFactory(restoredRegistry, "timer", pipelineOptions, store);
@@ -264,7 +267,7 @@ public class SamzaTimerInternalsFactoryTest {
final SamzaPipelineOptions pipelineOptions =
PipelineOptionsFactory.create().as(SamzaPipelineOptions.class);
- KeyValueStore<ByteArray, byte[]> store = createStore("store4");
+ KeyValueStore<ByteArray, byte[]> store = createStore();
final SamzaTimerInternalsFactory<String> timerInternalsFactory =
createTimerInternalsFactory(null, "timer", pipelineOptions, store);
diff --git a/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java
index a0e1536..f368fb4 100644
--- a/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java
+++ b/sdks/java/io/cassandra/src/test/java/org/apache/beam/sdk/io/cassandra/CassandraIOTest.java
@@ -41,7 +41,6 @@ import com.datastax.driver.mapping.annotations.PartitionKey;
import com.datastax.driver.mapping.annotations.Table;
import info.archinnov.achilles.embedded.CassandraEmbeddedServerBuilder;
import info.archinnov.achilles.embedded.CassandraShutDownHook;
-import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.math.BigInteger;
@@ -81,13 +80,14 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Objects;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.ListeningExecutorService;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.MoreExecutors;
import org.apache.cassandra.service.StorageServiceMBean;
-import org.apache.commons.io.FileUtils;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.slf4j.Logger;
@@ -111,8 +111,7 @@ public class CassandraIOTest implements Serializable {
private static Cluster cluster;
private static Session session;
- private static final String TEMPORARY_FOLDER =
- System.getProperty("java.io.tmpdir") + "/embedded-cassandra/";
+ @ClassRule public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder();
@Rule public transient TestPipeline pipeline = TestPipeline.create();
private static CassandraShutDownHook shutdownHook;
@@ -120,15 +119,11 @@ public class CassandraIOTest implements Serializable {
public static void beforeClass() throws Exception {
jmxPort = NetworkTestHelper.getAvailableLocalPort();
shutdownHook = new CassandraShutDownHook();
- String data = TEMPORARY_FOLDER + "/data";
- Files.createDirectories(Paths.get(data));
- String commitLog = TEMPORARY_FOLDER + "/commit-log";
- Files.createDirectories(Paths.get(commitLog));
- String cdcRaw = TEMPORARY_FOLDER + "/cdc-raw";
- Files.createDirectories(Paths.get(cdcRaw));
- String hints = TEMPORARY_FOLDER + "/hints";
- Files.createDirectories(Paths.get(hints));
- String savedCache = TEMPORARY_FOLDER + "/saved-cache";
+ String data = TEMPORARY_FOLDER.newFolder("data").getAbsolutePath();
+ String commitLog = TEMPORARY_FOLDER.newFolder("commit-log").getAbsolutePath();
+ String cdcRaw = TEMPORARY_FOLDER.newFolder("cdc-raw").getAbsolutePath();
+ String hints = TEMPORARY_FOLDER.newFolder("hints").getAbsolutePath();
+ String savedCache = TEMPORARY_FOLDER.newFolder("saved-cache").getAbsolutePath();
Files.createDirectories(Paths.get(savedCache));
CassandraEmbeddedServerBuilder builder =
CassandraEmbeddedServerBuilder.builder()
@@ -186,7 +181,6 @@ public class CassandraIOTest implements Serializable {
@AfterClass
public static void afterClass() throws InterruptedException, IOException {
shutdownHook.shutDownNow();
- FileUtils.deleteDirectory(new File(TEMPORARY_FOLDER));
}
private static void insertData() throws Exception {