You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sr...@apache.org on 2018/01/23 13:05:36 UTC
[3/3] flink git commit: [FLINK-8469] Unify RocksDB params in
RocksDBPerformanceTest
[FLINK-8469] Unify RocksDB params in RocksDBPerformanceTest
This closes #5325.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1c9c1e36
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1c9c1e36
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1c9c1e36
Branch: refs/heads/master
Commit: 1c9c1e36c1dd7b3f2a160216e405302d7854c148
Parents: e157cfa
Author: Bowen Li <bo...@gmail.com>
Authored: Sun Jan 21 01:23:06 2018 +0100
Committer: Stefan Richter <s....@data-artisans.com>
Committed: Tue Jan 23 14:05:08 2018 +0100
----------------------------------------------------------------------
.../state/benchmark/RocksDBPerformanceTest.java | 90 +++++++++-----------
1 file changed, 40 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1c9c1e36/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
index 533196e..e05e7ae 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/test/java/org/apache/flink/contrib/streaming/state/benchmark/RocksDBPerformanceTest.java
@@ -24,6 +24,8 @@ import org.apache.flink.testutils.junit.RetryOnFailure;
import org.apache.flink.testutils.junit.RetryRule;
import org.apache.flink.util.TestLogger;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -36,6 +38,7 @@ import org.rocksdb.WriteOptions;
import sun.misc.Unsafe;
import java.io.File;
+import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
@@ -51,37 +54,43 @@ public class RocksDBPerformanceTest extends TestLogger {
@Rule
public final RetryRule retry = new RetryRule();
- @Test(timeout = 2000)
- @RetryOnFailure(times = 3)
- public void testRocksDbMergePerformance() throws Exception {
- final File rocksDir = tmp.newFolder();
+ private File rocksDir;
+ private Options options;
+ private WriteOptions writeOptions;
- // ensure the RocksDB library is loaded to a distinct location each retry
- NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());
+ private final String key = "key";
+ private final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
- final String key = "key";
- final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
+ private final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
+ private final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
- final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
- final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
+ @Before
+ public void init() throws IOException {
+ rocksDir = tmp.newFolder();
- final int num = 50000;
+ // ensure the RocksDB library is loaded to a distinct location each retry
+ NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());
- try (
- final Options options = new Options()
- .setCompactionStyle(CompactionStyle.LEVEL)
- .setLevelCompactionDynamicLevelBytes(true)
- .setIncreaseParallelism(4)
- .setUseFsync(false)
- .setMaxOpenFiles(-1)
- .setCreateIfMissing(true)
- .setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);
+ options = new Options()
+ .setCompactionStyle(CompactionStyle.LEVEL)
+ .setLevelCompactionDynamicLevelBytes(true)
+ .setIncreaseParallelism(4)
+ .setUseFsync(false)
+ .setMaxOpenFiles(-1)
+ .setCreateIfMissing(true)
+ .setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);
+
+ writeOptions = new WriteOptions()
+ .setSync(false)
+ .setDisableWAL(true);
+ }
- final WriteOptions writeOptions = new WriteOptions()
- .setSync(false)
- .setDisableWAL(true);
+ @Test(timeout = 2000)
+ @RetryOnFailure(times = 3)
+ public void testRocksDbMergePerformance() throws Exception {
+ final int num = 50000;
- final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {
+ try (RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {
// ----- insert -----
log.info("begin insert");
@@ -131,34 +140,9 @@ public class RocksDBPerformanceTest extends TestLogger {
@Test(timeout = 2000)
@RetryOnFailure(times = 3)
public void testRocksDbRangeGetPerformance() throws Exception {
- final File rocksDir = tmp.newFolder();
-
- // ensure the RocksDB library is loaded to a distinct location each retry
- NativeLibraryLoader.getInstance().loadLibrary(rocksDir.getAbsolutePath());
-
- final String key = "key";
- final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
-
- final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
- final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
-
final int num = 50000;
- try (
- final Options options = new Options()
- .setCompactionStyle(CompactionStyle.LEVEL)
- .setLevelCompactionDynamicLevelBytes(true)
- .setIncreaseParallelism(4)
- .setUseFsync(false)
- .setMaxOpenFiles(-1)
- .setCreateIfMissing(true)
- .setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);
-
- final WriteOptions writeOptions = new WriteOptions()
- .setSync(false)
- .setDisableWAL(true);
-
- final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {
+ try (RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath())) {
final byte[] keyTemplate = Arrays.copyOf(keyBytes, keyBytes.length + 4);
@@ -211,4 +195,10 @@ public class RocksDBPerformanceTest extends TestLogger {
return true;
}
+
+ @After
+ public void close() {
+ options.close();
+ writeOptions.close();
+ }
}