You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gy...@apache.org on 2016/08/25 08:36:33 UTC

[1/2] flink git commit: [FLINK-4441] Make RocksDB backend return null on empty state + add test for all backends

Repository: flink
Updated Branches:
  refs/heads/release-1.1 b87165deb -> 016c381cd


[FLINK-4441] Make RocksDB backend return null on empty state + add test for all backends

Closes #2399


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a0ffa381
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a0ffa381
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a0ffa381

Branch: refs/heads/release-1.1
Commit: a0ffa3814de88a2ba4aea7f9f00836a2130cb01e
Parents: b87165d
Author: Gyula Fora <gy...@apache.org>
Authored: Mon Aug 22 07:38:18 2016 +0200
Committer: Gyula Fora <gy...@apache.org>
Committed: Wed Aug 24 18:02:37 2016 +0200

----------------------------------------------------------------------
 .../streaming/state/RocksDBStateBackend.java    |  4 ++--
 .../runtime/state/StateBackendTestBase.java     | 23 ++++++++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/a0ffa381/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
index 9496d12..2c9a5d2 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
@@ -386,7 +386,7 @@ public class RocksDBStateBackend extends AbstractStateBackend {
 	@Override
 	public HashMap<String, KvStateSnapshot<?, ?, ?, ?, ?>> snapshotPartitionedState(long checkpointId, long timestamp) throws Exception {
 		if (keyValueStatesByName == null || keyValueStatesByName.size() == 0) {
-			return new HashMap<>();
+			return null;
 		}
 
 		if (fullyAsyncBackup) {
@@ -482,7 +482,7 @@ public class RocksDBStateBackend extends AbstractStateBackend {
 
 	@Override
 	public final void injectKeyValueStateSnapshots(HashMap<String, KvStateSnapshot> keyValueStateSnapshots) throws Exception {
-		if (keyValueStateSnapshots.size() == 0) {
+		if (keyValueStateSnapshots == null) {
 			return;
 		}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/a0ffa381/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
index 80f1de3..2f85edf 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/StateBackendTestBase.java
@@ -729,6 +729,29 @@ public abstract class StateBackendTestBase<B extends AbstractStateBackend> {
 		assertFalse(default1 == default2);
 	}
 
+	@Test
+	public void testEmptyStateCheckpointing() {
+		try {
+			DummyEnvironment env = new DummyEnvironment("test", 1, 0);
+			backend.initializeForJob(env, "test_op", IntSerializer.INSTANCE);
+
+			HashMap<String, KvStateSnapshot<?, ?, ?, ?, ?>> snapshot = backend
+					.snapshotPartitionedState(682375462379L, 1);
+			
+			assertNull(snapshot);
+			backend.dispose();
+
+			// Make sure we can restore from empty state
+			backend.initializeForJob(env, "test_op", IntSerializer.INSTANCE);
+			backend.injectKeyValueStateSnapshots((HashMap) snapshot);
+			backend.dispose();
+		}
+		catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+
 	private static class AppendingReduce implements ReduceFunction<String> {
 		@Override
 		public String reduce(String value1, String value2) throws Exception {


[2/2] flink git commit: [FLINK-4471] Use user code classloader to deserialize state descriptor in RocksDb backend

Posted by gy...@apache.org.
[FLINK-4471] Use user code classloader to deserialize state descriptor in RocksDb backend


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/016c381c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/016c381c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/016c381c

Branch: refs/heads/release-1.1
Commit: 016c381cdb5a9d7fdcb03f0d95da1582f6b68783
Parents: a0ffa38
Author: Gyula Fora <gy...@apache.org>
Authored: Wed Aug 24 12:02:51 2016 +0200
Committer: Gyula Fora <gy...@apache.org>
Committed: Wed Aug 24 18:04:34 2016 +0200

----------------------------------------------------------------------
 .../flink/contrib/streaming/state/RocksDBStateBackend.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/016c381c/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
index 2c9a5d2..deba9f9 100644
--- a/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
+++ b/flink-contrib/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/RocksDBStateBackend.java
@@ -61,7 +61,7 @@ import org.apache.flink.runtime.state.filesystem.FsStateBackend;
 import org.apache.flink.runtime.util.SerializableObject;
 import org.apache.flink.streaming.util.HDFSCopyFromLocal;
 import org.apache.flink.streaming.util.HDFSCopyToLocal;
-
+import org.apache.flink.util.InstantiationUtil;
 import org.apache.hadoop.fs.FileSystem;
 
 import org.rocksdb.BackupEngine;
@@ -580,7 +580,7 @@ public class RocksDBStateBackend extends AbstractStateBackend {
 		for (int i = 0; i < numColumns; i++) {
 			byte mappingByte = inputView.readByte();
 
-			ObjectInputStream ooIn = new ObjectInputStream(new DataInputViewStream(inputView));
+			ObjectInputStream ooIn = new InstantiationUtil.ClassLoaderObjectInputStream(new DataInputViewStream(inputView), userCodeClassLoader);
 			StateDescriptor stateDescriptor = (StateDescriptor) ooIn.readObject();
 
 			columnFamilyMapping.put(mappingByte, stateDescriptor);