You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2017/10/27 16:26:24 UTC

[7/8] flink git commit: [hotfix] [checkpoints] fix warnings and make minor improvements to CheckpointCoordinatorTest and SharedStateRegistry

[hotfix] [checkpoints] fix warnings and make minor improvements to CheckpointCoordinatorTest and SharedStateRegistry


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b5e2551f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b5e2551f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b5e2551f

Branch: refs/heads/master
Commit: b5e2551fce7491de073b4448a53634bdc4e9e021
Parents: 990df8f
Author: Stephan Ewen <se...@apache.org>
Authored: Wed Oct 25 13:30:51 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Oct 27 17:37:23 2017 +0200

----------------------------------------------------------------------
 .../runtime/state/SharedStateRegistry.java      |  7 +-
 .../checkpoint/CheckpointCoordinatorTest.java   | 97 ++++++--------------
 2 files changed, 28 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/b5e2551f/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
index 347f30c..24e3d92 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
@@ -43,12 +43,7 @@ public class SharedStateRegistry implements AutoCloseable {
 	private static final Logger LOG = LoggerFactory.getLogger(SharedStateRegistry.class);
 
 	/** A singleton object for the default implementation of a {@link SharedStateRegistryFactory} */
-	public static final SharedStateRegistryFactory DEFAULT_FACTORY = new SharedStateRegistryFactory() {
-		@Override
-		public SharedStateRegistry create(Executor deleteExecutor) {
-			return new SharedStateRegistry(deleteExecutor);
-		}
-	};
+	public static final SharedStateRegistryFactory DEFAULT_FACTORY = SharedStateRegistry::new;
 
 	/** All registered state objects by an artificial key */
 	private final Map<SharedStateRegistryKey, SharedStateRegistry.SharedStateEntry> registeredStates;

http://git-wip-us.apache.org/repos/asf/flink/blob/b5e2551f/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
index 4193c2c..282cb56 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
@@ -45,7 +45,6 @@ import org.apache.flink.runtime.state.KeyedStateHandle;
 import org.apache.flink.runtime.state.OperatorStateHandle;
 import org.apache.flink.runtime.state.PlaceholderStreamStateHandle;
 import org.apache.flink.runtime.state.SharedStateRegistry;
-import org.apache.flink.runtime.state.SharedStateRegistryFactory;
 import org.apache.flink.runtime.state.StateHandleID;
 import org.apache.flink.runtime.state.StreamStateHandle;
 import org.apache.flink.runtime.state.filesystem.FileStateHandle;
@@ -54,7 +53,6 @@ import org.apache.flink.runtime.testutils.CommonTestUtils;
 import org.apache.flink.runtime.testutils.RecoverableCompletedCheckpointStore;
 import org.apache.flink.runtime.util.TestByteStreamStateHandleDeepCompare;
 import org.apache.flink.shaded.guava18.com.google.common.collect.Iterables;
-import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
 import org.apache.flink.util.InstantiationUtil;
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.util.TestLogger;
@@ -63,6 +61,7 @@ import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 import org.mockito.verification.VerificationMode;
@@ -178,7 +177,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
 			ExecutionVertex triggerVertex2 = mockExecutionVertex(
 				triggerAttemptID2,
 				jobVertexID2,
-				Lists.newArrayList(OperatorID.fromJobVertexID(jobVertexID2)),
+				Collections.singletonList(OperatorID.fromJobVertexID(jobVertexID2)),
 				1,
 				1,
 				ExecutionState.FINISHED);
@@ -341,8 +340,6 @@ public class CheckpointCoordinatorTest extends TestLogger {
 			verify(vertex1.getCurrentExecutionAttempt()).triggerCheckpoint(checkpointId, timestamp, CheckpointOptions.forFullCheckpoint());
 			verify(vertex2.getCurrentExecutionAttempt()).triggerCheckpoint(checkpointId, timestamp, CheckpointOptions.forFullCheckpoint());
 
-			CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 0L);
-
 			// acknowledge from one of the tasks
 			coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID2, checkpointId));
 			assertEquals(1, checkpoint.getNumberOfAcknowledgedTasks());
@@ -1342,12 +1339,9 @@ public class CheckpointCoordinatorTest extends TestLogger {
 
 		final BlockingQueue<Long> triggerCalls = new LinkedBlockingQueue<>();
 
-		doAnswer(new Answer<Void>() {
-			@Override
-			public Void answer(InvocationOnMock invocation) throws Throwable {
-				triggerCalls.add((Long) invocation.getArguments()[0]);
-				return null;
-			}
+		doAnswer(invocation -> {
+			triggerCalls.add((Long) invocation.getArguments()[0]);
+			return null;
 		}).when(executionAttempt).triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));
 
 		final long delay = 50;
@@ -1669,20 +1663,14 @@ public class CheckpointCoordinatorTest extends TestLogger {
 
 			final Execution execution = triggerVertex.getCurrentExecutionAttempt();
 
-			doAnswer(new Answer<Void>() {
-				@Override
-				public Void answer(InvocationOnMock invocation) throws Throwable {
-					numCalls.incrementAndGet();
-					return null;
-				}
+			doAnswer(invocation -> {
+				numCalls.incrementAndGet();
+				return null;
 			}).when(execution).triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));
 
-			doAnswer(new Answer<Void>() {
-				@Override
-				public Void answer(InvocationOnMock invocation) throws Throwable {
-					numCalls.incrementAndGet();
-					return null;
-				}
+			doAnswer(invocation -> {
+				numCalls.incrementAndGet();
+				return null;
 			}).when(execution).notifyCheckpointComplete(anyLong(), anyLong());
 
 			CheckpointCoordinator coord = new CheckpointCoordinator(
@@ -1834,13 +1822,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
 			ExecutionVertex commitVertex = mockExecutionVertex(commitAttemptID);
 
 			final AtomicReference<ExecutionState> currentState = new AtomicReference<>(ExecutionState.CREATED);
-			when(triggerVertex.getCurrentExecutionAttempt().getState()).thenAnswer(
-					new Answer<ExecutionState>() {
-						@Override
-						public ExecutionState answer(InvocationOnMock invocation){
-							return currentState.get();
-						}
-					});
+			when(triggerVertex.getCurrentExecutionAttempt().getState()).thenAnswer(invocation -> currentState.get());
 
 			CheckpointCoordinator coord = new CheckpointCoordinator(
 				jid,
@@ -2295,7 +2277,6 @@ public class CheckpointCoordinatorTest extends TestLogger {
 
 		assertTrue(coord.getPendingCheckpoints().keySet().size() == 1);
 		long checkpointId = Iterables.getOnlyElement(coord.getPendingCheckpoints().keySet());
-		CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 0L);
 
 		List<KeyGroupRange> keyGroupPartitions1 =
 				StateAssignmentOperation.createKeyGroupPartitions(maxParallelism1, parallelism1);
@@ -2451,7 +2432,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
 		Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
 
 		//prepare vertex1 state
-		for (Tuple2<JobVertexID, OperatorID> id : Lists.newArrayList(id1, id2)) {
+		for (Tuple2<JobVertexID, OperatorID> id : Arrays.asList(id1, id2)) {
 			OperatorState taskState = new OperatorState(id.f1, parallelism1, maxParallelism1);
 			operatorStates.put(id.f1, taskState);
 			for (int index = 0; index < taskState.getParallelism(); index++) {
@@ -2471,7 +2452,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
 		List<List<ChainedStateHandle<OperatorStateHandle>>> expectedManagedOperatorStates = new ArrayList<>();
 		List<List<ChainedStateHandle<OperatorStateHandle>>> expectedRawOperatorStates = new ArrayList<>();
 		//prepare vertex2 state
-		for (Tuple2<JobVertexID, OperatorID> id : Lists.newArrayList(id3, id4)) {
+		for (Tuple2<JobVertexID, OperatorID> id : Arrays.asList(id3, id4)) {
 			OperatorState operatorState = new OperatorState(id.f1, parallelism2, maxParallelism2);
 			operatorStates.put(id.f1, operatorState);
 			List<ChainedStateHandle<OperatorStateHandle>> expectedManagedOperatorState = new ArrayList<>();
@@ -2526,13 +2507,13 @@ public class CheckpointCoordinatorTest extends TestLogger {
 
 		final ExecutionJobVertex newJobVertex1 = mockExecutionJobVertex(
 			id5.f0,
-			Lists.newArrayList(id2.f1, id1.f1, id5.f1),
+			Arrays.asList(id2.f1, id1.f1, id5.f1),
 			newParallelism1,
 			maxParallelism1);
 
 		final ExecutionJobVertex newJobVertex2 = mockExecutionJobVertex(
 			id3.f0,
-			Lists.newArrayList(id6.f1, id3.f1),
+			Arrays.asList(id6.f1, id3.f1),
 			newParallelism2,
 			maxParallelism2);
 
@@ -2820,10 +2801,8 @@ public class CheckpointCoordinatorTest extends TestLogger {
 		ByteStreamStateHandle allSerializedStatesHandle = new TestByteStreamStateHandleDeepCompare(
 				String.valueOf(UUID.randomUUID()),
 				serializedDataWithOffsets.f0);
-		KeyGroupsStateHandle keyGroupsStateHandle = new KeyGroupsStateHandle(
-				keyGroupRangeOffsets,
-				allSerializedStatesHandle);
-		return keyGroupsStateHandle;
+
+		return new KeyGroupsStateHandle(keyGroupRangeOffsets, allSerializedStatesHandle);
 	}
 
 	public static Tuple2<byte[], List<long[]>> serializeTogetherAndTrackOffsets(
@@ -2861,25 +2840,6 @@ public class CheckpointCoordinatorTest extends TestLogger {
 		return new Tuple2<>(allSerializedValuesConcatenated, offsets);
 	}
 
-	public static StreamStateHandle generateStateForVertex(
-			JobVertexID jobVertexID,
-			int index) throws IOException {
-
-		Random random = new Random(jobVertexID.hashCode() + index);
-		int value = random.nextInt();
-		return generateStreamStateHandle(value);
-	}
-
-	public static StreamStateHandle generateStreamStateHandle(Serializable value) throws IOException {
-		return TestByteStreamStateHandleDeepCompare.fromSerializable(String.valueOf(UUID.randomUUID()), value);
-	}
-
-	public static ChainedStateHandle<StreamStateHandle> generateChainedStateHandle(
-			Serializable value) throws IOException {
-		return ChainedStateHandle.wrapSingleHandle(
-				generateStreamStateHandle(value));
-	}
-
 	public static OperatorStateHandle generatePartitionableStateHandle(
 		JobVertexID jobVertexID,
 		int index,
@@ -3014,7 +2974,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
 		return mockExecutionVertex(
 			attemptID,
 			jobVertexID,
-			Arrays.asList(OperatorID.fromJobVertexID(jobVertexID)),
+			Collections.singletonList(OperatorID.fromJobVertexID(jobVertexID)),
 			1,
 			1,
 			ExecutionState.RUNNING);
@@ -3310,9 +3270,9 @@ public class CheckpointCoordinatorTest extends TestLogger {
 				expectedTotalPartitions += replication * offs.length;
 				List<Long> offsList = new ArrayList<>(offs.length);
 
-				for (int i = 0; i < offs.length; ++i) {
-					for(int p = 0; p < replication; ++p) {
-						offsList.add(offs[i]);
+				for (long off : offs) {
+					for (int p = 0; p < replication; ++p) {
+						offsList.add(off);
 					}
 				}
 				offsMapWithList.put(e.getKey(), offsList);
@@ -3349,8 +3309,8 @@ public class CheckpointCoordinatorTest extends TestLogger {
 						stateToOffsets.put(namedState.getKey(), actualOffs);
 					}
 					long[] add = namedState.getValue().getOffsets();
-					for (int i = 0; i < add.length; ++i) {
-						actualOffs.add(add[i]);
+					for (long l : add) {
+						actualOffs.add(l);
 					}
 
 					partitionCount += namedState.getValue().getOffsets().length;
@@ -3586,14 +3546,11 @@ public class CheckpointCoordinatorTest extends TestLogger {
 			store,
 			null,
 			Executors.directExecutor(),
-			new SharedStateRegistryFactory() {
-				@Override
-				public SharedStateRegistry create(Executor deleteExecutor) {
+				deleteExecutor -> {
 					SharedStateRegistry instance = new SharedStateRegistry(deleteExecutor);
 					createdSharedStateRegistries.add(instance);
 					return instance;
-				}
-			});
+				});
 
 		final int numCheckpoints = 3;
 
@@ -3612,7 +3569,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
 		List<Map<StateHandleID, StreamStateHandle>> sharedHandlesByCheckpoint = new ArrayList<>(numCheckpoints);
 
 		for (int i = 0; i < numCheckpoints; ++i) {
-			sharedHandlesByCheckpoint.add(new HashMap<StateHandleID, StreamStateHandle>(2));
+			sharedHandlesByCheckpoint.add(new HashMap<>(2));
 		}
 
 		int cp = 0;