You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2017/10/27 16:26:24 UTC
[7/8] flink git commit: [hotfix] [checkpoints] fix warnings and make
minor improvements to CheckpointCoordinatorTest and SharedStateRegistry
[hotfix] [checkpoints] fix warnings and make minor improvements to CheckpointCoordinatorTest and SharedStateRegistry
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b5e2551f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b5e2551f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b5e2551f
Branch: refs/heads/master
Commit: b5e2551fce7491de073b4448a53634bdc4e9e021
Parents: 990df8f
Author: Stephan Ewen <se...@apache.org>
Authored: Wed Oct 25 13:30:51 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Oct 27 17:37:23 2017 +0200
----------------------------------------------------------------------
.../runtime/state/SharedStateRegistry.java | 7 +-
.../checkpoint/CheckpointCoordinatorTest.java | 97 ++++++--------------
2 files changed, 28 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b5e2551f/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
index 347f30c..24e3d92 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/SharedStateRegistry.java
@@ -43,12 +43,7 @@ public class SharedStateRegistry implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(SharedStateRegistry.class);
/** A singleton object for the default implementation of a {@link SharedStateRegistryFactory} */
- public static final SharedStateRegistryFactory DEFAULT_FACTORY = new SharedStateRegistryFactory() {
- @Override
- public SharedStateRegistry create(Executor deleteExecutor) {
- return new SharedStateRegistry(deleteExecutor);
- }
- };
+ public static final SharedStateRegistryFactory DEFAULT_FACTORY = SharedStateRegistry::new;
/** All registered state objects by an artificial key */
private final Map<SharedStateRegistryKey, SharedStateRegistry.SharedStateEntry> registeredStates;
http://git-wip-us.apache.org/repos/asf/flink/blob/b5e2551f/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
index 4193c2c..282cb56 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
@@ -45,7 +45,6 @@ import org.apache.flink.runtime.state.KeyedStateHandle;
import org.apache.flink.runtime.state.OperatorStateHandle;
import org.apache.flink.runtime.state.PlaceholderStreamStateHandle;
import org.apache.flink.runtime.state.SharedStateRegistry;
-import org.apache.flink.runtime.state.SharedStateRegistryFactory;
import org.apache.flink.runtime.state.StateHandleID;
import org.apache.flink.runtime.state.StreamStateHandle;
import org.apache.flink.runtime.state.filesystem.FileStateHandle;
@@ -54,7 +53,6 @@ import org.apache.flink.runtime.testutils.CommonTestUtils;
import org.apache.flink.runtime.testutils.RecoverableCompletedCheckpointStore;
import org.apache.flink.runtime.util.TestByteStreamStateHandleDeepCompare;
import org.apache.flink.shaded.guava18.com.google.common.collect.Iterables;
-import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.TestLogger;
@@ -63,6 +61,7 @@ import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.mockito.verification.VerificationMode;
@@ -178,7 +177,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
ExecutionVertex triggerVertex2 = mockExecutionVertex(
triggerAttemptID2,
jobVertexID2,
- Lists.newArrayList(OperatorID.fromJobVertexID(jobVertexID2)),
+ Collections.singletonList(OperatorID.fromJobVertexID(jobVertexID2)),
1,
1,
ExecutionState.FINISHED);
@@ -341,8 +340,6 @@ public class CheckpointCoordinatorTest extends TestLogger {
verify(vertex1.getCurrentExecutionAttempt()).triggerCheckpoint(checkpointId, timestamp, CheckpointOptions.forFullCheckpoint());
verify(vertex2.getCurrentExecutionAttempt()).triggerCheckpoint(checkpointId, timestamp, CheckpointOptions.forFullCheckpoint());
- CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 0L);
-
// acknowledge from one of the tasks
coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, attemptID2, checkpointId));
assertEquals(1, checkpoint.getNumberOfAcknowledgedTasks());
@@ -1342,12 +1339,9 @@ public class CheckpointCoordinatorTest extends TestLogger {
final BlockingQueue<Long> triggerCalls = new LinkedBlockingQueue<>();
- doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- triggerCalls.add((Long) invocation.getArguments()[0]);
- return null;
- }
+ doAnswer(invocation -> {
+ triggerCalls.add((Long) invocation.getArguments()[0]);
+ return null;
}).when(executionAttempt).triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));
final long delay = 50;
@@ -1669,20 +1663,14 @@ public class CheckpointCoordinatorTest extends TestLogger {
final Execution execution = triggerVertex.getCurrentExecutionAttempt();
- doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- numCalls.incrementAndGet();
- return null;
- }
+ doAnswer(invocation -> {
+ numCalls.incrementAndGet();
+ return null;
}).when(execution).triggerCheckpoint(anyLong(), anyLong(), any(CheckpointOptions.class));
- doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- numCalls.incrementAndGet();
- return null;
- }
+ doAnswer(invocation -> {
+ numCalls.incrementAndGet();
+ return null;
}).when(execution).notifyCheckpointComplete(anyLong(), anyLong());
CheckpointCoordinator coord = new CheckpointCoordinator(
@@ -1834,13 +1822,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
ExecutionVertex commitVertex = mockExecutionVertex(commitAttemptID);
final AtomicReference<ExecutionState> currentState = new AtomicReference<>(ExecutionState.CREATED);
- when(triggerVertex.getCurrentExecutionAttempt().getState()).thenAnswer(
- new Answer<ExecutionState>() {
- @Override
- public ExecutionState answer(InvocationOnMock invocation){
- return currentState.get();
- }
- });
+ when(triggerVertex.getCurrentExecutionAttempt().getState()).thenAnswer(invocation -> currentState.get());
CheckpointCoordinator coord = new CheckpointCoordinator(
jid,
@@ -2295,7 +2277,6 @@ public class CheckpointCoordinatorTest extends TestLogger {
assertTrue(coord.getPendingCheckpoints().keySet().size() == 1);
long checkpointId = Iterables.getOnlyElement(coord.getPendingCheckpoints().keySet());
- CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 0L);
List<KeyGroupRange> keyGroupPartitions1 =
StateAssignmentOperation.createKeyGroupPartitions(maxParallelism1, parallelism1);
@@ -2451,7 +2432,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
//prepare vertex1 state
- for (Tuple2<JobVertexID, OperatorID> id : Lists.newArrayList(id1, id2)) {
+ for (Tuple2<JobVertexID, OperatorID> id : Arrays.asList(id1, id2)) {
OperatorState taskState = new OperatorState(id.f1, parallelism1, maxParallelism1);
operatorStates.put(id.f1, taskState);
for (int index = 0; index < taskState.getParallelism(); index++) {
@@ -2471,7 +2452,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
List<List<ChainedStateHandle<OperatorStateHandle>>> expectedManagedOperatorStates = new ArrayList<>();
List<List<ChainedStateHandle<OperatorStateHandle>>> expectedRawOperatorStates = new ArrayList<>();
//prepare vertex2 state
- for (Tuple2<JobVertexID, OperatorID> id : Lists.newArrayList(id3, id4)) {
+ for (Tuple2<JobVertexID, OperatorID> id : Arrays.asList(id3, id4)) {
OperatorState operatorState = new OperatorState(id.f1, parallelism2, maxParallelism2);
operatorStates.put(id.f1, operatorState);
List<ChainedStateHandle<OperatorStateHandle>> expectedManagedOperatorState = new ArrayList<>();
@@ -2526,13 +2507,13 @@ public class CheckpointCoordinatorTest extends TestLogger {
final ExecutionJobVertex newJobVertex1 = mockExecutionJobVertex(
id5.f0,
- Lists.newArrayList(id2.f1, id1.f1, id5.f1),
+ Arrays.asList(id2.f1, id1.f1, id5.f1),
newParallelism1,
maxParallelism1);
final ExecutionJobVertex newJobVertex2 = mockExecutionJobVertex(
id3.f0,
- Lists.newArrayList(id6.f1, id3.f1),
+ Arrays.asList(id6.f1, id3.f1),
newParallelism2,
maxParallelism2);
@@ -2820,10 +2801,8 @@ public class CheckpointCoordinatorTest extends TestLogger {
ByteStreamStateHandle allSerializedStatesHandle = new TestByteStreamStateHandleDeepCompare(
String.valueOf(UUID.randomUUID()),
serializedDataWithOffsets.f0);
- KeyGroupsStateHandle keyGroupsStateHandle = new KeyGroupsStateHandle(
- keyGroupRangeOffsets,
- allSerializedStatesHandle);
- return keyGroupsStateHandle;
+
+ return new KeyGroupsStateHandle(keyGroupRangeOffsets, allSerializedStatesHandle);
}
public static Tuple2<byte[], List<long[]>> serializeTogetherAndTrackOffsets(
@@ -2861,25 +2840,6 @@ public class CheckpointCoordinatorTest extends TestLogger {
return new Tuple2<>(allSerializedValuesConcatenated, offsets);
}
- public static StreamStateHandle generateStateForVertex(
- JobVertexID jobVertexID,
- int index) throws IOException {
-
- Random random = new Random(jobVertexID.hashCode() + index);
- int value = random.nextInt();
- return generateStreamStateHandle(value);
- }
-
- public static StreamStateHandle generateStreamStateHandle(Serializable value) throws IOException {
- return TestByteStreamStateHandleDeepCompare.fromSerializable(String.valueOf(UUID.randomUUID()), value);
- }
-
- public static ChainedStateHandle<StreamStateHandle> generateChainedStateHandle(
- Serializable value) throws IOException {
- return ChainedStateHandle.wrapSingleHandle(
- generateStreamStateHandle(value));
- }
-
public static OperatorStateHandle generatePartitionableStateHandle(
JobVertexID jobVertexID,
int index,
@@ -3014,7 +2974,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
return mockExecutionVertex(
attemptID,
jobVertexID,
- Arrays.asList(OperatorID.fromJobVertexID(jobVertexID)),
+ Collections.singletonList(OperatorID.fromJobVertexID(jobVertexID)),
1,
1,
ExecutionState.RUNNING);
@@ -3310,9 +3270,9 @@ public class CheckpointCoordinatorTest extends TestLogger {
expectedTotalPartitions += replication * offs.length;
List<Long> offsList = new ArrayList<>(offs.length);
- for (int i = 0; i < offs.length; ++i) {
- for(int p = 0; p < replication; ++p) {
- offsList.add(offs[i]);
+ for (long off : offs) {
+ for (int p = 0; p < replication; ++p) {
+ offsList.add(off);
}
}
offsMapWithList.put(e.getKey(), offsList);
@@ -3349,8 +3309,8 @@ public class CheckpointCoordinatorTest extends TestLogger {
stateToOffsets.put(namedState.getKey(), actualOffs);
}
long[] add = namedState.getValue().getOffsets();
- for (int i = 0; i < add.length; ++i) {
- actualOffs.add(add[i]);
+ for (long l : add) {
+ actualOffs.add(l);
}
partitionCount += namedState.getValue().getOffsets().length;
@@ -3586,14 +3546,11 @@ public class CheckpointCoordinatorTest extends TestLogger {
store,
null,
Executors.directExecutor(),
- new SharedStateRegistryFactory() {
- @Override
- public SharedStateRegistry create(Executor deleteExecutor) {
+ deleteExecutor -> {
SharedStateRegistry instance = new SharedStateRegistry(deleteExecutor);
createdSharedStateRegistries.add(instance);
return instance;
- }
- });
+ });
final int numCheckpoints = 3;
@@ -3612,7 +3569,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
List<Map<StateHandleID, StreamStateHandle>> sharedHandlesByCheckpoint = new ArrayList<>(numCheckpoints);
for (int i = 0; i < numCheckpoints; ++i) {
- sharedHandlesByCheckpoint.add(new HashMap<StateHandleID, StreamStateHandle>(2));
+ sharedHandlesByCheckpoint.add(new HashMap<>(2));
}
int cp = 0;