You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by uc...@apache.org on 2017/01/10 08:48:45 UTC
[09/11] flink git commit: [FLINK-4410] [runtime] Rework checkpoint
stats tracking
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java
new file mode 100644
index 0000000..cf1e7f7
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsCountsTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class CheckpointStatsCountsTest {
+
+ /**
+ * Tests that counts are reported correctly.
+ */
+ @Test
+ public void testCounts() throws Exception {
+ CheckpointStatsCounts counts = new CheckpointStatsCounts();
+ assertEquals(0, counts.getNumberOfRestoredCheckpoints());
+ assertEquals(0, counts.getTotalNumberOfCheckpoints());
+ assertEquals(0, counts.getNumberOfInProgressCheckpoints());
+ assertEquals(0, counts.getNumberOfCompletedCheckpoints());
+ assertEquals(0, counts.getNumberOfFailedCheckpoints());
+
+ counts.incrementRestoredCheckpoints();
+ assertEquals(1, counts.getNumberOfRestoredCheckpoints());
+ assertEquals(0, counts.getTotalNumberOfCheckpoints());
+ assertEquals(0, counts.getNumberOfInProgressCheckpoints());
+ assertEquals(0, counts.getNumberOfCompletedCheckpoints());
+ assertEquals(0, counts.getNumberOfFailedCheckpoints());
+
+ // 1st checkpoint
+ counts.incrementInProgressCheckpoints();
+ assertEquals(1, counts.getNumberOfRestoredCheckpoints());
+ assertEquals(1, counts.getTotalNumberOfCheckpoints());
+ assertEquals(1, counts.getNumberOfInProgressCheckpoints());
+ assertEquals(0, counts.getNumberOfCompletedCheckpoints());
+ assertEquals(0, counts.getNumberOfFailedCheckpoints());
+
+ counts.incrementCompletedCheckpoints();
+ assertEquals(1, counts.getNumberOfRestoredCheckpoints());
+ assertEquals(1, counts.getTotalNumberOfCheckpoints());
+ assertEquals(0, counts.getNumberOfInProgressCheckpoints());
+ assertEquals(1, counts.getNumberOfCompletedCheckpoints());
+ assertEquals(0, counts.getNumberOfFailedCheckpoints());
+
+ // 2nd checkpoint
+ counts.incrementInProgressCheckpoints();
+ assertEquals(1, counts.getNumberOfRestoredCheckpoints());
+ assertEquals(2, counts.getTotalNumberOfCheckpoints());
+ assertEquals(1, counts.getNumberOfInProgressCheckpoints());
+ assertEquals(1, counts.getNumberOfCompletedCheckpoints());
+ assertEquals(0, counts.getNumberOfFailedCheckpoints());
+
+ counts.incrementFailedCheckpoints();
+ assertEquals(1, counts.getNumberOfRestoredCheckpoints());
+ assertEquals(2, counts.getTotalNumberOfCheckpoints());
+ assertEquals(0, counts.getNumberOfInProgressCheckpoints());
+ assertEquals(1, counts.getNumberOfCompletedCheckpoints());
+ assertEquals(1, counts.getNumberOfFailedCheckpoints());
+ }
+
+ /**
+ * Tests that increment the completed or failed number of checkpoints without
+ * incrementing the in progress checkpoints before throws an Exception.
+ */
+ @Test
+ public void testCompleteOrFailWithoutInProgressCheckpoint() throws Exception {
+ CheckpointStatsCounts counts = new CheckpointStatsCounts();
+ try {
+ counts.incrementCompletedCheckpoints();
+ fail("Did not throw expected Exception");
+ } catch (IllegalStateException ignored) {
+ }
+
+ try {
+ counts.incrementFailedCheckpoints();
+ fail("Did not throw expected Exception");
+ } catch (IllegalStateException ignored) {
+ }
+ }
+
+ /**
+ * Tests that that taking snapshots of the state are independent from the
+ * parent.
+ */
+ @Test
+ public void testCreateSnapshot() throws Exception {
+ CheckpointStatsCounts counts = new CheckpointStatsCounts();
+ counts.incrementRestoredCheckpoints();
+ counts.incrementRestoredCheckpoints();
+ counts.incrementRestoredCheckpoints();
+
+ counts.incrementInProgressCheckpoints();
+ counts.incrementCompletedCheckpoints();
+
+ counts.incrementInProgressCheckpoints();
+ counts.incrementCompletedCheckpoints();
+
+ counts.incrementInProgressCheckpoints();
+ counts.incrementCompletedCheckpoints();
+
+ counts.incrementInProgressCheckpoints();
+ counts.incrementCompletedCheckpoints();
+
+ counts.incrementInProgressCheckpoints();
+ counts.incrementFailedCheckpoints();
+
+ long restored = counts.getNumberOfRestoredCheckpoints();
+ long total = counts.getTotalNumberOfCheckpoints();
+ long inProgress = counts.getNumberOfInProgressCheckpoints();
+ long completed = counts.getNumberOfCompletedCheckpoints();
+ long failed = counts.getNumberOfFailedCheckpoints();
+
+ CheckpointStatsCounts snapshot = counts.createSnapshot();
+ assertEquals(restored, snapshot.getNumberOfRestoredCheckpoints());
+ assertEquals(total, snapshot.getTotalNumberOfCheckpoints());
+ assertEquals(inProgress, snapshot.getNumberOfInProgressCheckpoints());
+ assertEquals(completed, snapshot.getNumberOfCompletedCheckpoints());
+ assertEquals(failed, snapshot.getNumberOfFailedCheckpoints());
+
+ // Update the original
+ counts.incrementRestoredCheckpoints();
+ counts.incrementRestoredCheckpoints();
+
+ counts.incrementInProgressCheckpoints();
+ counts.incrementCompletedCheckpoints();
+
+ counts.incrementInProgressCheckpoints();
+ counts.incrementFailedCheckpoints();
+
+ assertEquals(restored, snapshot.getNumberOfRestoredCheckpoints());
+ assertEquals(total, snapshot.getTotalNumberOfCheckpoints());
+ assertEquals(inProgress, snapshot.getNumberOfInProgressCheckpoints());
+ assertEquals(completed, snapshot.getNumberOfCompletedCheckpoints());
+ assertEquals(failed, snapshot.getNumberOfFailedCheckpoints());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsHistoryTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsHistoryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsHistoryTest.java
new file mode 100644
index 0000000..098fe17
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsHistoryTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.junit.Test;
+
+import java.util.Iterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CheckpointStatsHistoryTest {
+
+ /**
+ * Tests a checkpoint history with allowed size 0.
+ */
+ @Test
+ public void testZeroMaxSizeHistory() throws Exception {
+ CheckpointStatsHistory history = new CheckpointStatsHistory(0);
+
+ history.addInProgressCheckpoint(createPendingCheckpointStats(0));
+ assertFalse(history.replacePendingCheckpointById(createCompletedCheckpointStats(0)));
+
+ CheckpointStatsHistory snapshot = history.createSnapshot();
+
+ int counter = 0;
+ for (AbstractCheckpointStats ignored : snapshot.getCheckpoints()) {
+ counter++;
+ }
+
+ assertEquals(0, counter);
+ assertNotNull(snapshot.getCheckpointById(0));
+ }
+
+ /**
+ * Tests a checkpoint history with allowed size 1.
+ */
+ @Test
+ public void testSizeOneHistory() throws Exception {
+ CheckpointStatsHistory history = new CheckpointStatsHistory(1);
+
+ history.addInProgressCheckpoint(createPendingCheckpointStats(0));
+ history.addInProgressCheckpoint(createPendingCheckpointStats(1));
+
+ assertFalse(history.replacePendingCheckpointById(createCompletedCheckpointStats(0)));
+ assertTrue(history.replacePendingCheckpointById(createCompletedCheckpointStats(1)));
+
+ CheckpointStatsHistory snapshot = history.createSnapshot();
+
+ for (AbstractCheckpointStats stats : snapshot.getCheckpoints()) {
+ assertEquals(1, stats.getCheckpointId());
+ assertTrue(stats.getStatus().isCompleted());
+ }
+ }
+
+ /**
+ * Tests the checkpoint history with multiple checkpoints.
+ */
+ @Test
+ public void testCheckpointHistory() throws Exception {
+ CheckpointStatsHistory history = new CheckpointStatsHistory(3);
+
+ history.addInProgressCheckpoint(createPendingCheckpointStats(0));
+
+ CheckpointStatsHistory snapshot = history.createSnapshot();
+ for (AbstractCheckpointStats stats : snapshot.getCheckpoints()) {
+ assertEquals(0, stats.getCheckpointId());
+ assertTrue(stats.getStatus().isInProgress());
+ }
+
+ history.addInProgressCheckpoint(createPendingCheckpointStats(1));
+ history.addInProgressCheckpoint(createPendingCheckpointStats(2));
+ history.addInProgressCheckpoint(createPendingCheckpointStats(3));
+
+ snapshot = history.createSnapshot();
+
+ // Check in progress stats.
+ Iterator<AbstractCheckpointStats> it = snapshot.getCheckpoints().iterator();
+ for (int i = 3; i > 0; i--) {
+ assertTrue(it.hasNext());
+ AbstractCheckpointStats stats = it.next();
+ assertEquals(i, stats.getCheckpointId());
+ assertTrue(stats.getStatus().isInProgress());
+ }
+ assertFalse(it.hasNext());
+
+ // Update checkpoints
+ history.replacePendingCheckpointById(createFailedCheckpointStats(1));
+ history.replacePendingCheckpointById(createCompletedCheckpointStats(3));
+ history.replacePendingCheckpointById(createFailedCheckpointStats(2));
+
+ snapshot = history.createSnapshot();
+ it = snapshot.getCheckpoints().iterator();
+
+ assertTrue(it.hasNext());
+ AbstractCheckpointStats stats = it.next();
+ assertEquals(3, stats.getCheckpointId());
+ assertNotNull(snapshot.getCheckpointById(3));
+ assertTrue(stats.getStatus().isCompleted());
+ assertTrue(snapshot.getCheckpointById(3).getStatus().isCompleted());
+
+ assertTrue(it.hasNext());
+ stats = it.next();
+ assertEquals(2, stats.getCheckpointId());
+ assertNotNull(snapshot.getCheckpointById(2));
+ assertTrue(stats.getStatus().isFailed());
+ assertTrue(snapshot.getCheckpointById(2).getStatus().isFailed());
+
+ assertTrue(it.hasNext());
+ stats = it.next();
+ assertEquals(1, stats.getCheckpointId());
+ assertNotNull(snapshot.getCheckpointById(1));
+ assertTrue(stats.getStatus().isFailed());
+ assertTrue(snapshot.getCheckpointById(1).getStatus().isFailed());
+
+ assertFalse(it.hasNext());
+ }
+
+ /**
+ * Tests that a snapshot cannot be modified or copied.
+ */
+ @Test
+ public void testModifySnapshot() throws Exception {
+ CheckpointStatsHistory history = new CheckpointStatsHistory(3);
+
+ history.addInProgressCheckpoint(createPendingCheckpointStats(0));
+ history.addInProgressCheckpoint(createPendingCheckpointStats(1));
+ history.addInProgressCheckpoint(createPendingCheckpointStats(2));
+
+ CheckpointStatsHistory snapshot = history.createSnapshot();
+
+ try {
+ snapshot.addInProgressCheckpoint(createPendingCheckpointStats(4));
+ fail("Did not throw expected Exception");
+ } catch (UnsupportedOperationException ignored) {
+ }
+
+ try {
+ snapshot.replacePendingCheckpointById(createCompletedCheckpointStats(2));
+ fail("Did not throw expected Exception");
+ } catch (UnsupportedOperationException ignored) {
+ }
+
+ try {
+ snapshot.createSnapshot();
+ fail("Did not throw expected Exception");
+ } catch (UnsupportedOperationException ignored) {
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ private PendingCheckpointStats createPendingCheckpointStats(long checkpointId) {
+ PendingCheckpointStats pending = mock(PendingCheckpointStats.class);
+ when(pending.getStatus()).thenReturn(CheckpointStatsStatus.IN_PROGRESS);
+ when(pending.getCheckpointId()).thenReturn(checkpointId);
+ return pending;
+ }
+
+ private CompletedCheckpointStats createCompletedCheckpointStats(long checkpointId) {
+ CompletedCheckpointStats completed = mock(CompletedCheckpointStats.class);
+ when(completed.getStatus()).thenReturn(CheckpointStatsStatus.COMPLETED);
+ when(completed.getCheckpointId()).thenReturn(checkpointId);
+ return completed;
+ }
+
+ private FailedCheckpointStats createFailedCheckpointStats(long checkpointId) {
+ FailedCheckpointStats failed = mock(FailedCheckpointStats.class);
+ when(failed.getStatus()).thenReturn(CheckpointStatsStatus.FAILED);
+ when(failed.getCheckpointId()).thenReturn(checkpointId);
+ return failed;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsStatusTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsStatusTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsStatusTest.java
new file mode 100644
index 0000000..0e16109
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsStatusTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class CheckpointStatsStatusTest {
+
+ /**
+ * Tests the getters of each status.
+ */
+ @Test
+ public void testStatusValues() throws Exception {
+ CheckpointStatsStatus inProgress = CheckpointStatsStatus.IN_PROGRESS;
+ assertTrue(inProgress.isInProgress());
+ assertFalse(inProgress.isCompleted());
+ assertFalse(inProgress.isFailed());
+
+ CheckpointStatsStatus completed = CheckpointStatsStatus.COMPLETED;
+ assertFalse(completed.isInProgress());
+ assertTrue(completed.isCompleted());
+ assertFalse(completed.isFailed());
+
+ CheckpointStatsStatus failed = CheckpointStatsStatus.FAILED;
+ assertFalse(failed.isInProgress());
+ assertFalse(failed.isCompleted());
+ assertTrue(failed.isFailed());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java
new file mode 100644
index 0000000..9a39182
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class CheckpointStatsTrackerTest {
+
+ /**
+ * Tests access to the snapshotting settings.
+ */
+ @Test
+ public void testGetSnapshottingSettings() throws Exception {
+ ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
+ when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
+ when(jobVertex.getParallelism()).thenReturn(1);
+
+ JobSnapshottingSettings snapshottingSettings = new JobSnapshottingSettings(
+ Collections.singletonList(new JobVertexID()),
+ Collections.singletonList(new JobVertexID()),
+ Collections.singletonList(new JobVertexID()),
+ 181238123L,
+ 19191992L,
+ 191929L,
+ 123,
+ ExternalizedCheckpointSettings.none(),
+ false);
+
+ CheckpointStatsTracker tracker = new CheckpointStatsTracker(
+ 0,
+ Collections.singletonList(jobVertex),
+ snapshottingSettings,
+ new UnregisteredMetricsGroup());
+
+ assertEquals(snapshottingSettings, tracker.getSnapshottingSettings());
+ }
+
+ /**
+ * Tests that the number of remembered checkpoints configuration
+ * is respected.
+ */
+ @Test
+ public void testTrackerWithoutHistory() throws Exception {
+ int numberOfSubtasks = 3;
+
+ ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
+ when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
+ when(jobVertex.getParallelism()).thenReturn(numberOfSubtasks);
+
+ CheckpointStatsTracker tracker = new CheckpointStatsTracker(
+ 0,
+ Collections.singletonList(jobVertex),
+ mock(JobSnapshottingSettings.class),
+ new UnregisteredMetricsGroup());
+
+ PendingCheckpointStats pending = tracker.reportPendingCheckpoint(
+ 0,
+ 1,
+ CheckpointProperties.forStandardCheckpoint());
+
+ pending.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(0));
+ pending.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(1));
+ pending.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(2));
+
+ pending.reportCompletedCheckpoint(null);
+
+ CheckpointStatsSnapshot snapshot = tracker.createSnapshot();
+ // History should be empty
+ assertFalse(snapshot.getHistory().getCheckpoints().iterator().hasNext());
+
+ // Counts should be available
+ CheckpointStatsCounts counts = snapshot.getCounts();
+ assertEquals(1, counts.getNumberOfCompletedCheckpoints());
+ assertEquals(1, counts.getTotalNumberOfCheckpoints());
+
+ // Summary should be available
+ CompletedCheckpointStatsSummary summary = snapshot.getSummaryStats();
+ assertEquals(1, summary.getStateSizeStats().getCount());
+ assertEquals(1, summary.getEndToEndDurationStats().getCount());
+ assertEquals(1, summary.getAlignmentBufferedStats().getCount());
+
+ // Latest completed checkpoint
+ assertNotNull(snapshot.getHistory().getLatestCompletedCheckpoint());
+ assertEquals(0, snapshot.getHistory().getLatestCompletedCheckpoint().getCheckpointId());
+ }
+
+ /**
+ * Tests tracking of checkpoints.
+ */
+ @Test
+ public void testCheckpointTracking() throws Exception {
+ int numberOfSubtasks = 3;
+
+ ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
+ when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
+ when(jobVertex.getParallelism()).thenReturn(numberOfSubtasks);
+
+ CheckpointStatsTracker tracker = new CheckpointStatsTracker(
+ 10,
+ Collections.singletonList(jobVertex),
+ mock(JobSnapshottingSettings.class),
+ new UnregisteredMetricsGroup());
+
+ // Completed checkpoint
+ PendingCheckpointStats completed1 = tracker.reportPendingCheckpoint(
+ 0,
+ 1,
+ CheckpointProperties.forStandardCheckpoint());
+
+ completed1.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(0));
+ completed1.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(1));
+ completed1.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(2));
+
+ completed1.reportCompletedCheckpoint(null);
+
+ // Failed checkpoint
+ PendingCheckpointStats failed = tracker.reportPendingCheckpoint(
+ 1,
+ 1,
+ CheckpointProperties.forStandardCheckpoint());
+
+ failed.reportFailedCheckpoint(12, null);
+
+ // Completed savepoint
+ PendingCheckpointStats savepoint = tracker.reportPendingCheckpoint(
+ 2,
+ 1,
+ CheckpointProperties.forStandardSavepoint());
+
+ savepoint.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(0));
+ savepoint.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(1));
+ savepoint.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(2));
+
+ savepoint.reportCompletedCheckpoint(null);
+
+ // In Progress
+ PendingCheckpointStats inProgress = tracker.reportPendingCheckpoint(
+ 3,
+ 1,
+ CheckpointProperties.forStandardCheckpoint());
+
+ RestoredCheckpointStats restored = new RestoredCheckpointStats(81, CheckpointProperties.forStandardCheckpoint(), 123, null);
+ tracker.reportRestoredCheckpoint(restored);
+
+ CheckpointStatsSnapshot snapshot = tracker.createSnapshot();
+
+ // Counts
+ CheckpointStatsCounts counts = snapshot.getCounts();
+ assertEquals(4, counts.getTotalNumberOfCheckpoints());
+ assertEquals(1, counts.getNumberOfInProgressCheckpoints());
+ assertEquals(2, counts.getNumberOfCompletedCheckpoints());
+ assertEquals(1, counts.getNumberOfFailedCheckpoints());
+
+ // Summary stats
+ CompletedCheckpointStatsSummary summary = snapshot.getSummaryStats();
+ assertEquals(2, summary.getStateSizeStats().getCount());
+ assertEquals(2, summary.getEndToEndDurationStats().getCount());
+ assertEquals(2, summary.getAlignmentBufferedStats().getCount());
+
+ // History
+ CheckpointStatsHistory history = snapshot.getHistory();
+ Iterator<AbstractCheckpointStats> it = history.getCheckpoints().iterator();
+
+ assertTrue(it.hasNext());
+ AbstractCheckpointStats stats = it.next();
+ assertEquals(3, stats.getCheckpointId());
+ assertTrue(stats.getStatus().isInProgress());
+
+ assertTrue(it.hasNext());
+ stats = it.next();
+ assertEquals(2, stats.getCheckpointId());
+ assertTrue(stats.getStatus().isCompleted());
+
+ assertTrue(it.hasNext());
+ stats = it.next();
+ assertEquals(1, stats.getCheckpointId());
+ assertTrue(stats.getStatus().isFailed());
+
+ assertTrue(it.hasNext());
+ stats = it.next();
+ assertEquals(0, stats.getCheckpointId());
+ assertTrue(stats.getStatus().isCompleted());
+
+ assertFalse(it.hasNext());
+
+ // Latest checkpoints
+ assertEquals(completed1.getCheckpointId(), snapshot.getHistory().getLatestCompletedCheckpoint().getCheckpointId());
+ assertEquals(savepoint.getCheckpointId(), snapshot.getHistory().getLatestSavepoint().getCheckpointId());
+ assertEquals(failed.getCheckpointId(), snapshot.getHistory().getLatestFailedCheckpoint().getCheckpointId());
+ assertEquals(restored, snapshot.getLatestRestoredCheckpoint());
+ }
+
+ /**
+ * Tests that snapshots are only created if a new snapshot has been reported
+ * or updated.
+ */
+ @Test
+ public void testCreateSnapshot() throws Exception {
+ ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
+ when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
+ when(jobVertex.getParallelism()).thenReturn(1);
+
+ CheckpointStatsTracker tracker = new CheckpointStatsTracker(
+ 10,
+ Collections.singletonList(jobVertex),
+ mock(JobSnapshottingSettings.class),
+ new UnregisteredMetricsGroup());
+
+ CheckpointStatsSnapshot snapshot1 = tracker.createSnapshot();
+
+ // Pending checkpoint => new snapshot
+ PendingCheckpointStats pending = tracker.reportPendingCheckpoint(
+ 0,
+ 1,
+ CheckpointProperties.forStandardCheckpoint());
+
+ pending.reportSubtaskStats(jobVertex.getJobVertexId(), createSubtaskStats(0));
+
+ CheckpointStatsSnapshot snapshot2 = tracker.createSnapshot();
+ assertNotEquals(snapshot1, snapshot2);
+
+ assertEquals(snapshot2, tracker.createSnapshot());
+
+ // Complete checkpoint => new snapshot
+ pending.reportCompletedCheckpoint(null);
+
+ CheckpointStatsSnapshot snapshot3 = tracker.createSnapshot();
+ assertNotEquals(snapshot2, snapshot3);
+
+ // Restore operation => new snapshot
+ tracker.reportRestoredCheckpoint(new RestoredCheckpointStats(12, CheckpointProperties.forStandardCheckpoint(), 12, null));
+
+ CheckpointStatsSnapshot snapshot4 = tracker.createSnapshot();
+ assertNotEquals(snapshot3, snapshot4);
+ assertEquals(snapshot4, tracker.createSnapshot());
+ }
+
+ /**
+ * Tests the registered metrics.
+ */
+ @Test
+ public void testMetrics() throws Exception {
+ MetricGroup metricGroup = mock(MetricGroup.class);
+
+ ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
+ when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
+ when(jobVertex.getParallelism()).thenReturn(1);
+
+ new CheckpointStatsTracker(
+ 0,
+ Collections.singletonList(jobVertex),
+ mock(JobSnapshottingSettings.class),
+ metricGroup);
+
+ verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.NUMBER_OF_CHECKPOINTS_METRIC), any(Gauge.class));
+ verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.NUMBER_OF_IN_PROGRESS_CHECKPOINTS_METRIC), any(Gauge.class));
+ verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.NUMBER_OF_COMPLETED_CHECKPOINTS_METRIC), any(Gauge.class));
+ verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.NUMBER_OF_FAILED_CHECKPOINTS_METRIC), any(Gauge.class));
+ verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.LATEST_RESTORED_CHECKPOINT_TIMESTAMP_METRIC), any(Gauge.class));
+ verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_SIZE_METRIC), any(Gauge.class));
+ verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_DURATION_METRIC), any(Gauge.class));
+ verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_ALIGNMENT_BUFFERED_METRIC), any(Gauge.class));
+ verify(metricGroup, times(1)).gauge(eq(CheckpointStatsTracker.LATEST_COMPLETED_CHECKPOINT_EXTERNAL_PATH_METRIC), any(Gauge.class));
+
+ // Make sure this test is adjusted when further metrics are added
+ verify(metricGroup, times(9)).gauge(any(String.class), any(Gauge.class));
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Creates a "disabled" checkpoint tracker for tests.
+ */
+ static CheckpointStatsTracker createTestTracker() {
+ ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
+ when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
+ when(jobVertex.getParallelism()).thenReturn(1);
+
+ return new CheckpointStatsTracker(
+ 0,
+ Collections.singletonList(jobVertex),
+ mock(JobSnapshottingSettings.class),
+ new UnregisteredMetricsGroup());
+ }
+
+ private SubtaskStateStats createSubtaskStats(int index) {
+ return new SubtaskStateStats(index, 0, 0, 0, 0, 0, 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStatsSummaryTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStatsSummaryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStatsSummaryTest.java
new file mode 100644
index 0000000..dd9c2c8
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStatsSummaryTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CompletedCheckpointStatsSummaryTest {
+
+ /**
+ * Tests simple updates of the completed checkpoint stats.
+ */
+ @Test
+ public void testSimpleUpdates() throws Exception {
+ long triggerTimestamp = 123123L;
+ long ackTimestamp = 123123 + 1212312399L;
+ long stateSize = Integer.MAX_VALUE + 17787L;
+ long alignmentBuffered = Integer.MAX_VALUE + 123123L;
+
+ CompletedCheckpointStatsSummary summary = new CompletedCheckpointStatsSummary();
+ assertEquals(0, summary.getStateSizeStats().getCount());
+ assertEquals(0, summary.getEndToEndDurationStats().getCount());
+ assertEquals(0, summary.getAlignmentBufferedStats().getCount());
+
+ int numCheckpoints = 10;
+
+ for (int i = 0; i < numCheckpoints; i++) {
+ CompletedCheckpointStats completed = createCompletedCheckpoint(
+ i,
+ triggerTimestamp,
+ ackTimestamp + i,
+ stateSize + i,
+ alignmentBuffered + i);
+
+ summary.updateSummary(completed);
+
+ assertEquals(i + 1, summary.getStateSizeStats().getCount());
+ assertEquals(i + 1, summary.getEndToEndDurationStats().getCount());
+ assertEquals(i + 1, summary.getAlignmentBufferedStats().getCount());
+ }
+
+ MinMaxAvgStats stateSizeStats = summary.getStateSizeStats();
+ assertEquals(stateSize, stateSizeStats.getMinimum());
+ assertEquals(stateSize + numCheckpoints - 1, stateSizeStats.getMaximum());
+
+ MinMaxAvgStats durationStats = summary.getEndToEndDurationStats();
+ assertEquals(ackTimestamp - triggerTimestamp, durationStats.getMinimum());
+ assertEquals(ackTimestamp - triggerTimestamp + numCheckpoints - 1, durationStats.getMaximum());
+
+ MinMaxAvgStats alignmentBufferedStats = summary.getAlignmentBufferedStats();
+ assertEquals(alignmentBuffered, alignmentBufferedStats.getMinimum());
+ assertEquals(alignmentBuffered + numCheckpoints - 1, alignmentBufferedStats.getMaximum());
+ }
+
+ private CompletedCheckpointStats createCompletedCheckpoint(
+ long checkpointId,
+ long triggerTimestamp,
+ long ackTimestamp,
+ long stateSize,
+ long alignmentBuffered) {
+
+ SubtaskStateStats latest = mock(SubtaskStateStats.class);
+ when(latest.getAckTimestamp()).thenReturn(ackTimestamp);
+
+ Map<JobVertexID, TaskStateStats> taskStats = new HashMap<>();
+ JobVertexID jobVertexId = new JobVertexID();
+ taskStats.put(jobVertexId, new TaskStateStats(jobVertexId, 1));
+
+ return new CompletedCheckpointStats(
+ checkpointId,
+ triggerTimestamp,
+ CheckpointProperties.forStandardCheckpoint(),
+ 1,
+ taskStats,
+ 1,
+ stateSize,
+ alignmentBuffered,
+ latest,
+ null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
index 9f4bdba..725b85f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
@@ -206,7 +206,7 @@ public abstract class CompletedCheckpointStoreTest extends TestLogger {
ChainedStateHandle<StreamStateHandle> stateHandle = CheckpointCoordinatorTest.generateChainedStateHandle(
new CheckpointMessagesTest.MyHandle());
- taskState.putState(i, new SubtaskState(stateHandle, null, null, null, null, 0L));
+ taskState.putState(i, new SubtaskState(stateHandle, null, null, null, null));
}
return new TestCompletedCheckpoint(new JobID(), id, 0, taskGroupStates, props);
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
index 25a4703..e466dc7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointTest.java
@@ -118,4 +118,29 @@ public class CompletedCheckpointTest {
verify(state, times(1)).discardState();
}
}
+
+ /**
+ * Tests that the stats callbacks happen if the callback is registered.
+ */
+ @Test
+ public void testCompletedCheckpointStatsCallbacks() throws Exception {
+ TaskState state = mock(TaskState.class);
+ Map<JobVertexID, TaskState> taskStates = new HashMap<>();
+ taskStates.put(new JobVertexID(), state);
+
+ CompletedCheckpoint completed = new CompletedCheckpoint(
+ new JobID(),
+ 0,
+ 0,
+ 1,
+ new HashMap<>(taskStates),
+ CheckpointProperties.forStandardCheckpoint(),
+ null);
+
+ CompletedCheckpointStats.DiscardCallback callback = mock(CompletedCheckpointStats.DiscardCallback.class);
+ completed.setDiscardCallback(callback);
+
+ completed.discard(JobStatus.FINISHED);
+ verify(callback, times(1)).notifyDiscardedCheckpoint();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
index 777ba9b..6c63f53 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
@@ -64,7 +64,7 @@ public class CoordinatorShutdownTest {
JobGraph testGraph = new JobGraph("test job", vertex);
testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList,
- 5000, 60000, 0L, Integer.MAX_VALUE, ExternalizedCheckpointSettings.none()));
+ 5000, 60000, 0L, Integer.MAX_VALUE, ExternalizedCheckpointSettings.none(), true));
ActorGateway jmGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
@@ -123,7 +123,7 @@ public class CoordinatorShutdownTest {
JobGraph testGraph = new JobGraph("test job", vertex);
testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList,
- 5000, 60000, 0L, Integer.MAX_VALUE, ExternalizedCheckpointSettings.none()));
+ 5000, 60000, 0L, Integer.MAX_VALUE, ExternalizedCheckpointSettings.none(), true));
ActorGateway jmGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
index c6c7ae5..bc95de7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/ExecutionGraphCheckpointCoordinatorTest.java
@@ -27,7 +27,6 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.blob.BlobKey;
-import org.apache.flink.runtime.checkpoint.stats.DisabledCheckpointStatsTracker;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
@@ -40,11 +39,9 @@ import org.apache.flink.util.SerializedValue;
import org.junit.AfterClass;
import org.junit.Test;
import org.mockito.Matchers;
-import scala.concurrent.duration.FiniteDuration;
import java.net.URL;
import java.util.Collections;
-import java.util.concurrent.TimeUnit;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -121,7 +118,7 @@ public class ExecutionGraphCheckpointCoordinatorTest {
counter,
store,
null,
- new DisabledCheckpointStatsTracker());
+ CheckpointStatsTrackerTest.createTestTracker());
JobVertex jobVertex = new JobVertex("MockVertex");
jobVertex.setInvokableClass(AbstractInvokable.class);
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/FailedCheckpointStatsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/FailedCheckpointStatsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/FailedCheckpointStatsTest.java
new file mode 100644
index 0000000..683c1c9
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/FailedCheckpointStatsTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class FailedCheckpointStatsTest {
+
+ /**
+ * Tests that the end to end duration of a failed checkpoint is the duration
+ * until the failure.
+ */
+ @Test
+ public void testEndToEndDuration() throws Exception {
+ long duration = 123912931293L;
+ long triggerTimestamp = 10123;
+ long failureTimestamp = triggerTimestamp + duration;
+
+ Map<JobVertexID, TaskStateStats> taskStats = new HashMap<>();
+ JobVertexID jobVertexId = new JobVertexID();
+ taskStats.put(jobVertexId, new TaskStateStats(jobVertexId, 1));
+
+ FailedCheckpointStats failed = new FailedCheckpointStats(
+ 0,
+ triggerTimestamp,
+ CheckpointProperties.forStandardCheckpoint(),
+ 1,
+ taskStats,
+ 0,
+ 0,
+ 0,
+ failureTimestamp,
+ null,
+ null);
+
+ assertEquals(duration, failed.getEndToEndDuration());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/MinMaxAvgStatsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/MinMaxAvgStatsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/MinMaxAvgStatsTest.java
new file mode 100644
index 0000000..5bb7492
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/MinMaxAvgStatsTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.junit.Test;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.junit.Assert.assertEquals;
+
+public class MinMaxAvgStatsTest {
+
+ /**
+ * Test the initial/empty state.
+ */
+ @Test
+ public void testInitialState() throws Exception {
+ MinMaxAvgStats mma = new MinMaxAvgStats();
+
+ assertEquals(0, mma.getMinimum());
+ assertEquals(0, mma.getMaximum());
+ assertEquals(0, mma.getSum());
+ assertEquals(0, mma.getCount());
+ assertEquals(0, mma.getAverage());
+ }
+
+ /**
+ * Test that non-positive numbers are not counted.
+ */
+ @Test
+ public void testAddNonPositiveStats() throws Exception {
+ MinMaxAvgStats mma = new MinMaxAvgStats();
+ mma.add(-1);
+
+ assertEquals(0, mma.getMinimum());
+ assertEquals(0, mma.getMaximum());
+ assertEquals(0, mma.getSum());
+ assertEquals(0, mma.getCount());
+ assertEquals(0, mma.getAverage());
+
+ mma.add(0);
+
+ assertEquals(0, mma.getMinimum());
+ assertEquals(0, mma.getMaximum());
+ assertEquals(0, mma.getSum());
+ assertEquals(1, mma.getCount());
+ assertEquals(0, mma.getAverage());
+ }
+
+ /**
+ * Test sequence of random numbers.
+ */
+ @Test
+ public void testAddRandomNumbers() throws Exception {
+ ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+ MinMaxAvgStats mma = new MinMaxAvgStats();
+
+ long count = 13;
+ long sum = 0;
+ long min = Integer.MAX_VALUE;
+ long max = Integer.MIN_VALUE;
+
+ for (int i = 0; i < count; i++) {
+ int number = rand.nextInt(124) + 1;
+ sum += number;
+ min = Math.min(min, number);
+ max = Math.max(max, number);
+
+ mma.add(number);
+ }
+
+ assertEquals(min, mma.getMinimum());
+ assertEquals(max, mma.getMaximum());
+ assertEquals(sum, mma.getSum());
+ assertEquals(count, mma.getCount());
+ assertEquals(sum / count, mma.getAverage());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointStatsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointStatsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointStatsTest.java
new file mode 100644
index 0000000..854e106
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointStatsTest.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class PendingCheckpointStatsTest {
+
+ /**
+ * Tests reporting of subtask stats.
+ */
+ @Test
+ public void testReportSubtaskStats() throws Exception {
+ long checkpointId = Integer.MAX_VALUE + 1222L;
+ long triggerTimestamp = Integer.MAX_VALUE - 1239L;
+ CheckpointProperties props = CheckpointProperties.forStandardCheckpoint();
+ TaskStateStats task1 = new TaskStateStats(new JobVertexID(), 3);
+ TaskStateStats task2 = new TaskStateStats(new JobVertexID(), 4);
+ int totalSubtaskCount = task1.getNumberOfSubtasks() + task2.getNumberOfSubtasks();
+
+ Map<JobVertexID, TaskStateStats> taskStats = new HashMap<>();
+ taskStats.put(task1.getJobVertexId(), task1);
+ taskStats.put(task2.getJobVertexId(), task2);
+
+ CheckpointStatsTracker.PendingCheckpointStatsCallback callback = mock(
+ CheckpointStatsTracker.PendingCheckpointStatsCallback.class);
+
+ PendingCheckpointStats pending = new PendingCheckpointStats(
+ checkpointId,
+ triggerTimestamp,
+ props,
+ totalSubtaskCount,
+ taskStats,
+ callback);
+
+ // Check initial state
+ assertEquals(checkpointId, pending.getCheckpointId());
+ assertEquals(triggerTimestamp, pending.getTriggerTimestamp());
+ assertEquals(props, pending.getProperties());
+ assertEquals(CheckpointStatsStatus.IN_PROGRESS, pending.getStatus());
+ assertEquals(0, pending.getNumberOfAcknowledgedSubtasks());
+ assertEquals(0, pending.getStateSize());
+ assertEquals(totalSubtaskCount, pending.getNumberOfSubtasks());
+ assertNull(pending.getLatestAcknowledgedSubtaskStats());
+ assertEquals(-1, pending.getLatestAckTimestamp());
+ assertEquals(-1, pending.getEndToEndDuration());
+ assertEquals(task1, pending.getTaskStateStats(task1.getJobVertexId()));
+ assertEquals(task2, pending.getTaskStateStats(task2.getJobVertexId()));
+ assertNull(pending.getTaskStateStats(new JobVertexID()));
+
+ // Report subtasks and check getters
+ assertFalse(pending.reportSubtaskStats(new JobVertexID(), createSubtaskStats(0)));
+
+ long stateSize = 0;
+ long alignmentBuffered = 0;
+
+ // Report 1st task
+ for (int i = 0; i < task1.getNumberOfSubtasks(); i++) {
+ SubtaskStateStats subtask = createSubtaskStats(i);
+ stateSize += subtask.getStateSize();
+ alignmentBuffered += subtask.getAlignmentBuffered();
+
+ pending.reportSubtaskStats(task1.getJobVertexId(), subtask);
+
+ assertEquals(subtask, pending.getLatestAcknowledgedSubtaskStats());
+ assertEquals(subtask.getAckTimestamp(), pending.getLatestAckTimestamp());
+ assertEquals(subtask.getAckTimestamp() - triggerTimestamp, pending.getEndToEndDuration());
+ assertEquals(stateSize, pending.getStateSize());
+ assertEquals(alignmentBuffered, pending.getAlignmentBuffered());
+ }
+
+ // Don't allow overwrite
+ assertFalse(pending.reportSubtaskStats(task1.getJobVertexId(), task1.getSubtaskStats()[0]));
+
+ // Report 2nd task
+ for (int i = 0; i < task2.getNumberOfSubtasks(); i++) {
+ SubtaskStateStats subtask = createSubtaskStats(i);
+ stateSize += subtask.getStateSize();
+ alignmentBuffered += subtask.getAlignmentBuffered();
+
+ pending.reportSubtaskStats(task2.getJobVertexId(), subtask);
+
+ assertEquals(subtask, pending.getLatestAcknowledgedSubtaskStats());
+ assertEquals(subtask.getAckTimestamp(), pending.getLatestAckTimestamp());
+ assertEquals(subtask.getAckTimestamp() - triggerTimestamp, pending.getEndToEndDuration());
+ assertEquals(stateSize, pending.getStateSize());
+ assertEquals(alignmentBuffered, pending.getAlignmentBuffered());
+ }
+
+ assertEquals(task1.getNumberOfSubtasks(), task1.getNumberOfAcknowledgedSubtasks());
+ assertEquals(task2.getNumberOfSubtasks(), task2.getNumberOfAcknowledgedSubtasks());
+ }
+
+ /**
+ * Test reporting of a completed checkpoint.
+ */
+ @Test
+ public void testReportCompletedCheckpoint() throws Exception {
+ TaskStateStats task1 = new TaskStateStats(new JobVertexID(), 3);
+ TaskStateStats task2 = new TaskStateStats(new JobVertexID(), 4);
+
+ Map<JobVertexID, TaskStateStats> taskStats = new HashMap<>();
+ taskStats.put(task1.getJobVertexId(), task1);
+ taskStats.put(task2.getJobVertexId(), task2);
+
+ CheckpointStatsTracker.PendingCheckpointStatsCallback callback = mock(
+ CheckpointStatsTracker.PendingCheckpointStatsCallback.class);
+
+ PendingCheckpointStats pending = new PendingCheckpointStats(
+ 0,
+ 1,
+ CheckpointProperties.forStandardCheckpoint(),
+ task1.getNumberOfSubtasks() + task2.getNumberOfSubtasks(),
+ taskStats,
+ callback);
+
+ // Report subtasks
+ for (int i = 0; i < task1.getNumberOfSubtasks(); i++) {
+ pending.reportSubtaskStats(task1.getJobVertexId(), createSubtaskStats(i));
+ }
+
+ for (int i = 0; i < task2.getNumberOfSubtasks(); i++) {
+ pending.reportSubtaskStats(task2.getJobVertexId(), createSubtaskStats(i));
+ }
+
+ // Report completed
+ String externalPath = "asdjkasdjkasd";
+
+ CompletedCheckpointStats.DiscardCallback discardCallback = pending.reportCompletedCheckpoint(externalPath);
+
+ ArgumentCaptor<CompletedCheckpointStats> args = ArgumentCaptor.forClass(CompletedCheckpointStats.class);
+ verify(callback).reportCompletedCheckpoint(args.capture());
+
+ CompletedCheckpointStats completed = args.getValue();
+
+ assertNotNull(completed);
+ assertEquals(CheckpointStatsStatus.COMPLETED, completed.getStatus());
+ assertFalse(completed.isDiscarded());
+ assertEquals(discardCallback, completed.getDiscardCallback());
+ discardCallback.notifyDiscardedCheckpoint();
+ assertTrue(completed.isDiscarded());
+ assertEquals(externalPath, completed.getExternalPath());
+
+ assertEquals(pending.getCheckpointId(), completed.getCheckpointId());
+ assertEquals(pending.getNumberOfAcknowledgedSubtasks(), completed.getNumberOfAcknowledgedSubtasks());
+ assertEquals(pending.getLatestAcknowledgedSubtaskStats(), completed.getLatestAcknowledgedSubtaskStats());
+ assertEquals(pending.getLatestAckTimestamp(), completed.getLatestAckTimestamp());
+ assertEquals(pending.getEndToEndDuration(), completed.getEndToEndDuration());
+ assertEquals(pending.getStateSize(), completed.getStateSize());
+ assertEquals(pending.getAlignmentBuffered(), completed.getAlignmentBuffered());
+ assertEquals(task1, completed.getTaskStateStats(task1.getJobVertexId()));
+ assertEquals(task2, completed.getTaskStateStats(task2.getJobVertexId()));
+ }
+
+ /**
+ * Test reporting of a failed checkpoint.
+ */
+ @Test
+ public void testReportFailedCheckpoint() throws Exception {
+ TaskStateStats task1 = new TaskStateStats(new JobVertexID(), 3);
+ TaskStateStats task2 = new TaskStateStats(new JobVertexID(), 4);
+
+ Map<JobVertexID, TaskStateStats> taskStats = new HashMap<>();
+ taskStats.put(task1.getJobVertexId(), task1);
+ taskStats.put(task2.getJobVertexId(), task2);
+
+ CheckpointStatsTracker.PendingCheckpointStatsCallback callback = mock(
+ CheckpointStatsTracker.PendingCheckpointStatsCallback.class);
+
+ long triggerTimestamp = 123123;
+ PendingCheckpointStats pending = new PendingCheckpointStats(
+ 0,
+ triggerTimestamp,
+ CheckpointProperties.forStandardCheckpoint(),
+ task1.getNumberOfSubtasks() + task2.getNumberOfSubtasks(),
+ taskStats,
+ callback);
+
+ // Report subtasks
+ for (int i = 0; i < task1.getNumberOfSubtasks(); i++) {
+ pending.reportSubtaskStats(task1.getJobVertexId(), createSubtaskStats(i));
+ }
+
+ for (int i = 0; i < task2.getNumberOfSubtasks(); i++) {
+ pending.reportSubtaskStats(task2.getJobVertexId(), createSubtaskStats(i));
+ }
+
+ // Report failed
+ Exception cause = new Exception("test exception");
+ long failureTimestamp = 112211137;
+ pending.reportFailedCheckpoint(failureTimestamp, cause);
+
+ ArgumentCaptor<FailedCheckpointStats> args = ArgumentCaptor.forClass(FailedCheckpointStats.class);
+ verify(callback).reportFailedCheckpoint(args.capture());
+
+ FailedCheckpointStats failed = args.getValue();
+
+ assertNotNull(failed);
+ assertEquals(CheckpointStatsStatus.FAILED, failed.getStatus());
+ assertEquals(failureTimestamp, failed.getFailureTimestamp());
+ assertEquals(cause.getMessage(), failed.getFailureMessage());
+
+ assertEquals(pending.getCheckpointId(), failed.getCheckpointId());
+ assertEquals(pending.getNumberOfAcknowledgedSubtasks(), failed.getNumberOfAcknowledgedSubtasks());
+ assertEquals(pending.getLatestAcknowledgedSubtaskStats(), failed.getLatestAcknowledgedSubtaskStats());
+ assertEquals(pending.getLatestAckTimestamp(), failed.getLatestAckTimestamp());
+ assertEquals(failureTimestamp - triggerTimestamp, failed.getEndToEndDuration());
+ assertEquals(pending.getStateSize(), failed.getStateSize());
+ assertEquals(pending.getAlignmentBuffered(), failed.getAlignmentBuffered());
+ assertEquals(task1, failed.getTaskStateStats(task1.getJobVertexId()));
+ assertEquals(task2, failed.getTaskStateStats(task2.getJobVertexId()));
+ }
+
+
+ // ------------------------------------------------------------------------
+
+ private SubtaskStateStats createSubtaskStats(int index) {
+ return new SubtaskStateStats(
+ index,
+ Integer.MAX_VALUE + (long) index,
+ Integer.MAX_VALUE + (long) index,
+ Integer.MAX_VALUE + (long) index,
+ Integer.MAX_VALUE + (long) index,
+ Integer.MAX_VALUE + (long) index,
+ Integer.MAX_VALUE + (long) index);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
index be7b033..71013bd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PendingCheckpointTest.java
@@ -38,7 +38,10 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -89,8 +92,7 @@ public class PendingCheckpointTest {
CheckpointProperties persisted = new CheckpointProperties(false, true, false, false, false, false, false);
PendingCheckpoint pending = createPendingCheckpoint(persisted, tmp.getAbsolutePath());
- pending.acknowledgeTask(ATTEMPT_ID, null);
-
+ pending.acknowledgeTask(ATTEMPT_ID, null, new CheckpointMetaData(pending.getCheckpointId(), pending.getCheckpointTimestamp()));
assertEquals(0, tmp.listFiles().length);
pending.finalizeCheckpoint();
assertEquals(1, tmp.listFiles().length);
@@ -98,7 +100,7 @@ public class PendingCheckpointTest {
// Ephemeral checkpoint
CheckpointProperties ephemeral = new CheckpointProperties(false, false, true, true, true, true, true);
pending = createPendingCheckpoint(ephemeral, null);
- pending.acknowledgeTask(ATTEMPT_ID, null);
+ pending.acknowledgeTask(ATTEMPT_ID, null, new CheckpointMetaData(pending.getCheckpointId(), pending.getCheckpointTimestamp()));
assertEquals(1, tmp.listFiles().length);
pending.finalizeCheckpoint();
@@ -143,7 +145,7 @@ public class PendingCheckpointTest {
future = pending.getCompletionFuture();
assertFalse(future.isDone());
- pending.acknowledgeTask(ATTEMPT_ID, null);
+ pending.acknowledgeTask(ATTEMPT_ID, null, new CheckpointMetaData(pending.getCheckpointId(), pending.getCheckpointTimestamp()));
pending.finalizeCheckpoint();
assertTrue(future.isDone());
@@ -206,6 +208,65 @@ public class PendingCheckpointTest {
verify(state, times(1)).discardState();
}
+ /**
+ * Tests that the stats callbacks happen if the callback is registered.
+ */
+ @Test
+ public void testPendingCheckpointStatsCallbacks() throws Exception {
+ {
+ // Complete sucessfully
+ PendingCheckpointStats callback = mock(PendingCheckpointStats.class);
+ PendingCheckpoint pending = createPendingCheckpoint(CheckpointProperties.forStandardCheckpoint(), null);
+ pending.setStatsCallback(callback);
+
+ pending.acknowledgeTask(ATTEMPT_ID, null, new CheckpointMetaData(pending.getCheckpointId(), pending.getCheckpointTimestamp()));
+ verify(callback, times(1)).reportSubtaskStats(any(JobVertexID.class), any(SubtaskStateStats.class));
+
+ pending.finalizeCheckpoint();
+ verify(callback, times(1)).reportCompletedCheckpoint(any(String.class));
+ }
+
+ {
+ // Fail subsumed
+ PendingCheckpointStats callback = mock(PendingCheckpointStats.class);
+ PendingCheckpoint pending = createPendingCheckpoint(CheckpointProperties.forStandardCheckpoint(), null);
+ pending.setStatsCallback(callback);
+
+ pending.abortSubsumed();
+ verify(callback, times(1)).reportFailedCheckpoint(anyLong(), any(Exception.class));
+ }
+
+ {
+ // Fail subsumed
+ PendingCheckpointStats callback = mock(PendingCheckpointStats.class);
+ PendingCheckpoint pending = createPendingCheckpoint(CheckpointProperties.forStandardCheckpoint(), null);
+ pending.setStatsCallback(callback);
+
+ pending.abortDeclined();
+ verify(callback, times(1)).reportFailedCheckpoint(anyLong(), any(Exception.class));
+ }
+
+ {
+ // Fail subsumed
+ PendingCheckpointStats callback = mock(PendingCheckpointStats.class);
+ PendingCheckpoint pending = createPendingCheckpoint(CheckpointProperties.forStandardCheckpoint(), null);
+ pending.setStatsCallback(callback);
+
+ pending.abortError(new Exception("Expected test error"));
+ verify(callback, times(1)).reportFailedCheckpoint(anyLong(), any(Exception.class));
+ }
+
+ {
+ // Fail subsumed
+ PendingCheckpointStats callback = mock(PendingCheckpointStats.class);
+ PendingCheckpoint pending = createPendingCheckpoint(CheckpointProperties.forStandardCheckpoint(), null);
+ pending.setStatsCallback(callback);
+
+ pending.abortExpired();
+ verify(callback, times(1)).reportFailedCheckpoint(anyLong(), any(Exception.class));
+ }
+ }
+
// ------------------------------------------------------------------------
private static PendingCheckpoint createPendingCheckpoint(CheckpointProperties props, String targetDirectory) {
@@ -215,7 +276,6 @@ public class PendingCheckpointTest {
0,
1,
ackTasks,
- false,
props,
targetDirectory,
Executors.directExecutor());
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/RestoredCheckpointStatsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/RestoredCheckpointStatsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/RestoredCheckpointStatsTest.java
new file mode 100644
index 0000000..85b1516
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/RestoredCheckpointStatsTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class RestoredCheckpointStatsTest {
+
+ /**
+ * Tests simple access to restore properties.
+ */
+ @Test
+ public void testSimpleAccess() throws Exception {
+ long checkpointId = Integer.MAX_VALUE + 1L;
+ long triggerTimestamp = Integer.MAX_VALUE + 1L;
+ CheckpointProperties props = new CheckpointProperties(true, true, false, false, true, false, true);
+ long restoreTimestamp = Integer.MAX_VALUE + 1L;
+ String externalPath = "external-path";
+
+ RestoredCheckpointStats restored = new RestoredCheckpointStats(
+ checkpointId,
+ props,
+ restoreTimestamp,
+ externalPath);
+
+ assertEquals(checkpointId, restored.getCheckpointId());
+ assertEquals(props, restored.getProperties());
+ assertEquals(restoreTimestamp, restored.getRestoreTimestamp());
+ assertEquals(externalPath, restored.getExternalPath());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/SubtaskStateStatsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/SubtaskStateStatsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/SubtaskStateStatsTest.java
new file mode 100644
index 0000000..75c40c5
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/SubtaskStateStatsTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubtaskStateStatsTest {
+
+ /**
+ * Tests simple access via the getters.
+ */
+ @Test
+ public void testSimpleAccess() throws Exception {
+ SubtaskStateStats stats = new SubtaskStateStats(
+ 0,
+ Integer.MAX_VALUE + 1L,
+ Integer.MAX_VALUE + 2L,
+ Integer.MAX_VALUE + 3L,
+ Integer.MAX_VALUE + 4L,
+ Integer.MAX_VALUE + 5L,
+ Integer.MAX_VALUE + 6L);
+
+ assertEquals(0, stats.getSubtaskIndex());
+ assertEquals(Integer.MAX_VALUE + 1L, stats.getAckTimestamp());
+ assertEquals(Integer.MAX_VALUE + 2L, stats.getStateSize());
+ assertEquals(Integer.MAX_VALUE + 3L, stats.getSyncCheckpointDuration());
+ assertEquals(Integer.MAX_VALUE + 4L, stats.getAsyncCheckpointDuration());
+ assertEquals(Integer.MAX_VALUE + 5L, stats.getAlignmentBuffered());
+ assertEquals(Integer.MAX_VALUE + 6L, stats.getAlignmentDuration());
+
+ // Check duration helper
+ long ackTimestamp = stats.getAckTimestamp();
+ long triggerTimestamp = ackTimestamp - 10123;
+ assertEquals(10123, stats.getEndToEndDuration(triggerTimestamp));
+
+ // Trigger timestamp < ack timestamp
+ assertEquals(0, stats.getEndToEndDuration(ackTimestamp + 1));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/TaskStateStatsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/TaskStateStatsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/TaskStateStatsTest.java
new file mode 100644
index 0000000..94edd9e
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/TaskStateStatsTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.junit.Test;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class TaskStateStatsTest {
+
+ /**
+ * Tests that subtask stats are correctly collected.
+ */
+ @Test
+ public void testHandInSubtasks() throws Exception {
+ JobVertexID jobVertexId = new JobVertexID();
+ SubtaskStateStats[] subtasks = new SubtaskStateStats[7];
+
+ TaskStateStats taskStats = new TaskStateStats(jobVertexId, subtasks.length);
+
+ assertEquals(jobVertexId, taskStats.getJobVertexId());
+ assertEquals(subtasks.length, taskStats.getNumberOfSubtasks());
+ assertEquals(0, taskStats.getNumberOfAcknowledgedSubtasks());
+ assertNull(taskStats.getLatestAcknowledgedSubtaskStats());
+ assertEquals(-1, taskStats.getLatestAckTimestamp());
+ assertArrayEquals(subtasks, taskStats.getSubtaskStats());
+
+ ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+ long stateSize = 0;
+ long alignmentBuffered = 0;
+
+ // Hand in some subtasks
+ for (int i = 0; i < subtasks.length; i++) {
+ subtasks[i] = new SubtaskStateStats(
+ i,
+ rand.nextInt(128),
+ rand.nextInt(128),
+ rand.nextInt(128),
+ rand.nextInt(128),
+ rand.nextInt(128),
+ rand.nextInt(128));
+
+ stateSize += subtasks[i].getStateSize();
+ alignmentBuffered += subtasks[i].getAlignmentBuffered();
+
+ assertTrue(taskStats.reportSubtaskStats(subtasks[i]));
+ assertEquals(i + 1, taskStats.getNumberOfAcknowledgedSubtasks());
+ assertEquals(subtasks[i], taskStats.getLatestAcknowledgedSubtaskStats());
+ assertEquals(subtasks[i].getAckTimestamp(), taskStats.getLatestAckTimestamp());
+ int duration = rand.nextInt(128);
+ assertEquals(duration, taskStats.getEndToEndDuration(subtasks[i].getAckTimestamp() - duration));
+ assertEquals(stateSize, taskStats.getStateSize());
+ assertEquals(alignmentBuffered, taskStats.getAlignmentBuffered());
+ }
+
+ assertFalse(taskStats.reportSubtaskStats(new SubtaskStateStats(0, 0, 0, 0, 0, 0, 0)));
+
+ // Test that all subtasks are taken into the account for the summary.
+ // The correctness of the actual results is checked in the test of the
+ // MinMaxAvgStats.
+ TaskStateStats.TaskStateStatsSummary summary = taskStats.getSummaryStats();
+ assertEquals(subtasks.length, summary.getStateSizeStats().getCount());
+ assertEquals(subtasks.length, summary.getAckTimestampStats().getCount());
+ assertEquals(subtasks.length, summary.getSyncCheckpointDurationStats().getCount());
+ assertEquals(subtasks.length, summary.getAsyncCheckpointDurationStats().getCount());
+ assertEquals(subtasks.length, summary.getAlignmentBufferedStats().getCount());
+ assertEquals(subtasks.length, summary.getAlignmentDurationStats().getCount());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Test.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Test.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Test.java
index 1ae74ff..db5c35b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Test.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/savepoint/SavepointV1Test.java
@@ -140,8 +140,7 @@ public class SavepointV1Test {
new ChainedStateHandle<>(operatorStatesBackend),
new ChainedStateHandle<>(operatorStatesStream),
keyedStateStream,
- keyedStateBackend,
- subtaskIdx * 10L));
+ keyedStateBackend));
}
taskStates.add(taskState);
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
index 4cba4e3..6d7427a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.flink.runtime.executiongraph;
+import org.apache.flink.api.common.ArchivedExecutionConfig;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.ExecutionMode;
import org.apache.flink.api.common.JobID;
@@ -25,28 +26,25 @@ import org.apache.flink.api.common.accumulators.LongCounter;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.testutils.CommonTestUtils;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
import org.apache.flink.runtime.akka.AkkaUtils;
-import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
+import org.apache.flink.runtime.checkpoint.CheckpointStatsSnapshot;
+import org.apache.flink.runtime.checkpoint.CheckpointStatsTracker;
import org.apache.flink.runtime.checkpoint.StandaloneCheckpointIDCounter;
import org.apache.flink.runtime.checkpoint.StandaloneCompletedCheckpointStore;
-import org.apache.flink.runtime.checkpoint.stats.CheckpointStats;
-import org.apache.flink.runtime.checkpoint.stats.CheckpointStatsTracker;
-import org.apache.flink.runtime.checkpoint.stats.JobCheckpointStats;
-import org.apache.flink.runtime.checkpoint.stats.OperatorCheckpointStats;
import org.apache.flink.runtime.execution.ExecutionState;
-import org.apache.flink.api.common.ArchivedExecutionConfig;
import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertex;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
+import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
import org.apache.flink.runtime.testingUtils.TestingUtils;
import org.apache.flink.util.SerializedValue;
import org.junit.BeforeClass;
import org.junit.Test;
-import scala.Option;
import java.io.IOException;
import java.util.ArrayList;
@@ -61,6 +59,7 @@ import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
public class ArchivedExecutionGraphTest {
private static JobVertexID v1ID = new JobVertexID();
@@ -107,6 +106,12 @@ public class ArchivedExecutionGraphTest {
new NoRestartStrategy());
runtimeGraph.attachJobGraph(vertices);
+ CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(
+ 0,
+ Collections.<ExecutionJobVertex>emptyList(),
+ mock(JobSnapshottingSettings.class),
+ new UnregisteredMetricsGroup());
+
runtimeGraph.enableSnapshotCheckpointing(
100,
100,
@@ -119,7 +124,7 @@ public class ArchivedExecutionGraphTest {
new StandaloneCheckpointIDCounter(),
new StandaloneCompletedCheckpointStore(1),
null,
- new TestCheckpointStatsTracker());
+ statsTracker);
Map<String, Accumulator<?, ?>> userAccumulators = new HashMap<>();
userAccumulators.put("userAcc", new LongCounter(64));
@@ -165,19 +170,25 @@ public class ArchivedExecutionGraphTest {
assertEquals(runtimeGraph.isStoppable(), archivedGraph.isStoppable());
// -------------------------------------------------------------------------------------------------------------
- // JobCheckpointStats
+ // CheckpointStats
// -------------------------------------------------------------------------------------------------------------
- JobCheckpointStats runtimeStats = runtimeGraph.getCheckpointStatsTracker().getJobStats().get();
- JobCheckpointStats archivedStats = archivedGraph.getCheckpointStatsTracker().getJobStats().get();
-
- assertEquals(runtimeStats.getAverageDuration(), archivedStats.getAverageDuration());
- assertEquals(runtimeStats.getMinDuration(), archivedStats.getMinDuration());
- assertEquals(runtimeStats.getMaxDuration(), archivedStats.getMaxDuration());
- assertEquals(runtimeStats.getAverageStateSize(), archivedStats.getAverageStateSize());
- assertEquals(runtimeStats.getMinStateSize(), archivedStats.getMinStateSize());
- assertEquals(runtimeStats.getMaxStateSize(), archivedStats.getMaxStateSize());
- assertEquals(runtimeStats.getCount(), archivedStats.getCount());
- assertEquals(runtimeStats.getRecentHistory(), archivedStats.getRecentHistory());
+ CheckpointStatsTracker runtimeStats = runtimeGraph.getCheckpointStatsTracker();
+ CheckpointStatsTracker archivedStats = archivedGraph.getCheckpointStatsTracker();
+
+ CheckpointStatsSnapshot runtimeSnapshot = runtimeStats.createSnapshot();
+ CheckpointStatsSnapshot archivedSnapshot = archivedStats.createSnapshot();
+
+ assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getAverage(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getAverage());
+ assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getMinimum(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getMinimum());
+ assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getMaximum(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getMaximum());
+
+ assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getAverage(), archivedSnapshot.getSummaryStats().getStateSizeStats().getAverage());
+ assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getMinimum(), archivedSnapshot.getSummaryStats().getStateSizeStats().getMinimum());
+ assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getMaximum(), archivedSnapshot.getSummaryStats().getStateSizeStats().getMaximum());
+
+ assertEquals(runtimeSnapshot.getCounts().getTotalNumberOfCheckpoints(), archivedSnapshot.getCounts().getTotalNumberOfCheckpoints());
+ assertEquals(runtimeSnapshot.getCounts().getNumberOfCompletedCheckpoints(), archivedSnapshot.getCounts().getNumberOfCompletedCheckpoints());
+ assertEquals(runtimeSnapshot.getCounts().getNumberOfInProgressCheckpoints(), archivedSnapshot.getCounts().getNumberOfInProgressCheckpoints());
// -------------------------------------------------------------------------------------------------------------
// ArchivedExecutionConfig
@@ -217,14 +228,6 @@ public class ArchivedExecutionGraphTest {
}
// -------------------------------------------------------------------------------------------------------------
- // OperatorCheckpointStats
- // -------------------------------------------------------------------------------------------------------------
- CheckpointStatsTracker runtimeTracker = runtimeGraph.getCheckpointStatsTracker();
- CheckpointStatsTracker archivedTracker = archivedGraph.getCheckpointStatsTracker();
- compareOperatorCheckpointStats(runtimeTracker.getOperatorStats(v1ID).get(), archivedTracker.getOperatorStats(v1ID).get());
- compareOperatorCheckpointStats(runtimeTracker.getOperatorStats(v2ID).get(), archivedTracker.getOperatorStats(v2ID).get());
-
- // -------------------------------------------------------------------------------------------------------------
// ExecutionVertices
// -------------------------------------------------------------------------------------------------------------
Iterator<? extends AccessExecutionVertex> runtimeExecutionVertices = runtimeGraph.getAllExecutionVertices().iterator();
@@ -243,8 +246,6 @@ public class ArchivedExecutionGraphTest {
assertEquals(runtimeJobVertex.getJobVertexId(), archivedJobVertex.getJobVertexId());
assertEquals(runtimeJobVertex.getAggregateState(), archivedJobVertex.getAggregateState());
- compareOperatorCheckpointStats(runtimeJobVertex.getCheckpointStats().get(), archivedJobVertex.getCheckpointStats().get());
-
compareStringifiedAccumulators(runtimeJobVertex.getAggregatedUserAccumulatorsStringified(), archivedJobVertex.getAggregatedUserAccumulatorsStringified());
AccessExecutionVertex[] runtimeExecutionVertices = runtimeJobVertex.getTaskVertices();
@@ -315,96 +316,11 @@ public class ArchivedExecutionGraphTest {
}
}
- private static void compareOperatorCheckpointStats(OperatorCheckpointStats runtimeStats, OperatorCheckpointStats archivedStats) {
- assertEquals(runtimeStats.getNumberOfSubTasks(), archivedStats.getNumberOfSubTasks());
- assertEquals(runtimeStats.getCheckpointId(), archivedStats.getCheckpointId());
- assertEquals(runtimeStats.getDuration(), archivedStats.getDuration());
- assertEquals(runtimeStats.getStateSize(), archivedStats.getStateSize());
- assertEquals(runtimeStats.getTriggerTimestamp(), archivedStats.getTriggerTimestamp());
- assertEquals(runtimeStats.getSubTaskDuration(0), archivedStats.getSubTaskDuration(0));
- assertEquals(runtimeStats.getSubTaskStateSize(0), archivedStats.getSubTaskStateSize(0));
- }
-
private static void verifySerializability(ArchivedExecutionGraph graph) throws IOException, ClassNotFoundException {
ArchivedExecutionGraph copy = CommonTestUtils.createCopySerializable(graph);
compareExecutionGraph(graph, copy);
}
-
- private static class TestCheckpointStatsTracker implements CheckpointStatsTracker {
-
- @Override
- public void onCompletedCheckpoint(CompletedCheckpoint checkpoint) {
- }
-
- @Override
- public Option<JobCheckpointStats> getJobStats() {
- return Option.<JobCheckpointStats>apply(new TestJobCheckpointStats());
- }
-
- @Override
- public Option<OperatorCheckpointStats> getOperatorStats(JobVertexID operatorId) {
- return Option.<OperatorCheckpointStats>apply(new TestOperatorCheckpointStats(operatorId.getUpperPart()));
- }
- }
-
- private static class TestJobCheckpointStats implements JobCheckpointStats {
- private static final long serialVersionUID = -2630234917947292836L;
-
- @Override
- public List<CheckpointStats> getRecentHistory() {
- return Collections.emptyList();
- }
-
- @Override
- public long getCount() {
- return 1;
- }
-
- @Override
- public String getExternalPath() {
- return null;
- }
-
- @Override
- public long getMinDuration() {
- return 2;
- }
-
- @Override
- public long getMaxDuration() {
- return 4;
- }
-
- @Override
- public long getAverageDuration() {
- return 3;
- }
-
- @Override
- public long getMinStateSize() {
- return 5;
- }
-
- @Override
- public long getMaxStateSize() {
- return 7;
- }
-
- @Override
- public long getAverageStateSize() {
- return 6;
- }
- }
-
- private static class TestOperatorCheckpointStats extends OperatorCheckpointStats {
- private static final long serialVersionUID = -2798640928349528644L;
-
- public TestOperatorCheckpointStats(long offset) {
- super(1 + offset, 2 + offset, 3 + offset, 4 + offset, new long[][]{new long[]{5 + offset, 6 + offset}});
- }
- }
-
private static class TestJobParameters extends ExecutionConfig.GlobalJobParameters {
private static final long serialVersionUID = -8118611781035212808L;
private Map<String, String> parameters;
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
index 36412f5..398505f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerHARecoveryTest.java
@@ -228,7 +228,8 @@ public class JobManagerHARecoveryTest {
10 * 60 * 1000,
0,
1,
- ExternalizedCheckpointSettings.none()));
+ ExternalizedCheckpointSettings.none(),
+ true));
BlockingStatefulInvokable.initializeStaticHelpers(slots);
@@ -617,7 +618,7 @@ public class JobManagerHARecoveryTest {
ChainedStateHandle<StreamStateHandle> chainedStateHandle =
new ChainedStateHandle<StreamStateHandle>(Collections.singletonList(byteStreamStateHandle));
SubtaskState checkpointStateHandles =
- new SubtaskState(chainedStateHandle, null, null, null, null, 0L);
+ new SubtaskState(chainedStateHandle, null, null, null, null);
getEnvironment().acknowledgeCheckpoint(
new CheckpointMetaData(checkpointMetaData.getCheckpointId(), -1, 0L, 0L, 0L, 0L),
http://git-wip-us.apache.org/repos/asf/flink/blob/0d1f4bcb/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
index 6d8c70b..1fea6f6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/JobManagerTest.java
@@ -824,7 +824,8 @@ public class JobManagerTest {
3600000,
0,
Integer.MAX_VALUE,
- ExternalizedCheckpointSettings.none());
+ ExternalizedCheckpointSettings.none(),
+ true);
jobGraph.setSnapshotSettings(snapshottingSettings);
@@ -952,7 +953,8 @@ public class JobManagerTest {
360000,
0,
Integer.MAX_VALUE,
- ExternalizedCheckpointSettings.none());
+ ExternalizedCheckpointSettings.none(),
+ true);
jobGraph.setSnapshotSettings(snapshottingSettings);
@@ -1053,7 +1055,8 @@ public class JobManagerTest {
360000,
0,
Integer.MAX_VALUE,
- ExternalizedCheckpointSettings.none());
+ ExternalizedCheckpointSettings.none(),
+ true);
jobGraph.setSnapshotSettings(snapshottingSettings);
@@ -1098,7 +1101,8 @@ public class JobManagerTest {
360000,
0,
Integer.MAX_VALUE,
- ExternalizedCheckpointSettings.none());
+ ExternalizedCheckpointSettings.none(),
+ true);
newJobGraph.setSnapshotSettings(newSnapshottingSettings);