You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2018/02/06 19:02:25 UTC
[01/31] flink git commit: [FLINK-8472] [DataStream,
test] Refactor StatefulJobSavepointFrom*MigrationITCase to single
ITCase
Repository: flink
Updated Branches:
refs/heads/master 62a777bc8 -> 97f0cac2a
[FLINK-8472] [DataStream, test] Refactor StatefulJobSavepointFrom*MigrationITCase to single ITCase
This commit refactors the StatefulJobSavepointFrom12MigrationITCase and
StatefulJobSavepointFrom13MigrationITCase to a single class,
StatefulJobSavepointMigrationITCase. The new ITCase is parameterized to
ensure that all previous versions and state backend variants are
covered.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8d3a74f9
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8d3a74f9
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8d3a74f9
Branch: refs/heads/master
Commit: 8d3a74f91aae43ba560f725072c0ee28ead75704
Parents: 45c4b66
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 18:25:45 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
...atefulJobSavepointFrom12MigrationITCase.java | 694 -------------------
...atefulJobSavepointFrom13MigrationITCase.java | 58 --
.../StatefulJobSavepointMigrationITCase.java | 661 ++++++++++++++++++
3 files changed, 661 insertions(+), 752 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8d3a74f9/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java
deleted file mode 100644
index 6859c2d..0000000
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom12MigrationITCase.java
+++ /dev/null
@@ -1,694 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.checkpointing.utils;
-
-import org.apache.flink.api.common.accumulators.IntCounter;
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.api.common.state.ValueState;
-import org.apache.flink.api.common.state.ValueStateDescriptor;
-import org.apache.flink.api.common.typeinfo.TypeHint;
-import org.apache.flink.api.common.typeutils.base.LongSerializer;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
-import org.apache.flink.runtime.state.memory.MemoryStateBackend;
-import org.apache.flink.streaming.api.TimeCharacteristic;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
-import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
-import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
-import org.apache.flink.streaming.api.operators.InternalTimer;
-import org.apache.flink.streaming.api.operators.InternalTimerService;
-import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
-import org.apache.flink.streaming.api.operators.TimestampedCollector;
-import org.apache.flink.streaming.api.operators.Triggerable;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-import org.apache.flink.util.Collector;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * This verifies that we can restore a complete job from a Flink 1.2 savepoint.
- *
- * <p>The test for checkpointed (legacy state) was removed from this test for Flink 1.4 because compatibility with
- * Flink 1.1 is removed. The legacy state in the binary savepoints is ignored by the tests now.
- *
- * <p>The tests will time out if they don't see the required number of successful checks within
- * a time limit.
- *
- *
- */
-public class StatefulJobSavepointFrom12MigrationITCase extends SavepointMigrationTestBase {
- private static final int NUM_SOURCE_ELEMENTS = 4;
-
- /**
- * This has to be manually executed to create the savepoint on Flink 1.2.
- */
- @Test
- @Ignore
- public void testCreateSavepointOnFlink12() throws Exception {
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setStateBackend(new MemoryStateBackend());
- env.enableCheckpointing(500);
- env.setParallelism(4);
- env.setMaxParallelism(4);
-
- env
- .addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
- .flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
- .keyBy(0)
- .flatMap(new LegacyCheckpointedFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
- .keyBy(0)
- .flatMap(new KeyedStateSettingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
- .keyBy(0)
- .transform(
- "custom_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckpointedUdfOperator(new LegacyCheckpointedFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator")
- .keyBy(0)
- .transform(
- "timely_stateful_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new TimelyStatefulOperator()).uid("TimelyStatefulOperator")
- .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
-
- executeAndSavepoint(
- env,
- "src/test/resources/" + getSavepointPath(),
- new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
- }
-
- /**
- * This has to be manually executed to create the savepoint on Flink 1.2.
- */
- @Test
- @Ignore
- public void testCreateSavepointOnFlink12WithRocksDB() throws Exception {
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- RocksDBStateBackend rocksBackend =
- new RocksDBStateBackend(new MemoryStateBackend());
- env.setStateBackend(rocksBackend);
- env.enableCheckpointing(500);
- env.setParallelism(4);
- env.setMaxParallelism(4);
-
- env
- .addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
- .flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
- .keyBy(0)
- .flatMap(new LegacyCheckpointedFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
- .keyBy(0)
- .flatMap(new KeyedStateSettingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
- .keyBy(0)
- .transform(
- "custom_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckpointedUdfOperator(new LegacyCheckpointedFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator")
- .keyBy(0)
- .transform(
- "timely_stateful_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new TimelyStatefulOperator()).uid("TimelyStatefulOperator")
- .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
-
- executeAndSavepoint(
- env,
- "src/test/resources/" + getRocksDBSavepointPath(),
- new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
- }
-
- @Test
- public void testSavepointRestoreFromFlink12() throws Exception {
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setRestartStrategy(RestartStrategies.noRestart());
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setStateBackend(new MemoryStateBackend());
- env.enableCheckpointing(500);
- env.setParallelism(4);
- env.setMaxParallelism(4);
-
- env
- .addSource(new CheckingRestoringSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
- .flatMap(new CheckingRestoringFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
- .keyBy(0)
- .flatMap(new CheckingRestoringFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
- .keyBy(0)
- .flatMap(new CheckingKeyedStateFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
- .keyBy(0)
- .transform(
- "custom_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckingRestoringUdfOperator(new CheckingRestoringFlatMapWithKeyedStateInOperator())).uid("LegacyCheckpointedOperator")
- .keyBy(0)
- .transform(
- "timely_stateful_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckingTimelyStatefulOperator()).uid("TimelyStatefulOperator")
- .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
-
- restoreAndExecute(
- env,
- getResourceFilename(getSavepointPath()),
- new Tuple2<>(CheckingRestoringSource.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
- new Tuple2<>(CheckingRestoringFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringFlatMapWithKeyedState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringUdfOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringFlatMapWithKeyedStateInOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
- }
-
- @Test
- public void testSavepointRestoreFromFlink12FromRocksDB() throws Exception {
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setRestartStrategy(RestartStrategies.noRestart());
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
- env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
- env.enableCheckpointing(500);
- env.setParallelism(4);
- env.setMaxParallelism(4);
-
- env
- .addSource(new CheckingRestoringSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
- .flatMap(new CheckingRestoringFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
- .keyBy(0)
- .flatMap(new CheckingRestoringFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
- .keyBy(0)
- .flatMap(new CheckingKeyedStateFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
- .keyBy(0)
- .transform(
- "custom_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckingRestoringUdfOperator(new CheckingRestoringFlatMapWithKeyedStateInOperator())).uid("LegacyCheckpointedOperator")
- .keyBy(0)
- .transform(
- "timely_stateful_operator",
- new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
- new CheckingTimelyStatefulOperator()).uid("TimelyStatefulOperator")
- .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
-
- restoreAndExecute(
- env,
- getResourceFilename(getRocksDBSavepointPath()),
- new Tuple2<>(CheckingRestoringSource.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
- new Tuple2<>(CheckingRestoringFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringFlatMapWithKeyedState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringUdfOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingRestoringFlatMapWithKeyedStateInOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
- new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
- }
-
- protected String getSavepointPath() {
- return "stateful-udf-migration-itcase-flink1.2-savepoint";
- }
-
- protected String getRocksDBSavepointPath() {
- return "stateful-udf-migration-itcase-flink1.2-rocksdb-savepoint";
- }
-
- private static class LegacyCheckpointedSource
- implements SourceFunction<Tuple2<Long, Long>> {
-
- public static String checkpointedString = "Here be dragons!";
-
- private static final long serialVersionUID = 1L;
-
- private volatile boolean isRunning = true;
-
- private final int numElements;
-
- public LegacyCheckpointedSource(int numElements) {
- this.numElements = numElements;
- }
-
- @Override
- public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
-
- ctx.emitWatermark(new Watermark(0));
-
- synchronized (ctx.getCheckpointLock()) {
- for (long i = 0; i < numElements; i++) {
- ctx.collect(new Tuple2<>(i, i));
- }
- }
-
- // don't emit a final watermark so that we don't trigger the registered event-time
- // timers
- while (isRunning) {
- Thread.sleep(20);
- }
- }
-
- @Override
- public void cancel() {
- isRunning = false;
- }
- }
-
- private static class CheckingRestoringSource
- extends RichSourceFunction<Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringSource.class + "_RESTORE_CHECK";
-
- private volatile boolean isRunning = true;
-
- private final int numElements;
-
- private String restoredState;
-
- public CheckingRestoringSource(int numElements) {
- this.numElements = numElements;
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
-
- // immediately trigger any set timers
- ctx.emitWatermark(new Watermark(1000));
-
- synchronized (ctx.getCheckpointLock()) {
- for (long i = 0; i < numElements; i++) {
- ctx.collect(new Tuple2<>(i, i));
- }
- }
-
- while (isRunning) {
- Thread.sleep(20);
- }
- }
-
- @Override
- public void cancel() {
- isRunning = false;
- }
- }
-
- private static class LegacyCheckpointedFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static Tuple2<String, Long> checkpointedTuple =
- new Tuple2<>("hello", 42L);
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
- }
- }
-
- private static class CheckingRestoringFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMap.class + "_RESTORE_CHECK";
-
- private transient Tuple2<String, Long> restoredState;
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
-
- }
-
- }
-
- private static class LegacyCheckpointedFlatMapWithKeyedState
- extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static Tuple2<String, Long> checkpointedTuple =
- new Tuple2<>("hello", 42L);
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- getRuntimeContext().getState(stateDescriptor).update(value.f1);
-
- assertEquals(value.f1, getRuntimeContext().getState(stateDescriptor).value());
- }
- }
-
- private static class CheckingRestoringFlatMapWithKeyedState
- extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMapWithKeyedState.class + "_RESTORE_CHECK";
-
- private transient Tuple2<String, Long> restoredState;
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
- if (state == null) {
- throw new RuntimeException("Missing key value state for " + value);
- }
-
- assertEquals(value.f1, state.value());
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
- }
- }
-
- private static class CheckingRestoringFlatMapWithKeyedStateInOperator
- extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMapWithKeyedStateInOperator.class + "_RESTORE_CHECK";
-
- private transient Tuple2<String, Long> restoredState;
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
- if (state == null) {
- throw new RuntimeException("Missing key value state for " + value);
- }
-
- assertEquals(value.f1, state.value());
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
- }
- }
-
- private static class KeyedStateSettingFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- getRuntimeContext().getState(stateDescriptor).update(value.f1);
- }
- }
-
- private static class CheckingKeyedStateFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingKeyedStateFlatMap.class + "_RESTORE_CHECK";
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(value);
-
- ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
- if (state == null) {
- throw new RuntimeException("Missing key value state for " + value);
- }
-
- assertEquals(value.f1, state.value());
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
- }
- }
-
- private static class CheckpointedUdfOperator
- extends AbstractUdfStreamOperator<Tuple2<Long, Long>, FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>>
- implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> {
- private static final long serialVersionUID = 1L;
-
- private static final String CHECKPOINTED_STRING = "Oh my, that's nice!";
-
- public CheckpointedUdfOperator(FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> userFunction) {
- super(userFunction);
- }
-
- @Override
- public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
- userFunction.flatMap(element.getValue(), new TimestampedCollector<>(output));
- }
-
- @Override
- public void processWatermark(Watermark mark) throws Exception {
- output.emitWatermark(mark);
- }
- }
-
- private static class CheckingRestoringUdfOperator
- extends AbstractUdfStreamOperator<Tuple2<Long, Long>, FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>>
- implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringUdfOperator.class + "_RESTORE_CHECK";
-
- private String restoredState;
-
- public CheckingRestoringUdfOperator(FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> userFunction) {
- super(userFunction);
- }
-
- @Override
- public void open() throws Exception {
- super.open();
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
- userFunction.flatMap(element.getValue(), new TimestampedCollector<>(output));
- getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
- }
-
- @Override
- public void processWatermark(Watermark mark) throws Exception {
- output.emitWatermark(mark);
- }
- }
-
- private static class TimelyStatefulOperator
- extends AbstractStreamOperator<Tuple2<Long, Long>>
- implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>>, Triggerable<Long, Long> {
- private static final long serialVersionUID = 1L;
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- private transient InternalTimerService<Long> timerService;
-
- @Override
- public void open() throws Exception {
- super.open();
-
- timerService = getInternalTimerService(
- "timer",
- LongSerializer.INSTANCE,
- this);
-
- }
-
- @Override
- public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
- ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
- element.getValue().f0,
- LongSerializer.INSTANCE,
- stateDescriptor);
-
- state.update(element.getValue().f1);
-
- timerService.registerEventTimeTimer(element.getValue().f0, timerService.currentWatermark() + 10);
- timerService.registerProcessingTimeTimer(element.getValue().f0, timerService.currentProcessingTime() + 30_000);
-
- output.collect(element);
- }
-
- @Override
- public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
-
- }
-
- @Override
- public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
-
- }
-
- @Override
- public void processWatermark(Watermark mark) throws Exception {
- output.emitWatermark(mark);
- }
- }
-
- private static class CheckingTimelyStatefulOperator
- extends AbstractStreamOperator<Tuple2<Long, Long>>
- implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>>, Triggerable<Long, Long> {
- private static final long serialVersionUID = 1L;
-
- public static final String SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_PROCESS_CHECKS";
- public static final String SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_ET_CHECKS";
- public static final String SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_PT_CHECKS";
-
- private final ValueStateDescriptor<Long> stateDescriptor =
- new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
-
- private transient InternalTimerService<Long> timerService;
-
- @Override
- public void open() throws Exception {
- super.open();
-
- timerService = getInternalTimerService(
- "timer",
- LongSerializer.INSTANCE,
- this);
-
- getRuntimeContext().addAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, new IntCounter());
- getRuntimeContext().addAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, new IntCounter());
- getRuntimeContext().addAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
- ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
- element.getValue().f0,
- LongSerializer.INSTANCE,
- stateDescriptor);
-
- assertEquals(state.value(), element.getValue().f1);
- getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR).add(1);
-
- output.collect(element);
- }
-
- @Override
- public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
- ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
- timer.getNamespace(),
- LongSerializer.INSTANCE,
- stateDescriptor);
-
- assertEquals(state.value(), timer.getNamespace());
- getRuntimeContext().getAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR).add(1);
- }
-
- @Override
- public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
- ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
- timer.getNamespace(),
- LongSerializer.INSTANCE,
- stateDescriptor);
-
- assertEquals(state.value(), timer.getNamespace());
- getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR).add(1);
- }
- }
-
- private static class AccumulatorCountingSink<T> extends RichSinkFunction<T> {
- private static final long serialVersionUID = 1L;
-
- public static final String NUM_ELEMENTS_ACCUMULATOR = AccumulatorCountingSink.class + "_NUM_ELEMENTS";
-
- int count = 0;
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
-
- getRuntimeContext().addAccumulator(NUM_ELEMENTS_ACCUMULATOR, new IntCounter());
- }
-
- @Override
- public void invoke(T value) throws Exception {
- count++;
- getRuntimeContext().getAccumulator(NUM_ELEMENTS_ACCUMULATOR).add(1);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/8d3a74f9/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom13MigrationITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom13MigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom13MigrationITCase.java
deleted file mode 100644
index a2d3201..0000000
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointFrom13MigrationITCase.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.test.checkpointing.utils;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * This verifies that we can restore a complete job from a Flink 1.3 savepoint.
- *
- * <p>The test pipeline contains both "Checkpointed" state and keyed user state.
- */
-public class StatefulJobSavepointFrom13MigrationITCase extends StatefulJobSavepointFrom12MigrationITCase {
-
- /**
- * This has to be manually executed to create the savepoint on Flink 1.3.
- */
- @Test
- @Ignore
- public void testCreateSavepointOnFlink13() throws Exception {
- testCreateSavepointOnFlink12();
- }
-
- /**
- * This has to be manually executed to create the savepoint on Flink 1.3.
- */
- @Test
- @Ignore
- public void testCreateSavepointOnFlink13WithRocksDB() throws Exception {
- testCreateSavepointOnFlink12WithRocksDB();
- }
-
- @Override
- protected String getSavepointPath() {
- return "stateful-udf-migration-itcase-flink1.3-savepoint";
- }
-
- @Override
- protected String getRocksDBSavepointPath() {
- return "stateful-udf-migration-itcase-flink1.3-rocksdb-savepoint";
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/8d3a74f9/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
new file mode 100644
index 0000000..f83a3c6
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
@@ -0,0 +1,661 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.checkpointing.utils;
+
+import org.apache.flink.api.common.accumulators.IntCounter;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.TypeHint;
+import org.apache.flink.api.common.typeutils.base.LongSerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
+import org.apache.flink.runtime.state.StateBackendLoader;
+import org.apache.flink.runtime.state.memory.MemoryStateBackend;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
+import org.apache.flink.streaming.api.operators.InternalTimer;
+import org.apache.flink.streaming.api.operators.InternalTimerService;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.TimestampedCollector;
+import org.apache.flink.streaming.api.operators.Triggerable;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
+import org.apache.flink.util.Collector;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Migration ITCases for a stateful job. The tests are parameterized to cover
+ * migrating for multiple previous Flink versions, as well as for different state backends.
+ */
+@RunWith(Parameterized.class)
+public class StatefulJobSavepointMigrationITCase extends SavepointMigrationTestBase {
+
+ private static final int NUM_SOURCE_ELEMENTS = 4;
+
+ @Parameterized.Parameters(name = "Migrate Savepoint / Backend: {0}")
+ public static Collection<Tuple2<MigrationVersion, String>> parameters () {
+ return Arrays.asList(
+ Tuple2.of(MigrationVersion.v1_2, StateBackendLoader.MEMORY_STATE_BACKEND_NAME),
+ Tuple2.of(MigrationVersion.v1_2, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME),
+ Tuple2.of(MigrationVersion.v1_3, StateBackendLoader.MEMORY_STATE_BACKEND_NAME),
+ Tuple2.of(MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME));
+ }
+
+ /**
+ * TODO to generate savepoints for a specific Flink version / backend type,
+ * TODO change these values accordingly, e.g. to generate for 1.3 with RocksDB,
+ * TODO set as (MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME)
+ */
+ private final MigrationVersion flinkGenerateSavepointVersion = MigrationVersion.v1_3;
+ private final String flinkGenerateSavepointBackendType = StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME;
+
+ private final MigrationVersion testMigrateVersion;
+ private final String testStateBackend;
+
+ public StatefulJobSavepointMigrationITCase(Tuple2<MigrationVersion, String> testMigrateVersionAndBackend) {
+ this.testMigrateVersion = testMigrateVersionAndBackend.f0;
+ this.testStateBackend = testMigrateVersionAndBackend.f1;
+ }
+
+ /**
+ * Manually run this to write binary snapshot data.
+ */
+ @Test
+ @Ignore
+ public void writeSavepoint() throws Exception {
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+
+ switch (flinkGenerateSavepointBackendType) {
+ case StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME:
+ env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
+ break;
+ case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
+ env.setStateBackend(new MemoryStateBackend());
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ env.enableCheckpointing(500);
+ env.setParallelism(4);
+ env.setMaxParallelism(4);
+
+ env
+ .addSource(new LegacyCheckpointedSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
+ .flatMap(new LegacyCheckpointedFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
+ .keyBy(0)
+ .flatMap(new LegacyCheckpointedFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
+ .keyBy(0)
+ .flatMap(new KeyedStateSettingFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
+ .keyBy(0)
+ .transform(
+ "custom_operator",
+ new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+ new CheckpointedUdfOperator(new LegacyCheckpointedFlatMapWithKeyedState())).uid("LegacyCheckpointedOperator")
+ .keyBy(0)
+ .transform(
+ "timely_stateful_operator",
+ new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+ new TimelyStatefulOperator()).uid("TimelyStatefulOperator")
+ .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
+
+ executeAndSavepoint(
+ env,
+ "src/test/resources/" + getSavepointPath(flinkGenerateSavepointVersion, flinkGenerateSavepointBackendType),
+ new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
+ }
+
+ @Test
+ public void testSavepointRestore() throws Exception {
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setRestartStrategy(RestartStrategies.noRestart());
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+
+ switch (testStateBackend) {
+ case StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME:
+ env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
+ break;
+ case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
+ env.setStateBackend(new MemoryStateBackend());
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ env.enableCheckpointing(500);
+ env.setParallelism(4);
+ env.setMaxParallelism(4);
+
+ env
+ .addSource(new CheckingRestoringSource(NUM_SOURCE_ELEMENTS)).setMaxParallelism(1).uid("LegacyCheckpointedSource")
+ .flatMap(new CheckingRestoringFlatMap()).startNewChain().uid("LegacyCheckpointedFlatMap")
+ .keyBy(0)
+ .flatMap(new CheckingRestoringFlatMapWithKeyedState()).startNewChain().uid("LegacyCheckpointedFlatMapWithKeyedState")
+ .keyBy(0)
+ .flatMap(new CheckingKeyedStateFlatMap()).startNewChain().uid("KeyedStateSettingFlatMap")
+ .keyBy(0)
+ .transform(
+ "custom_operator",
+ new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+ new CheckingRestoringUdfOperator(new CheckingRestoringFlatMapWithKeyedStateInOperator())).uid("LegacyCheckpointedOperator")
+ .keyBy(0)
+ .transform(
+ "timely_stateful_operator",
+ new TypeHint<Tuple2<Long, Long>>() {}.getTypeInfo(),
+ new CheckingTimelyStatefulOperator()).uid("TimelyStatefulOperator")
+ .addSink(new AccumulatorCountingSink<Tuple2<Long, Long>>());
+
+ restoreAndExecute(
+ env,
+ getResourceFilename(getSavepointPath(testMigrateVersion, testStateBackend)),
+ new Tuple2<>(CheckingRestoringSource.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, 1),
+ new Tuple2<>(CheckingRestoringFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingRestoringFlatMapWithKeyedState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingKeyedStateFlatMap.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingRestoringUdfOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingRestoringFlatMapWithKeyedStateInOperator.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(CheckingTimelyStatefulOperator.SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, NUM_SOURCE_ELEMENTS),
+ new Tuple2<>(AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS));
+ }
+
+ private String getSavepointPath(MigrationVersion savepointVersion, String backendType) {
+ switch (backendType) {
+ case StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME:
+ return "stateful-udf-migration-itcase-flink" + savepointVersion + "-rocksdb-savepoint";
+ case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
+ return "stateful-udf-migration-itcase-flink" + savepointVersion + "-savepoint";
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private static class LegacyCheckpointedSource
+ implements SourceFunction<Tuple2<Long, Long>> {
+
+ public static String checkpointedString = "Here be dragons!";
+
+ private static final long serialVersionUID = 1L;
+
+ private volatile boolean isRunning = true;
+
+ private final int numElements;
+
+ public LegacyCheckpointedSource(int numElements) {
+ this.numElements = numElements;
+ }
+
+ @Override
+ public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
+
+ ctx.emitWatermark(new Watermark(0));
+
+ synchronized (ctx.getCheckpointLock()) {
+ for (long i = 0; i < numElements; i++) {
+ ctx.collect(new Tuple2<>(i, i));
+ }
+ }
+
+ // don't emit a final watermark so that we don't trigger the registered event-time
+ // timers
+ while (isRunning) {
+ Thread.sleep(20);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ isRunning = false;
+ }
+ }
+
+ private static class CheckingRestoringSource
+ extends RichSourceFunction<Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringSource.class + "_RESTORE_CHECK";
+
+ private volatile boolean isRunning = true;
+
+ private final int numElements;
+
+ private String restoredState;
+
+ public CheckingRestoringSource(int numElements) {
+ this.numElements = numElements;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+
+ // immediately trigger any set timers
+ ctx.emitWatermark(new Watermark(1000));
+
+ synchronized (ctx.getCheckpointLock()) {
+ for (long i = 0; i < numElements; i++) {
+ ctx.collect(new Tuple2<>(i, i));
+ }
+ }
+
+ while (isRunning) {
+ Thread.sleep(20);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ isRunning = false;
+ }
+ }
+
+ private static class LegacyCheckpointedFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static Tuple2<String, Long> checkpointedTuple =
+ new Tuple2<>("hello", 42L);
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+ }
+ }
+
+ private static class CheckingRestoringFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMap.class + "_RESTORE_CHECK";
+
+ private transient Tuple2<String, Long> restoredState;
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+
+ }
+
+ }
+
+ private static class LegacyCheckpointedFlatMapWithKeyedState
+ extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static Tuple2<String, Long> checkpointedTuple =
+ new Tuple2<>("hello", 42L);
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ getRuntimeContext().getState(stateDescriptor).update(value.f1);
+
+ assertEquals(value.f1, getRuntimeContext().getState(stateDescriptor).value());
+ }
+ }
+
+ private static class CheckingRestoringFlatMapWithKeyedState
+ extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMapWithKeyedState.class + "_RESTORE_CHECK";
+
+ private transient Tuple2<String, Long> restoredState;
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
+ if (state == null) {
+ throw new RuntimeException("Missing key value state for " + value);
+ }
+
+ assertEquals(value.f1, state.value());
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+ }
+ }
+
+ private static class CheckingRestoringFlatMapWithKeyedStateInOperator
+ extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringFlatMapWithKeyedStateInOperator.class + "_RESTORE_CHECK";
+
+ private transient Tuple2<String, Long> restoredState;
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
+ if (state == null) {
+ throw new RuntimeException("Missing key value state for " + value);
+ }
+
+ assertEquals(value.f1, state.value());
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+ }
+ }
+
+ private static class KeyedStateSettingFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ getRuntimeContext().getState(stateDescriptor).update(value.f1);
+ }
+ }
+
+ private static class CheckingKeyedStateFlatMap extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingKeyedStateFlatMap.class + "_RESTORE_CHECK";
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void flatMap(Tuple2<Long, Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(value);
+
+ ValueState<Long> state = getRuntimeContext().getState(stateDescriptor);
+ if (state == null) {
+ throw new RuntimeException("Missing key value state for " + value);
+ }
+
+ assertEquals(value.f1, state.value());
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+ }
+ }
+
+ private static class CheckpointedUdfOperator
+ extends AbstractUdfStreamOperator<Tuple2<Long, Long>, FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>>
+ implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ private static final long serialVersionUID = 1L;
+
+ private static final String CHECKPOINTED_STRING = "Oh my, that's nice!";
+
+ public CheckpointedUdfOperator(FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> userFunction) {
+ super(userFunction);
+ }
+
+ @Override
+ public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+ userFunction.flatMap(element.getValue(), new TimestampedCollector<>(output));
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {
+ output.emitWatermark(mark);
+ }
+ }
+
+ private static class CheckingRestoringUdfOperator
+ extends AbstractUdfStreamOperator<Tuple2<Long, Long>, FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>>
+ implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR = CheckingRestoringUdfOperator.class + "_RESTORE_CHECK";
+
+ private String restoredState;
+
+ public CheckingRestoringUdfOperator(FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> userFunction) {
+ super(userFunction);
+ }
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+ userFunction.flatMap(element.getValue(), new TimestampedCollector<>(output));
+ getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1);
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {
+ output.emitWatermark(mark);
+ }
+ }
+
+ private static class TimelyStatefulOperator
+ extends AbstractStreamOperator<Tuple2<Long, Long>>
+ implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>>, Triggerable<Long, Long> {
+ private static final long serialVersionUID = 1L;
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ private transient InternalTimerService<Long> timerService;
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+
+ timerService = getInternalTimerService(
+ "timer",
+ LongSerializer.INSTANCE,
+ this);
+
+ }
+
+ @Override
+ public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+ ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+ element.getValue().f0,
+ LongSerializer.INSTANCE,
+ stateDescriptor);
+
+ state.update(element.getValue().f1);
+
+ timerService.registerEventTimeTimer(element.getValue().f0, timerService.currentWatermark() + 10);
+ timerService.registerProcessingTimeTimer(element.getValue().f0, timerService.currentProcessingTime() + 30_000);
+
+ output.collect(element);
+ }
+
+ @Override
+ public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
+
+ }
+
+ @Override
+ public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
+
+ }
+
+ @Override
+ public void processWatermark(Watermark mark) throws Exception {
+ output.emitWatermark(mark);
+ }
+ }
+
+ private static class CheckingTimelyStatefulOperator
+ extends AbstractStreamOperator<Tuple2<Long, Long>>
+ implements OneInputStreamOperator<Tuple2<Long, Long>, Tuple2<Long, Long>>, Triggerable<Long, Long> {
+ private static final long serialVersionUID = 1L;
+
+ public static final String SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_PROCESS_CHECKS";
+ public static final String SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_ET_CHECKS";
+ public static final String SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR = CheckingTimelyStatefulOperator.class + "_PT_CHECKS";
+
+ private final ValueStateDescriptor<Long> stateDescriptor =
+ new ValueStateDescriptor<Long>("state-name", LongSerializer.INSTANCE);
+
+ private transient InternalTimerService<Long> timerService;
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+
+ timerService = getInternalTimerService(
+ "timer",
+ LongSerializer.INSTANCE,
+ this);
+
+ getRuntimeContext().addAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR, new IntCounter());
+ getRuntimeContext().addAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR, new IntCounter());
+ getRuntimeContext().addAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void processElement(StreamRecord<Tuple2<Long, Long>> element) throws Exception {
+ ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+ element.getValue().f0,
+ LongSerializer.INSTANCE,
+ stateDescriptor);
+
+ assertEquals(state.value(), element.getValue().f1);
+ getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESS_CHECK_ACCUMULATOR).add(1);
+
+ output.collect(element);
+ }
+
+ @Override
+ public void onEventTime(InternalTimer<Long, Long> timer) throws Exception {
+ ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+ timer.getNamespace(),
+ LongSerializer.INSTANCE,
+ stateDescriptor);
+
+ assertEquals(state.value(), timer.getNamespace());
+ getRuntimeContext().getAccumulator(SUCCESSFUL_EVENT_TIME_CHECK_ACCUMULATOR).add(1);
+ }
+
+ @Override
+ public void onProcessingTime(InternalTimer<Long, Long> timer) throws Exception {
+ ValueState<Long> state = getKeyedStateBackend().getPartitionedState(
+ timer.getNamespace(),
+ LongSerializer.INSTANCE,
+ stateDescriptor);
+
+ assertEquals(state.value(), timer.getNamespace());
+ getRuntimeContext().getAccumulator(SUCCESSFUL_PROCESSING_TIME_CHECK_ACCUMULATOR).add(1);
+ }
+ }
+
+ private static class AccumulatorCountingSink<T> extends RichSinkFunction<T> {
+ private static final long serialVersionUID = 1L;
+
+ public static final String NUM_ELEMENTS_ACCUMULATOR = AccumulatorCountingSink.class + "_NUM_ELEMENTS";
+
+ int count = 0;
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ getRuntimeContext().addAccumulator(NUM_ELEMENTS_ACCUMULATOR, new IntCounter());
+ }
+
+ @Override
+ public void invoke(T value) throws Exception {
+ count++;
+ getRuntimeContext().getAccumulator(NUM_ELEMENTS_ACCUMULATOR).add(1);
+ }
+ }
+
+}
[06/31] flink git commit: [FLINK-6004] [kinesis] Allow
FlinkKinesisConsumer to skip records
Posted by tz...@apache.org.
[FLINK-6004] [kinesis] Allow FlinkKinesisConsumer to skip records
This commit acknowledges that null can be returned from the
deserialization schema, if the message cannot be deserialized. If null
is returned for a Kinesis record, no output is produced for that record,
while the sequence number in the shard state is still advanced so that
the record is effectively accounted as processed.
This closes #5269.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/30677772
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/30677772
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/30677772
Branch: refs/heads/master
Commit: 30677772e29a8bbb667d098c43a0b225d5602049
Parents: 3f0d6c6
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 10 14:04:10 2018 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../kinesis/internals/KinesisDataFetcher.java | 9 +++-
.../KinesisDeserializationSchema.java | 7 ++-
.../internals/KinesisDataFetcherTest.java | 52 ++++++++++++++++++++
.../kinesis/testutils/TestSourceContext.java | 4 ++
4 files changed, 69 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/30677772/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
index d20a449..274ae70 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
@@ -501,7 +501,14 @@ public class KinesisDataFetcher<T> {
*/
protected final void emitRecordAndUpdateState(T record, long recordTimestamp, int shardStateIndex, SequenceNumber lastSequenceNumber) {
synchronized (checkpointLock) {
- sourceContext.collectWithTimestamp(record, recordTimestamp);
+ if (record != null) {
+ sourceContext.collectWithTimestamp(record, recordTimestamp);
+ } else {
+ LOG.warn("Skipping non-deserializable record at sequence number {} of shard {}.",
+ lastSequenceNumber,
+ subscribedShardsState.get(shardStateIndex).getStreamShardHandle());
+ }
+
updateState(shardStateIndex, lastSequenceNumber);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/30677772/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/serialization/KinesisDeserializationSchema.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/serialization/KinesisDeserializationSchema.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/serialization/KinesisDeserializationSchema.java
index 5f40a0f..423477e 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/serialization/KinesisDeserializationSchema.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/serialization/KinesisDeserializationSchema.java
@@ -35,7 +35,9 @@ import java.io.Serializable;
public interface KinesisDeserializationSchema<T> extends Serializable, ResultTypeQueryable<T> {
/**
- * Deserializes a Kinesis record's bytes.
+ * Deserializes a Kinesis record's bytes. If the record cannot be deserialized, {@code null}
+ * may be returned. This informs the Flink Kinesis Consumer to process the Kinesis record
+ * without producing any output for it, i.e. effectively "skipping" the record.
*
* @param recordValue the record's value as a byte array
* @param partitionKey the record's partition key at the time of writing
@@ -43,7 +45,8 @@ public interface KinesisDeserializationSchema<T> extends Serializable, ResultTyp
* @param approxArrivalTimestamp the server-side timestamp of when Kinesis received and stored the record
* @param stream the name of the Kinesis stream that this record was sent to
* @param shardId The identifier of the shard the record was sent to
- * @return the deserialized message as an Java object
+ *
+ * @return the deserialized message as an Java object ({@code null} if the message cannot be deserialized).
* @throws IOException
*/
T deserialize(byte[] recordValue, String partitionKey, String seqNum, long approxArrivalTimestamp, String stream, String shardId) throws IOException;
http://git-wip-us.apache.org/repos/asf/flink/blob/30677772/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
index 95e87c6..1620366 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
@@ -37,9 +37,11 @@ import org.apache.flink.streaming.connectors.kinesis.testutils.TestableKinesisDa
import com.amazonaws.services.kinesis.model.HashKeyRange;
import com.amazonaws.services.kinesis.model.SequenceNumberRange;
import com.amazonaws.services.kinesis.model.Shard;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.util.TestLogger;
import org.junit.Test;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -87,6 +89,56 @@ public class KinesisDataFetcherTest extends TestLogger {
}
@Test
+ public void testSkipCorruptedRecord() throws Exception {
+ final String stream = "fakeStream";
+ final int numShards = 3;
+
+ final LinkedList<KinesisStreamShardState> testShardStates = new LinkedList<>();
+ final TestSourceContext<String> sourceContext = new TestSourceContext<>();
+
+ final TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>(
+ Collections.singletonList(stream),
+ sourceContext,
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ 1,
+ 0,
+ new AtomicReference<>(),
+ testShardStates,
+ new HashMap<>(),
+ FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(Collections.singletonMap(stream, numShards)));
+
+ // FlinkKinesisConsumer is responsible for setting up the fetcher before it can be run;
+ // run the consumer until it reaches the point where the fetcher starts to run
+ final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(TestUtils.getStandardProperties(), fetcher, 1, 0);
+
+ CheckedThread consumerThread = new CheckedThread() {
+ @Override
+ public void go() throws Exception {
+ consumer.run(new TestSourceContext<>());
+ }
+ };
+ consumerThread.start();
+
+ fetcher.waitUntilRun();
+ consumer.cancel();
+ consumerThread.sync();
+
+ assertEquals(numShards, testShardStates.size());
+
+ for (int i = 0; i < numShards; i++) {
+ fetcher.emitRecordAndUpdateState("record-" + i, 10L, i, new SequenceNumber("seq-num-1"));
+ assertEquals(new SequenceNumber("seq-num-1"), testShardStates.get(i).getLastProcessedSequenceNum());
+ assertEquals(new StreamRecord<>("record-" + i, 10L), sourceContext.removeLatestOutput());
+ }
+
+ // emitting a null (i.e., a corrupt record) should not produce any output, but still have the shard state updated
+ fetcher.emitRecordAndUpdateState(null, 10L, 1, new SequenceNumber("seq-num-2"));
+ assertEquals(new SequenceNumber("seq-num-2"), testShardStates.get(1).getLastProcessedSequenceNum());
+ assertEquals(null, sourceContext.removeLatestOutput()); // no output should have been collected
+ }
+
+ @Test
public void testStreamToLastSeenShardStateIsCorrectlySetWhenNotRestoringFromFailure() throws Exception {
List<String> fakeStreams = new LinkedList<>();
fakeStreams.add("fakeStream1");
http://git-wip-us.apache.org/repos/asf/flink/blob/30677772/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java
index 4fe5c54..151436b 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java
@@ -58,6 +58,10 @@ public class TestSourceContext<T> implements SourceFunction.SourceContext<T> {
@Override
public void close() {}
+ public StreamRecord<T> removeLatestOutput() {
+ return collectedOutputs.poll();
+ }
+
public ConcurrentLinkedQueue<StreamRecord<T>> getCollectedOutputs() {
return collectedOutputs;
}
[21/31] flink git commit: [FLINK-8472] [kafka,
test] Extend FlinkKafkaConsumerBaseMigrationTest for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [kafka, test] Extend FlinkKafkaConsumerBaseMigrationTest for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/535d1392
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/535d1392
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/535d1392
Branch: refs/heads/master
Commit: 535d139260faf73b289ae6bbed812d831cd75805
Parents: e609f81
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 17:23:22 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../kafka/FlinkKafkaConsumerBaseMigrationTest.java | 2 +-
...mer-migration-test-flink1.4-empty-state-snapshot | Bin 0 -> 6471 bytes
.../kafka-consumer-migration-test-flink1.4-snapshot | Bin 0 -> 6525 bytes
3 files changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/535d1392/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
index a533cca..1980ccc 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
@@ -93,7 +93,7 @@ public class FlinkKafkaConsumerBaseMigrationTest {
@Parameterized.Parameters(name = "Migration Savepoint: {0}")
public static Collection<MigrationVersion> parameters () {
- return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3);
+ return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3, MigrationVersion.v1_4);
}
public FlinkKafkaConsumerBaseMigrationTest(MigrationVersion testMigrateVersion) {
http://git-wip-us.apache.org/repos/asf/flink/blob/535d1392/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-empty-state-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-empty-state-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-empty-state-snapshot
new file mode 100644
index 0000000..2a06fb7
Binary files /dev/null and b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-empty-state-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/535d1392/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-snapshot
new file mode 100644
index 0000000..d778ca0
Binary files /dev/null and b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.4-snapshot differ
[11/31] flink git commit: [hotfix] [kinesis,
test] Use real test classes instead of Mockito in
FlinkKinesisConsumerMigrationTest
Posted by tz...@apache.org.
[hotfix] [kinesis, test] Use real test classes instead of Mockito in FlinkKinesisConsumerMigrationTest
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ffd7995b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ffd7995b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ffd7995b
Branch: refs/heads/master
Commit: ffd7995b932ffcf03731c4e423676e370639507b
Parents: b5b95eb
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Feb 1 11:11:48 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../FlinkKinesisConsumerMigrationTest.java | 145 +++++++++++++++----
1 file changed, 118 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/ffd7995b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
index ab9826e..9277cbc 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
@@ -18,15 +18,20 @@
package org.apache.flink.streaming.connectors.kinesis;
import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.operators.StreamSource;
import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
+import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardMetadata;
import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
+import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchemaWrapper;
import org.apache.flink.streaming.connectors.kinesis.testutils.KinesisShardIdGenerator;
+import org.apache.flink.streaming.connectors.kinesis.testutils.TestRuntimeContext;
+import org.apache.flink.streaming.connectors.kinesis.testutils.TestSourceContext;
import org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils;
import org.apache.flink.streaming.runtime.tasks.OperatorStateHandles;
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness;
@@ -34,15 +39,17 @@ import org.apache.flink.streaming.util.OperatorSnapshotUtil;
import org.apache.flink.streaming.util.migration.MigrationTestUtil;
import org.apache.flink.streaming.util.migration.MigrationVersion;
+import com.amazonaws.services.kinesis.model.Shard;
+
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
@@ -51,9 +58,6 @@ import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
/**
* Tests for checking whether {@link FlinkKinesisConsumer} can restore from snapshots that were
@@ -71,10 +75,12 @@ public class FlinkKinesisConsumerMigrationTest {
*/
private final MigrationVersion flinkGenerateSavepointVersion = null;
+ private static final String TEST_STREAM_NAME = "fakeStream1";
+
private static final HashMap<StreamShardMetadata, SequenceNumber> TEST_STATE = new HashMap<>();
static {
StreamShardMetadata shardMetadata = new StreamShardMetadata();
- shardMetadata.setStreamName("fakeStream1");
+ shardMetadata.setStreamName(TEST_STREAM_NAME);
shardMetadata.setShardId(KinesisShardIdGenerator.generateFromShardOrder(0));
TEST_STATE.put(shardMetadata, new SequenceNumber("987654321"));
@@ -105,7 +111,25 @@ public class FlinkKinesisConsumerMigrationTest {
@Test
public void testRestoreWithEmptyState() throws Exception {
- final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(mock(KinesisDataFetcher.class));
+ final List<StreamShardHandle> initialDiscoveryShards = new ArrayList<>(TEST_STATE.size());
+ for (StreamShardMetadata shardMetadata : TEST_STATE.keySet()) {
+ Shard shard = new Shard();
+ shard.setShardId(shardMetadata.getShardId());
+
+ initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), shard));
+ }
+
+ final TestFetcher<String> fetcher = new TestFetcher<>(
+ Collections.singletonList(TEST_STREAM_NAME),
+ new TestSourceContext<>(),
+ new TestRuntimeContext(true, 1, 0),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ null,
+ initialDiscoveryShards);
+
+ final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(
+ fetcher, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()));
StreamSource<String, DummyFlinkKinesisConsumer<String>> consumerOperator = new StreamSource<>(consumerFunction);
@@ -118,6 +142,8 @@ public class FlinkKinesisConsumerMigrationTest {
"src/test/resources/kinesis-consumer-migration-test-flink" + testMigrateVersion + "-empty-snapshot", testMigrateVersion);
testHarness.open();
+ consumerFunction.run(new TestSourceContext<>());
+
// assert that no state was restored
assertTrue(consumerFunction.getRestoredState().isEmpty());
@@ -127,7 +153,25 @@ public class FlinkKinesisConsumerMigrationTest {
@Test
public void testRestore() throws Exception {
- final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(mock(KinesisDataFetcher.class));
+ final List<StreamShardHandle> initialDiscoveryShards = new ArrayList<>(TEST_STATE.size());
+ for (StreamShardMetadata shardMetadata : TEST_STATE.keySet()) {
+ Shard shard = new Shard();
+ shard.setShardId(shardMetadata.getShardId());
+
+ initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), shard));
+ }
+
+ final TestFetcher<String> fetcher = new TestFetcher<>(
+ Collections.singletonList(TEST_STREAM_NAME),
+ new TestSourceContext<>(),
+ new TestRuntimeContext(true, 1, 0),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ null,
+ initialDiscoveryShards);
+
+ final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(
+ fetcher, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()));
StreamSource<String, DummyFlinkKinesisConsumer<String>> consumerOperator =
new StreamSource<>(consumerFunction);
@@ -141,6 +185,8 @@ public class FlinkKinesisConsumerMigrationTest {
"src/test/resources/kinesis-consumer-migration-test-flink" + testMigrateVersion + "-snapshot", testMigrateVersion);
testHarness.open();
+ consumerFunction.run(new TestSourceContext<>());
+
// assert that state is correctly restored
assertNotEquals(null, consumerFunction.getRestoredState());
assertEquals(1, consumerFunction.getRestoredState().size());
@@ -154,19 +200,17 @@ public class FlinkKinesisConsumerMigrationTest {
@SuppressWarnings("unchecked")
private void writeSnapshot(String path, HashMap<StreamShardMetadata, SequenceNumber> state) throws Exception {
- final OneShotLatch latch = new OneShotLatch();
-
- final KinesisDataFetcher<String> fetcher = mock(KinesisDataFetcher.class);
- doAnswer(new Answer() {
- @Override
- public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
- latch.trigger();
- return null;
- }
- }).when(fetcher).runFetcher();
- when(fetcher.snapshotState()).thenReturn(state);
-
- final DummyFlinkKinesisConsumer<String> consumer = new DummyFlinkKinesisConsumer<>(fetcher);
+ final TestFetcher<String> fetcher = new TestFetcher<>(
+ Collections.singletonList(TEST_STREAM_NAME),
+ new TestSourceContext<>(),
+ new TestRuntimeContext(true, 1, 0),
+ new Properties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ state,
+ null);
+
+ final DummyFlinkKinesisConsumer<String> consumer = new DummyFlinkKinesisConsumer<>(
+ fetcher, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()));
StreamSource<String, DummyFlinkKinesisConsumer<String>> consumerOperator = new StreamSource<>(consumer);
@@ -185,7 +229,7 @@ public class FlinkKinesisConsumerMigrationTest {
@Override
public void run() {
try {
- consumer.run(mock(SourceFunction.SourceContext.class));
+ consumer.run(new TestSourceContext<>());
} catch (Throwable t) {
t.printStackTrace();
error.set(t);
@@ -194,9 +238,7 @@ public class FlinkKinesisConsumerMigrationTest {
};
runner.start();
- if (!latch.isTriggered()) {
- latch.await();
- }
+ fetcher.waitUntilRun();
final OperatorStateHandles snapshot;
synchronized (testHarness.getCheckpointLock()) {
@@ -211,12 +253,14 @@ public class FlinkKinesisConsumerMigrationTest {
private static class DummyFlinkKinesisConsumer<T> extends FlinkKinesisConsumer<T> {
+ private static final long serialVersionUID = -1573896262106029446L;
+
private KinesisDataFetcher<T> mockFetcher;
private static Properties dummyConfig = TestUtils.getStandardProperties();
- DummyFlinkKinesisConsumer(KinesisDataFetcher<T> mockFetcher) {
- super("dummy-topic", mock(KinesisDeserializationSchema.class), dummyConfig);
+ DummyFlinkKinesisConsumer(KinesisDataFetcher<T> mockFetcher, KinesisDeserializationSchema<T> schema) {
+ super(TEST_STREAM_NAME, schema, dummyConfig);
this.mockFetcher = mockFetcher;
}
@@ -230,4 +274,51 @@ public class FlinkKinesisConsumerMigrationTest {
return mockFetcher;
}
}
+
+ private static class TestFetcher<T> extends KinesisDataFetcher<T> {
+
+ final OneShotLatch runLatch = new OneShotLatch();
+
+ final HashMap<StreamShardMetadata, SequenceNumber> testStateSnapshot;
+ final List<StreamShardHandle> testInitialDiscoveryShards;
+
+ public TestFetcher(
+ List<String> streams,
+ SourceFunction.SourceContext<T> sourceContext,
+ RuntimeContext runtimeContext,
+ Properties configProps,
+ KinesisDeserializationSchema<T> deserializationSchema,
+ HashMap<StreamShardMetadata, SequenceNumber> testStateSnapshot,
+ List<StreamShardHandle> testInitialDiscoveryShards) {
+
+ super(streams, sourceContext, runtimeContext, configProps, deserializationSchema);
+
+ this.testStateSnapshot = testStateSnapshot;
+ this.testInitialDiscoveryShards = testInitialDiscoveryShards;
+ }
+
+ @Override
+ public void runFetcher() throws Exception {
+ runLatch.trigger();
+ }
+
+ @Override
+ public HashMap<StreamShardMetadata, SequenceNumber> snapshotState() {
+ return testStateSnapshot;
+ }
+
+ public void waitUntilRun() throws InterruptedException {
+ runLatch.await();
+ }
+
+ @Override
+ public List<StreamShardHandle> discoverNewShardsToSubscribe() throws InterruptedException {
+ return testInitialDiscoveryShards;
+ }
+
+ @Override
+ public void awaitTermination() throws InterruptedException {
+ // do nothing
+ }
+ }
}
[23/31] flink git commit: [FLINK-8275] [security,
yarn] Remove test-specific code path in YarnTaskManagerRunner
Posted by tz...@apache.org.
[FLINK-8275] [security, yarn] Remove test-specific code path in YarnTaskManagerRunner
Previously, the YarnTaskManagerRunner contained a code path that exists
for the sole purpose of injecting mock runners. Having code paths just
to utilize tests in production code is in general a bad idea.
This commit fixes this be making YarnTaskManagerRunner a factory-like
class, which creates a Runner that contains all the runner’s properties,
such as configuration. Unit tests can than test against the contained
configuration in the created Runner to validate that everything is
configured properly.
This closes #5172.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/97f0cac2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/97f0cac2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/97f0cac2
Branch: refs/heads/master
Commit: 97f0cac2af3a1140fa68090d94d83e009ad1e684
Parents: 2bbf4e5
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Fri Feb 2 15:53:13 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../yarn/TestingYarnTaskManagerRunner.java | 12 +-
.../flink/yarn/TestingYarnTaskManager.scala | 14 +-
.../flink/yarn/YarnApplicationMasterRunner.java | 8 +-
.../flink/yarn/YarnTaskManagerRunner.java | 165 ----------------
.../yarn/YarnTaskManagerRunnerFactory.java | 186 +++++++++++++++++++
.../org/apache/flink/yarn/YarnTaskManager.scala | 30 ++-
.../yarn/YarnTaskManagerRunnerFactoryTest.java | 76 ++++++++
.../flink/yarn/YarnTaskManagerRunnerTest.java | 74 --------
8 files changed, 312 insertions(+), 253 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/97f0cac2/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
index 90cdf9f..ff030be 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
@@ -18,6 +18,8 @@
package org.apache.flink.yarn;
+import org.apache.flink.runtime.security.SecurityUtils;
+
import java.io.IOException;
/**
@@ -25,7 +27,13 @@ import java.io.IOException;
*/
public class TestingYarnTaskManagerRunner {
public static void main(String[] args) throws IOException {
- YarnTaskManagerRunner.runYarnTaskManager(
- args, TestingYarnTaskManager.class, System.getenv(), null);
+ YarnTaskManagerRunnerFactory.Runner tmRunner = YarnTaskManagerRunnerFactory.create(
+ args, TestingYarnTaskManager.class, System.getenv());
+
+ try {
+ SecurityUtils.getInstalledContext().runSecured(tmRunner);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/97f0cac2/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala b/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
index da95e01..e33aa49 100644
--- a/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
+++ b/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
@@ -24,6 +24,7 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager
import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup
+import org.apache.flink.runtime.security.SecurityUtils
import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration
import org.apache.flink.runtime.taskmanager.TaskManagerLocation
import org.apache.flink.runtime.testingUtils.TestingTaskManagerLike
@@ -68,12 +69,19 @@ class TestingYarnTaskManager(
object YarnTaskManager {
/** Entry point (main method) to run the TaskManager on YARN.
- *
+ *
* @param args The command line arguments.
*/
def main(args: Array[String]): Unit = {
- YarnTaskManagerRunner.runYarnTaskManager(
- args, classOf[TestingYarnTaskManager], System.getenv(), null)
+ val tmRunner = YarnTaskManagerRunnerFactory.create(
+ args, classOf[TestingYarnTaskManager], System.getenv())
+
+ try {
+ SecurityUtils.getInstalledContext.runSecured(tmRunner)
+ } catch {
+ case e: Exception =>
+ throw new RuntimeException(e)
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/97f0cac2/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
index 1813034..9d1af35 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
@@ -163,6 +163,9 @@ public class YarnApplicationMasterRunner {
File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
if (remoteKeytabPrincipal != null && f.exists()) {
+ String keytabPath = f.getAbsolutePath();
+ LOG.debug("keytabPath: {}", keytabPath);
+
// set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
@@ -248,7 +251,10 @@ public class YarnApplicationMasterRunner {
File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
if (remoteKeytabPrincipal != null && f.exists()) {
- config.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
+ String keytabPath = f.getAbsolutePath();
+ LOG.debug("keytabPath: {}", keytabPath);
+
+ config.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
config.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/97f0cac2/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
deleted file mode 100644
index 9d69fbd..0000000
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.yarn;
-
-import org.apache.flink.configuration.AkkaOptions;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.CoreOptions;
-import org.apache.flink.configuration.SecurityOptions;
-import org.apache.flink.runtime.clusterframework.types.ResourceID;
-import org.apache.flink.runtime.security.SecurityConfiguration;
-import org.apache.flink.runtime.security.SecurityUtils;
-import org.apache.flink.runtime.security.modules.HadoopModule;
-import org.apache.flink.runtime.taskmanager.TaskManager;
-import org.apache.flink.runtime.util.EnvironmentInformation;
-import org.apache.flink.runtime.util.JvmShutdownSafeguard;
-import org.apache.flink.runtime.util.SignalHandler;
-import org.apache.flink.util.Preconditions;
-
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.Callable;
-
-/**
- * The entry point for running a TaskManager in a YARN container.
- */
-public class YarnTaskManagerRunner {
-
- private static final Logger LOG = LoggerFactory.getLogger(YarnTaskManagerRunner.class);
-
- private static Callable<Object> createMainRunner(
- Configuration configuration,
- ResourceID resourceId,
- final Class<? extends YarnTaskManager> taskManager) {
- return new Callable<Object>() {
- @Override
- public Integer call() {
- try {
- TaskManager.selectNetworkInterfaceAndRunTaskManager(
- configuration, resourceId, taskManager);
- } catch (Throwable t) {
- LOG.error("Error while starting the TaskManager", t);
- System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
- }
- return null;
- }
- };
- }
-
- public static void runYarnTaskManager(String[] args,
- final Class<? extends YarnTaskManager> taskManager,
- Map<String, String> envs,
- Callable<Object> mainRunner) throws IOException {
- EnvironmentInformation.logEnvironmentInfo(LOG, "YARN TaskManager", args);
- SignalHandler.register(LOG);
- JvmShutdownSafeguard.installAsShutdownHook(LOG);
-
- // try to parse the command line arguments
- final Configuration configuration;
- try {
- configuration = TaskManager.parseArgsAndLoadConfig(args);
- }
- catch (Throwable t) {
- LOG.error(t.getMessage(), t);
- System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
- return;
- }
-
- // read the environment variables for YARN
- final String yarnClientUsername = envs.get(YarnConfigKeys.ENV_HADOOP_USER_NAME);
- final String localDirs = envs.get(Environment.LOCAL_DIRS.key());
- LOG.info("Current working/local Directory: {}", localDirs);
-
- final String currDir = envs.get(Environment.PWD.key());
- LOG.info("Current working Directory: {}", currDir);
-
- final String remoteKeytabPrincipal = envs.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
- LOG.info("TM: remoteKeytabPrincipal obtained {}", remoteKeytabPrincipal);
-
- // configure local directory
- if (configuration.contains(CoreOptions.TMP_DIRS)) {
- LOG.info("Overriding YARN's temporary file directories with those " +
- "specified in the Flink config: " + configuration.getValue(CoreOptions.TMP_DIRS));
- }
- else {
- LOG.info("Setting directories for temporary files to: {}", localDirs);
- configuration.setString(CoreOptions.TMP_DIRS, localDirs);
- }
-
- // tell akka to die in case of an error
- configuration.setBoolean(AkkaOptions.JVM_EXIT_ON_FATAL_ERROR, true);
-
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
-
- LOG.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
- currentUser.getShortUserName(), yarnClientUsername);
-
- // Infer the resource identifier from the environment variable
- String containerID = Preconditions.checkNotNull(envs.get(YarnFlinkResourceManager.ENV_FLINK_CONTAINER_ID));
- final ResourceID resourceId = new ResourceID(containerID);
- LOG.info("ResourceID assigned for this container: {}", resourceId);
-
- File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
- if (remoteKeytabPrincipal != null && f.exists()) {
- // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
- configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
- configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
- }
- try {
-
- SecurityConfiguration sc;
-
- //To support Yarn Secure Integration Test Scenario
- File krb5Conf = new File(currDir, Utils.KRB5_FILE_NAME);
- if (krb5Conf.exists() && krb5Conf.canRead()) {
- String krb5Path = krb5Conf.getAbsolutePath();
- LOG.info("KRB5 Conf: {}", krb5Path);
- org.apache.hadoop.conf.Configuration hadoopConfiguration = new org.apache.hadoop.conf.Configuration();
- hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true");
-
- sc = new SecurityConfiguration(configuration,
- Collections.singletonList(securityConfig -> new HadoopModule(securityConfig, hadoopConfiguration)));
-
- } else {
- sc = new SecurityConfiguration(configuration);
-
- }
-
- SecurityUtils.install(sc);
-
- if (mainRunner == null) {
- mainRunner = createMainRunner(configuration, resourceId, taskManager);
- }
- SecurityUtils.getInstalledContext().runSecured(mainRunner);
- } catch (Exception e) {
- LOG.error("Exception occurred while launching Task Manager", e);
- throw new RuntimeException(e);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/97f0cac2/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactory.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactory.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactory.java
new file mode 100644
index 0000000..d14248c
--- /dev/null
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactory.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.configuration.AkkaOptions;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.CoreOptions;
+import org.apache.flink.configuration.SecurityOptions;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
+import org.apache.flink.runtime.security.SecurityConfiguration;
+import org.apache.flink.runtime.security.SecurityUtils;
+import org.apache.flink.runtime.security.modules.HadoopModule;
+import org.apache.flink.runtime.taskmanager.TaskManager;
+import org.apache.flink.runtime.util.EnvironmentInformation;
+import org.apache.flink.runtime.util.JvmShutdownSafeguard;
+import org.apache.flink.runtime.util.SignalHandler;
+import org.apache.flink.util.Preconditions;
+
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+/**
+ * The entry point for running a TaskManager in a YARN container.
+ */
+public class YarnTaskManagerRunnerFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(YarnTaskManagerRunnerFactory.class);
+
+ /**
+ * Runner for the {@link YarnTaskManager}.
+ */
+ public static class Runner implements Callable<Object> {
+
+ private final Configuration configuration;
+ private final ResourceID resourceId;
+ private final Class<? extends YarnTaskManager> taskManager;
+
+ Runner(Configuration configuration, ResourceID resourceId, Class<? extends YarnTaskManager> taskManager) {
+ this.configuration = Preconditions.checkNotNull(configuration);
+ this.resourceId = Preconditions.checkNotNull(resourceId);
+ this.taskManager = Preconditions.checkNotNull(taskManager);
+ }
+
+ @Override
+ public Object call() throws Exception {
+ try {
+ TaskManager.selectNetworkInterfaceAndRunTaskManager(
+ configuration, resourceId, taskManager);
+ } catch (Throwable t) {
+ LOG.error("Error while starting the TaskManager", t);
+ System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
+ }
+ return null;
+ }
+
+ @VisibleForTesting
+ Configuration getConfiguration() {
+ return configuration;
+ }
+
+ @VisibleForTesting
+ ResourceID getResourceId() {
+ return resourceId;
+ }
+ }
+
+ /**
+ * Creates a {@link YarnTaskManagerRunnerFactory.Runner}.
+ */
+ public static Runner create(
+ String[] args,
+ final Class<? extends YarnTaskManager> taskManager,
+ Map<String, String> envs) throws IOException {
+
+ EnvironmentInformation.logEnvironmentInfo(LOG, "YARN TaskManager", args);
+ SignalHandler.register(LOG);
+ JvmShutdownSafeguard.installAsShutdownHook(LOG);
+
+ // try to parse the command line arguments
+ final Configuration configuration;
+ try {
+ configuration = TaskManager.parseArgsAndLoadConfig(args);
+ }
+ catch (Throwable t) {
+ LOG.error(t.getMessage(), t);
+ System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
+ return null;
+ }
+
+ // read the environment variables for YARN
+ final String yarnClientUsername = envs.get(YarnConfigKeys.ENV_HADOOP_USER_NAME);
+ final String localDirs = envs.get(Environment.LOCAL_DIRS.key());
+ LOG.info("Current working/local Directory: {}", localDirs);
+
+ final String currDir = envs.get(Environment.PWD.key());
+ LOG.info("Current working Directory: {}", currDir);
+
+ final String remoteKeytabPrincipal = envs.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
+ LOG.info("TM: remoteKeytabPrincipal obtained {}", remoteKeytabPrincipal);
+
+ // configure local directory
+ if (configuration.contains(CoreOptions.TMP_DIRS)) {
+ LOG.info("Overriding YARN's temporary file directories with those " +
+ "specified in the Flink config: " + configuration.getValue(CoreOptions.TMP_DIRS));
+ }
+ else {
+ LOG.info("Setting directories for temporary files to: {}", localDirs);
+ configuration.setString(CoreOptions.TMP_DIRS, localDirs);
+ }
+
+ // tell akka to die in case of an error
+ configuration.setBoolean(AkkaOptions.JVM_EXIT_ON_FATAL_ERROR, true);
+
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+
+ LOG.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
+ currentUser.getShortUserName(), yarnClientUsername);
+
+ // Infer the resource identifier from the environment variable
+ String containerID = Preconditions.checkNotNull(envs.get(YarnFlinkResourceManager.ENV_FLINK_CONTAINER_ID));
+ final ResourceID resourceId = new ResourceID(containerID);
+ LOG.info("ResourceID assigned for this container: {}", resourceId);
+
+ File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
+ if (remoteKeytabPrincipal != null && f.exists()) {
+ // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
+ configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
+ configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
+ }
+
+ try {
+ SecurityConfiguration sc;
+
+ //To support Yarn Secure Integration Test Scenario
+ File krb5Conf = new File(currDir, Utils.KRB5_FILE_NAME);
+ if (krb5Conf.exists() && krb5Conf.canRead()) {
+ String krb5Path = krb5Conf.getAbsolutePath();
+ LOG.info("KRB5 Conf: {}", krb5Path);
+ org.apache.hadoop.conf.Configuration hadoopConfiguration = new org.apache.hadoop.conf.Configuration();
+ hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true");
+
+ sc = new SecurityConfiguration(configuration,
+ Collections.singletonList(securityConfig -> new HadoopModule(securityConfig, hadoopConfiguration)));
+
+ } else {
+ sc = new SecurityConfiguration(configuration);
+
+ }
+
+ SecurityUtils.install(sc);
+
+ return new Runner(configuration, resourceId, taskManager);
+ } catch (Exception e) {
+ LOG.error("Exception occurred while building Task Manager runner", e);
+ throw new RuntimeException(e);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/97f0cac2/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
index 5b8f4ca..08d24b4 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
@@ -24,9 +24,12 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager
import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup
+import org.apache.flink.runtime.security.SecurityUtils
import org.apache.flink.runtime.taskexecutor.TaskManagerConfiguration
import org.apache.flink.runtime.taskmanager.{TaskManager, TaskManagerLocation}
+import grizzled.slf4j.Logger
+
/** An extension of the TaskManager that listens for additional YARN related
* messages.
*/
@@ -56,14 +59,25 @@ class YarnTaskManager(
}
}
- object YarnTaskManager {
- /** Entry point (main method) to run the TaskManager on YARN.
- *
- * @param args The command line arguments.
- */
- def main(args: Array[String]): Unit = {
- YarnTaskManagerRunner.runYarnTaskManager(
- args, classOf[YarnTaskManager], System.getenv(), null)
+object YarnTaskManager {
+
+ val LOG = Logger(classOf[TaskManager])
+
+ /** Entry point (main method) to run the TaskManager on YARN.
+ *
+ * @param args The command line arguments.
+ */
+ def main(args: Array[String]): Unit = {
+ val tmRunner = YarnTaskManagerRunnerFactory.create(
+ args, classOf[YarnTaskManager], System.getenv())
+
+ try {
+ SecurityUtils.getInstalledContext.runSecured(tmRunner)
+ } catch {
+ case e: Exception =>
+ LOG.error("Exception occurred while launching Task Manager runner", e)
+ throw new RuntimeException(e)
}
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/97f0cac2/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactoryTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactoryTest.java
new file mode 100644
index 0000000..7f518f1
--- /dev/null
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerFactoryTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.SecurityOptions;
+import org.apache.flink.runtime.security.SecurityUtils;
+import org.apache.flink.runtime.security.modules.HadoopModule;
+import org.apache.flink.runtime.security.modules.SecurityModule;
+
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Tests for {@link YarnTaskManagerRunnerFactory}.
+ */
+public class YarnTaskManagerRunnerFactoryTest {
+
+ @Test
+ public void testKerberosKeytabConfiguration() throws IOException {
+ final String resourceDirPath =
+ Paths.get("src", "test", "resources").toAbsolutePath().toString();
+ final Map<String, String> envs = new HashMap<>();
+ envs.put(YarnFlinkResourceManager.ENV_FLINK_CONTAINER_ID, "test_container_00001");
+ envs.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "testuser1@domain");
+ envs.put(ApplicationConstants.Environment.PWD.key(), resourceDirPath);
+
+ final YarnTaskManagerRunnerFactory.Runner tmRunner = YarnTaskManagerRunnerFactory.create(
+ new String[]{"--configDir", resourceDirPath},
+ YarnTaskManager.class,
+ envs);
+
+ final List<SecurityModule> modules = SecurityUtils.getInstalledModules();
+ Optional<SecurityModule> moduleOpt =
+ modules.stream().filter(s -> s instanceof HadoopModule).findFirst();
+ if (moduleOpt.isPresent()) {
+ HadoopModule hadoopModule = (HadoopModule) moduleOpt.get();
+ assertEquals("testuser1@domain", hadoopModule.getSecurityConfig().getPrincipal());
+ assertEquals(resourceDirPath + "/" + Utils.KEYTAB_FILE_NAME,
+ hadoopModule.getSecurityConfig().getKeytab());
+ } else {
+ fail("Can not find HadoopModule!");
+ }
+
+ final Configuration configuration = tmRunner.getConfiguration();
+ assertEquals(resourceDirPath + "/" + Utils.KEYTAB_FILE_NAME, configuration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB));
+ assertEquals("testuser1@domain", configuration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL));
+ assertEquals("test_container_00001", tmRunner.getResourceId().toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/97f0cac2/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
deleted file mode 100644
index cdacfc0..0000000
--- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.yarn;
-
-import org.apache.flink.runtime.security.SecurityUtils;
-import org.apache.flink.runtime.security.modules.HadoopModule;
-import org.apache.flink.runtime.security.modules.SecurityModule;
-
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.Callable;
-
-/**
- * Tests for {@link YarnTaskManagerRunner}.
- */
-public class YarnTaskManagerRunnerTest {
-
- @Test
- public void testKerberosKeytabConfiguration() throws IOException {
- final String resourceDirPath =
- Paths.get("src", "test", "resources").toAbsolutePath().toString();
- final Map<String, String> envs = new HashMap<>();
- envs.put(YarnFlinkResourceManager.ENV_FLINK_CONTAINER_ID, "test_container_00001");
- envs.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "testuser1@domain");
- envs.put(ApplicationConstants.Environment.PWD.key(), resourceDirPath);
- YarnTaskManagerRunner.runYarnTaskManager(
- new String[]{"--configDir", resourceDirPath},
- YarnTaskManager.class,
- envs,
- new Callable<Object>() {
- @Override
- public Integer call() {
- final List<SecurityModule> modules = SecurityUtils.getInstalledModules();
- Optional<SecurityModule> moduleOpt =
- modules.stream().filter(s -> s instanceof HadoopModule).findFirst();
- if (moduleOpt.isPresent()) {
- HadoopModule hadoopModule = (HadoopModule) moduleOpt.get();
- assertEquals("testuser1@domain", hadoopModule.getSecurityConfig().getPrincipal());
- assertEquals(resourceDirPath + "/krb5.keytab",
- hadoopModule.getSecurityConfig().getKeytab());
- } else {
- fail("Can not find HadoopModule!");
- }
- return null;
- }
- });
- }
-}
[05/31] flink git commit: [FLINK-8398] [kinesis,
tests] Cleanup confusing implementations in KinesisDataFetcherTest
and related classes
Posted by tz...@apache.org.
[FLINK-8398] [kinesis, tests] Cleanup confusing implementations in KinesisDataFetcherTest and related classes
The previous implementation of the TestableKinesisDataFetcher was
confusing in various ways, causing it hard to be re-used for other
tests. This commit contains the following various cleaups:
- Remove confusing mocks of source context and checkpoint lock. We now
allow users of the TestableKinesisDataFetcher to provide a source
context, which should provide the checkpoint lock.
- Remove override of emitRecordAndUpdateState(). Strictly speaking, that
method should be final. It was previously overriden to allow
verifying how many records were output by the fetcher. That
verification would be better implemented within a mock source context.
- Properly parameterize the output type for the
TestableKinesisDataFetcher.
- Remove use of PowerMockito in KinesisDataFetcherTest.
- Use CheckedThreads to properly capture any exceptions in fetcher /
consumer threads in unit tests.
- Use assertEquals / assertNull instead of assertTrue where-ever
appropriate.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1767898d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1767898d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1767898d
Branch: refs/heads/master
Commit: 1767898dec0fe832865c7b0b6f53f8069518430e
Parents: 62a777b
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 10 10:11:31 2018 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../kinesis/internals/KinesisDataFetcher.java | 10 +-
.../internals/KinesisDataFetcherTest.java | 239 +++++++++----------
.../kinesis/internals/ShardConsumerTest.java | 48 ++--
.../kinesis/testutils/TestSourceContext.java | 64 +++++
.../testutils/TestableKinesisDataFetcher.java | 98 +++-----
5 files changed, 255 insertions(+), 204 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1767898d/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
index 83bc57b..d20a449 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
@@ -18,6 +18,7 @@
package org.apache.flink.streaming.connectors.kinesis.internals;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
@@ -195,7 +196,7 @@ public class KinesisDataFetcher<T> {
KinesisProxy.create(configProps));
}
- /** This constructor is exposed for testing purposes. */
+ @VisibleForTesting
protected KinesisDataFetcher(List<String> streams,
SourceFunction.SourceContext<T> sourceContext,
Object checkpointLock,
@@ -498,7 +499,7 @@ public class KinesisDataFetcher<T> {
* when the shard state was registered.
* @param lastSequenceNumber the last sequence number value to update
*/
- protected void emitRecordAndUpdateState(T record, long recordTimestamp, int shardStateIndex, SequenceNumber lastSequenceNumber) {
+ protected final void emitRecordAndUpdateState(T record, long recordTimestamp, int shardStateIndex, SequenceNumber lastSequenceNumber) {
synchronized (checkpointLock) {
sourceContext.collectWithTimestamp(record, recordTimestamp);
updateState(shardStateIndex, lastSequenceNumber);
@@ -515,7 +516,7 @@ public class KinesisDataFetcher<T> {
* when the shard state was registered.
* @param lastSequenceNumber the last sequence number value to update
*/
- protected void updateState(int shardStateIndex, SequenceNumber lastSequenceNumber) {
+ protected final void updateState(int shardStateIndex, SequenceNumber lastSequenceNumber) {
synchronized (checkpointLock) {
subscribedShardsState.get(shardStateIndex).setLastProcessedSequenceNum(lastSequenceNumber);
@@ -597,7 +598,8 @@ public class KinesisDataFetcher<T> {
return (Math.abs(shard.hashCode() % totalNumberOfConsumerSubtasks)) == indexOfThisConsumerSubtask;
}
- private static ExecutorService createShardConsumersThreadPool(final String subtaskName) {
+ @VisibleForTesting
+ protected ExecutorService createShardConsumersThreadPool(final String subtaskName) {
return Executors.newCachedThreadPool(new ThreadFactory() {
@Override
public Thread newThread(Runnable runnable) {
http://git-wip-us.apache.org/repos/asf/flink/blob/1767898d/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
index 2e1adb6..56566c0 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
@@ -18,27 +18,26 @@
package org.apache.flink.streaming.connectors.kinesis.internals;
import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.core.testutils.CheckedThread;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kinesis.FlinkKinesisConsumer;
-import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants;
import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardMetadata;
import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
+import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchemaWrapper;
import org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisBehavioursFactory;
import org.apache.flink.streaming.connectors.kinesis.testutils.KinesisShardIdGenerator;
+import org.apache.flink.streaming.connectors.kinesis.testutils.TestSourceContext;
+import org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils;
import org.apache.flink.streaming.connectors.kinesis.testutils.TestableKinesisDataFetcher;
import com.amazonaws.services.kinesis.model.HashKeyRange;
import com.amazonaws.services.kinesis.model.SequenceNumberRange;
import com.amazonaws.services.kinesis.model.Shard;
import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
import java.util.HashMap;
import java.util.LinkedList;
@@ -51,6 +50,7 @@ import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -58,8 +58,6 @@ import static org.mockito.Mockito.when;
/**
* Tests for the {@link KinesisDataFetcher}.
*/
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(TestableKinesisDataFetcher.class)
public class KinesisDataFetcherTest {
@Test(expected = RuntimeException.class)
@@ -71,14 +69,16 @@ public class KinesisDataFetcherTest {
HashMap<String, String> subscribedStreamsToLastSeenShardIdsUnderTest =
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
- TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.noShardsFoundForRequestedStreamsBehaviour());
@@ -102,52 +102,44 @@ public class KinesisDataFetcherTest {
streamToShardCount.put(fakeStream, rand.nextInt(5) + 1);
}
- final TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ final TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
- Properties testConfig = new Properties();
- testConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");
- testConfig.setProperty(ConsumerConfigConstants.AWS_CREDENTIALS_PROVIDER, "BASIC");
- testConfig.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, "accessKeyId");
- testConfig.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, "secretKey");
+ final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(
+ TestUtils.getStandardProperties(), fetcher, 1, 0);
- final DummyFlinkKafkaConsumer<String> consumer = new DummyFlinkKafkaConsumer<>(testConfig, fetcher);
-
- PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
- Thread consumerThread = new Thread(new Runnable() {
+ CheckedThread consumerThread = new CheckedThread() {
@Override
- public void run() {
- try {
- consumer.run(mock(SourceFunction.SourceContext.class));
- } catch (Exception e) {
- //
- }
+ public void go() throws Exception {
+ consumer.run(new TestSourceContext<>());
}
- });
+ };
consumerThread.start();
fetcher.waitUntilRun();
consumer.cancel();
- consumerThread.join();
+ consumerThread.sync();
// assert that the streams tracked in the state are identical to the subscribed streams
Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
- assertTrue(streamsInState.size() == fakeStreams.size());
+ assertEquals(fakeStreams.size(), streamsInState.size());
assertTrue(streamsInState.containsAll(fakeStreams));
// assert that the last seen shards in state is correctly set
for (Map.Entry<String, String> streamToLastSeenShard : subscribedStreamsToLastSeenShardIdsUnderTest.entrySet()) {
- assertTrue(
- streamToLastSeenShard.getValue().equals(
- KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1)));
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1),
+ streamToLastSeenShard.getValue());
}
}
@@ -195,14 +187,16 @@ public class KinesisDataFetcherTest {
HashMap<String, String> subscribedStreamsToLastSeenShardIdsUnderTest =
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
- final TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ final TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
@@ -213,31 +207,27 @@ public class KinesisDataFetcherTest {
restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
}
- PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
- Thread runFetcherThread = new Thread(new Runnable() {
+ CheckedThread runFetcherThread = new CheckedThread() {
@Override
- public void run() {
- try {
- fetcher.runFetcher();
- } catch (Exception e) {
- //
- }
+ public void go() throws Exception {
+ fetcher.runFetcher();
}
- });
+ };
runFetcherThread.start();
Thread.sleep(1000); // sleep a while before closing
fetcher.shutdownFetcher();
+ runFetcherThread.sync();
// assert that the streams tracked in the state are identical to the subscribed streams
Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
- assertTrue(streamsInState.size() == fakeStreams.size());
+ assertEquals(fakeStreams.size(), streamsInState.size());
assertTrue(streamsInState.containsAll(fakeStreams));
// assert that the last seen shards in state is correctly set
for (Map.Entry<String, String> streamToLastSeenShard : subscribedStreamsToLastSeenShardIdsUnderTest.entrySet()) {
- assertTrue(
- streamToLastSeenShard.getValue().equals(
- KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1)));
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1),
+ streamToLastSeenShard.getValue());
}
}
@@ -286,14 +276,16 @@ public class KinesisDataFetcherTest {
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
// using a non-resharded streams kinesis behaviour to represent that Kinesis is not resharded AFTER the restore
- final TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ final TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
@@ -304,31 +296,27 @@ public class KinesisDataFetcherTest {
restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
}
- PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
- Thread runFetcherThread = new Thread(new Runnable() {
+ CheckedThread runFetcherThread = new CheckedThread() {
@Override
- public void run() {
- try {
- fetcher.runFetcher();
- } catch (Exception e) {
- //
- }
+ public void go() throws Exception {
+ fetcher.runFetcher();
}
- });
+ };
runFetcherThread.start();
Thread.sleep(1000); // sleep a while before closing
fetcher.shutdownFetcher();
+ runFetcherThread.sync();
// assert that the streams tracked in the state are identical to the subscribed streams
Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
- assertTrue(streamsInState.size() == fakeStreams.size());
+ assertEquals(fakeStreams.size(), streamsInState.size());
assertTrue(streamsInState.containsAll(fakeStreams));
// assert that the last seen shards in state is correctly set
for (Map.Entry<String, String> streamToLastSeenShard : subscribedStreamsToLastSeenShardIdsUnderTest.entrySet()) {
- assertTrue(
- streamToLastSeenShard.getValue().equals(
- KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1)));
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1),
+ streamToLastSeenShard.getValue());
}
}
@@ -381,14 +369,16 @@ public class KinesisDataFetcherTest {
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
// using a non-resharded streams kinesis behaviour to represent that Kinesis is not resharded AFTER the restore
- final TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ final TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
@@ -399,33 +389,31 @@ public class KinesisDataFetcherTest {
restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
}
- PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
- Thread runFetcherThread = new Thread(new Runnable() {
+ CheckedThread runFetcherThread = new CheckedThread() {
@Override
- public void run() {
- try {
- fetcher.runFetcher();
- } catch (Exception e) {
- //
- }
+ public void go() throws Exception {
+ fetcher.runFetcher();
}
- });
+ };
runFetcherThread.start();
Thread.sleep(1000); // sleep a while before closing
fetcher.shutdownFetcher();
+ runFetcherThread.sync();
// assert that the streams tracked in the state are identical to the subscribed streams
Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
- assertTrue(streamsInState.size() == fakeStreams.size());
+ assertEquals(fakeStreams.size(), streamsInState.size());
assertTrue(streamsInState.containsAll(fakeStreams));
// assert that the last seen shards in state is correctly set
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream1").equals(
- KinesisShardIdGenerator.generateFromShardOrder(2)));
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream2").equals(
- KinesisShardIdGenerator.generateFromShardOrder(1)));
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream3") == null);
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream4") == null);
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(2),
+ subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream1"));
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(1),
+ subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream2"));
+ assertNull(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream3"));
+ assertNull(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream4"));
}
@Test
@@ -477,14 +465,16 @@ public class KinesisDataFetcherTest {
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
// using a non-resharded streams kinesis behaviour to represent that Kinesis is not resharded AFTER the restore
- final TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ final TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
fakeStreams,
- new Properties(),
+ new TestSourceContext<>(),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new LinkedList<>(),
subscribedStreamsToLastSeenShardIdsUnderTest,
FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
@@ -495,33 +485,31 @@ public class KinesisDataFetcherTest {
restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
}
- PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
- Thread runFetcherThread = new Thread(new Runnable() {
+ CheckedThread runFetcherThread = new CheckedThread() {
@Override
- public void run() {
- try {
- fetcher.runFetcher();
- } catch (Exception e) {
- //
- }
+ public void go() throws Exception {
+ fetcher.runFetcher();
}
- });
+ };
runFetcherThread.start();
Thread.sleep(1000); // sleep a while before closing
fetcher.shutdownFetcher();
+ runFetcherThread.sync();
// assert that the streams tracked in the state are identical to the subscribed streams
Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
- assertTrue(streamsInState.size() == fakeStreams.size());
+ assertEquals(fakeStreams.size(), streamsInState.size());
assertTrue(streamsInState.containsAll(fakeStreams));
// assert that the last seen shards in state is correctly set
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream1").equals(
- KinesisShardIdGenerator.generateFromShardOrder(3)));
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream2").equals(
- KinesisShardIdGenerator.generateFromShardOrder(4)));
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream3") == null);
- assertTrue(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream4") == null);
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(3),
+ subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream1"));
+ assertEquals(
+ KinesisShardIdGenerator.generateFromShardOrder(4),
+ subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream2"));
+ assertNull(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream3"));
+ assertNull(subscribedStreamsToLastSeenShardIdsUnderTest.get("fakeStream4"));
}
@Test
@@ -564,12 +552,21 @@ public class KinesisDataFetcherTest {
private static class DummyFlinkKafkaConsumer<T> extends FlinkKinesisConsumer<T> {
private static final long serialVersionUID = 1L;
- private KinesisDataFetcher<T> fetcher;
+ private final KinesisDataFetcher<T> fetcher;
+
+ private final int numParallelSubtasks;
+ private final int subtaskIndex;
@SuppressWarnings("unchecked")
- DummyFlinkKafkaConsumer(Properties properties, KinesisDataFetcher<T> fetcher) {
+ DummyFlinkKafkaConsumer(
+ Properties properties,
+ KinesisDataFetcher<T> fetcher,
+ int numParallelSubtasks,
+ int subtaskIndex) {
super("test", mock(KinesisDeserializationSchema.class), properties);
this.fetcher = fetcher;
+ this.numParallelSubtasks = numParallelSubtasks;
+ this.subtaskIndex = subtaskIndex;
}
@Override
@@ -585,8 +582,8 @@ public class KinesisDataFetcherTest {
@Override
public RuntimeContext getRuntimeContext() {
RuntimeContext context = mock(RuntimeContext.class);
- when(context.getIndexOfThisSubtask()).thenReturn(0);
- when(context.getNumberOfParallelSubtasks()).thenReturn(1);
+ when(context.getIndexOfThisSubtask()).thenReturn(subtaskIndex);
+ when(context.getNumberOfParallelSubtasks()).thenReturn(numParallelSubtasks);
return context;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1767898d/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/ShardConsumerTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/ShardConsumerTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/ShardConsumerTest.java
index 7ab3735..4478b2f 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/ShardConsumerTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/ShardConsumerTest.java
@@ -17,14 +17,17 @@
package org.apache.flink.streaming.connectors.kinesis.internals;
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.connectors.kinesis.metrics.ShardMetricsReporter;
import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
import org.apache.flink.streaming.connectors.kinesis.model.SentinelSequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
import org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxyInterface;
+import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchemaWrapper;
import org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisBehavioursFactory;
import org.apache.flink.streaming.connectors.kinesis.testutils.KinesisShardIdGenerator;
+import org.apache.flink.streaming.connectors.kinesis.testutils.TestSourceContext;
import org.apache.flink.streaming.connectors.kinesis.testutils.TestableKinesisDataFetcher;
import com.amazonaws.services.kinesis.model.HashKeyRange;
@@ -40,7 +43,6 @@ import java.util.Properties;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
/**
* Tests for the {@link ShardConsumer}.
@@ -58,13 +60,17 @@ public class ShardConsumerTest {
fakeToBeConsumedShard,
new SequenceNumber("fakeStartingState")));
- TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ TestSourceContext<String> sourceContext = new TestSourceContext<>();
+
+ TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
+ sourceContext,
new Properties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
+ new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
@@ -92,13 +98,17 @@ public class ShardConsumerTest {
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
- TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ TestSourceContext<String> sourceContext = new TestSourceContext<>();
+
+ TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
+ sourceContext,
new Properties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
+ new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
@@ -111,9 +121,10 @@ public class ShardConsumerTest {
FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, 500L),
new ShardMetricsReporter()).run();
- assertTrue(fetcher.getNumOfElementsCollected() == 1000);
- assertTrue(subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum().equals(
- SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get()));
+ assertEquals(1000, sourceContext.getCollectedOutputs().size());
+ assertEquals(
+ SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
+ subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
@Test
@@ -125,13 +136,17 @@ public class ShardConsumerTest {
new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard),
fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
- TestableKinesisDataFetcher fetcher =
- new TestableKinesisDataFetcher(
+ TestSourceContext<String> sourceContext = new TestSourceContext<>();
+
+ TestableKinesisDataFetcher<String> fetcher =
+ new TestableKinesisDataFetcher<>(
Collections.singletonList("fakeStream"),
+ sourceContext,
new Properties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
10,
2,
- new AtomicReference<Throwable>(),
+ new AtomicReference<>(),
subscribedShardsStateUnderTest,
KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")),
Mockito.mock(KinesisProxyInterface.class));
@@ -147,9 +162,10 @@ public class ShardConsumerTest {
1000, 9, 7, 500L),
new ShardMetricsReporter()).run();
- assertTrue(fetcher.getNumOfElementsCollected() == 1000);
- assertTrue(subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum().equals(
- SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get()));
+ assertEquals(1000, sourceContext.getCollectedOutputs().size());
+ assertEquals(
+ SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(),
+ subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
}
private static StreamShardHandle getMockStreamShard(String streamName, int shardId) {
http://git-wip-us.apache.org/repos/asf/flink/blob/1767898d/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java
new file mode 100644
index 0000000..4fe5c54
--- /dev/null
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestSourceContext.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kinesis.testutils;
+
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+/**
+ * A testable {@link SourceFunction.SourceContext}.
+ */
+public class TestSourceContext<T> implements SourceFunction.SourceContext<T> {
+
+ private final Object checkpointLock = new Object();
+
+ private ConcurrentLinkedQueue<StreamRecord<T>> collectedOutputs = new ConcurrentLinkedQueue<>();
+
+ @Override
+ public void collect(T element) {
+ this.collectedOutputs.add(new StreamRecord<>(element));
+ }
+
+ @Override
+ public void collectWithTimestamp(T element, long timestamp) {
+ this.collectedOutputs.add(new StreamRecord<>(element, timestamp));
+ }
+
+ @Override
+ public void emitWatermark(Watermark mark) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void markAsTemporarilyIdle() {}
+
+ @Override
+ public Object getCheckpointLock() {
+ return checkpointLock;
+ }
+
+ @Override
+ public void close() {}
+
+ public ConcurrentLinkedQueue<StreamRecord<T>> getCollectedOutputs() {
+ return collectedOutputs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1767898d/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
index f63a30c..7b3e1bf 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
@@ -18,77 +18,60 @@
package org.apache.flink.streaming.connectors.kinesis.testutils;
import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
-import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxyInterface;
import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
-import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchemaWrapper;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicReference;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
/**
* Extension of the {@link KinesisDataFetcher} for testing.
*/
-public class TestableKinesisDataFetcher extends KinesisDataFetcher<String> {
-
- private static final Object fakeCheckpointLock = new Object();
-
- private long numElementsCollected;
+public class TestableKinesisDataFetcher<T> extends KinesisDataFetcher<T> {
private OneShotLatch runWaiter;
+ private volatile boolean running;
public TestableKinesisDataFetcher(
List<String> fakeStreams,
+ SourceFunction.SourceContext<T> sourceContext,
Properties fakeConfiguration,
+ KinesisDeserializationSchema<T> deserializationSchema,
int fakeTotalCountOfSubtasks,
int fakeIndexOfThisSubtask,
AtomicReference<Throwable> thrownErrorUnderTest,
LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
KinesisProxyInterface fakeKinesis) {
- super(fakeStreams,
- getMockedSourceContext(),
- fakeCheckpointLock,
+ super(
+ fakeStreams,
+ sourceContext,
+ sourceContext.getCheckpointLock(),
getMockedRuntimeContext(fakeTotalCountOfSubtasks, fakeIndexOfThisSubtask),
fakeConfiguration,
- new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ deserializationSchema,
thrownErrorUnderTest,
subscribedShardsStateUnderTest,
subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
fakeKinesis);
- this.numElementsCollected = 0;
this.runWaiter = new OneShotLatch();
- }
-
- public long getNumOfElementsCollected() {
- return numElementsCollected;
- }
-
- @Override
- protected KinesisDeserializationSchema<String> getClonedDeserializationSchema() {
- return new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema());
- }
-
- @Override
- protected void emitRecordAndUpdateState(String record, long recordTimestamp, int shardStateIndex, SequenceNumber lastSequenceNumber) {
- synchronized (fakeCheckpointLock) {
- this.numElementsCollected++;
- updateState(shardStateIndex, lastSequenceNumber);
- }
+ this.running = true;
}
@Override
@@ -101,41 +84,30 @@ public class TestableKinesisDataFetcher extends KinesisDataFetcher<String> {
runWaiter.await();
}
- @SuppressWarnings("unchecked")
- private static SourceFunction.SourceContext<String> getMockedSourceContext() {
- return Mockito.mock(SourceFunction.SourceContext.class);
+ @Override
+ protected ExecutorService createShardConsumersThreadPool(String subtaskName) {
+ // this is just a dummy fetcher, so no need to create a thread pool for shard consumers
+ ExecutorService mockExecutor = mock(ExecutorService.class);
+ when(mockExecutor.isTerminated()).thenAnswer((InvocationOnMock invocation) -> !running);
+ return mockExecutor;
+ }
+
+ @Override
+ public void awaitTermination() throws InterruptedException {
+ this.running = false;
+ super.awaitTermination();
}
private static RuntimeContext getMockedRuntimeContext(final int fakeTotalCountOfSubtasks, final int fakeIndexOfThisSubtask) {
- RuntimeContext mockedRuntimeContext = Mockito.mock(RuntimeContext.class);
-
- Mockito.when(mockedRuntimeContext.getNumberOfParallelSubtasks()).thenAnswer(new Answer<Integer>() {
- @Override
- public Integer answer(InvocationOnMock invocationOnMock) throws Throwable {
- return fakeTotalCountOfSubtasks;
- }
- });
-
- Mockito.when(mockedRuntimeContext.getIndexOfThisSubtask()).thenAnswer(new Answer<Integer>() {
- @Override
- public Integer answer(InvocationOnMock invocationOnMock) throws Throwable {
- return fakeIndexOfThisSubtask;
- }
- });
-
- Mockito.when(mockedRuntimeContext.getTaskName()).thenAnswer(new Answer<String>() {
- @Override
- public String answer(InvocationOnMock invocationOnMock) throws Throwable {
- return "Fake Task";
- }
- });
-
- Mockito.when(mockedRuntimeContext.getTaskNameWithSubtasks()).thenAnswer(new Answer<String>() {
- @Override
- public String answer(InvocationOnMock invocationOnMock) throws Throwable {
- return "Fake Task (" + fakeIndexOfThisSubtask + "/" + fakeTotalCountOfSubtasks + ")";
- }
- });
+ RuntimeContext mockedRuntimeContext = mock(RuntimeContext.class);
+
+ Mockito.when(mockedRuntimeContext.getNumberOfParallelSubtasks()).thenReturn(fakeTotalCountOfSubtasks);
+ Mockito.when(mockedRuntimeContext.getIndexOfThisSubtask()).thenReturn(fakeIndexOfThisSubtask);
+ Mockito.when(mockedRuntimeContext.getTaskName()).thenReturn("Fake Task");
+ Mockito.when(mockedRuntimeContext.getTaskNameWithSubtasks()).thenReturn(
+ "Fake Task (" + fakeIndexOfThisSubtask + "/" + fakeTotalCountOfSubtasks + ")");
+ Mockito.when(mockedRuntimeContext.getUserCodeClassLoader()).thenReturn(
+ Thread.currentThread().getContextClassLoader());
Mockito.when(mockedRuntimeContext.getMetricGroup()).thenReturn(new UnregisteredMetricsGroup());
[07/31] flink git commit: [FLINK-8419] [kafka] Register metrics for
dynamically discovered Kafka partitions
Posted by tz...@apache.org.
[FLINK-8419] [kafka] Register metrics for dynamically discovered Kafka partitions
This closes #5335.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/40f26c80
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/40f26c80
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/40f26c80
Branch: refs/heads/master
Commit: 40f26c80476b884a6b7f3431562a78018fa014c5
Parents: 0a1ce00
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Mon Jan 22 14:14:44 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../kafka/internals/AbstractFetcher.java | 58 ++++++++++++++++----
1 file changed, 48 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/40f26c80/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
index 27afbca..b6ce65a 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
@@ -26,6 +26,7 @@ import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks
import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.connectors.kafka.config.OffsetCommitMode;
+import org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaConsumerMetricConstants;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
import org.apache.flink.util.SerializedValue;
@@ -112,6 +113,27 @@ public abstract class AbstractFetcher<T, KPH> {
private volatile long maxWatermarkSoFar = Long.MIN_VALUE;
// ------------------------------------------------------------------------
+ // Metrics
+ // ------------------------------------------------------------------------
+
+ /**
+ * Flag indicating whether or not metrics should be exposed.
+ * If {@code true}, offset metrics (e.g. current offset, committed offset) and
+ * Kafka-shipped metrics will be registered.
+ */
+ private final boolean useMetrics;
+
+ /**
+ * The metric group which all metrics for the consumer should be registered to.
+ * This metric group is defined under the user scope {@link KafkaConsumerMetricConstants#KAFKA_CONSUMER_METRICS_GROUP}.
+ */
+ private final MetricGroup consumerMetricGroup;
+
+ @Deprecated
+ private final MetricGroup legacyCurrentOffsetsMetricGroup;
+
+ @Deprecated
+ private final MetricGroup legacyCommittedOffsetsMetricGroup;
protected AbstractFetcher(
SourceContext<T> sourceContext,
@@ -127,6 +149,11 @@ public abstract class AbstractFetcher<T, KPH> {
this.checkpointLock = sourceContext.getCheckpointLock();
this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
+ this.useMetrics = useMetrics;
+ this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
+ this.legacyCurrentOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_CURRENT_OFFSETS_METRICS_GROUP);
+ this.legacyCommittedOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_COMMITTED_OFFSETS_METRICS_GROUP);
+
// figure out what we watermark mode we will be using
this.watermarksPeriodic = watermarksPeriodic;
this.watermarksPunctuated = watermarksPunctuated;
@@ -168,8 +195,9 @@ public abstract class AbstractFetcher<T, KPH> {
unassignedPartitionsQueue.add(partition);
}
+ // register metrics for the initial seed partitions
if (useMetrics) {
- addOffsetStateGauge(checkNotNull(consumerMetricGroup));
+ registerOffsetMetrics(consumerMetricGroup, subscribedPartitionStates);
}
// if we have periodic watermarks, kick off the interval scheduler
@@ -207,6 +235,10 @@ public abstract class AbstractFetcher<T, KPH> {
watermarksPunctuated,
userCodeClassLoader);
+ if (useMetrics) {
+ registerOffsetMetrics(consumerMetricGroup, newPartitionStates);
+ }
+
for (KafkaTopicPartitionState<KPH> newPartitionState : newPartitionStates) {
subscribedPartitionStates.add(newPartitionState);
unassignedPartitionsQueue.add(newPartitionState);
@@ -568,23 +600,29 @@ public abstract class AbstractFetcher<T, KPH> {
// ------------------------- Metrics ----------------------------------
/**
- * Add current and committed offsets to metric group.
+ * For each partition, register a new metric group to expose current offsets and committed offsets.
+ * Per-partition metric groups can be scoped by user variables {@link KafkaConsumerMetricConstants#OFFSETS_BY_TOPIC_METRICS_GROUP}
+ * and {@link KafkaConsumerMetricConstants#OFFSETS_BY_PARTITION_METRICS_GROUP}.
+ *
+ * <p>Note: this method also registers gauges for deprecated offset metrics, to maintain backwards compatibility.
*
- * @param metricGroup The metric group to use
+ * @param consumerMetricGroup The consumer metric group
+ * @param partitionOffsetStates The partition offset state holders, whose values will be used to update metrics
*/
- protected void addOffsetStateGauge(MetricGroup metricGroup) {
- MetricGroup legacyCurrentOffsetsGroup = metricGroup.addGroup(LEGACY_CURRENT_OFFSETS_METRICS_GROUP);
- MetricGroup legacyCommittedOffsetsGroup = metricGroup.addGroup(LEGACY_COMMITTED_OFFSETS_METRICS_GROUP);
+ private void registerOffsetMetrics(
+ MetricGroup consumerMetricGroup,
+ List<KafkaTopicPartitionState<KPH>> partitionOffsetStates) {
- for (KafkaTopicPartitionState<KPH> ktp : subscribedPartitionStates) {
- MetricGroup topicPartitionGroup = metricGroup
+ for (KafkaTopicPartitionState<KPH> ktp : partitionOffsetStates) {
+ MetricGroup topicPartitionGroup = consumerMetricGroup
.addGroup(OFFSETS_BY_TOPIC_METRICS_GROUP, ktp.getTopic())
.addGroup(OFFSETS_BY_PARTITION_METRICS_GROUP, Integer.toString(ktp.getPartition()));
+
topicPartitionGroup.gauge(CURRENT_OFFSETS_METRICS_GAUGE, new OffsetGauge(ktp, OffsetGaugeType.CURRENT_OFFSET));
topicPartitionGroup.gauge(COMMITTED_OFFSETS_METRICS_GAUGE, new OffsetGauge(ktp, OffsetGaugeType.COMMITTED_OFFSET));
- legacyCurrentOffsetsGroup.gauge(getLegacyOffsetsMetricsGaugeName(ktp), new OffsetGauge(ktp, OffsetGaugeType.CURRENT_OFFSET));
- legacyCommittedOffsetsGroup.gauge(getLegacyOffsetsMetricsGaugeName(ktp), new OffsetGauge(ktp, OffsetGaugeType.COMMITTED_OFFSET));
+ legacyCurrentOffsetsMetricGroup.gauge(getLegacyOffsetsMetricsGaugeName(ktp), new OffsetGauge(ktp, OffsetGaugeType.CURRENT_OFFSET));
+ legacyCommittedOffsetsMetricGroup.gauge(getLegacyOffsetsMetricsGaugeName(ktp), new OffsetGauge(ktp, OffsetGaugeType.COMMITTED_OFFSET));
}
}
[09/31] flink git commit: [hotfix] [kafka] Make
AbortedReassignmentException a static class
Posted by tz...@apache.org.
[hotfix] [kafka] Make AbortedReassignmentException a static class
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2330e8d2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2330e8d2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2330e8d2
Branch: refs/heads/master
Commit: 2330e8d2774f2c6d9c21f2d24630196aaca14c29
Parents: 3655200
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Sat Jan 13 20:35:03 2018 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../streaming/connectors/kafka/internal/KafkaConsumerThread.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2330e8d2/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
index 8895b42..c995910 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
@@ -507,7 +507,7 @@ public class KafkaConsumerThread extends Thread {
* Utility exception that serves as a signal for the main loop to continue through the loop
* if a reassignment attempt was aborted due to an pre-reassignment wakeup call on the consumer.
*/
- private class AbortedReassignmentException extends Exception {
+ private static class AbortedReassignmentException extends Exception {
private static final long serialVersionUID = 1L;
}
}
[08/31] flink git commit: [FLINK-8419] [kafka] Move consumer metric
group registration to FlinkKafkaConsumerBase
Posted by tz...@apache.org.
[FLINK-8419] [kafka] Move consumer metric group registration to FlinkKafkaConsumerBase
This commit is a refactor to move the registration of the consumer
metric group (user scope "KafkaConsumer") to FlinkKafkaConsumerBase.
Previously, the registration was scattered around in Kafka
version-specific subclasses.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0a1ce006
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0a1ce006
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0a1ce006
Branch: refs/heads/master
Commit: 0a1ce0060ef3af29b196ab6ad58f97e49a40a4cf
Parents: 2330e8d
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Mon Jan 22 13:36:20 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../connectors/kafka/FlinkKafkaConsumer010.java | 11 +++++----
.../kafka/internal/Kafka010Fetcher.java | 6 +++--
.../kafka/internal/Kafka010FetcherTest.java | 9 ++++---
.../connectors/kafka/FlinkKafkaConsumer08.java | 12 ++++++---
.../kafka/internals/Kafka08Fetcher.java | 9 ++-----
.../connectors/kafka/FlinkKafkaConsumer09.java | 16 +++++++-----
.../kafka/internal/Kafka09Fetcher.java | 15 ++++++-----
.../kafka/internal/KafkaConsumerThread.java | 26 ++++++++++++++------
.../kafka/internal/Kafka09FetcherTest.java | 9 ++++---
.../kafka/internal/KafkaConsumerThreadTest.java | 5 ++--
.../kafka/FlinkKafkaConsumerBase.java | 22 ++++++++++++++---
.../kafka/internals/AbstractFetcher.java | 9 ++++---
.../FlinkKafkaConsumerBaseMigrationTest.java | 8 ++++--
.../kafka/FlinkKafkaConsumerBaseTest.java | 8 ++++--
.../kafka/internals/AbstractFetcherTest.java | 2 ++
15 files changed, 109 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java
index d0b61e2..394d123 100644
--- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer010.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
@@ -31,7 +32,6 @@ import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition
import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicsDescriptor;
import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchema;
import org.apache.flink.streaming.util.serialization.KeyedDeserializationSchemaWrapper;
-import org.apache.flink.util.PropertiesUtil;
import org.apache.flink.util.SerializedValue;
import org.apache.kafka.clients.consumer.ConsumerConfig;
@@ -179,9 +179,9 @@ public class FlinkKafkaConsumer010<T> extends FlinkKafkaConsumer09<T> {
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception {
-
- boolean useMetrics = !PropertiesUtil.getBoolean(properties, KEY_DISABLE_METRICS, false);
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup consumerMetricGroup,
+ boolean useMetrics) throws Exception {
// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
// this overwrites whatever setting the user configured in the properties
@@ -198,10 +198,11 @@ public class FlinkKafkaConsumer010<T> extends FlinkKafkaConsumer09<T> {
runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
runtimeContext.getUserCodeClassLoader(),
runtimeContext.getTaskNameWithSubtasks(),
- runtimeContext.getMetricGroup(),
deserializer,
properties,
pollTimeout,
+ runtimeContext.getMetricGroup(),
+ consumerMetricGroup,
useMetrics);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java
index cddccdc..9b9b217 100644
--- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010Fetcher.java
@@ -55,10 +55,11 @@ public class Kafka010Fetcher<T> extends Kafka09Fetcher<T> {
long autoWatermarkInterval,
ClassLoader userCodeClassLoader,
String taskNameWithSubtasks,
- MetricGroup metricGroup,
KeyedDeserializationSchema<T> deserializer,
Properties kafkaProperties,
long pollTimeout,
+ MetricGroup subtaskMetricGroup,
+ MetricGroup consumerMetricGroup,
boolean useMetrics) throws Exception {
super(
sourceContext,
@@ -69,10 +70,11 @@ public class Kafka010Fetcher<T> extends Kafka09Fetcher<T> {
autoWatermarkInterval,
userCodeClassLoader,
taskNameWithSubtasks,
- metricGroup,
deserializer,
kafkaProperties,
pollTimeout,
+ subtaskMetricGroup,
+ consumerMetricGroup,
useMetrics);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java
index 45ceadc..f57fbea 100644
--- a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java
+++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka010FetcherTest.java
@@ -126,10 +126,11 @@ public class Kafka010FetcherTest {
10,
getClass().getClassLoader(),
"taskname-with-subtask",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
@@ -262,10 +263,11 @@ public class Kafka010FetcherTest {
10,
getClass().getClassLoader(),
"taskname-with-subtask",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
@@ -376,10 +378,11 @@ public class Kafka010FetcherTest {
10, /* watermark interval */
this.getClass().getClassLoader(),
"task_name",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
index 3718476..312c8c4 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer08.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
@@ -43,6 +44,7 @@ import java.util.Properties;
import java.util.regex.Pattern;
import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.PropertiesUtil.getBoolean;
import static org.apache.flink.util.PropertiesUtil.getLong;
/**
@@ -217,7 +219,8 @@ public class FlinkKafkaConsumer08<T> extends FlinkKafkaConsumerBase<T> {
deserializer,
getLong(
checkNotNull(props, "props"),
- KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS, PARTITION_DISCOVERY_DISABLED));
+ KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS, PARTITION_DISCOVERY_DISABLED),
+ !getBoolean(props, KEY_DISABLE_METRICS, false));
this.kafkaProperties = props;
@@ -235,9 +238,9 @@ public class FlinkKafkaConsumer08<T> extends FlinkKafkaConsumerBase<T> {
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception {
-
- boolean useMetrics = !PropertiesUtil.getBoolean(kafkaProperties, KEY_DISABLE_METRICS, false);
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup consumerMetricGroup,
+ boolean useMetrics) throws Exception {
long autoCommitInterval = (offsetCommitMode == OffsetCommitMode.KAFKA_PERIODIC)
? PropertiesUtil.getLong(kafkaProperties, "auto.commit.interval.ms", 60000)
@@ -252,6 +255,7 @@ public class FlinkKafkaConsumer08<T> extends FlinkKafkaConsumerBase<T> {
deserializer,
kafkaProperties,
autoCommitInterval,
+ consumerMetricGroup,
useMetrics);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
index 23d87d7..a2edb72 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
@@ -48,7 +48,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import static org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaConsumerMetricConstants.KAFKA_CONSUMER_METRICS_GROUP;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
@@ -98,6 +97,7 @@ public class Kafka08Fetcher<T> extends AbstractFetcher<T, TopicAndPartition> {
KeyedDeserializationSchema<T> deserializer,
Properties kafkaProperties,
long autoCommitInterval,
+ MetricGroup consumerMetricGroup,
boolean useMetrics) throws Exception {
super(
sourceContext,
@@ -107,6 +107,7 @@ public class Kafka08Fetcher<T> extends AbstractFetcher<T, TopicAndPartition> {
runtimeContext.getProcessingTimeService(),
runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
runtimeContext.getUserCodeClassLoader(),
+ consumerMetricGroup,
useMetrics);
this.deserializer = checkNotNull(deserializer);
@@ -178,12 +179,6 @@ public class Kafka08Fetcher<T> extends AbstractFetcher<T, TopicAndPartition> {
periodicCommitter.start();
}
- // register offset metrics
- if (useMetrics) {
- final MetricGroup kafkaMetricGroup = runtimeContext.getMetricGroup().addGroup(KAFKA_CONSUMER_METRICS_GROUP);
- addOffsetStateGauge(kafkaMetricGroup);
- }
-
// Main loop polling elements from the unassignedPartitions queue to the threads
while (running) {
// re-throw any exception from the concurrent fetcher threads
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java
index 578a2d2..6e6051b 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumer09.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
@@ -46,6 +47,7 @@ import java.util.Properties;
import java.util.regex.Pattern;
import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.PropertiesUtil.getBoolean;
import static org.apache.flink.util.PropertiesUtil.getLong;
/**
@@ -208,7 +210,8 @@ public class FlinkKafkaConsumer09<T> extends FlinkKafkaConsumerBase<T> {
deserializer,
getLong(
checkNotNull(props, "props"),
- KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS, PARTITION_DISCOVERY_DISABLED));
+ KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS, PARTITION_DISCOVERY_DISABLED),
+ !getBoolean(props, KEY_DISABLE_METRICS, false));
this.properties = props;
setDeserializer(this.properties);
@@ -233,9 +236,9 @@ public class FlinkKafkaConsumer09<T> extends FlinkKafkaConsumerBase<T> {
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception {
-
- boolean useMetrics = !PropertiesUtil.getBoolean(properties, KEY_DISABLE_METRICS, false);
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup consumerMetricGroup,
+ boolean useMetrics) throws Exception {
// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
// this overwrites whatever setting the user configured in the properties
@@ -252,10 +255,11 @@ public class FlinkKafkaConsumer09<T> extends FlinkKafkaConsumerBase<T> {
runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
runtimeContext.getUserCodeClassLoader(),
runtimeContext.getTaskNameWithSubtasks(),
- runtimeContext.getMetricGroup(),
deserializer,
properties,
pollTimeout,
+ runtimeContext.getMetricGroup(),
+ consumerMetricGroup,
useMetrics);
}
@@ -270,7 +274,7 @@ public class FlinkKafkaConsumer09<T> extends FlinkKafkaConsumerBase<T> {
@Override
protected boolean getIsAutoCommitEnabled() {
- return PropertiesUtil.getBoolean(properties, ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true) &&
+ return getBoolean(properties, ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true) &&
PropertiesUtil.getLong(properties, ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 5000) > 0;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
index 162cc09..5839360 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
@@ -45,7 +45,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-import static org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaConsumerMetricConstants.KAFKA_CONSUMER_METRICS_GROUP;
import static org.apache.flink.util.Preconditions.checkState;
/**
@@ -83,10 +82,11 @@ public class Kafka09Fetcher<T> extends AbstractFetcher<T, TopicPartition> {
long autoWatermarkInterval,
ClassLoader userCodeClassLoader,
String taskNameWithSubtasks,
- MetricGroup metricGroup,
KeyedDeserializationSchema<T> deserializer,
Properties kafkaProperties,
long pollTimeout,
+ MetricGroup subtaskMetricGroup,
+ MetricGroup consumerMetricGroup,
boolean useMetrics) throws Exception {
super(
sourceContext,
@@ -95,25 +95,24 @@ public class Kafka09Fetcher<T> extends AbstractFetcher<T, TopicPartition> {
watermarksPunctuated,
processingTimeProvider,
autoWatermarkInterval,
- userCodeClassLoader,
+ userCodeClassLoader.getParent(),
+ consumerMetricGroup,
useMetrics);
this.deserializer = deserializer;
this.handover = new Handover();
- final MetricGroup kafkaMetricGroup = metricGroup.addGroup(KAFKA_CONSUMER_METRICS_GROUP);
- addOffsetStateGauge(kafkaMetricGroup);
-
this.consumerThread = new KafkaConsumerThread(
LOG,
handover,
kafkaProperties,
unassignedPartitionsQueue,
- kafkaMetricGroup,
createCallBridge(),
getFetcherName() + " for " + taskNameWithSubtasks,
pollTimeout,
- useMetrics);
+ useMetrics,
+ consumerMetricGroup,
+ subtaskMetricGroup);
}
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
index c995910..38e8a41 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
@@ -80,9 +80,6 @@ public class KafkaConsumerThread extends Thread {
/** The queue of unassigned partitions that we need to assign to the Kafka consumer. */
private final ClosableBlockingQueue<KafkaTopicPartitionState<TopicPartition>> unassignedPartitionsQueue;
- /** We get this from the outside to publish metrics. **/
- private final MetricGroup kafkaMetricGroup;
-
/** The indirections on KafkaConsumer methods, for cases where KafkaConsumer compatibility is broken. */
private final KafkaConsumerCallBridge consumerCallBridge;
@@ -92,6 +89,16 @@ public class KafkaConsumerThread extends Thread {
/** Flag whether to add Kafka's metrics to the Flink metrics. */
private final boolean useMetrics;
+ /**
+ * @deprecated We should only be publishing to the {{@link #consumerMetricGroup}}.
+ * This is kept to retain compatibility for metrics.
+ **/
+ @Deprecated
+ private final MetricGroup subtaskMetricGroup;
+
+ /** We get this from the outside to publish metrics. */
+ private final MetricGroup consumerMetricGroup;
+
/** Reference to the Kafka consumer, once it is created. */
private volatile KafkaConsumer<byte[], byte[]> consumer;
@@ -118,11 +125,12 @@ public class KafkaConsumerThread extends Thread {
Handover handover,
Properties kafkaProperties,
ClosableBlockingQueue<KafkaTopicPartitionState<TopicPartition>> unassignedPartitionsQueue,
- MetricGroup kafkaMetricGroup,
KafkaConsumerCallBridge consumerCallBridge,
String threadName,
long pollTimeout,
- boolean useMetrics) {
+ boolean useMetrics,
+ MetricGroup consumerMetricGroup,
+ MetricGroup subtaskMetricGroup) {
super(threadName);
setDaemon(true);
@@ -130,7 +138,8 @@ public class KafkaConsumerThread extends Thread {
this.log = checkNotNull(log);
this.handover = checkNotNull(handover);
this.kafkaProperties = checkNotNull(kafkaProperties);
- this.kafkaMetricGroup = checkNotNull(kafkaMetricGroup);
+ this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
+ this.subtaskMetricGroup = checkNotNull(subtaskMetricGroup);
this.consumerCallBridge = checkNotNull(consumerCallBridge);
this.unassignedPartitionsQueue = checkNotNull(unassignedPartitionsQueue);
@@ -178,7 +187,10 @@ public class KafkaConsumerThread extends Thread {
} else {
// we have Kafka metrics, register them
for (Map.Entry<MetricName, ? extends Metric> metric: metrics.entrySet()) {
- kafkaMetricGroup.gauge(metric.getKey().name(), new KafkaMetricWrapper(metric.getValue()));
+ consumerMetricGroup.gauge(metric.getKey().name(), new KafkaMetricWrapper(metric.getValue()));
+
+ // TODO this metric is kept for compatibility purposes; should remove in the future
+ subtaskMetricGroup.gauge(metric.getKey().name(), new KafkaMetricWrapper(metric.getValue()));
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java
index e4e276a..27b67f1 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09FetcherTest.java
@@ -126,10 +126,11 @@ public class Kafka09FetcherTest {
10, /* watermark interval */
this.getClass().getClassLoader(),
"task_name",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
@@ -261,10 +262,11 @@ public class Kafka09FetcherTest {
10, /* watermark interval */
this.getClass().getClassLoader(),
"task_name",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
@@ -375,10 +377,11 @@ public class Kafka09FetcherTest {
10, /* watermark interval */
this.getClass().getClassLoader(),
"task_name",
- new UnregisteredMetricsGroup(),
schema,
new Properties(),
0L,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup(),
false);
// ----- run the fetcher -----
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThreadTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThreadTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThreadTest.java
index 383eb13..3d40187 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThreadTest.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThreadTest.java
@@ -716,11 +716,12 @@ public class KafkaConsumerThreadTest {
handover,
new Properties(),
unassignedPartitionsQueue,
- new UnregisteredMetricsGroup(),
new KafkaConsumerCallBridge(),
"test-kafka-consumer-thread",
0,
- false);
+ false,
+ new UnregisteredMetricsGroup(),
+ new UnregisteredMetricsGroup());
this.mockConsumer = mockConsumer;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
index 660af21..d126301 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
@@ -29,6 +29,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.state.CheckpointListener;
import org.apache.flink.runtime.state.DefaultOperatorStateBackend;
import org.apache.flink.runtime.state.FunctionInitializationContext;
@@ -65,6 +66,7 @@ import java.util.regex.Pattern;
import static org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaConsumerMetricConstants.COMMITS_FAILED_METRICS_COUNTER;
import static org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaConsumerMetricConstants.COMMITS_SUCCEEDED_METRICS_COUNTER;
+import static org.apache.flink.streaming.connectors.kafka.internals.metrics.KafkaConsumerMetricConstants.KAFKA_CONSUMER_METRICS_GROUP;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -192,6 +194,13 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
// internal metrics
// ------------------------------------------------------------------------
+ /**
+ * Flag indicating whether or not metrics should be exposed.
+ * If {@code true}, offset metrics (e.g. current offset, committed offset) and
+ * Kafka-shipped metrics will be registered.
+ */
+ private final boolean useMetrics;
+
/** Counter for successful Kafka offset commits. */
private transient Counter successfulCommits;
@@ -221,7 +230,8 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
List<String> topics,
Pattern topicPattern,
KeyedDeserializationSchema<T> deserializer,
- long discoveryIntervalMillis) {
+ long discoveryIntervalMillis,
+ boolean useMetrics) {
this.topicsDescriptor = new KafkaTopicsDescriptor(topics, topicPattern);
this.deserializer = checkNotNull(deserializer, "valueDeserializer");
@@ -229,6 +239,8 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
discoveryIntervalMillis == PARTITION_DISCOVERY_DISABLED || discoveryIntervalMillis >= 0,
"Cannot define a negative value for the topic / partition discovery interval.");
this.discoveryIntervalMillis = discoveryIntervalMillis;
+
+ this.useMetrics = useMetrics;
}
// ------------------------------------------------------------------------
@@ -560,7 +572,9 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
periodicWatermarkAssigner,
punctuatedWatermarkAssigner,
(StreamingRuntimeContext) getRuntimeContext(),
- offsetCommitMode);
+ offsetCommitMode,
+ getRuntimeContext().getMetricGroup().addGroup(KAFKA_CONSUMER_METRICS_GROUP),
+ useMetrics);
if (!running) {
return;
@@ -839,7 +853,9 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception;
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup kafkaMetricGroup,
+ boolean useMetrics) throws Exception;
/**
* Creates the partition discoverer that is used to find new partitions for this subtask.
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
index c48a2b5..27afbca 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
@@ -91,9 +91,6 @@ public abstract class AbstractFetcher<T, KPH> {
/** The mode describing whether the fetcher also generates timestamps and watermarks. */
protected final int timestampWatermarkMode;
- /** Flag whether to register metrics for the fetcher. */
- protected final boolean useMetrics;
-
/**
* Optional timestamp extractor / watermark generator that will be run per Kafka partition,
* to exploit per-partition timestamp characteristics.
@@ -124,10 +121,10 @@ public abstract class AbstractFetcher<T, KPH> {
ProcessingTimeService processingTimeProvider,
long autoWatermarkInterval,
ClassLoader userCodeClassLoader,
+ MetricGroup consumerMetricGroup,
boolean useMetrics) throws Exception {
this.sourceContext = checkNotNull(sourceContext);
this.checkpointLock = sourceContext.getCheckpointLock();
- this.useMetrics = useMetrics;
this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
// figure out what we watermark mode we will be using
@@ -171,6 +168,10 @@ public abstract class AbstractFetcher<T, KPH> {
unassignedPartitionsQueue.add(partition);
}
+ if (useMetrics) {
+ addOffsetStateGauge(checkNotNull(consumerMetricGroup));
+ }
+
// if we have periodic watermarks, kick off the interval scheduler
if (timestampWatermarkMode == PERIODIC_WATERMARKS) {
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
index 84f0e38..a533cca 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
@@ -19,6 +19,7 @@
package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
@@ -376,7 +377,8 @@ public class FlinkKafkaConsumerBaseMigrationTest {
Arrays.asList("dummy-topic"),
null,
(KeyedDeserializationSchema< T >) mock(KeyedDeserializationSchema.class),
- discoveryInterval);
+ discoveryInterval,
+ false);
this.fetcher = fetcher;
this.partitions = partitions;
@@ -393,7 +395,9 @@ public class FlinkKafkaConsumerBaseMigrationTest {
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception {
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup consumerMetricGroup,
+ boolean useMetrics) throws Exception {
return fetcher;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java
index e6fd3fe..41b609e 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseTest.java
@@ -640,7 +640,8 @@ public class FlinkKafkaConsumerBaseTest {
Collections.singletonList("dummy-topic"),
null,
(KeyedDeserializationSchema < T >) mock(KeyedDeserializationSchema.class),
- PARTITION_DISCOVERY_DISABLED);
+ PARTITION_DISCOVERY_DISABLED,
+ false);
this.testFetcher = testFetcher;
this.testPartitionDiscoverer = testPartitionDiscoverer;
@@ -655,7 +656,9 @@ public class FlinkKafkaConsumerBaseTest {
SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
StreamingRuntimeContext runtimeContext,
- OffsetCommitMode offsetCommitMode) throws Exception {
+ OffsetCommitMode offsetCommitMode,
+ MetricGroup consumerMetricGroup,
+ boolean useMetrics) throws Exception {
return this.testFetcher;
}
@@ -752,6 +755,7 @@ public class FlinkKafkaConsumerBaseTest {
new TestProcessingTimeService(),
0,
MockFetcher.class.getClassLoader(),
+ new UnregisteredMetricsGroup(),
false);
this.stateSnapshotsToReturn.addAll(Arrays.asList(stateSnapshotsToReturn));
http://git-wip-us.apache.org/repos/asf/flink/blob/0a1ce006/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcherTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcherTest.java
index 6fe1d6f..d276cfb 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcherTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcherTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.streaming.connectors.kafka.internals;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext;
@@ -411,6 +412,7 @@ public class AbstractFetcherTest {
processingTimeProvider,
autoWatermarkInterval,
TestFetcher.class.getClassLoader(),
+ new UnregisteredMetricsGroup(),
false);
}
[16/31] flink git commit: [hotfix] [kinesis] Use ArrayList for faster
shard state updates
Posted by tz...@apache.org.
[hotfix] [kinesis] Use ArrayList for faster shard state updates
Previously, the Kinesis Consumer uses LinkedLists as the underlying
implementation for subscribedShardsState. This list is accessed on every
record, updating a shard's state using a shard state index (i.e., the
order of the shard state in the list). With LinkedLists, this access has
linear time complexity, and since this operation happens per record, can
have a big performance hit on executions with a huge number of Kinesis
shards.
This commit changes the list implementation to be an ArrayList for
constant-time shard state access. The downside is that when new shards
are discovered, there will be a costly array re-size. However, since
resharding is not expected to happen so often, this is acceptable.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b5d459a5
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b5d459a5
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b5d459a5
Branch: refs/heads/master
Commit: b5d459a5107dc606473ea40849460e17e6afe5ea
Parents: 140bb0c
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Feb 1 13:05:56 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../connectors/kinesis/internals/KinesisDataFetcher.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b5d459a5/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
index 7928fe3..4d03ba8 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
@@ -43,6 +43,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -190,8 +191,8 @@ public class KinesisDataFetcher<T> {
runtimeContext,
configProps,
deserializationSchema,
- new AtomicReference<Throwable>(),
- new LinkedList<KinesisStreamShardState>(),
+ new AtomicReference<>(),
+ new ArrayList<>(),
createInitialSubscribedStreamsToLastDiscoveredShardsState(streams),
KinesisProxy.create(configProps));
}
@@ -204,7 +205,7 @@ public class KinesisDataFetcher<T> {
Properties configProps,
KinesisDeserializationSchema<T> deserializationSchema,
AtomicReference<Throwable> error,
- LinkedList<KinesisStreamShardState> subscribedShardsState,
+ List<KinesisStreamShardState> subscribedShardsState,
HashMap<String, String> subscribedStreamsToLastDiscoveredShardIds,
KinesisProxyInterface kinesis) {
this.streams = checkNotNull(streams);
[28/31] flink git commit: [FLINK-8275] [security,
yarn] Fix keytab local path in YarnTaskManagerRunner
Posted by tz...@apache.org.
[FLINK-8275] [security, yarn] Fix keytab local path in YarnTaskManagerRunner
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2bbf4e50
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2bbf4e50
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2bbf4e50
Branch: refs/heads/master
Commit: 2bbf4e5002d7657018ba0b53b7a3ed8ee3124da8
Parents: a79916b
Author: Shuyi Chen <sh...@uber.com>
Authored: Sun Dec 17 23:50:07 2017 -0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../flink/runtime/security/SecurityUtils.java | 4 +-
.../runtime/security/modules/HadoopModule.java | 6 ++
.../yarn/TestingYarnTaskManagerRunner.java | 3 +-
.../flink/yarn/TestingYarnTaskManager.scala | 3 +-
.../flink/yarn/YarnApplicationMasterRunner.java | 31 ++------
.../flink/yarn/YarnTaskManagerRunner.java | 64 +++++++++--------
.../org/apache/flink/yarn/YarnTaskManager.scala | 3 +-
.../flink/yarn/YarnTaskManagerRunnerTest.java | 74 ++++++++++++++++++++
flink-yarn/src/test/resources/flink-conf.yaml | 23 ++++++
flink-yarn/src/test/resources/krb5.keytab | 0
pom.xml | 1 +
11 files changed, 151 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
index c73e966..cb68da4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.security;
-import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.security.modules.SecurityModule;
import org.apache.flink.runtime.security.modules.SecurityModuleFactory;
@@ -47,8 +46,7 @@ public class SecurityUtils {
return installedContext;
}
- @VisibleForTesting
- static List<SecurityModule> getInstalledModules() {
+ public static List<SecurityModule> getInstalledModules() {
return installedModules;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/HadoopModule.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/HadoopModule.java b/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/HadoopModule.java
index a39d879..0738708 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/HadoopModule.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/security/modules/HadoopModule.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.security.modules;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.security.SecurityConfiguration;
import org.apache.flink.runtime.util.HadoopUtils;
@@ -58,6 +59,11 @@ public class HadoopModule implements SecurityModule {
this.hadoopConfiguration = checkNotNull(hadoopConfiguration);
}
+ @VisibleForTesting
+ public SecurityConfiguration getSecurityConfig() {
+ return securityConfig;
+ }
+
@Override
public void install() throws SecurityInstallException {
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
index 8586a77..90cdf9f 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/TestingYarnTaskManagerRunner.java
@@ -25,6 +25,7 @@ import java.io.IOException;
*/
public class TestingYarnTaskManagerRunner {
public static void main(String[] args) throws IOException {
- YarnTaskManagerRunner.runYarnTaskManager(args, TestingYarnTaskManager.class);
+ YarnTaskManagerRunner.runYarnTaskManager(
+ args, TestingYarnTaskManager.class, System.getenv(), null);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala b/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
index f54528f..da95e01 100644
--- a/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
+++ b/flink-yarn-tests/src/test/scala/org/apache/flink/yarn/TestingYarnTaskManager.scala
@@ -72,7 +72,8 @@ class TestingYarnTaskManager(
* @param args The command line arguments.
*/
def main(args: Array[String]): Unit = {
- YarnTaskManagerRunner.runYarnTaskManager(args, classOf[TestingYarnTaskManager])
+ YarnTaskManagerRunner.runYarnTaskManager(
+ args, classOf[TestingYarnTaskManager], System.getenv(), null)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
index 68c0aec..1813034 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnApplicationMasterRunner.java
@@ -146,19 +146,9 @@ public class YarnApplicationMasterRunner {
require(currDir != null, "Current working directory variable (%s) not set", Environment.PWD.key());
LOG.debug("Current working Directory: {}", currDir);
- final String remoteKeytabPath = ENV.get(YarnConfigKeys.KEYTAB_PATH);
- LOG.debug("remoteKeytabPath obtained {}", remoteKeytabPath);
-
final String remoteKeytabPrincipal = ENV.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
LOG.info("remoteKeytabPrincipal obtained {}", remoteKeytabPrincipal);
- String keytabPath = null;
- if (remoteKeytabPath != null) {
- File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
- keytabPath = f.getAbsolutePath();
- LOG.debug("keytabPath: {}", keytabPath);
- }
-
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
LOG.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
@@ -171,9 +161,10 @@ public class YarnApplicationMasterRunner {
final Configuration flinkConfig = createConfiguration(currDir, dynamicProperties, LOG);
- // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
- if (keytabPath != null && remoteKeytabPrincipal != null) {
- flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
+ File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
+ if (remoteKeytabPrincipal != null && f.exists()) {
+ // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
+ flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
}
@@ -253,19 +244,11 @@ public class YarnApplicationMasterRunner {
LOG.info("YARN assigned hostname for application master: {}", appMasterHostname);
- //Update keytab and principal path to reflect YARN container path location
- final String remoteKeytabPath = ENV.get(YarnConfigKeys.KEYTAB_PATH);
-
final String remoteKeytabPrincipal = ENV.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
- String keytabPath = null;
- if (remoteKeytabPath != null) {
- File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
- keytabPath = f.getAbsolutePath();
- LOG.info("keytabPath: {}", keytabPath);
- }
- if (keytabPath != null && remoteKeytabPrincipal != null) {
- config.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
+ File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
+ if (remoteKeytabPrincipal != null && f.exists()) {
+ config.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
config.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
index a6e34c5..9d69fbd 100644
--- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
+++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnTaskManagerRunner.java
@@ -51,7 +51,29 @@ public class YarnTaskManagerRunner {
private static final Logger LOG = LoggerFactory.getLogger(YarnTaskManagerRunner.class);
- public static void runYarnTaskManager(String[] args, final Class<? extends YarnTaskManager> taskManager) throws IOException {
+ private static Callable<Object> createMainRunner(
+ Configuration configuration,
+ ResourceID resourceId,
+ final Class<? extends YarnTaskManager> taskManager) {
+ return new Callable<Object>() {
+ @Override
+ public Integer call() {
+ try {
+ TaskManager.selectNetworkInterfaceAndRunTaskManager(
+ configuration, resourceId, taskManager);
+ } catch (Throwable t) {
+ LOG.error("Error while starting the TaskManager", t);
+ System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
+ }
+ return null;
+ }
+ };
+ }
+
+ public static void runYarnTaskManager(String[] args,
+ final Class<? extends YarnTaskManager> taskManager,
+ Map<String, String> envs,
+ Callable<Object> mainRunner) throws IOException {
EnvironmentInformation.logEnvironmentInfo(LOG, "YARN TaskManager", args);
SignalHandler.register(LOG);
JvmShutdownSafeguard.installAsShutdownHook(LOG);
@@ -68,7 +90,6 @@ public class YarnTaskManagerRunner {
}
// read the environment variables for YARN
- final Map<String, String> envs = System.getenv();
final String yarnClientUsername = envs.get(YarnConfigKeys.ENV_HADOOP_USER_NAME);
final String localDirs = envs.get(Environment.LOCAL_DIRS.key());
LOG.info("Current working/local Directory: {}", localDirs);
@@ -76,9 +97,6 @@ public class YarnTaskManagerRunner {
final String currDir = envs.get(Environment.PWD.key());
LOG.info("Current working Directory: {}", currDir);
- final String remoteKeytabPath = envs.get(YarnConfigKeys.KEYTAB_PATH);
- LOG.info("TM: remoteKeytabPath obtained {}", remoteKeytabPath);
-
final String remoteKeytabPrincipal = envs.get(YarnConfigKeys.KEYTAB_PRINCIPAL);
LOG.info("TM: remoteKeytabPrincipal obtained {}", remoteKeytabPrincipal);
@@ -95,13 +113,6 @@ public class YarnTaskManagerRunner {
// tell akka to die in case of an error
configuration.setBoolean(AkkaOptions.JVM_EXIT_ON_FATAL_ERROR, true);
- String localKeytabPath = null;
- if (remoteKeytabPath != null) {
- File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
- localKeytabPath = f.getAbsolutePath();
- LOG.info("localKeytabPath: {}", localKeytabPath);
- }
-
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
LOG.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
@@ -112,6 +123,12 @@ public class YarnTaskManagerRunner {
final ResourceID resourceId = new ResourceID(containerID);
LOG.info("ResourceID assigned for this container: {}", resourceId);
+ File f = new File(currDir, Utils.KEYTAB_FILE_NAME);
+ if (remoteKeytabPrincipal != null && f.exists()) {
+ // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
+ configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, f.getAbsolutePath());
+ configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
+ }
try {
SecurityConfiguration sc;
@@ -125,12 +142,6 @@ public class YarnTaskManagerRunner {
hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
hadoopConfiguration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true");
- // set keytab principal and replace path with the local path of the shipped keytab file in NodeManager
- if (localKeytabPath != null && remoteKeytabPrincipal != null) {
- configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, localKeytabPath);
- configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
- }
-
sc = new SecurityConfiguration(configuration,
Collections.singletonList(securityConfig -> new HadoopModule(securityConfig, hadoopConfiguration)));
@@ -141,19 +152,10 @@ public class YarnTaskManagerRunner {
SecurityUtils.install(sc);
- SecurityUtils.getInstalledContext().runSecured(new Callable<Object>() {
- @Override
- public Integer call() {
- try {
- TaskManager.selectNetworkInterfaceAndRunTaskManager(configuration, resourceId, taskManager);
- }
- catch (Throwable t) {
- LOG.error("Error while starting the TaskManager", t);
- System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
- }
- return null;
- }
- });
+ if (mainRunner == null) {
+ mainRunner = createMainRunner(configuration, resourceId, taskManager);
+ }
+ SecurityUtils.getInstalledContext().runSecured(mainRunner);
} catch (Exception e) {
LOG.error("Exception occurred while launching Task Manager", e);
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
index 615466d..5b8f4ca 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/YarnTaskManager.scala
@@ -62,7 +62,8 @@ class YarnTaskManager(
* @param args The command line arguments.
*/
def main(args: Array[String]): Unit = {
- YarnTaskManagerRunner.runYarnTaskManager(args, classOf[YarnTaskManager])
+ YarnTaskManagerRunner.runYarnTaskManager(
+ args, classOf[YarnTaskManager], System.getenv(), null)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
new file mode 100644
index 0000000..cdacfc0
--- /dev/null
+++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnTaskManagerRunnerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.yarn;
+
+import org.apache.flink.runtime.security.SecurityUtils;
+import org.apache.flink.runtime.security.modules.HadoopModule;
+import org.apache.flink.runtime.security.modules.SecurityModule;
+
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+
+/**
+ * Tests for {@link YarnTaskManagerRunner}.
+ */
+public class YarnTaskManagerRunnerTest {
+
+ @Test
+ public void testKerberosKeytabConfiguration() throws IOException {
+ final String resourceDirPath =
+ Paths.get("src", "test", "resources").toAbsolutePath().toString();
+ final Map<String, String> envs = new HashMap<>();
+ envs.put(YarnFlinkResourceManager.ENV_FLINK_CONTAINER_ID, "test_container_00001");
+ envs.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "testuser1@domain");
+ envs.put(ApplicationConstants.Environment.PWD.key(), resourceDirPath);
+ YarnTaskManagerRunner.runYarnTaskManager(
+ new String[]{"--configDir", resourceDirPath},
+ YarnTaskManager.class,
+ envs,
+ new Callable<Object>() {
+ @Override
+ public Integer call() {
+ final List<SecurityModule> modules = SecurityUtils.getInstalledModules();
+ Optional<SecurityModule> moduleOpt =
+ modules.stream().filter(s -> s instanceof HadoopModule).findFirst();
+ if (moduleOpt.isPresent()) {
+ HadoopModule hadoopModule = (HadoopModule) moduleOpt.get();
+ assertEquals("testuser1@domain", hadoopModule.getSecurityConfig().getPrincipal());
+ assertEquals(resourceDirPath + "/krb5.keytab",
+ hadoopModule.getSecurityConfig().getKeytab());
+ } else {
+ fail("Can not find HadoopModule!");
+ }
+ return null;
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/flink-yarn/src/test/resources/flink-conf.yaml
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/resources/flink-conf.yaml b/flink-yarn/src/test/resources/flink-conf.yaml
new file mode 100644
index 0000000..2c95194
--- /dev/null
+++ b/flink-yarn/src/test/resources/flink-conf.yaml
@@ -0,0 +1,23 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+#
+# This is a test configuration for validation of YarnTaskManagerRunner.
+#
+
+taskmanager.tmp.dirs: /tmp
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/flink-yarn/src/test/resources/krb5.keytab
----------------------------------------------------------------------
diff --git a/flink-yarn/src/test/resources/krb5.keytab b/flink-yarn/src/test/resources/krb5.keytab
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/flink/blob/2bbf4e50/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3065abf..83445dd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1015,6 +1015,7 @@ under the License.
<exclude>flink-formats/flink-avro/src/test/resources/avro/*.avsc</exclude>
<exclude>out/test/flink-avro/avro/user.avsc</exclude>
<exclude>flink-libraries/flink-table/src/test/scala/resources/*.out</exclude>
+ <exclude>flink-yarn/src/test/resources/krb5.keytab</exclude>
<exclude>test-infra/end-to-end-test/test-data/*</exclude>
<!-- snapshots -->
[15/31] flink git commit: [FLINK-8472] [DataStream,
test] Extend WindowOperatorMigrationTest for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [DataStream, test] Extend WindowOperatorMigrationTest for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c71fb422
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c71fb422
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c71fb422
Branch: refs/heads/master
Commit: c71fb422a0fe8d8a6f550b63dcbd4d5be6fbee2c
Parents: b5d459a
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 16:08:53 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../windowing/WindowOperatorMigrationTest.java | 2 +-
...igration-test-apply-event-time-flink1.4-snapshot | Bin 0 -> 4630 bytes
...ion-test-apply-processing-time-flink1.4-snapshot | Bin 0 -> 4522 bytes
...gration-test-reduce-event-time-flink1.4-snapshot | Bin 0 -> 3497 bytes
...on-test-reduce-processing-time-flink1.4-snapshot | Bin 0 -> 3438 bytes
...-session-with-stateful-trigger-flink1.4-snapshot | Bin 0 -> 8680 bytes
...ion-with-stateful-trigger-mint-flink1.4-snapshot | Bin 0 -> 7651 bytes
7 files changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c71fb422/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
index 7c5767a..99bea96 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
@@ -83,7 +83,7 @@ public class WindowOperatorMigrationTest {
@Parameterized.Parameters(name = "Migration Savepoint: {0}")
public static Collection<MigrationVersion> parameters () {
- return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3);
+ return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3, MigrationVersion.v1_4);
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/c71fb422/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.4-snapshot
new file mode 100644
index 0000000..5caddc2
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/c71fb422/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.4-snapshot
new file mode 100644
index 0000000..a2c1148
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/c71fb422/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.4-snapshot
new file mode 100644
index 0000000..b59d9ca
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/c71fb422/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.4-snapshot
new file mode 100644
index 0000000..94909fc
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/c71fb422/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.4-snapshot
new file mode 100644
index 0000000..9337718
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/c71fb422/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.4-snapshot
new file mode 100644
index 0000000..ea48bdd
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.4-snapshot differ
[18/31] flink git commit: [FLINK-8421] [DataStream,
tests] Add WindowOperator migration test for Kryo-serialized window
keys
Posted by tz...@apache.org.
[FLINK-8421] [DataStream, tests] Add WindowOperator migration test for Kryo-serialized window keys
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7a9d7e7a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7a9d7e7a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7a9d7e7a
Branch: refs/heads/master
Commit: 7a9d7e7a243c9170635c6a7cd6c414e16d17b8f8
Parents: 72f0eb1
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 24 17:08:13 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../operators/windowing/NonPojoType.java | 68 ++++++
.../windowing/WindowOperatorMigrationTest.java | 219 +++++++++++++++----
...n-test-kryo-serialized-key-flink1.2-snapshot | Bin 0 -> 3686 bytes
...n-test-kryo-serialized-key-flink1.3-snapshot | Bin 0 -> 11298 bytes
...n-test-kryo-serialized-key-flink1.4-snapshot | Bin 0 -> 9539 bytes
5 files changed, 241 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/7a9d7e7a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonPojoType.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonPojoType.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonPojoType.java
new file mode 100644
index 0000000..e70b847
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/NonPojoType.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.operators.windowing;
+
+/**
+ * A non-Pojo type used in {@link WindowOperatorMigrationTest} to test
+ * migration behaviours for window operators that have a Kryo-serialized key.
+ *
+ * <p>NOTE: modifying this class would cause tests in {@link WindowOperatorMigrationTest} to fail,
+ * since it is serialized in older version test savepoints.
+ */
+public class NonPojoType implements Comparable<NonPojoType> {
+
+ private final String data;
+
+ NonPojoType(String data) {
+ this.data = data;
+ }
+
+ public String getData() {
+ return data;
+ }
+
+ @Override
+ public int compareTo(NonPojoType o) {
+ return data.compareTo(o.data);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj == null) {
+ return false;
+ }
+
+ if (!(obj instanceof NonPojoType)) {
+ return false;
+ }
+
+ NonPojoType other = (NonPojoType) obj;
+
+ return data.equals(other.data);
+ }
+
+ @Override
+ public int hashCode() {
+ return data.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/7a9d7e7a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
index 99bea96..ec537c6 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorMigrationTest.java
@@ -23,11 +23,14 @@ import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.state.ListStateDescriptor;
import org.apache.flink.api.common.state.ReducingStateDescriptor;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.typeutils.TypeInfoParser;
+import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.windowing.PassThroughWindowFunction;
import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
@@ -66,6 +69,7 @@ import java.util.Comparator;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
/**
@@ -114,7 +118,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
EventTimeSessionWindows.withGap(Time.seconds(sessionSize)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<String>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
@@ -123,7 +127,7 @@ public class WindowOperatorMigrationTest {
null /* late data output tag */);
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -160,7 +164,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
EventTimeSessionWindows.withGap(Time.seconds(sessionSize)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<String>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
@@ -171,7 +175,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -218,7 +222,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
EventTimeSessionWindows.withGap(Time.seconds(sessionSize)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<String>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
@@ -227,7 +231,7 @@ public class WindowOperatorMigrationTest {
null /* late data output tag */);
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -258,7 +262,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(
EventTimeSessionWindows.withGap(Time.seconds(sessionSize)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<String>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
@@ -269,7 +273,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -322,13 +326,13 @@ public class WindowOperatorMigrationTest {
TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
- new SumReducer(),
+ new SumReducer<>(),
inputType.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
@@ -339,7 +343,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -358,11 +362,11 @@ public class WindowOperatorMigrationTest {
testHarness.processWatermark(new Watermark(999));
expectedOutput.add(new Watermark(999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.processWatermark(new Watermark(1999));
expectedOutput.add(new Watermark(1999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
// do snapshot and save to file
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
@@ -380,13 +384,13 @@ public class WindowOperatorMigrationTest {
TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
- new SumReducer(),
+ new SumReducer<>(),
inputType.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
@@ -397,7 +401,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -424,7 +428,7 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999));
expectedOutput.add(new Watermark(5999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.close();
}
@@ -444,7 +448,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
@@ -455,7 +459,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -474,11 +478,11 @@ public class WindowOperatorMigrationTest {
testHarness.processWatermark(new Watermark(999));
expectedOutput.add(new Watermark(999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.processWatermark(new Watermark(1999));
expectedOutput.add(new Watermark(1999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
// do snapshot and save to file
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
@@ -501,7 +505,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
@@ -512,7 +516,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -539,7 +543,7 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999));
expectedOutput.add(new Watermark(5999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.close();
}
@@ -554,13 +558,13 @@ public class WindowOperatorMigrationTest {
TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
- new SumReducer(),
+ new SumReducer<>(),
inputType.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
@@ -571,7 +575,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -587,7 +591,7 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 1), 2999));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 1), 2999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
// do snapshot and save to file
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
@@ -606,13 +610,13 @@ public class WindowOperatorMigrationTest {
TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
- new SumReducer(),
+ new SumReducer<>(),
inputType.createSerializer(new ExecutionConfig()));
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
@@ -623,7 +627,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -645,7 +649,7 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), 5999));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key3", 1), 5999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.close();
}
@@ -665,7 +669,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
@@ -676,7 +680,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
testHarness.open();
@@ -692,7 +696,7 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 1), 2999));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 1), 2999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
// do snapshot and save to file
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
@@ -715,7 +719,7 @@ public class WindowOperatorMigrationTest {
WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>(
TumblingProcessingTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
new TimeWindow.Serializer(),
- new TupleKeySelector(),
+ new TupleKeySelector<>(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
@@ -726,7 +730,7 @@ public class WindowOperatorMigrationTest {
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
- new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.setup();
@@ -748,28 +752,151 @@ public class WindowOperatorMigrationTest {
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), 5999));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key3", 1), 5999));
- TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
testHarness.close();
}
- private static class TupleKeySelector implements KeySelector<Tuple2<String, Integer>, String> {
+ /**
+ * Manually run this to write binary snapshot data.
+ */
+ @Ignore
+ @Test
+ public void writeWindowsWithKryoSerializedKeysSnapshot() throws Exception {
+ final int windowSize = 3;
+
+ TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo();
+
+ ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer<>(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig());
+ assertTrue(keySerializer instanceof KryoSerializer);
+
+ WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>(
+ TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector<>(),
+ keySerializer,
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()),
+ EventTimeTrigger.create(),
+ 0,
+ null /* late data output tag */);
+
+ ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+ OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness =
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class));
+
+ testHarness.setup();
+ testHarness.open();
+
+ // add elements out-of-order
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 3999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 3000));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 20));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 0));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 1), 999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 1), 1000));
+
+ testHarness.processWatermark(new Watermark(999));
+ expectedOutput.add(new Watermark(999));
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
+
+ testHarness.processWatermark(new Watermark(1999));
+ expectedOutput.add(new Watermark(1999));
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
+
+ // do snapshot and save to file
+ OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
+ OperatorSnapshotUtil.writeStateHandle(
+ snapshot,
+ "src/test/resources/win-op-migration-test-kryo-serialized-key-flink" + flinkGenerateSavepointVersion + "-snapshot");
+
+ testHarness.close();
+ }
+
+ @Test
+ public void testRestoreKryoSerializedKeysWindows() throws Exception {
+ final int windowSize = 3;
+
+ TypeInformation<Tuple2<NonPojoType, Integer>> inputType = new TypeHint<Tuple2<NonPojoType, Integer>>() {}.getTypeInfo();
+
+ ReducingStateDescriptor<Tuple2<NonPojoType, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer<>(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ TypeSerializer<NonPojoType> keySerializer = TypeInformation.of(NonPojoType.class).createSerializer(new ExecutionConfig());
+ assertTrue(keySerializer instanceof KryoSerializer);
+
+ WindowOperator<NonPojoType, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>, TimeWindow> operator = new WindowOperator<>(
+ TumblingEventTimeWindows.of(Time.of(windowSize, TimeUnit.SECONDS)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector<>(),
+ keySerializer,
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<>()),
+ EventTimeTrigger.create(),
+ 0,
+ null /* late data output tag */);
+
+ ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+ OneInputStreamOperatorTestHarness<Tuple2<NonPojoType, Integer>, Tuple2<NonPojoType, Integer>> testHarness =
+ new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), TypeInformation.of(NonPojoType.class));
+
+ testHarness.setup();
+
+ MigrationTestUtil.restoreFromSnapshot(
+ testHarness,
+ OperatorSnapshotUtil.getResourceFilename(
+ "win-op-migration-test-kryo-serialized-key-flink" + testMigrateVersion + "-snapshot"),
+ testMigrateVersion);
+
+ testHarness.open();
+
+ testHarness.processWatermark(new Watermark(2999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key1"), 3), 2999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 3), 2999));
+ expectedOutput.add(new Watermark(2999));
+
+ testHarness.processWatermark(new Watermark(3999));
+ expectedOutput.add(new Watermark(3999));
+
+ testHarness.processWatermark(new Watermark(4999));
+ expectedOutput.add(new Watermark(4999));
+
+ testHarness.processWatermark(new Watermark(5999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>(new NonPojoType("key2"), 2), 5999));
+ expectedOutput.add(new Watermark(5999));
+
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator<>());
+ testHarness.close();
+ }
+
+ private static class TupleKeySelector<K> implements KeySelector<Tuple2<K, Integer>, K> {
private static final long serialVersionUID = 1L;
@Override
- public String getKey(Tuple2<String, Integer> value) throws Exception {
+ public K getKey(Tuple2<K, Integer> value) throws Exception {
return value.f0;
}
}
@SuppressWarnings("unchecked")
- private static class Tuple2ResultSortComparator implements Comparator<Object> {
+ private static class Tuple2ResultSortComparator<K extends Comparable> implements Comparator<Object> {
@Override
public int compare(Object o1, Object o2) {
if (o1 instanceof Watermark || o2 instanceof Watermark) {
return 0;
} else {
- StreamRecord<Tuple2<String, Integer>> sr0 = (StreamRecord<Tuple2<String, Integer>>) o1;
- StreamRecord<Tuple2<String, Integer>> sr1 = (StreamRecord<Tuple2<String, Integer>>) o2;
+ StreamRecord<Tuple2<K, Integer>> sr0 = (StreamRecord<Tuple2<K, Integer>>) o1;
+ StreamRecord<Tuple2<K, Integer>> sr1 = (StreamRecord<Tuple2<K, Integer>>) o2;
if (sr0.getTimestamp() != sr1.getTimestamp()) {
return (int) (sr0.getTimestamp() - sr1.getTimestamp());
}
@@ -809,11 +936,11 @@ public class WindowOperatorMigrationTest {
}
}
- private static class SumReducer implements ReduceFunction<Tuple2<String, Integer>> {
+ private static class SumReducer<K> implements ReduceFunction<Tuple2<K, Integer>> {
private static final long serialVersionUID = 1L;
@Override
- public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1,
- Tuple2<String, Integer> value2) throws Exception {
+ public Tuple2<K, Integer> reduce(Tuple2<K, Integer> value1,
+ Tuple2<K, Integer> value2) throws Exception {
return new Tuple2<>(value2.f0, value1.f1 + value2.f1);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7a9d7e7a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.2-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.2-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.2-snapshot
new file mode 100644
index 0000000..3a4cc98
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.2-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/7a9d7e7a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.3-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.3-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.3-snapshot
new file mode 100644
index 0000000..aa81eb4
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.3-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/7a9d7e7a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.4-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.4-snapshot
new file mode 100644
index 0000000..8449113
Binary files /dev/null and b/flink-streaming-java/src/test/resources/win-op-migration-test-kryo-serialized-key-flink1.4-snapshot differ
[04/31] flink git commit: [FLINK-8398] [kinesis,
tests] Stabilize flaky KinesisDataFetcherTests
Posted by tz...@apache.org.
[FLINK-8398] [kinesis, tests] Stabilize flaky KinesisDataFetcherTests
Prior to this commit, several unit tests in KinesisDataFetcherTest
relied on sleeps to wait until a certain operation happens, in order for
the test to pass.
This commit removes those sleeps and replaces the test behaviours with
OneShotLatches.
This closes #5268.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3f0d6c61
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3f0d6c61
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3f0d6c61
Branch: refs/heads/master
Commit: 3f0d6c618302e4b341e5a442126e6ba2889cd2f4
Parents: 1767898
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 10 13:41:49 2018 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../kinesis/internals/KinesisDataFetcherTest.java | 15 ++++++++++-----
.../testutils/TestableKinesisDataFetcher.java | 16 ++++++++++++++++
2 files changed, 26 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/3f0d6c61/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
index 56566c0..95e87c6 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcherTest.java
@@ -37,6 +37,7 @@ import org.apache.flink.streaming.connectors.kinesis.testutils.TestableKinesisDa
import com.amazonaws.services.kinesis.model.HashKeyRange;
import com.amazonaws.services.kinesis.model.SequenceNumberRange;
import com.amazonaws.services.kinesis.model.Shard;
+import org.apache.flink.util.TestLogger;
import org.junit.Test;
import java.util.HashMap;
@@ -58,7 +59,7 @@ import static org.mockito.Mockito.when;
/**
* Tests for the {@link KinesisDataFetcher}.
*/
-public class KinesisDataFetcherTest {
+public class KinesisDataFetcherTest extends TestLogger {
@Test(expected = RuntimeException.class)
public void testIfNoShardsAreFoundShouldThrowException() throws Exception {
@@ -214,7 +215,8 @@ public class KinesisDataFetcherTest {
}
};
runFetcherThread.start();
- Thread.sleep(1000); // sleep a while before closing
+
+ fetcher.waitUntilInitialDiscovery();
fetcher.shutdownFetcher();
runFetcherThread.sync();
@@ -303,7 +305,8 @@ public class KinesisDataFetcherTest {
}
};
runFetcherThread.start();
- Thread.sleep(1000); // sleep a while before closing
+
+ fetcher.waitUntilInitialDiscovery();
fetcher.shutdownFetcher();
runFetcherThread.sync();
@@ -396,7 +399,8 @@ public class KinesisDataFetcherTest {
}
};
runFetcherThread.start();
- Thread.sleep(1000); // sleep a while before closing
+
+ fetcher.waitUntilInitialDiscovery();
fetcher.shutdownFetcher();
runFetcherThread.sync();
@@ -492,7 +496,8 @@ public class KinesisDataFetcherTest {
}
};
runFetcherThread.start();
- Thread.sleep(1000); // sleep a while before closing
+
+ fetcher.waitUntilInitialDiscovery();
fetcher.shutdownFetcher();
runFetcherThread.sync();
http://git-wip-us.apache.org/repos/asf/flink/blob/3f0d6c61/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
index 7b3e1bf..93f3d38 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestableKinesisDataFetcher.java
@@ -23,6 +23,7 @@ import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
+import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
import org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxyInterface;
import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;
@@ -45,6 +46,8 @@ import static org.mockito.Mockito.when;
public class TestableKinesisDataFetcher<T> extends KinesisDataFetcher<T> {
private OneShotLatch runWaiter;
+ private OneShotLatch initialDiscoveryWaiter;
+
private volatile boolean running;
public TestableKinesisDataFetcher(
@@ -71,6 +74,8 @@ public class TestableKinesisDataFetcher<T> extends KinesisDataFetcher<T> {
fakeKinesis);
this.runWaiter = new OneShotLatch();
+ this.initialDiscoveryWaiter = new OneShotLatch();
+
this.running = true;
}
@@ -98,6 +103,17 @@ public class TestableKinesisDataFetcher<T> extends KinesisDataFetcher<T> {
super.awaitTermination();
}
+ @Override
+ public List<StreamShardHandle> discoverNewShardsToSubscribe() throws InterruptedException {
+ List<StreamShardHandle> newShards = super.discoverNewShardsToSubscribe();
+ initialDiscoveryWaiter.trigger();
+ return newShards;
+ }
+
+ public void waitUntilInitialDiscovery() throws InterruptedException {
+ initialDiscoveryWaiter.await();
+ }
+
private static RuntimeContext getMockedRuntimeContext(final int fakeTotalCountOfSubtasks, final int fakeIndexOfThisSubtask) {
RuntimeContext mockedRuntimeContext = mock(RuntimeContext.class);
[14/31] flink git commit: [FLINK-8484] [kinesis] Synchronize local
shard metainfo's on restore
Posted by tz...@apache.org.
[FLINK-8484] [kinesis] Synchronize local shard metainfo's on restore
Ensure that a metainfo change in the StreamShardMetadata other than
`StreamShardMetadata.shardId` or `StreamShardMetadata.streamName` does
not result in the shard not being able to be restored. This handles the
corner case where a shard might have been closed (ending sequence number
set to not-null) since the last savepoint or checkpoint when a job is
restarted from a snapshot state.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b5b95eb1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b5b95eb1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b5b95eb1
Branch: refs/heads/master
Commit: b5b95eb146cd59454e9d3900280d989f72ba9de3
Parents: a03fbf9
Author: Philip Luppens <ph...@gmail.com>
Authored: Tue Jan 23 10:00:23 2018 +0200
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../kinesis/FlinkKinesisConsumer.java | 49 ++++++++
.../kinesis/FlinkKinesisConsumerTest.java | 125 +++++++++++++++++++
2 files changed, 174 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b5b95eb1/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
index 06b70b6..977eb05 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
@@ -210,6 +210,16 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
for (StreamShardHandle shard : allShards) {
StreamShardMetadata kinesisStreamShard = KinesisDataFetcher.convertToStreamShardMetadata(shard);
if (sequenceNumsToRestore != null) {
+
+ // We need to do this to make sure that a shard that was closed after this restored state was taken will be properly
+ // detected and have its sequence numbers restored. A shard will be closed when re-sharding, which can happen when
+ // scaling up & down the Kinesis stream, and if the state is not synchronized, then the equality check of the current
+ // Kinesis shard will not match the stored state, which will cause us to re-read the entire shard from the event horizon.
+ if (updateKinesisShardStateWithMissingEndingSequenceNumber(kinesisStreamShard, sequenceNumsToRestore)) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Updated local stored state for shard {} with a new ending number: {}", kinesisStreamShard.getShardId(), sequenceNumsToRestore.get(kinesisStreamShard));
+ }
+ }
if (sequenceNumsToRestore.containsKey(kinesisStreamShard)) {
// if the shard was already seen and is contained in the state,
// just use the sequence number stored in the state
@@ -267,6 +277,45 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
sourceContext.close();
}
+ /**
+ * Synchronizes the Kinesis shard information from the current Kinesis shard with the restored state, if we find
+ * a shard that match the shardId and streamName. If we find one, and its ending key is different that what we
+ * have in our stored state, then we update the stored's shard's metadata's ending number.
+ *
+ * @param current the current Kinesis shard we're trying to synchronize.
+ * @param sequenceNumsToRestore the (re)stored shard metadata and their sequence numbers.
+ * @return {@code true} if the local state was updated with the current Kinesis shard's ending number.
+ */
+ @VisibleForTesting
+ boolean updateKinesisShardStateWithMissingEndingSequenceNumber(StreamShardMetadata current, HashMap<StreamShardMetadata, SequenceNumber> sequenceNumsToRestore) {
+ checkNotNull(current.getStreamName(), "Stream name not set on the current metadata shard");
+ checkNotNull(current.getShardId(), "Shard id not set on the current metadata shard");
+
+ // short-circuit: if the current shard doesn't have an ending sequence number, then there's no point in trying to update the local state
+ // since that's the only property that can change.
+ if (current.getEndingSequenceNumber() == null) {
+ return false;
+ }
+
+ // try to find the matching shard based on the id & stream name
+ for (Map.Entry<StreamShardMetadata, SequenceNumber> entry : sequenceNumsToRestore.entrySet()) {
+ if (current.getStreamName().equals(entry.getKey().getStreamName())
+ && current.getShardId().equals(entry.getKey().getShardId())) {
+ // synchronize the local state if the ending sequence number is different
+ if (!current.getEndingSequenceNumber().equals(entry.getKey().getEndingSequenceNumber())) {
+ // ugly, but since the hashcode will change, we'll need to remove it and add it back
+ sequenceNumsToRestore.remove(entry.getKey());
+ entry.getKey().setEndingSequenceNumber(current.getEndingSequenceNumber());
+ sequenceNumsToRestore.put(entry.getKey(), entry.getValue());
+ return true;
+ }
+ // we already found the matching shard
+ break;
+ }
+ }
+ return false;
+ }
+
@Override
public void cancel() {
running = false;
http://git-wip-us.apache.org/repos/asf/flink/blob/b5b95eb1/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
index 379b640..3fc501c 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
@@ -62,12 +62,14 @@ import org.powermock.modules.junit4.PowerMockRunner;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -515,6 +517,129 @@ public class FlinkKinesisConsumerTest {
assertTrue(typeInformation.createSerializer(new ExecutionConfig()) instanceof PojoSerializer);
}
+ /**
+ * FLINK-8484: ensure that a state change in the StreamShardMetadata other than {@link StreamShardMetadata#shardId} or
+ * {@link StreamShardMetadata#streamName} does not result in the shard not being able to be restored.
+ * This handles the corner case where the stored shard metadata is open (no ending sequence number), but after the
+ * job restore, the shard has been closed (ending number set) due to re-sharding, and we can no longer rely on
+ * {@link StreamShardMetadata#equals(Object)} to find back the sequence number in the collection of restored shard metadata.
+ * <p></p>
+ * Therefore, we will rely on synchronizing the snapshot's state with the Kinesis shard before attempting to find back
+ * the sequence number to restore.
+ */
+ @Test
+ public void testFindSequenceNumberToRestoreFromIfTheShardHasBeenClosedSinceTheStateWasStored() throws Exception {
+ // ----------------------------------------------------------------------
+ // setup initial state
+ // ----------------------------------------------------------------------
+
+ HashMap<StreamShardHandle, SequenceNumber> fakeRestoredState = getFakeRestoredStore("all");
+
+ // ----------------------------------------------------------------------
+ // mock operator state backend and initial state for initializeState()
+ // ----------------------------------------------------------------------
+
+ TestingListState<Tuple2<StreamShardMetadata, SequenceNumber>> listState = new TestingListState<>();
+ for (Map.Entry<StreamShardHandle, SequenceNumber> state : fakeRestoredState.entrySet()) {
+ listState.add(Tuple2.of(KinesisDataFetcher.convertToStreamShardMetadata(state.getKey()), state.getValue()));
+ }
+
+ OperatorStateStore operatorStateStore = mock(OperatorStateStore.class);
+ when(operatorStateStore.getUnionListState(Matchers.any(ListStateDescriptor.class))).thenReturn(listState);
+
+ StateInitializationContext initializationContext = mock(StateInitializationContext.class);
+ when(initializationContext.getOperatorStateStore()).thenReturn(operatorStateStore);
+ when(initializationContext.isRestored()).thenReturn(true);
+
+ // ----------------------------------------------------------------------
+ // mock fetcher
+ // ----------------------------------------------------------------------
+
+ KinesisDataFetcher mockedFetcher = Mockito.mock(KinesisDataFetcher.class);
+ List<StreamShardHandle> shards = new ArrayList<>();
+
+ // create a fake stream shard handle based on the first entry in the restored state
+ final StreamShardHandle originalStreamShardHandle = fakeRestoredState.keySet().iterator().next();
+ final StreamShardHandle closedStreamShardHandle = new StreamShardHandle(originalStreamShardHandle.getStreamName(), originalStreamShardHandle.getShard());
+ // close the shard handle by setting an ending sequence number
+ final SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
+ sequenceNumberRange.setEndingSequenceNumber("1293844");
+ closedStreamShardHandle.getShard().setSequenceNumberRange(sequenceNumberRange);
+
+ shards.add(closedStreamShardHandle);
+
+ when(mockedFetcher.discoverNewShardsToSubscribe()).thenReturn(shards);
+ PowerMockito.whenNew(KinesisDataFetcher.class).withAnyArguments().thenReturn(mockedFetcher);
+
+ // assume the given config is correct
+ PowerMockito.mockStatic(KinesisConfigUtil.class);
+ PowerMockito.doNothing().when(KinesisConfigUtil.class);
+
+ // ----------------------------------------------------------------------
+ // start to test fetcher's initial state seeding
+ // ----------------------------------------------------------------------
+
+ TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer(
+ "fakeStream", new Properties(), 10, 2);
+ consumer.initializeState(initializationContext);
+ consumer.open(new Configuration());
+ consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
+
+ Mockito.verify(mockedFetcher).registerNewSubscribedShardState(
+ new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(closedStreamShardHandle),
+ closedStreamShardHandle, fakeRestoredState.get(closedStreamShardHandle)));
+ }
+
+ @Test
+ public void testUpdateKinesisShardStateWithMissingEndingSequenceNumber() {
+ final String streamName = "fakeStream1";
+ final String shardId = "shard-000001";
+
+ Properties config = TestUtils.getStandardProperties();
+ FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>(streamName, new SimpleStringSchema(), config);
+
+ // having the current Kinesis shard we're trying to find the restored state for
+ final StreamShardMetadata current = new StreamShardMetadata();
+ current.setShardId(shardId);
+ current.setStreamName(streamName);
+ current.setEndingSequenceNumber(null);
+
+ final HashMap<StreamShardMetadata, SequenceNumber> sequenceNumsToRestore = new LinkedHashMap<>();
+ assertFalse("Current shard is open, expecting a short-circuit", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
+
+ // create some non-matching shards
+ final StreamShardMetadata differentStreamName = new StreamShardMetadata();
+ differentStreamName.setStreamName("fakeStream2");
+ differentStreamName.setShardId(shardId);
+
+ final StreamShardMetadata differentShardId = new StreamShardMetadata();
+ differentShardId.setStreamName(streamName);
+ differentShardId.setShardId("shard-000002");
+
+ // create the matching shard
+ final StreamShardMetadata match = new StreamShardMetadata();
+ match.setStreamName(streamName);
+ match.setShardId(shardId);
+ // ensure the sequence number isn't set (shard is considered in open state)
+ match.setEndingSequenceNumber(null);
+
+ sequenceNumsToRestore.put(differentStreamName, new SequenceNumber("123"));
+ sequenceNumsToRestore.put(differentShardId, new SequenceNumber("456"));
+ sequenceNumsToRestore.put(match, new SequenceNumber("789"));
+
+ assertFalse("No ending sequence number was set, so no synchronisation was done.", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
+
+ // alter the ending sequence number (indicating the shard is now closed)
+ final String endingSequenceNumber = "99999";
+ current.setEndingSequenceNumber(endingSequenceNumber);
+ assertTrue("Shard was closed (ending number set).", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
+ assertEquals(endingSequenceNumber, match.getEndingSequenceNumber());
+ assertEquals(current, match);
+ assertTrue("Make sure we can still find back our match in the restored sequences ", sequenceNumsToRestore.containsKey(match));
+
+ assertFalse("Ending number was already set, no more need for synchronisation", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
+ }
+
private static final class TestingListState<T> implements ListState<T> {
private final List<T> list = new ArrayList<>();
[02/31] flink git commit: [hotfix] [kafka] Fix inapproriate access
modifiers in AbstractFetcher
Posted by tz...@apache.org.
[hotfix] [kafka] Fix inapproriate access modifiers in AbstractFetcher
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d0e2daed
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d0e2daed
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d0e2daed
Branch: refs/heads/master
Commit: d0e2daed7dc0dd686b44970b1c32ea51b9d43ed3
Parents: 40f26c8
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Mon Jan 22 14:17:54 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../connectors/kafka/internals/AbstractFetcher.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d0e2daed/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
index b6ce65a..6f5d02a 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/AbstractFetcher.java
@@ -62,9 +62,9 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
@Internal
public abstract class AbstractFetcher<T, KPH> {
- protected static final int NO_TIMESTAMPS_WATERMARKS = 0;
- protected static final int PERIODIC_WATERMARKS = 1;
- protected static final int PUNCTUATED_WATERMARKS = 2;
+ private static final int NO_TIMESTAMPS_WATERMARKS = 0;
+ private static final int PERIODIC_WATERMARKS = 1;
+ private static final int PUNCTUATED_WATERMARKS = 2;
// ------------------------------------------------------------------------
@@ -73,7 +73,7 @@ public abstract class AbstractFetcher<T, KPH> {
/** The lock that guarantees that record emission and state updates are atomic,
* from the view of taking a checkpoint. */
- protected final Object checkpointLock;
+ private final Object checkpointLock;
/** All partitions (and their state) that this fetcher is subscribed to. */
private final List<KafkaTopicPartitionState<KPH>> subscribedPartitionStates;
@@ -90,7 +90,7 @@ public abstract class AbstractFetcher<T, KPH> {
protected final ClosableBlockingQueue<KafkaTopicPartitionState<KPH>> unassignedPartitionsQueue;
/** The mode describing whether the fetcher also generates timestamps and watermarks. */
- protected final int timestampWatermarkMode;
+ private final int timestampWatermarkMode;
/**
* Optional timestamp extractor / watermark generator that will be run per Kafka partition,
@@ -413,7 +413,7 @@ public abstract class AbstractFetcher<T, KPH> {
* Record emission, if a timestamp will be attached from an assigner that is
* also a periodic watermark generator.
*/
- protected void emitRecordWithTimestampAndPeriodicWatermark(
+ private void emitRecordWithTimestampAndPeriodicWatermark(
T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) {
@SuppressWarnings("unchecked")
final KafkaTopicPartitionStateWithPeriodicWatermarks<T, KPH> withWatermarksState =
@@ -440,7 +440,7 @@ public abstract class AbstractFetcher<T, KPH> {
* Record emission, if a timestamp will be attached from an assigner that is
* also a punctuated watermark generator.
*/
- protected void emitRecordWithTimestampAndPunctuatedWatermark(
+ private void emitRecordWithTimestampAndPunctuatedWatermark(
T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp) {
@SuppressWarnings("unchecked")
final KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH> withWatermarksState =
@@ -646,7 +646,7 @@ public abstract class AbstractFetcher<T, KPH> {
private final KafkaTopicPartitionState<?> ktp;
private final OffsetGaugeType gaugeType;
- public OffsetGauge(KafkaTopicPartitionState<?> ktp, OffsetGaugeType gaugeType) {
+ OffsetGauge(KafkaTopicPartitionState<?> ktp, OffsetGaugeType gaugeType) {
this.ktp = ktp;
this.gaugeType = gaugeType;
}
[29/31] flink git commit: [FLINK-8421] [core] Introduce
PostVersionedIOReadableWritable
Posted by tz...@apache.org.
[FLINK-8421] [core] Introduce PostVersionedIOReadableWritable
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ba8ffdd6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ba8ffdd6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ba8ffdd6
Branch: refs/heads/master
Commit: ba8ffdd6bc5222b1b3e12aea572417ea3a455d7b
Parents: 9659ffd
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 24 18:08:52 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../io/PostVersionedIOReadableWritable.java | 88 +++++++++++++
.../io/PostVersionedIOReadableWritableTest.java | 132 +++++++++++++++++++
2 files changed, 220 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/ba8ffdd6/flink-core/src/main/java/org/apache/flink/core/io/PostVersionedIOReadableWritable.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/core/io/PostVersionedIOReadableWritable.java b/flink-core/src/main/java/org/apache/flink/core/io/PostVersionedIOReadableWritable.java
new file mode 100644
index 0000000..6edc983
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/core/io/PostVersionedIOReadableWritable.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.io;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputView;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PushbackInputStream;
+import java.util.Arrays;
+
+/**
+ * A {@link VersionedIOReadableWritable} which allows to differentiate whether the previous
+ * data was versioned with a {@link VersionedIOReadableWritable}. This can be used if previously
+ * written data was not versioned, and is to be migrated to a versioned format.
+ */
+@Internal
+public abstract class PostVersionedIOReadableWritable extends VersionedIOReadableWritable {
+
+ /** NOTE: CANNOT CHANGE! */
+ private static final byte[] VERSIONED_IDENTIFIER = new byte[] {-15, -51, -123, -97};
+
+ /**
+ * Read from the provided {@link DataInputView in}. A flag {@code wasVersioned} can be
+ * used to determine whether or not the data to read was previously written
+ * by a {@link VersionedIOReadableWritable}.
+ */
+ protected abstract void read(DataInputView in, boolean wasVersioned) throws IOException;
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ out.write(VERSIONED_IDENTIFIER);
+ super.write(out);
+ }
+
+ /**
+ * This read attempts to first identify if the input view contains the special
+ * {@link #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes.
+ * If identified to be versioned, the usual version resolution read path
+ * in {@link VersionedIOReadableWritable#read(DataInputView)} is invoked.
+ * Otherwise, we "reset" the input stream by pushing back the read buffered bytes
+ * into the stream.
+ */
+ public final void read(InputStream inputStream) throws IOException {
+ byte[] tmp = new byte[VERSIONED_IDENTIFIER.length];
+ inputStream.read(tmp);
+
+ if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) {
+ DataInputView inputView = new DataInputViewStreamWrapper(inputStream);
+
+ super.read(inputView);
+ read(inputView, true);
+ } else {
+ PushbackInputStream resetStream = new PushbackInputStream(inputStream, VERSIONED_IDENTIFIER.length);
+ resetStream.unread(tmp);
+
+ read(new DataInputViewStreamWrapper(resetStream), false);
+ }
+ }
+
+ /**
+ * We do not support reading from a {@link DataInputView}, because it does not
+ * support pushing back already read bytes.
+ */
+ @Override
+ public final void read(DataInputView in) throws IOException {
+ throw new UnsupportedOperationException("PostVersionedIOReadableWritable cannot read from a DataInputView.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/ba8ffdd6/flink-core/src/test/java/org/apache/flink/core/io/PostVersionedIOReadableWritableTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/core/io/PostVersionedIOReadableWritableTest.java b/flink-core/src/test/java/org/apache/flink/core/io/PostVersionedIOReadableWritableTest.java
new file mode 100644
index 0000000..2954536
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/core/io/PostVersionedIOReadableWritableTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.io;
+
+import org.apache.flink.core.memory.ByteArrayInputStreamWithPos;
+import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Suite of tests for {@link PostVersionedIOReadableWritable}.
+ */
+public class PostVersionedIOReadableWritableTest {
+
+ @Test
+ public void testReadVersioned() throws IOException {
+
+ String payload = "test-data";
+ TestPostVersionedReadableWritable versionedReadableWritable = new TestPostVersionedReadableWritable(payload);
+
+ byte[] serialized;
+ try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
+ versionedReadableWritable.write(new DataOutputViewStreamWrapper(out));
+ serialized = out.toByteArray();
+ }
+
+ TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable();
+ try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
+ restoredVersionedReadableWritable.read(in);
+ }
+
+ Assert.assertEquals(payload, restoredVersionedReadableWritable.getData());
+ }
+
+ @Test
+ public void testReadNonVersioned() throws IOException {
+ int preVersionedPayload = 563;
+
+ TestNonVersionedReadableWritable nonVersionedReadableWritable = new TestNonVersionedReadableWritable(preVersionedPayload);
+
+ byte[] serialized;
+ try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
+ nonVersionedReadableWritable.write(new DataOutputViewStreamWrapper(out));
+ serialized = out.toByteArray();
+ }
+
+ TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable();
+ try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
+ restoredVersionedReadableWritable.read(in);
+ }
+
+ Assert.assertEquals(String.valueOf(preVersionedPayload), restoredVersionedReadableWritable.getData());
+ }
+
+ static class TestPostVersionedReadableWritable extends PostVersionedIOReadableWritable {
+
+ private static final int VERSION = 1;
+ private String data;
+
+ TestPostVersionedReadableWritable() {}
+
+ TestPostVersionedReadableWritable(String data) {
+ this.data = data;
+ }
+
+ @Override
+ public int getVersion() {
+ return VERSION;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ super.write(out);
+ out.writeUTF(data);
+ }
+
+ @Override
+ protected void read(DataInputView in, boolean wasVersioned) throws IOException {
+ if (wasVersioned) {
+ this.data = in.readUTF();
+ } else {
+ // in the previous non-versioned format, we wrote integers instead
+ this.data = String.valueOf(in.readInt());
+ }
+ }
+
+ public String getData() {
+ return data;
+ }
+ }
+
+ static class TestNonVersionedReadableWritable implements IOReadableWritable {
+
+ private int data;
+
+ TestNonVersionedReadableWritable(int data) {
+ this.data = data;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ out.writeInt(data);
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+ this.data = in.readInt();
+ }
+ }
+
+}
[31/31] flink git commit: [FLINK-8472] [tests] Extend
AbstractOperatorRestoreTestBases for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [tests] Extend AbstractOperatorRestoreTestBases for Flink 1.4
This closes #5364.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/130ca4ec
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/130ca4ec
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/130ca4ec
Branch: refs/heads/master
Commit: 130ca4ec3895825022c2a05c0022684ab8c1a3bf
Parents: a12aca3
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Feb 1 15:27:46 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../AbstractKeyedOperatorRestoreTestBase.java | 15 +++++++--------
.../restore/keyed/KeyedComplexChainTest.java | 5 +++--
.../state/operator/restore/keyed/KeyedJob.java | 14 ++++----------
...AbstractNonKeyedOperatorRestoreTestBase.java | 19 +++++++++----------
.../restore/unkeyed/ChainBreakTest.java | 5 +++--
.../unkeyed/ChainLengthDecreaseTest.java | 5 +++--
.../unkeyed/ChainLengthIncreaseTest.java | 5 +++--
.../ChainLengthStatelessDecreaseTest.java | 5 +++--
.../restore/unkeyed/ChainOrderTest.java | 5 +++--
.../restore/unkeyed/ChainUnionTest.java | 5 +++--
.../operator/restore/unkeyed/NonKeyedJob.java | 8 ++------
.../complexKeyed-flink1.4/_metadata | Bin 0 -> 163530 bytes
.../operatorstate/nonKeyed-flink1.4/_metadata | Bin 0 -> 6248 bytes
13 files changed, 43 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/AbstractKeyedOperatorRestoreTestBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/AbstractKeyedOperatorRestoreTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/AbstractKeyedOperatorRestoreTestBase.java
index 1b66c21..36301df 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/AbstractKeyedOperatorRestoreTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/AbstractKeyedOperatorRestoreTestBase.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.keyed;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.AbstractOperatorRestoreTestBase;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
@@ -36,17 +37,15 @@ import java.util.Collection;
@RunWith(Parameterized.class)
public abstract class AbstractKeyedOperatorRestoreTestBase extends AbstractOperatorRestoreTestBase {
- private final String savepointPath;
+ private final MigrationVersion migrationVersion;
@Parameterized.Parameters(name = "Migrate Savepoint: {0}")
- public static Collection<String> parameters () {
- return Arrays.asList(
- "complexKeyed-flink1.2",
- "complexKeyed-flink1.3");
+ public static Collection<MigrationVersion> parameters () {
+ return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3, MigrationVersion.v1_4);
}
- public AbstractKeyedOperatorRestoreTestBase(String savepointPath) {
- this.savepointPath = savepointPath;
+ public AbstractKeyedOperatorRestoreTestBase(MigrationVersion migrationVersion) {
+ this.migrationVersion = migrationVersion;
}
@Override
@@ -65,6 +64,6 @@ public abstract class AbstractKeyedOperatorRestoreTestBase extends AbstractOpera
@Override
protected String getMigrationSavepointName() {
- return savepointPath;
+ return "complexKeyed-flink" + migrationVersion;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
index 45dbc78..04cd956 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedComplexChainTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.keyed;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
/**
@@ -28,8 +29,8 @@ import org.apache.flink.test.state.operator.restore.ExecutionMode;
*/
public class KeyedComplexChainTest extends AbstractKeyedOperatorRestoreTestBase {
- public KeyedComplexChainTest(String savepointPath) {
- super(savepointPath);
+ public KeyedComplexChainTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
index c53f80f..d54f9f2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/keyed/KeyedJob.java
@@ -98,11 +98,8 @@ public class KeyedJob {
public static SingleOutputStreamOperator<Integer> createFirstStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
SingleOutputStreamOperator<Integer> map = input
.map(new StatefulStringStoringMap(mode, "first"))
- .setParallelism(4);
-
- if (mode == ExecutionMode.MIGRATE || mode == ExecutionMode.RESTORE) {
- map.uid("first");
- }
+ .setParallelism(4)
+ .uid("first");
return map;
}
@@ -110,11 +107,8 @@ public class KeyedJob {
public static SingleOutputStreamOperator<Integer> createSecondStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
SingleOutputStreamOperator<Integer> map = input
.map(new StatefulStringStoringMap(mode, "second"))
- .setParallelism(4);
-
- if (mode == ExecutionMode.MIGRATE || mode == ExecutionMode.RESTORE) {
- map.uid("second");
- }
+ .setParallelism(4)
+ .uid("second");
return map;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
index c100dc9..6c9767d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/AbstractNonKeyedOperatorRestoreTestBase.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.AbstractOperatorRestoreTestBase;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
@@ -42,22 +43,20 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
@RunWith(Parameterized.class)
public abstract class AbstractNonKeyedOperatorRestoreTestBase extends AbstractOperatorRestoreTestBase {
- private final String savepointPath;
+ private final MigrationVersion migrationVersion;
@Parameterized.Parameters(name = "Migrate Savepoint: {0}")
- public static Collection<String> parameters () {
- return Arrays.asList(
- "nonKeyed-flink1.2",
- "nonKeyed-flink1.3");
+ public static Collection<MigrationVersion> parameters () {
+ return Arrays.asList(MigrationVersion.v1_2, MigrationVersion.v1_3, MigrationVersion.v1_4);
}
- protected AbstractNonKeyedOperatorRestoreTestBase(String savepointPath) {
- this.savepointPath = savepointPath;
+ protected AbstractNonKeyedOperatorRestoreTestBase(MigrationVersion migrationVersion) {
+ this.migrationVersion = migrationVersion;
}
- protected AbstractNonKeyedOperatorRestoreTestBase(String savepointPath, boolean allowNonRestoredState) {
+ protected AbstractNonKeyedOperatorRestoreTestBase(MigrationVersion migrationVersion, boolean allowNonRestoredState) {
super(allowNonRestoredState);
- this.savepointPath = savepointPath;
+ this.migrationVersion = migrationVersion;
}
@Override
@@ -80,6 +79,6 @@ public abstract class AbstractNonKeyedOperatorRestoreTestBase extends AbstractOp
@Override
protected String getMigrationSavepointName() {
- return savepointPath;
+ return "nonKeyed-flink" + migrationVersion;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
index 4e074ea..b061847 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainBreakTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -34,8 +35,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainBreakTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainBreakTest(String savepointPath) {
- super(savepointPath);
+ public ChainBreakTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
index af81c0e..0190ca0 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthDecreaseTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -33,8 +34,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainLengthDecreaseTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainLengthDecreaseTest(String savepointPath) {
- super(savepointPath);
+ public ChainLengthDecreaseTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
index 1f07e0b..d0648ad 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthIncreaseTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -34,8 +35,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainLengthIncreaseTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainLengthIncreaseTest(String savepointPath) {
- super(savepointPath);
+ public ChainLengthIncreaseTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthStatelessDecreaseTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthStatelessDecreaseTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthStatelessDecreaseTest.java
index fb4dcf5..b0f0945 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthStatelessDecreaseTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainLengthStatelessDecreaseTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -36,8 +37,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainLengthStatelessDecreaseTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainLengthStatelessDecreaseTest(String savepointPath) {
- super(savepointPath, false);
+ public ChainLengthStatelessDecreaseTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
index 61e366d..cb2252e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainOrderTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -34,8 +35,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainOrderTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainOrderTest(String savepointPath) {
- super(savepointPath);
+ public ChainOrderTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
index 456e746..d8134d8 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/ChainUnionTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.state.operator.restore.unkeyed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.migration.MigrationVersion;
import org.apache.flink.test.state.operator.restore.ExecutionMode;
import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.createFirstStatefulMap;
@@ -34,8 +35,8 @@ import static org.apache.flink.test.state.operator.restore.unkeyed.NonKeyedJob.c
*/
public class ChainUnionTest extends AbstractNonKeyedOperatorRestoreTestBase {
- public ChainUnionTest(String savepointPath) {
- super(savepointPath);
+ public ChainUnionTest(MigrationVersion migrationVersion) {
+ super(migrationVersion);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
index 0b5a21f..0660a66 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/state/operator/restore/unkeyed/NonKeyedJob.java
@@ -100,12 +100,8 @@ public class NonKeyedJob {
public static SingleOutputStreamOperator<Integer> createThirdStatefulMap(ExecutionMode mode, DataStream<Integer> input) {
SingleOutputStreamOperator<Integer> map = input
.map(new StatefulStringStoringMap(mode, "third"))
- .setParallelism(4);
-
- // we cannot set the uid on a chained operator in 1.2
- if (mode == ExecutionMode.MIGRATE || mode == ExecutionMode.RESTORE) {
- map.uid("third");
- }
+ .setParallelism(4)
+ .uid("third");
return map;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/resources/operatorstate/complexKeyed-flink1.4/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/operatorstate/complexKeyed-flink1.4/_metadata b/flink-tests/src/test/resources/operatorstate/complexKeyed-flink1.4/_metadata
new file mode 100644
index 0000000..8153044
Binary files /dev/null and b/flink-tests/src/test/resources/operatorstate/complexKeyed-flink1.4/_metadata differ
http://git-wip-us.apache.org/repos/asf/flink/blob/130ca4ec/flink-tests/src/test/resources/operatorstate/nonKeyed-flink1.4/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/operatorstate/nonKeyed-flink1.4/_metadata b/flink-tests/src/test/resources/operatorstate/nonKeyed-flink1.4/_metadata
new file mode 100644
index 0000000..0b2a6a7
Binary files /dev/null and b/flink-tests/src/test/resources/operatorstate/nonKeyed-flink1.4/_metadata differ
[13/31] flink git commit: [FLINK-8022][kafka-tests] Disable
at-least-once tests for Kafka 0.9
Posted by tz...@apache.org.
[FLINK-8022][kafka-tests] Disable at-least-once tests for Kafka 0.9
For some reasons this test is sometimes failing in Kafka09 while the same code works in Kafka010.
Disabling this test because everything indicates those failures might be caused by unfixed bugs
in Kafka 0.9 branch
This closes #5316.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4197b0da
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4197b0da
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4197b0da
Branch: refs/heads/master
Commit: 4197b0da100ab2da4b811847a401f17ce8bb1182
Parents: d0e2dae
Author: Piotr Nowojski <pi...@gmail.com>
Authored: Fri Jan 19 13:40:01 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../streaming/connectors/kafka/Kafka09ProducerITCase.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4197b0da/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09ProducerITCase.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09ProducerITCase.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09ProducerITCase.java
index b34132f..c619c3e 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09ProducerITCase.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09ProducerITCase.java
@@ -25,12 +25,18 @@ package org.apache.flink.streaming.connectors.kafka;
public class Kafka09ProducerITCase extends KafkaProducerTestBase {
@Override
public void testExactlyOnceRegularSink() throws Exception {
- // Kafka08 does not support exactly once semantic
+ // Kafka09 does not support exactly once semantic
}
@Override
public void testExactlyOnceCustomOperator() throws Exception {
- // Kafka08 does not support exactly once semantic
+ // Kafka09 does not support exactly once semantic
+ }
+
+ @Override
+ public void testOneToOneAtLeastOnceRegularSink() throws Exception {
+ // For some reasons this test is sometimes failing in Kafka09 while the same code works in Kafka010. Disabling
+ // this test because everything indicates those failures might be caused by unfixed bugs in Kafka 0.9 branch
}
@Override
[25/31] flink git commit: [FLINK-8472] [fs,
test] Extend BucketingSinkMigrationTest for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [fs, test] Extend BucketingSinkMigrationTest for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a12aca38
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a12aca38
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a12aca38
Branch: refs/heads/master
Commit: a12aca3834830feccb067ae04e4159d41c24f45e
Parents: c100936
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 17:32:28 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../fs/bucketing/BucketingSinkMigrationTest.java | 3 ++-
.../bucketing-sink-migration-test-flink1.4-snapshot | Bin 0 -> 1862 bytes
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a12aca38/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java b/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java
index bfb326c..3b7c7d4 100644
--- a/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java
+++ b/flink-connectors/flink-connector-filesystem/src/test/java/org/apache/flink/streaming/connectors/fs/bucketing/BucketingSinkMigrationTest.java
@@ -86,7 +86,8 @@ public class BucketingSinkMigrationTest {
public static Collection<Tuple2<MigrationVersion, String>> parameters () {
return Arrays.asList(
Tuple2.of(MigrationVersion.v1_2, "/var/folders/v_/ry2wp5fx0y7c1rvr41xy9_700000gn/T/junit9160378385359106772/junit479663758539998903/1970-01-01--01/part-0-"),
- Tuple2.of(MigrationVersion.v1_3, "/var/folders/tv/b_1d8fvx23dgk1_xs8db_95h0000gn/T/junit4273542175898623023/junit3801102997056424640/1970-01-01--01/part-0-"));
+ Tuple2.of(MigrationVersion.v1_3, "/var/folders/tv/b_1d8fvx23dgk1_xs8db_95h0000gn/T/junit4273542175898623023/junit3801102997056424640/1970-01-01--01/part-0-"),
+ Tuple2.of(MigrationVersion.v1_4, "/var/folders/tv/b_1d8fvx23dgk1_xs8db_95h0000gn/T/junit3198043255809479705/junit8947526563966405708/1970-01-01--01/part-0-"));
}
private final MigrationVersion testMigrateVersion;
http://git-wip-us.apache.org/repos/asf/flink/blob/a12aca38/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.4-snapshot b/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.4-snapshot
new file mode 100644
index 0000000..45e7bcc
Binary files /dev/null and b/flink-connectors/flink-connector-filesystem/src/test/resources/bucketing-sink-migration-test-flink1.4-snapshot differ
[30/31] flink git commit: [FLINK-8472] [fs,
test] Extend ContinuousFileProcessingMigrationTest for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [fs, test] Extend ContinuousFileProcessingMigrationTest for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c100936f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c100936f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c100936f
Branch: refs/heads/master
Commit: c100936fc54add940e024a96a1b111fc9ae75a4c
Parents: 535d139
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 17:28:17 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../ContinuousFileProcessingMigrationTest.java | 3 ++-
...n-migration-test-1516897628000-flink1.4-snapshot | Bin 0 -> 537 bytes
.../reader-migration-test-flink1.4-snapshot | Bin 0 -> 2823 bytes
3 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c100936f/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
index 602ad3e..1c2ea6f 100644
--- a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
+++ b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
@@ -77,7 +77,8 @@ public class ContinuousFileProcessingMigrationTest {
public static Collection<Tuple2<MigrationVersion, Long>> parameters () {
return Arrays.asList(
Tuple2.of(MigrationVersion.v1_2, 1493116191000L),
- Tuple2.of(MigrationVersion.v1_3, 1496532000000L));
+ Tuple2.of(MigrationVersion.v1_3, 1496532000000L),
+ Tuple2.of(MigrationVersion.v1_4, 1516897628000L));
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/c100936f/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1516897628000-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1516897628000-flink1.4-snapshot b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1516897628000-flink1.4-snapshot
new file mode 100644
index 0000000..7b98325
Binary files /dev/null and b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1516897628000-flink1.4-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/c100936f/flink-fs-tests/src/test/resources/reader-migration-test-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/resources/reader-migration-test-flink1.4-snapshot b/flink-fs-tests/src/test/resources/reader-migration-test-flink1.4-snapshot
new file mode 100644
index 0000000..c5d0ecf
Binary files /dev/null and b/flink-fs-tests/src/test/resources/reader-migration-test-flink1.4-snapshot differ
[26/31] flink git commit: [FLINK-8421] [DataStream] Make timer
serializers reconfigurable on restore
Posted by tz...@apache.org.
[FLINK-8421] [DataStream] Make timer serializers reconfigurable on restore
Previously, the key and namespace serializers for the
HeapInternalTimerService were not reconfigured on restore to be compatible
with previously written serializers.
This caused an immediate error to restore savepoints in Flink 1.4.0,
since in Flink 1.4.0 we changed the base registrations in the Kryo
serializer. That change requires serializer reconfiguration.
This commit fixes this by writing also the serializer configuration
snapshots of the key and namespace serializer into savepoints, and use
them to reconfigure the new serializers on rrestore. This improvement also
comes along with making the written data for timer service snapshots
versioned. Backwards compatibility with previous non-versioned formats
is not broken.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/07528133
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/07528133
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/07528133
Branch: refs/heads/master
Commit: 07528133e35aea71953ea9397e2fecd847e56325
Parents: ba8ffdd
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 24 18:09:44 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../api/operators/HeapInternalTimerService.java | 122 ++++----
.../operators/InternalTimeServiceManager.java | 51 ++--
.../InternalTimerServiceSerializationProxy.java | 132 +++++++++
.../api/operators/InternalTimersSnapshot.java | 120 ++++++++
.../InternalTimersSnapshotReaderWriters.java | 291 +++++++++++++++++++
.../StreamTaskStateInitializerImpl.java | 3 +-
.../operators/HeapInternalTimerServiceTest.java | 124 +++++---
7 files changed, 696 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/07528133/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
index b031dcf..fb38075 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java
@@ -19,14 +19,13 @@
package org.apache.flink.streaming.api.operators;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.typeutils.CompatibilityResult;
+import org.apache.flink.api.common.typeutils.CompatibilityUtil;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
import org.apache.flink.runtime.state.KeyGroupsList;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
-import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.Preconditions;
import java.io.IOException;
@@ -84,8 +83,6 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
private TypeSerializer<N> namespaceSerializer;
- private InternalTimer.TimerSerializer<K, N> timerSerializer;
-
private Triggerable<K, N> triggerTarget;
private volatile boolean isInitialized;
@@ -94,6 +91,9 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
private TypeSerializer<N> namespaceDeserializer;
+ /** The restored timers snapshot, if any. */
+ private InternalTimersSnapshot<K, N> restoredTimersSnapshot;
+
public HeapInternalTimerService(
int totalKeyGroups,
KeyGroupsList localKeyGroupRange,
@@ -148,10 +148,23 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
}
// the following is the case where we restore
- if ((this.keyDeserializer != null && !this.keyDeserializer.equals(keySerializer)) ||
- (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(namespaceSerializer))) {
- throw new IllegalStateException("Tried to initialize restored TimerService " +
- "with different serializers than those used to snapshot its state.");
+ if (restoredTimersSnapshot != null) {
+ CompatibilityResult<K> keySerializerCompatibility = CompatibilityUtil.resolveCompatibilityResult(
+ this.keyDeserializer,
+ null,
+ restoredTimersSnapshot.getKeySerializerConfigSnapshot(),
+ keySerializer);
+
+ CompatibilityResult<N> namespaceSerializerCompatibility = CompatibilityUtil.resolveCompatibilityResult(
+ this.namespaceDeserializer,
+ null,
+ restoredTimersSnapshot.getNamespaceSerializerConfigSnapshot(),
+ namespaceSerializer);
+
+ if (keySerializerCompatibility.isRequiresMigration() || namespaceSerializerCompatibility.isRequiresMigration()) {
+ throw new IllegalStateException("Tried to initialize restored TimerService " +
+ "with incompatible serializers than those used to snapshot its state.");
+ }
}
this.keySerializer = keySerializer;
@@ -161,8 +174,6 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
this.triggerTarget = Preconditions.checkNotNull(triggerTarget);
- this.timerSerializer = new InternalTimer.TimerSerializer<>(this.keySerializer, this.namespaceSerializer);
-
// re-register the restored timers (if any)
if (processingTimeTimersQueue.size() > 0) {
nextTimer = processingTimeService.registerTimer(processingTimeTimersQueue.peek().getTimestamp(), this);
@@ -280,86 +291,53 @@ public class HeapInternalTimerService<K, N> implements InternalTimerService<N>,
/**
* Snapshots the timers (both processing and event time ones) for a given {@code keyGroupIdx}.
- * @param stream the stream to write to.
+ *
* @param keyGroupIdx the id of the key-group to be put in the snapshot.
+ * @return a snapshot containing the timers for the given key-group, and the serializers for them
*/
- public void snapshotTimersForKeyGroup(DataOutputViewStreamWrapper stream, int keyGroupIdx) throws Exception {
- InstantiationUtil.serializeObject(stream, keySerializer);
- InstantiationUtil.serializeObject(stream, namespaceSerializer);
-
- // write the event time timers
- Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx);
- if (eventTimers != null) {
- stream.writeInt(eventTimers.size());
- for (InternalTimer<K, N> timer : eventTimers) {
- this.timerSerializer.serialize(timer, stream);
- }
- } else {
- stream.writeInt(0);
- }
-
- // write the processing time timers
- Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
- if (processingTimers != null) {
- stream.writeInt(processingTimers.size());
- for (InternalTimer<K, N> timer : processingTimers) {
- this.timerSerializer.serialize(timer, stream);
- }
- } else {
- stream.writeInt(0);
- }
+ public InternalTimersSnapshot<K, N> snapshotTimersForKeyGroup(int keyGroupIdx) {
+ return new InternalTimersSnapshot<>(
+ keySerializer,
+ keySerializer.snapshotConfiguration(),
+ namespaceSerializer,
+ namespaceSerializer.snapshotConfiguration(),
+ getEventTimeTimerSetForKeyGroup(keyGroupIdx),
+ getProcessingTimeTimerSetForKeyGroup(keyGroupIdx));
}
/**
* Restore the timers (both processing and event time ones) for a given {@code keyGroupIdx}.
- * @param stream the stream to read from.
+ *
+ * @param restoredTimersSnapshot the restored snapshot containing the key-group's timers,
+ * and the serializers that were used to write them
* @param keyGroupIdx the id of the key-group to be put in the snapshot.
- * @param userCodeClassLoader the class loader that will be used to deserialize
- * the local key and namespace serializers.
*/
- public void restoreTimersForKeyGroup(DataInputViewStreamWrapper stream, int keyGroupIdx,
- ClassLoader userCodeClassLoader) throws IOException, ClassNotFoundException {
-
- TypeSerializer<K> tmpKeyDeserializer = InstantiationUtil.deserializeObject(stream, userCodeClassLoader);
- TypeSerializer<N> tmpNamespaceDeserializer = InstantiationUtil.deserializeObject(stream, userCodeClassLoader);
+ @SuppressWarnings("unchecked")
+ public void restoreTimersForKeyGroup(InternalTimersSnapshot<?, ?> restoredTimersSnapshot, int keyGroupIdx) throws IOException {
+ this.restoredTimersSnapshot = (InternalTimersSnapshot<K, N>) restoredTimersSnapshot;
- if ((this.keyDeserializer != null && !this.keyDeserializer.equals(tmpKeyDeserializer)) ||
- (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(tmpNamespaceDeserializer))) {
+ if ((this.keyDeserializer != null && !this.keyDeserializer.equals(restoredTimersSnapshot.getKeySerializer())) ||
+ (this.namespaceDeserializer != null && !this.namespaceDeserializer.equals(restoredTimersSnapshot.getNamespaceSerializer()))) {
throw new IllegalArgumentException("Tried to restore timers " +
"for the same service with different serializers.");
}
- this.keyDeserializer = tmpKeyDeserializer;
- this.namespaceDeserializer = tmpNamespaceDeserializer;
-
- InternalTimer.TimerSerializer<K, N> timerSerializer =
- new InternalTimer.TimerSerializer<>(this.keyDeserializer, this.namespaceDeserializer);
+ this.keyDeserializer = this.restoredTimersSnapshot.getKeySerializer();
+ this.namespaceDeserializer = this.restoredTimersSnapshot.getNamespaceSerializer();
checkArgument(localKeyGroupRange.contains(keyGroupIdx),
"Key Group " + keyGroupIdx + " does not belong to the local range.");
- // read the event time timers
- int sizeOfEventTimeTimers = stream.readInt();
- if (sizeOfEventTimeTimers > 0) {
- Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx);
- for (int i = 0; i < sizeOfEventTimeTimers; i++) {
- InternalTimer<K, N> timer = timerSerializer.deserialize(stream);
- eventTimers.add(timer);
- eventTimeTimersQueue.add(timer);
- }
- }
+ // restore the event time timers
+ Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx);
+ eventTimers.addAll(this.restoredTimersSnapshot.getEventTimeTimers());
+ eventTimeTimersQueue.addAll(this.restoredTimersSnapshot.getEventTimeTimers());
- // read the processing time timers
- int sizeOfProcessingTimeTimers = stream.readInt();
- if (sizeOfProcessingTimeTimers > 0) {
- Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
- for (int i = 0; i < sizeOfProcessingTimeTimers; i++) {
- InternalTimer<K, N> timer = timerSerializer.deserialize(stream);
- processingTimers.add(timer);
- processingTimeTimersQueue.add(timer);
- }
- }
+ // restore the processing time timers
+ Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
+ processingTimers.addAll(this.restoredTimersSnapshot.getProcessingTimeTimers());
+ processingTimeTimersQueue.addAll(this.restoredTimersSnapshot.getProcessingTimeTimers());
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/07528133/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
index 7d5cb91..84c7bf1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
@@ -21,8 +21,7 @@ package org.apache.flink.streaming.api.operators;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.runtime.state.KeyGroupsList;
import org.apache.flink.runtime.state.VoidNamespaceSerializer;
import org.apache.flink.streaming.api.watermark.Watermark;
@@ -30,6 +29,7 @@ import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
import org.apache.flink.util.Preconditions;
import java.io.IOException;
+import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
@@ -111,36 +111,29 @@ public class InternalTimeServiceManager<K, N> {
////////////////// Fault Tolerance Methods ///////////////////
- public void snapshotStateForKeyGroup(DataOutputViewStreamWrapper stream, int keyGroupIdx) throws Exception {
- stream.writeInt(timerServices.size());
+ public void snapshotStateForKeyGroup(DataOutputView stream, int keyGroupIdx) throws IOException {
+ InternalTimerServiceSerializationProxy<K, N> serializationProxy =
+ new InternalTimerServiceSerializationProxy<>(timerServices, keyGroupIdx);
- for (Map.Entry<String, HeapInternalTimerService<K, N>> entry : timerServices.entrySet()) {
- String serviceName = entry.getKey();
- HeapInternalTimerService<?, ?> timerService = entry.getValue();
-
- stream.writeUTF(serviceName);
- timerService.snapshotTimersForKeyGroup(stream, keyGroupIdx);
- }
+ serializationProxy.write(stream);
}
- public void restoreStateForKeyGroup(DataInputViewStreamWrapper stream, int keyGroupIdx,
- ClassLoader userCodeClassLoader) throws IOException, ClassNotFoundException {
-
- int noOfTimerServices = stream.readInt();
- for (int i = 0; i < noOfTimerServices; i++) {
- String serviceName = stream.readUTF();
-
- HeapInternalTimerService<K, N> timerService = timerServices.get(serviceName);
- if (timerService == null) {
- timerService = new HeapInternalTimerService<>(
- totalKeyGroups,
- localKeyGroupRange,
- keyContext,
- processingTimeService);
- timerServices.put(serviceName, timerService);
- }
- timerService.restoreTimersForKeyGroup(stream, keyGroupIdx, userCodeClassLoader);
- }
+ public void restoreStateForKeyGroup(
+ InputStream stream,
+ int keyGroupIdx,
+ ClassLoader userCodeClassLoader) throws IOException {
+
+ InternalTimerServiceSerializationProxy<K, N> serializationProxy =
+ new InternalTimerServiceSerializationProxy<>(
+ timerServices,
+ userCodeClassLoader,
+ totalKeyGroups,
+ localKeyGroupRange,
+ keyContext,
+ processingTimeService,
+ keyGroupIdx);
+
+ serializationProxy.read(stream);
}
//////////////////// Methods used ONLY IN TESTS ////////////////////
http://git-wip-us.apache.org/repos/asf/flink/blob/07528133/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java
new file mode 100644
index 0000000..53d1507
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimerServiceSerializationProxy.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.io.PostVersionedIOReadableWritable;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.runtime.state.KeyGroupsList;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Serialization proxy for the timer services for a given key-group.
+ */
+@Internal
+public class InternalTimerServiceSerializationProxy<K, N> extends PostVersionedIOReadableWritable {
+
+ public static final int VERSION = 1;
+
+ /** The key-group timer services to write / read. */
+ private Map<String, HeapInternalTimerService<K, N>> timerServices;
+
+ /** The user classloader; only relevant if the proxy is used to restore timer services. */
+ private ClassLoader userCodeClassLoader;
+
+ /** Properties of restored timer services. */
+ private int keyGroupIdx;
+ private int totalKeyGroups;
+ private KeyGroupsList localKeyGroupRange;
+ private KeyContext keyContext;
+ private ProcessingTimeService processingTimeService;
+
+ /**
+ * Constructor to use when restoring timer services.
+ */
+ public InternalTimerServiceSerializationProxy(
+ Map<String, HeapInternalTimerService<K, N>> timerServicesMapToPopulate,
+ ClassLoader userCodeClassLoader,
+ int totalKeyGroups,
+ KeyGroupsList localKeyGroupRange,
+ KeyContext keyContext,
+ ProcessingTimeService processingTimeService,
+ int keyGroupIdx) {
+
+ this.timerServices = checkNotNull(timerServicesMapToPopulate);
+ this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
+ this.totalKeyGroups = totalKeyGroups;
+ this.localKeyGroupRange = checkNotNull(localKeyGroupRange);
+ this.keyContext = checkNotNull(keyContext);
+ this.processingTimeService = checkNotNull(processingTimeService);
+ this.keyGroupIdx = keyGroupIdx;
+ }
+
+ /**
+ * Constructor to use when writing timer services.
+ */
+ public InternalTimerServiceSerializationProxy(
+ Map<String, HeapInternalTimerService<K, N>> timerServices,
+ int keyGroupIdx) {
+
+ this.timerServices = checkNotNull(timerServices);
+ this.keyGroupIdx = keyGroupIdx;
+ }
+
+ @Override
+ public int getVersion() {
+ return VERSION;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+ super.write(out);
+
+ out.writeInt(timerServices.size());
+ for (Map.Entry<String, HeapInternalTimerService<K, N>> entry : timerServices.entrySet()) {
+ String serviceName = entry.getKey();
+ HeapInternalTimerService<K, N> timerService = entry.getValue();
+
+ out.writeUTF(serviceName);
+ InternalTimersSnapshotReaderWriters
+ .getWriterForVersion(VERSION, timerService.snapshotTimersForKeyGroup(keyGroupIdx))
+ .writeTimersSnapshot(out);
+ }
+ }
+
+ @Override
+ protected void read(DataInputView in, boolean wasVersioned) throws IOException {
+ int noOfTimerServices = in.readInt();
+
+ for (int i = 0; i < noOfTimerServices; i++) {
+ String serviceName = in.readUTF();
+
+ HeapInternalTimerService<K, N> timerService = timerServices.get(serviceName);
+ if (timerService == null) {
+ timerService = new HeapInternalTimerService<>(
+ totalKeyGroups,
+ localKeyGroupRange,
+ keyContext,
+ processingTimeService);
+ timerServices.put(serviceName, timerService);
+ }
+
+ int readerVersion = wasVersioned ? getReadVersion() : InternalTimersSnapshotReaderWriters.NO_VERSION;
+ InternalTimersSnapshot<?, ?> restoredTimersSnapshot = InternalTimersSnapshotReaderWriters
+ .getReaderForVersion(readerVersion, userCodeClassLoader)
+ .readTimersSnapshot(in);
+
+ timerService.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIdx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/07528133/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java
new file mode 100644
index 0000000..ddaee6b
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshot.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.Nullable;
+
+import java.util.Set;
+
+/**
+ * A snapshot of internal timers, containing event and processing timers and
+ * the serializers to use to write / read them.
+ */
+public class InternalTimersSnapshot<K, N> {
+
+ private TypeSerializer<K> keySerializer;
+ private TypeSerializerConfigSnapshot keySerializerConfigSnapshot;
+ private TypeSerializer<N> namespaceSerializer;
+ private TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot;
+
+ private Set<InternalTimer<K, N>> eventTimeTimers;
+ private Set<InternalTimer<K, N>> processingTimeTimers;
+
+ /** Empty constructor used when restoring the timers. */
+ public InternalTimersSnapshot() {}
+
+ /** Constructor to use when snapshotting the timers. */
+ public InternalTimersSnapshot(
+ TypeSerializer<K> keySerializer,
+ TypeSerializerConfigSnapshot keySerializerConfigSnapshot,
+ TypeSerializer<N> namespaceSerializer,
+ TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot,
+ @Nullable Set<InternalTimer<K, N>> eventTimeTimers,
+ @Nullable Set<InternalTimer<K, N>> processingTimeTimers) {
+
+ this.keySerializer = Preconditions.checkNotNull(keySerializer);
+ this.keySerializerConfigSnapshot = Preconditions.checkNotNull(keySerializerConfigSnapshot);
+ this.namespaceSerializer = Preconditions.checkNotNull(namespaceSerializer);
+ this.namespaceSerializerConfigSnapshot = Preconditions.checkNotNull(namespaceSerializerConfigSnapshot);
+ this.eventTimeTimers = eventTimeTimers;
+ this.processingTimeTimers = processingTimeTimers;
+ }
+
+ public TypeSerializer<K> getKeySerializer() {
+ return keySerializer;
+ }
+
+ public void setKeySerializer(TypeSerializer<K> keySerializer) {
+ this.keySerializer = keySerializer;
+ }
+
+ public TypeSerializerConfigSnapshot getKeySerializerConfigSnapshot() {
+ return keySerializerConfigSnapshot;
+ }
+
+ public void setKeySerializerConfigSnapshot(TypeSerializerConfigSnapshot keySerializerConfigSnapshot) {
+ this.keySerializerConfigSnapshot = keySerializerConfigSnapshot;
+ }
+
+ public TypeSerializer<N> getNamespaceSerializer() {
+ return namespaceSerializer;
+ }
+
+ public void setNamespaceSerializer(TypeSerializer<N> namespaceSerializer) {
+ this.namespaceSerializer = namespaceSerializer;
+ }
+
+ public TypeSerializerConfigSnapshot getNamespaceSerializerConfigSnapshot() {
+ return namespaceSerializerConfigSnapshot;
+ }
+
+ public void setNamespaceSerializerConfigSnapshot(TypeSerializerConfigSnapshot namespaceSerializerConfigSnapshot) {
+ this.namespaceSerializerConfigSnapshot = namespaceSerializerConfigSnapshot;
+ }
+
+ public Set<InternalTimer<K, N>> getEventTimeTimers() {
+ return eventTimeTimers;
+ }
+
+ public void setEventTimeTimers(Set<InternalTimer<K, N>> eventTimeTimers) {
+ this.eventTimeTimers = eventTimeTimers;
+ }
+
+ public Set<InternalTimer<K, N>> getProcessingTimeTimers() {
+ return processingTimeTimers;
+ }
+
+ public void setProcessingTimeTimers(Set<InternalTimer<K, N>> processingTimeTimers) {
+ this.processingTimeTimers = processingTimeTimers;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return super.equals(obj);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/07528133/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java
new file mode 100644
index 0000000..3173625
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimersSnapshotReaderWriters.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Readers and writers for different versions of the {@link InternalTimersSnapshot}.
+ * Outdated formats are also kept here for documentation of history backlog.
+ */
+@Internal
+public class InternalTimersSnapshotReaderWriters {
+
+ public static final int NO_VERSION = Integer.MIN_VALUE;
+
+ // -------------------------------------------------------------------------------
+ // Writers
+ // - pre-versioned: Flink 1.4.0
+ // - v1: Flink 1.4.1
+ // -------------------------------------------------------------------------------
+
+ public static <K, N> InternalTimersSnapshotWriter getWriterForVersion(int version, InternalTimersSnapshot<K, N> timersSnapshot) {
+
+ switch (version) {
+ case NO_VERSION:
+ return new InternalTimersSnapshotWriterPreVersioned<>(timersSnapshot);
+
+ case InternalTimerServiceSerializationProxy.VERSION:
+ return new InternalTimersSnapshotWriterV1<>(timersSnapshot);
+
+ default:
+ // guard for future
+ throw new IllegalStateException(
+ "Unrecognized internal timers snapshot writer version: " + version);
+ }
+ }
+
+ /**
+ * A writer for a {@link InternalTimersSnapshot}.
+ */
+ public interface InternalTimersSnapshotWriter {
+
+ /**
+ * Writes the timers snapshot to the output view.
+ *
+ * @param out the output view to write to
+ * @throws IOException
+ */
+ void writeTimersSnapshot(DataOutputView out) throws IOException;
+ }
+
+ private abstract static class AbstractInternalTimersSnapshotWriter<K, N> implements InternalTimersSnapshotWriter {
+
+ protected final InternalTimersSnapshot<K, N> timersSnapshot;
+
+ public AbstractInternalTimersSnapshotWriter(InternalTimersSnapshot<K, N> timersSnapshot) {
+ this.timersSnapshot = checkNotNull(timersSnapshot);
+ }
+
+ protected abstract void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException;
+
+ @Override
+ public final void writeTimersSnapshot(DataOutputView out) throws IOException {
+ writeKeyAndNamespaceSerializers(out);
+
+ InternalTimer.TimerSerializer<K, N> timerSerializer = new InternalTimer.TimerSerializer<>(
+ timersSnapshot.getKeySerializer(),
+ timersSnapshot.getNamespaceSerializer());
+
+ // write the event time timers
+ Set<InternalTimer<K, N>> eventTimers = timersSnapshot.getEventTimeTimers();
+ if (eventTimers != null) {
+ out.writeInt(eventTimers.size());
+ for (InternalTimer<K, N> eventTimer : eventTimers) {
+ timerSerializer.serialize(eventTimer, out);
+ }
+ } else {
+ out.writeInt(0);
+ }
+
+ // write the processing time timers
+ Set<InternalTimer<K, N>> processingTimers = timersSnapshot.getProcessingTimeTimers();
+ if (processingTimers != null) {
+ out.writeInt(processingTimers.size());
+ for (InternalTimer<K, N> processingTimer : processingTimers) {
+ timerSerializer.serialize(processingTimer, out);
+ }
+ } else {
+ out.writeInt(0);
+ }
+ }
+ }
+
+ private static class InternalTimersSnapshotWriterPreVersioned<K, N> extends AbstractInternalTimersSnapshotWriter<K, N> {
+
+ public InternalTimersSnapshotWriterPreVersioned(InternalTimersSnapshot<K, N> timersSnapshot) {
+ super(timersSnapshot);
+ }
+
+ @Override
+ protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException {
+ // the pre-versioned format only serializes the serializers, without their configuration snapshots
+ try (ByteArrayOutputStreamWithPos stream = new ByteArrayOutputStreamWithPos()) {
+ InstantiationUtil.serializeObject(stream, timersSnapshot.getKeySerializer());
+ InstantiationUtil.serializeObject(stream, timersSnapshot.getNamespaceSerializer());
+
+ out.write(stream.getBuf(), 0, stream.getPosition());
+ }
+ }
+ }
+
+ private static class InternalTimersSnapshotWriterV1<K, N> extends AbstractInternalTimersSnapshotWriter<K, N> {
+
+ public InternalTimersSnapshotWriterV1(InternalTimersSnapshot<K, N> timersSnapshot) {
+ super(timersSnapshot);
+ }
+
+ @Override
+ protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException {
+ // write key / namespace serializers, and their configuration snapshots
+ TypeSerializerSerializationUtil.writeSerializersAndConfigsWithResilience(
+ out,
+ Arrays.asList(
+ Tuple2.of(timersSnapshot.getKeySerializer(), timersSnapshot.getKeySerializerConfigSnapshot()),
+ Tuple2.of(timersSnapshot.getNamespaceSerializer(), timersSnapshot.getNamespaceSerializerConfigSnapshot())));
+ }
+ }
+
+ // -------------------------------------------------------------------------------
+ // Readers
+ // - pre-versioned: Flink 1.4.0
+ // - v1: Flink 1.4.1
+ // -------------------------------------------------------------------------------
+
+ public static <K, N> InternalTimersSnapshotReader<K, N> getReaderForVersion(
+ int version, ClassLoader userCodeClassLoader) {
+
+ switch (version) {
+ case NO_VERSION:
+ return new InternalTimersSnapshotReaderPreVersioned<>(userCodeClassLoader);
+
+ case InternalTimerServiceSerializationProxy.VERSION:
+ return new InternalTimersSnapshotReaderV1<>(userCodeClassLoader);
+
+ default:
+ // guard for future
+ throw new IllegalStateException(
+ "Unrecognized internal timers snapshot writer version: " + version);
+ }
+ }
+
+ /**
+ * A reader for a {@link InternalTimersSnapshot}.
+ */
+ public interface InternalTimersSnapshotReader<K, N> {
+
+ /**
+ * Reads a timers snapshot from the provided input view.
+ *
+ * @param in the input view
+ * @return the read timers snapshot
+ * @throws IOException
+ */
+ InternalTimersSnapshot<K, N> readTimersSnapshot(DataInputView in) throws IOException;
+ }
+
+ private abstract static class AbstractInternalTimersSnapshotReader<K, N> implements InternalTimersSnapshotReader<K, N> {
+
+ protected final ClassLoader userCodeClassLoader;
+
+ public AbstractInternalTimersSnapshotReader(ClassLoader userCodeClassLoader) {
+ this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
+ }
+
+ protected abstract void restoreKeyAndNamespaceSerializers(
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot,
+ DataInputView in) throws IOException;
+
+ @Override
+ public final InternalTimersSnapshot<K, N> readTimersSnapshot(DataInputView in) throws IOException {
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot = new InternalTimersSnapshot<>();
+
+ restoreKeyAndNamespaceSerializers(restoredTimersSnapshot, in);
+
+ InternalTimer.TimerSerializer<K, N> timerSerializer = new InternalTimer.TimerSerializer<>(
+ restoredTimersSnapshot.getKeySerializer(),
+ restoredTimersSnapshot.getNamespaceSerializer());
+
+ // read the event time timers
+ int sizeOfEventTimeTimers = in.readInt();
+ Set<InternalTimer<K, N>> restoredEventTimers = new HashSet<>(sizeOfEventTimeTimers);
+ if (sizeOfEventTimeTimers > 0) {
+ for (int i = 0; i < sizeOfEventTimeTimers; i++) {
+ InternalTimer<K, N> timer = timerSerializer.deserialize(in);
+ restoredEventTimers.add(timer);
+ }
+ }
+ restoredTimersSnapshot.setEventTimeTimers(restoredEventTimers);
+
+ // read the processing time timers
+ int sizeOfProcessingTimeTimers = in.readInt();
+ Set<InternalTimer<K, N>> restoredProcessingTimers = new HashSet<>(sizeOfProcessingTimeTimers);
+ if (sizeOfProcessingTimeTimers > 0) {
+ for (int i = 0; i < sizeOfProcessingTimeTimers; i++) {
+ InternalTimer<K, N> timer = timerSerializer.deserialize(in);
+ restoredProcessingTimers.add(timer);
+ }
+ }
+ restoredTimersSnapshot.setProcessingTimeTimers(restoredProcessingTimers);
+
+ return restoredTimersSnapshot;
+ }
+ }
+
+ private static class InternalTimersSnapshotReaderPreVersioned<K, N> extends AbstractInternalTimersSnapshotReader<K, N> {
+
+ public InternalTimersSnapshotReaderPreVersioned(ClassLoader userCodeClassLoader) {
+ super(userCodeClassLoader);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void restoreKeyAndNamespaceSerializers(
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot,
+ DataInputView in) throws IOException {
+
+ DataInputViewStream dis = new DataInputViewStream(in);
+ try {
+ restoredTimersSnapshot.setKeySerializer(InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true));
+ restoredTimersSnapshot.setNamespaceSerializer(InstantiationUtil.deserializeObject(dis, userCodeClassLoader, true));
+ } catch (ClassNotFoundException exception) {
+ throw new IOException(exception);
+ }
+ }
+ }
+
+ private static class InternalTimersSnapshotReaderV1<K, N> extends AbstractInternalTimersSnapshotReader<K, N> {
+
+ public InternalTimersSnapshotReaderV1(ClassLoader userCodeClassLoader) {
+ super(userCodeClassLoader);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void restoreKeyAndNamespaceSerializers(
+ InternalTimersSnapshot<K, N> restoredTimersSnapshot,
+ DataInputView in) throws IOException {
+
+ List<Tuple2<TypeSerializer<?>, TypeSerializerConfigSnapshot>> serializersAndConfigs =
+ TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader);
+
+ restoredTimersSnapshot.setKeySerializer((TypeSerializer<K>) serializersAndConfigs.get(0).f0);
+ restoredTimersSnapshot.setKeySerializerConfigSnapshot(serializersAndConfigs.get(0).f1);
+ restoredTimersSnapshot.setNamespaceSerializer((TypeSerializer<N>) serializersAndConfigs.get(1).f0);
+ restoredTimersSnapshot.setNamespaceSerializerConfigSnapshot(serializersAndConfigs.get(1).f1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/07528133/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
index 9afd2b7..9887e45 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
@@ -23,7 +23,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.core.fs.FSDataInputStream;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.jobgraph.OperatorID;
@@ -214,7 +213,7 @@ public class StreamTaskStateInitializerImpl implements StreamTaskStateInitialize
"Key Group " + keyGroupIdx + " does not belong to the local range.");
timeServiceManager.restoreStateForKeyGroup(
- new DataInputViewStreamWrapper(streamProvider.getStream()),
+ streamProvider.getStream(),
keyGroupIdx, environment.getUserClassLoader());
}
http://git-wip-us.apache.org/repos/asf/flink/blob/07528133/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
index 5a4f1c4..aade876 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/HeapInternalTimerServiceTest.java
@@ -548,13 +548,36 @@ public class HeapInternalTimerServiceTest {
@Test
public void testSnapshotAndRestore() throws Exception {
+ testSnapshotAndRestore(InternalTimerServiceSerializationProxy.VERSION);
+ }
+
+ @Test
+ public void testSnapshotAndRestorePreVersioned() throws Exception {
+ testSnapshotAndRestore(InternalTimersSnapshotReaderWriters.NO_VERSION);
+ }
+
+ /**
+ * This test checks whether timers are assigned to correct key groups
+ * and whether snapshot/restore respects key groups.
+ */
+ @Test
+ public void testSnapshotAndRebalancingRestore() throws Exception {
+ testSnapshotAndRebalancingRestore(InternalTimerServiceSerializationProxy.VERSION);
+ }
+
+ @Test
+ public void testSnapshotAndRebalancingRestorePreVersioned() throws Exception {
+ testSnapshotAndRebalancingRestore(InternalTimersSnapshotReaderWriters.NO_VERSION);
+ }
+
+ private void testSnapshotAndRestore(int snapshotVersion) throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService =
- createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
+ createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
// get two different keys
int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
@@ -582,10 +605,15 @@ public class HeapInternalTimerServiceTest {
Map<Integer, byte[]> snapshot = new HashMap<>();
for (Integer keyGroupIndex : testKeyGroupRange) {
- ByteArrayOutputStream outStream = new ByteArrayOutputStream();
- timerService.snapshotTimersForKeyGroup(new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
- outStream.close();
- snapshot.put(keyGroupIndex, outStream.toByteArray());
+ try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
+ InternalTimersSnapshot<?, ?> timersSnapshot = timerService.snapshotTimersForKeyGroup(keyGroupIndex);
+
+ InternalTimersSnapshotReaderWriters
+ .getWriterForVersion(snapshotVersion, timersSnapshot)
+ .writeTimersSnapshot(new DataOutputViewStreamWrapper(outStream));
+
+ snapshot.put(keyGroupIndex, outStream.toByteArray());
+ }
}
@SuppressWarnings("unchecked")
@@ -595,12 +623,13 @@ public class HeapInternalTimerServiceTest {
processingTimeService = new TestProcessingTimeService();
timerService = restoreTimerService(
- snapshot,
- mockTriggerable2,
- keyContext,
- processingTimeService,
- testKeyGroupRange,
- maxParallelism);
+ snapshot,
+ snapshotVersion,
+ mockTriggerable2,
+ keyContext,
+ processingTimeService,
+ testKeyGroupRange,
+ maxParallelism);
processingTimeService.setCurrentTime(10);
timerService.advanceWatermark(10);
@@ -615,22 +644,17 @@ public class HeapInternalTimerServiceTest {
assertEquals(0, timerService.numEventTimeTimers());
}
- /**
- * This test checks whether timers are assigned to correct key groups
- * and whether snapshot/restore respects key groups.
- */
- @Test
- public void testSnapshotAndRebalancingRestore() throws Exception {
+ private void testSnapshotAndRebalancingRestore(int snapshotVersion) throws Exception {
@SuppressWarnings("unchecked")
Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
TestKeyContext keyContext = new TestKeyContext();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService =
- createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
+ createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
int midpoint = testKeyGroupRange.getStartKeyGroup() +
- (testKeyGroupRange.getEndKeyGroup() - testKeyGroupRange.getStartKeyGroup()) / 2;
+ (testKeyGroupRange.getEndKeyGroup() - testKeyGroupRange.getStartKeyGroup()) / 2;
// get two sub key-ranges so that we can restore two ranges separately
KeyGroupRange subKeyGroupRange1 = new KeyGroupRange(testKeyGroupRange.getStartKeyGroup(), midpoint);
@@ -661,15 +685,20 @@ public class HeapInternalTimerServiceTest {
Map<Integer, byte[]> snapshot1 = new HashMap<>();
Map<Integer, byte[]> snapshot2 = new HashMap<>();
for (Integer keyGroupIndex : testKeyGroupRange) {
- ByteArrayOutputStream outStream = new ByteArrayOutputStream();
- timerService.snapshotTimersForKeyGroup(new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
- outStream.close();
- if (subKeyGroupRange1.contains(keyGroupIndex)) {
- snapshot1.put(keyGroupIndex, outStream.toByteArray());
- } else if (subKeyGroupRange2.contains(keyGroupIndex)) {
- snapshot2.put(keyGroupIndex, outStream.toByteArray());
- } else {
- throw new IllegalStateException("Key-Group index doesn't belong to any sub range.");
+ try (ByteArrayOutputStream outStream = new ByteArrayOutputStream()) {
+ InternalTimersSnapshot<?, ?> timersSnapshot = timerService.snapshotTimersForKeyGroup(keyGroupIndex);
+
+ InternalTimersSnapshotReaderWriters
+ .getWriterForVersion(snapshotVersion, timersSnapshot)
+ .writeTimersSnapshot(new DataOutputViewStreamWrapper(outStream));
+
+ if (subKeyGroupRange1.contains(keyGroupIndex)) {
+ snapshot1.put(keyGroupIndex, outStream.toByteArray());
+ } else if (subKeyGroupRange2.contains(keyGroupIndex)) {
+ snapshot2.put(keyGroupIndex, outStream.toByteArray());
+ } else {
+ throw new IllegalStateException("Key-Group index doesn't belong to any sub range.");
+ }
}
}
@@ -687,20 +716,22 @@ public class HeapInternalTimerServiceTest {
TestProcessingTimeService processingTimeService2 = new TestProcessingTimeService();
HeapInternalTimerService<Integer, String> timerService1 = restoreTimerService(
- snapshot1,
- mockTriggerable1,
- keyContext1,
- processingTimeService1,
- subKeyGroupRange1,
- maxParallelism);
+ snapshot1,
+ snapshotVersion,
+ mockTriggerable1,
+ keyContext1,
+ processingTimeService1,
+ subKeyGroupRange1,
+ maxParallelism);
HeapInternalTimerService<Integer, String> timerService2 = restoreTimerService(
- snapshot2,
- mockTriggerable2,
- keyContext2,
- processingTimeService2,
- subKeyGroupRange2,
- maxParallelism);
+ snapshot2,
+ snapshotVersion,
+ mockTriggerable2,
+ keyContext2,
+ processingTimeService2,
+ subKeyGroupRange2,
+ maxParallelism);
processingTimeService1.setCurrentTime(10);
timerService1.advanceWatermark(10);
@@ -779,6 +810,7 @@ public class HeapInternalTimerServiceTest {
private static HeapInternalTimerService<Integer, String> restoreTimerService(
Map<Integer, byte[]> state,
+ int snapshotVersion,
Triggerable<Integer, String> triggerable,
KeyContext keyContext,
ProcessingTimeService processingTimeService,
@@ -796,10 +828,14 @@ public class HeapInternalTimerServiceTest {
// restore the timers
for (Integer keyGroupIndex : keyGroupsList) {
if (state.containsKey(keyGroupIndex)) {
- service.restoreTimersForKeyGroup(
- new DataInputViewStreamWrapper(new ByteArrayInputStream(state.get(keyGroupIndex))),
- keyGroupIndex,
- HeapInternalTimerServiceTest.class.getClassLoader());
+ try (ByteArrayInputStream inputStream = new ByteArrayInputStream(state.get(keyGroupIndex))) {
+ InternalTimersSnapshot<?, ?> restoredTimersSnapshot =
+ InternalTimersSnapshotReaderWriters
+ .getReaderForVersion(snapshotVersion, HeapInternalTimerServiceTest.class.getClassLoader())
+ .readTimersSnapshot(new DataInputViewStreamWrapper(inputStream));
+
+ service.restoreTimersForKeyGroup(restoredTimersSnapshot, keyGroupIndex);
+ }
}
}
[27/31] flink git commit: [FLINK-8472] [kinesis,
test] Extend FlinkKinesisConsumerMigrationTest for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [kinesis, test] Extend FlinkKinesisConsumerMigrationTest for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e609f811
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e609f811
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e609f811
Branch: refs/heads/master
Commit: e609f81122905b33610f7cc9fcc1163cf750d98d
Parents: 74daaef
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 16:23:17 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../kinesis/FlinkKinesisConsumerMigrationTest.java | 2 +-
...consumer-migration-test-flink1.4-empty-snapshot | Bin 0 -> 13147 bytes
...nesis-consumer-migration-test-flink1.4-snapshot | Bin 0 -> 13215 bytes
3 files changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e609f811/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
index 799739d..fd2d880 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
@@ -97,7 +97,7 @@ public class FlinkKinesisConsumerMigrationTest {
@Parameterized.Parameters(name = "Migration Savepoint: {0}")
public static Collection<MigrationVersion> parameters () {
- return Arrays.asList(MigrationVersion.v1_3);
+ return Arrays.asList(MigrationVersion.v1_3, MigrationVersion.v1_4);
}
public FlinkKinesisConsumerMigrationTest(MigrationVersion testMigrateVersion) {
http://git-wip-us.apache.org/repos/asf/flink/blob/e609f811/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-empty-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-empty-snapshot b/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-empty-snapshot
new file mode 100644
index 0000000..fba6258
Binary files /dev/null and b/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-empty-snapshot differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e609f811/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-snapshot b/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-snapshot
new file mode 100644
index 0000000..5ea4009
Binary files /dev/null and b/flink-connectors/flink-connector-kinesis/src/test/resources/kinesis-consumer-migration-test-flink1.4-snapshot differ
[19/31] flink git commit: [FLINK-8421] [test] Increase
MAX_MEM_STATE_SIZE and Akka framesize in
AbstractEventTimeWindowCheckpointingITCase
Posted by tz...@apache.org.
[FLINK-8421] [test] Increase MAX_MEM_STATE_SIZE and Akka framesize in AbstractEventTimeWindowCheckpointingITCase
After adding the TypeSerializerConfigSnapshots of timer serializers to
the timers snapshots, the size of the timer snapshots have potentially
doubled. This caused the AbstractEventTimeWindowCheckpointingITCase to
be failing, because the configured max memory state size and Akka
framesize were too small. This commit doubles those sizes.
This closes #5362.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a79916b8
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a79916b8
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a79916b8
Branch: refs/heads/master
Commit: a79916b88ef3d97c11980c25d3887d43964c152d
Parents: 0752813
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 31 11:13:00 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../AbstractEventTimeWindowCheckpointingITCase.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a79916b8/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
index 829ac93..53d329b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/AbstractEventTimeWindowCheckpointingITCase.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.configuration.AkkaOptions;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.HighAvailabilityOptions;
@@ -87,7 +88,7 @@ import static org.junit.Assert.fail;
@SuppressWarnings("serial")
public abstract class AbstractEventTimeWindowCheckpointingITCase extends TestLogger {
- private static final int MAX_MEM_STATE_SIZE = 10 * 1024 * 1024;
+ private static final int MAX_MEM_STATE_SIZE = 20 * 1024 * 1024;
private static final int PARALLELISM = 4;
private static LocalFlinkMiniCluster cluster;
@@ -137,6 +138,7 @@ public abstract class AbstractEventTimeWindowCheckpointingITCase extends TestLog
config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 48L);
// the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
config.setLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, 80L << 20); // 80 MB
+ config.setString(AkkaOptions.FRAMESIZE, String.valueOf(MAX_MEM_STATE_SIZE) + "b");
if (zkServer != null) {
config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
[17/31] flink git commit: [FLINK-8484] [kinesis] Use an equivalence
wrapper around restored StreamShardMetadatas to bypass out-of-sync shard
infos
Posted by tz...@apache.org.
[FLINK-8484] [kinesis] Use an equivalence wrapper around restored StreamShardMetadatas to bypass out-of-sync shard infos
This closes #5337.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/140bb0c6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/140bb0c6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/140bb0c6
Branch: refs/heads/master
Commit: 140bb0c66257d5d09d61845731aef1dbdb90a0bd
Parents: ffd7995
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Feb 1 12:53:42 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../kinesis/FlinkKinesisConsumer.java | 76 +++-------
.../kinesis/internals/KinesisDataFetcher.java | 5 +
.../kinesis/model/StreamShardMetadata.java | 44 ++++++
.../FlinkKinesisConsumerMigrationTest.java | 145 ++++++++++++++++++-
.../kinesis/FlinkKinesisConsumerTest.java | 52 -------
.../kinesis/testutils/TestRuntimeContext.java | 90 ++++++++++++
6 files changed, 299 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/140bb0c6/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
index 977eb05..7d97b7d 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java
@@ -101,7 +101,7 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
private transient KinesisDataFetcher<T> fetcher;
/** The sequence numbers to restore to upon restore from failure. */
- private transient HashMap<StreamShardMetadata, SequenceNumber> sequenceNumsToRestore;
+ private transient HashMap<StreamShardMetadata.EquivalenceWrapper, SequenceNumber> sequenceNumsToRestore;
private volatile boolean running = true;
@@ -208,23 +208,16 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
List<StreamShardHandle> allShards = fetcher.discoverNewShardsToSubscribe();
for (StreamShardHandle shard : allShards) {
- StreamShardMetadata kinesisStreamShard = KinesisDataFetcher.convertToStreamShardMetadata(shard);
+ StreamShardMetadata.EquivalenceWrapper kinesisStreamShard =
+ new StreamShardMetadata.EquivalenceWrapper(KinesisDataFetcher.convertToStreamShardMetadata(shard));
+
if (sequenceNumsToRestore != null) {
- // We need to do this to make sure that a shard that was closed after this restored state was taken will be properly
- // detected and have its sequence numbers restored. A shard will be closed when re-sharding, which can happen when
- // scaling up & down the Kinesis stream, and if the state is not synchronized, then the equality check of the current
- // Kinesis shard will not match the stored state, which will cause us to re-read the entire shard from the event horizon.
- if (updateKinesisShardStateWithMissingEndingSequenceNumber(kinesisStreamShard, sequenceNumsToRestore)) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Updated local stored state for shard {} with a new ending number: {}", kinesisStreamShard.getShardId(), sequenceNumsToRestore.get(kinesisStreamShard));
- }
- }
if (sequenceNumsToRestore.containsKey(kinesisStreamShard)) {
// if the shard was already seen and is contained in the state,
// just use the sequence number stored in the state
fetcher.registerNewSubscribedShardState(
- new KinesisStreamShardState(kinesisStreamShard, shard, sequenceNumsToRestore.get(kinesisStreamShard)));
+ new KinesisStreamShardState(kinesisStreamShard.getShardMetadata(), shard, sequenceNumsToRestore.get(kinesisStreamShard)));
if (LOG.isInfoEnabled()) {
LOG.info("Subtask {} is seeding the fetcher with restored shard {}," +
@@ -234,7 +227,7 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
} else {
// the shard wasn't discovered in the previous run, therefore should be consumed from the beginning
fetcher.registerNewSubscribedShardState(
- new KinesisStreamShardState(kinesisStreamShard, shard, SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get()));
+ new KinesisStreamShardState(kinesisStreamShard.getShardMetadata(), shard, SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get()));
if (LOG.isInfoEnabled()) {
LOG.info("Subtask {} is seeding the fetcher with new discovered shard {}," +
@@ -250,7 +243,7 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
ConsumerConfigConstants.DEFAULT_STREAM_INITIAL_POSITION)).toSentinelSequenceNumber();
fetcher.registerNewSubscribedShardState(
- new KinesisStreamShardState(kinesisStreamShard, shard, startingSeqNum.get()));
+ new KinesisStreamShardState(kinesisStreamShard.getShardMetadata(), shard, startingSeqNum.get()));
if (LOG.isInfoEnabled()) {
LOG.info("Subtask {} will be seeded with initial shard {}, starting state set as sequence number {}",
@@ -277,45 +270,6 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
sourceContext.close();
}
- /**
- * Synchronizes the Kinesis shard information from the current Kinesis shard with the restored state, if we find
- * a shard that match the shardId and streamName. If we find one, and its ending key is different that what we
- * have in our stored state, then we update the stored's shard's metadata's ending number.
- *
- * @param current the current Kinesis shard we're trying to synchronize.
- * @param sequenceNumsToRestore the (re)stored shard metadata and their sequence numbers.
- * @return {@code true} if the local state was updated with the current Kinesis shard's ending number.
- */
- @VisibleForTesting
- boolean updateKinesisShardStateWithMissingEndingSequenceNumber(StreamShardMetadata current, HashMap<StreamShardMetadata, SequenceNumber> sequenceNumsToRestore) {
- checkNotNull(current.getStreamName(), "Stream name not set on the current metadata shard");
- checkNotNull(current.getShardId(), "Shard id not set on the current metadata shard");
-
- // short-circuit: if the current shard doesn't have an ending sequence number, then there's no point in trying to update the local state
- // since that's the only property that can change.
- if (current.getEndingSequenceNumber() == null) {
- return false;
- }
-
- // try to find the matching shard based on the id & stream name
- for (Map.Entry<StreamShardMetadata, SequenceNumber> entry : sequenceNumsToRestore.entrySet()) {
- if (current.getStreamName().equals(entry.getKey().getStreamName())
- && current.getShardId().equals(entry.getKey().getShardId())) {
- // synchronize the local state if the ending sequence number is different
- if (!current.getEndingSequenceNumber().equals(entry.getKey().getEndingSequenceNumber())) {
- // ugly, but since the hashcode will change, we'll need to remove it and add it back
- sequenceNumsToRestore.remove(entry.getKey());
- entry.getKey().setEndingSequenceNumber(current.getEndingSequenceNumber());
- sequenceNumsToRestore.put(entry.getKey(), entry.getValue());
- return true;
- }
- // we already found the matching shard
- break;
- }
- }
- return false;
- }
-
@Override
public void cancel() {
running = false;
@@ -364,7 +318,13 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
if (sequenceNumsToRestore == null) {
sequenceNumsToRestore = new HashMap<>();
for (Tuple2<StreamShardMetadata, SequenceNumber> kinesisSequenceNumber : sequenceNumsStateForCheckpoint.get()) {
- sequenceNumsToRestore.put(kinesisSequenceNumber.f0, kinesisSequenceNumber.f1);
+ sequenceNumsToRestore.put(
+ // we wrap the restored metadata inside an equivalence wrapper that checks only stream name and shard id,
+ // so that if a shard had been closed (due to a Kinesis reshard operation, for example) since
+ // the savepoint and has a different metadata than what we last stored,
+ // we will still be able to match it in sequenceNumsToRestore. Please see FLINK-8484 for details.
+ new StreamShardMetadata.EquivalenceWrapper(kinesisSequenceNumber.f0),
+ kinesisSequenceNumber.f1);
}
LOG.info("Setting restore state in the FlinkKinesisConsumer. Using the following offsets: {}",
@@ -388,16 +348,16 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
if (fetcher == null) {
if (sequenceNumsToRestore != null) {
- for (Map.Entry<StreamShardMetadata, SequenceNumber> entry : sequenceNumsToRestore.entrySet()) {
+ for (Map.Entry<StreamShardMetadata.EquivalenceWrapper, SequenceNumber> entry : sequenceNumsToRestore.entrySet()) {
// sequenceNumsToRestore is the restored global union state;
// should only snapshot shards that actually belong to us
if (KinesisDataFetcher.isThisSubtaskShouldSubscribeTo(
- KinesisDataFetcher.convertToStreamShardHandle(entry.getKey()),
+ KinesisDataFetcher.convertToStreamShardHandle(entry.getKey().getShardMetadata()),
getRuntimeContext().getNumberOfParallelSubtasks(),
getRuntimeContext().getIndexOfThisSubtask())) {
- sequenceNumsStateForCheckpoint.add(Tuple2.of(entry.getKey(), entry.getValue()));
+ sequenceNumsStateForCheckpoint.add(Tuple2.of(entry.getKey().getShardMetadata(), entry.getValue()));
}
}
}
@@ -428,7 +388,7 @@ public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T> imple
}
@VisibleForTesting
- HashMap<StreamShardMetadata, SequenceNumber> getRestoredState() {
+ HashMap<StreamShardMetadata.EquivalenceWrapper, SequenceNumber> getRestoredState() {
return sequenceNumsToRestore;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/140bb0c6/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
index 274ae70..7928fe3 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/internals/KinesisDataFetcher.java
@@ -619,6 +619,11 @@ public class KinesisDataFetcher<T> {
});
}
+ @VisibleForTesting
+ public List<KinesisStreamShardState> getSubscribedShardsState() {
+ return subscribedShardsState;
+ }
+
/**
* Utility function to create an initial map of the last discovered shard id of each subscribed stream, set to null;
* This is called in the constructor; correct values will be set later on by calling advanceLastDiscoveredShardOfStream().
http://git-wip-us.apache.org/repos/asf/flink/blob/140bb0c6/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/model/StreamShardMetadata.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/model/StreamShardMetadata.java b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/model/StreamShardMetadata.java
index 571a38b..30e6cac 100644
--- a/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/model/StreamShardMetadata.java
+++ b/flink-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/model/StreamShardMetadata.java
@@ -22,6 +22,8 @@ import org.apache.flink.annotation.Internal;
import java.io.Serializable;
import java.util.Objects;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* A serializable representation of a AWS Kinesis Stream shard. It is basically a wrapper class around the information
* disintegrated from {@link com.amazonaws.services.kinesis.model.Shard} and its nested classes. The disintegration
@@ -173,4 +175,46 @@ public class StreamShardMetadata implements Serializable {
return hash;
}
+ /** An equivalence wrapper that only checks for the stream name and shard id for equality. */
+ public static class EquivalenceWrapper {
+
+ private final StreamShardMetadata shardMetadata;
+
+ public EquivalenceWrapper(StreamShardMetadata shardMetadata) {
+ this.shardMetadata = checkNotNull(shardMetadata);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof EquivalenceWrapper)) {
+ return false;
+ }
+
+ if (obj == this) {
+ return true;
+ }
+
+ EquivalenceWrapper other = (EquivalenceWrapper) obj;
+
+ return shardMetadata.getStreamName().equals(other.shardMetadata.getStreamName())
+ && shardMetadata.getShardId().equals(other.shardMetadata.getShardId());
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 17;
+
+ if (shardMetadata.getStreamName() != null) {
+ hash = 37 * hash + shardMetadata.getStreamName().hashCode();
+ }
+ if (shardMetadata.getShardId() != null) {
+ hash = 37 * hash + shardMetadata.getShardId().hashCode();
+ }
+ return hash;
+ }
+
+ public StreamShardMetadata getShardMetadata() {
+ return shardMetadata;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/140bb0c6/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
index 9277cbc..799739d 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerMigrationTest.java
@@ -17,6 +17,7 @@
package org.apache.flink.streaming.connectors.kinesis;
+import com.amazonaws.services.kinesis.model.SequenceNumberRange;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.core.testutils.OneShotLatch;
@@ -24,6 +25,8 @@ import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.operators.StreamSource;
import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;
+import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState;
+import org.apache.flink.streaming.connectors.kinesis.model.SentinelSequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.model.SequenceNumber;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardHandle;
import org.apache.flink.streaming.connectors.kinesis.model.StreamShardMetadata;
@@ -52,10 +55,12 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
@@ -76,14 +81,16 @@ public class FlinkKinesisConsumerMigrationTest {
private final MigrationVersion flinkGenerateSavepointVersion = null;
private static final String TEST_STREAM_NAME = "fakeStream1";
+ private static final SequenceNumber TEST_SEQUENCE_NUMBER = new SequenceNumber("987654321");
+ private static final String TEST_SHARD_ID = KinesisShardIdGenerator.generateFromShardOrder(0);
private static final HashMap<StreamShardMetadata, SequenceNumber> TEST_STATE = new HashMap<>();
static {
StreamShardMetadata shardMetadata = new StreamShardMetadata();
shardMetadata.setStreamName(TEST_STREAM_NAME);
- shardMetadata.setShardId(KinesisShardIdGenerator.generateFromShardOrder(0));
+ shardMetadata.setShardId(TEST_SHARD_ID);
- TEST_STATE.put(shardMetadata, new SequenceNumber("987654321"));
+ TEST_STATE.put(shardMetadata, TEST_SEQUENCE_NUMBER);
}
private final MigrationVersion testMigrateVersion;
@@ -116,6 +123,10 @@ public class FlinkKinesisConsumerMigrationTest {
Shard shard = new Shard();
shard.setShardId(shardMetadata.getShardId());
+ SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
+ sequenceNumberRange.withStartingSequenceNumber("1");
+ shard.setSequenceNumberRange(sequenceNumberRange);
+
initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), shard));
}
@@ -147,6 +158,15 @@ public class FlinkKinesisConsumerMigrationTest {
// assert that no state was restored
assertTrue(consumerFunction.getRestoredState().isEmpty());
+ // although the restore state is empty, the fetcher should still have been registered the initial discovered shard;
+ // furthermore, the discovered shard should be considered a newly created shard while the job wasn't running,
+ // and therefore should be consumed from the earliest sequence number
+ KinesisStreamShardState restoredShardState = fetcher.getSubscribedShardsState().get(0);
+ assertEquals(TEST_STREAM_NAME, restoredShardState.getStreamShardHandle().getStreamName());
+ assertEquals(TEST_SHARD_ID, restoredShardState.getStreamShardHandle().getShard().getShardId());
+ assertFalse(restoredShardState.getStreamShardHandle().isClosed());
+ assertEquals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get(), restoredShardState.getLastProcessedSequenceNum());
+
consumerOperator.close();
consumerOperator.cancel();
}
@@ -158,6 +178,10 @@ public class FlinkKinesisConsumerMigrationTest {
Shard shard = new Shard();
shard.setShardId(shardMetadata.getShardId());
+ SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
+ sequenceNumberRange.withStartingSequenceNumber("1");
+ shard.setSequenceNumberRange(sequenceNumberRange);
+
initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), shard));
}
@@ -190,7 +214,111 @@ public class FlinkKinesisConsumerMigrationTest {
// assert that state is correctly restored
assertNotEquals(null, consumerFunction.getRestoredState());
assertEquals(1, consumerFunction.getRestoredState().size());
- assertEquals(TEST_STATE, consumerFunction.getRestoredState());
+ assertEquals(TEST_STATE, removeEquivalenceWrappers(consumerFunction.getRestoredState()));
+ assertEquals(1, fetcher.getSubscribedShardsState().size());
+ assertEquals(TEST_SEQUENCE_NUMBER, fetcher.getSubscribedShardsState().get(0).getLastProcessedSequenceNum());
+
+ KinesisStreamShardState restoredShardState = fetcher.getSubscribedShardsState().get(0);
+ assertEquals(TEST_STREAM_NAME, restoredShardState.getStreamShardHandle().getStreamName());
+ assertEquals(TEST_SHARD_ID, restoredShardState.getStreamShardHandle().getShard().getShardId());
+ assertFalse(restoredShardState.getStreamShardHandle().isClosed());
+ assertEquals(TEST_SEQUENCE_NUMBER, restoredShardState.getLastProcessedSequenceNum());
+
+ consumerOperator.close();
+ consumerOperator.cancel();
+ }
+
+ @Test
+ public void testRestoreWithReshardedStream() throws Exception {
+ final List<StreamShardHandle> initialDiscoveryShards = new ArrayList<>(TEST_STATE.size());
+ for (StreamShardMetadata shardMetadata : TEST_STATE.keySet()) {
+ // setup the closed shard
+ Shard closedShard = new Shard();
+ closedShard.setShardId(shardMetadata.getShardId());
+
+ SequenceNumberRange closedSequenceNumberRange = new SequenceNumberRange();
+ closedSequenceNumberRange.withStartingSequenceNumber("1");
+ closedSequenceNumberRange.withEndingSequenceNumber("1087654321"); // this represents a closed shard
+ closedShard.setSequenceNumberRange(closedSequenceNumberRange);
+
+ initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), closedShard));
+
+ // setup the new shards
+ Shard newSplitShard1 = new Shard();
+ newSplitShard1.setShardId(KinesisShardIdGenerator.generateFromShardOrder(1));
+
+ SequenceNumberRange newSequenceNumberRange1 = new SequenceNumberRange();
+ newSequenceNumberRange1.withStartingSequenceNumber("1087654322");
+ newSplitShard1.setSequenceNumberRange(newSequenceNumberRange1);
+
+ newSplitShard1.setParentShardId(TEST_SHARD_ID);
+
+ Shard newSplitShard2 = new Shard();
+ newSplitShard2.setShardId(KinesisShardIdGenerator.generateFromShardOrder(2));
+
+ SequenceNumberRange newSequenceNumberRange2 = new SequenceNumberRange();
+ newSequenceNumberRange2.withStartingSequenceNumber("2087654322");
+ newSplitShard2.setSequenceNumberRange(newSequenceNumberRange2);
+
+ newSplitShard2.setParentShardId(TEST_SHARD_ID);
+
+ initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), newSplitShard1));
+ initialDiscoveryShards.add(new StreamShardHandle(shardMetadata.getStreamName(), newSplitShard2));
+ }
+
+ final TestFetcher<String> fetcher = new TestFetcher<>(
+ Collections.singletonList(TEST_STREAM_NAME),
+ new TestSourceContext<>(),
+ new TestRuntimeContext(true, 1, 0),
+ TestUtils.getStandardProperties(),
+ new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()),
+ null,
+ initialDiscoveryShards);
+
+ final DummyFlinkKinesisConsumer<String> consumerFunction = new DummyFlinkKinesisConsumer<>(
+ fetcher, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()));
+
+ StreamSource<String, DummyFlinkKinesisConsumer<String>> consumerOperator =
+ new StreamSource<>(consumerFunction);
+
+ final AbstractStreamOperatorTestHarness<String> testHarness =
+ new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
+
+ testHarness.setup();
+ MigrationTestUtil.restoreFromSnapshot(
+ testHarness,
+ "src/test/resources/kinesis-consumer-migration-test-flink" + testMigrateVersion + "-snapshot", testMigrateVersion);
+ testHarness.open();
+
+ consumerFunction.run(new TestSourceContext<>());
+
+ // assert that state is correctly restored
+ assertNotEquals(null, consumerFunction.getRestoredState());
+ assertEquals(1, consumerFunction.getRestoredState().size());
+ assertEquals(TEST_STATE, removeEquivalenceWrappers(consumerFunction.getRestoredState()));
+
+ // assert that the fetcher is registered with all shards, including new shards
+ assertEquals(3, fetcher.getSubscribedShardsState().size());
+
+ KinesisStreamShardState restoredClosedShardState = fetcher.getSubscribedShardsState().get(0);
+ assertEquals(TEST_STREAM_NAME, restoredClosedShardState.getStreamShardHandle().getStreamName());
+ assertEquals(TEST_SHARD_ID, restoredClosedShardState.getStreamShardHandle().getShard().getShardId());
+ assertTrue(restoredClosedShardState.getStreamShardHandle().isClosed());
+ assertEquals(TEST_SEQUENCE_NUMBER, restoredClosedShardState.getLastProcessedSequenceNum());
+
+ KinesisStreamShardState restoredNewSplitShard1 = fetcher.getSubscribedShardsState().get(1);
+ assertEquals(TEST_STREAM_NAME, restoredNewSplitShard1.getStreamShardHandle().getStreamName());
+ assertEquals(KinesisShardIdGenerator.generateFromShardOrder(1), restoredNewSplitShard1.getStreamShardHandle().getShard().getShardId());
+ assertFalse(restoredNewSplitShard1.getStreamShardHandle().isClosed());
+ // new shards should be consumed from the beginning
+ assertEquals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get(), restoredNewSplitShard1.getLastProcessedSequenceNum());
+
+ KinesisStreamShardState restoredNewSplitShard2 = fetcher.getSubscribedShardsState().get(2);
+ assertEquals(TEST_STREAM_NAME, restoredNewSplitShard2.getStreamShardHandle().getStreamName());
+ assertEquals(KinesisShardIdGenerator.generateFromShardOrder(2), restoredNewSplitShard2.getStreamShardHandle().getShard().getShardId());
+ assertFalse(restoredNewSplitShard2.getStreamShardHandle().isClosed());
+ // new shards should be consumed from the beginning
+ assertEquals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get(), restoredNewSplitShard2.getLastProcessedSequenceNum());
consumerOperator.close();
consumerOperator.cancel();
@@ -321,4 +449,15 @@ public class FlinkKinesisConsumerMigrationTest {
// do nothing
}
}
+
+ private static Map<StreamShardMetadata, SequenceNumber> removeEquivalenceWrappers(
+ Map<StreamShardMetadata.EquivalenceWrapper, SequenceNumber> equivalenceWrappedMap) {
+
+ Map<StreamShardMetadata, SequenceNumber> unwrapped = new HashMap<>();
+ for (Map.Entry<StreamShardMetadata.EquivalenceWrapper, SequenceNumber> wrapped : equivalenceWrappedMap.entrySet()) {
+ unwrapped.put(wrapped.getKey().getShardMetadata(), wrapped.getValue());
+ }
+
+ return unwrapped;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/140bb0c6/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
index 3fc501c..05c1cce 100644
--- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumerTest.java
@@ -62,14 +62,12 @@ import org.powermock.modules.junit4.PowerMockRunner;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -590,56 +588,6 @@ public class FlinkKinesisConsumerTest {
closedStreamShardHandle, fakeRestoredState.get(closedStreamShardHandle)));
}
- @Test
- public void testUpdateKinesisShardStateWithMissingEndingSequenceNumber() {
- final String streamName = "fakeStream1";
- final String shardId = "shard-000001";
-
- Properties config = TestUtils.getStandardProperties();
- FlinkKinesisConsumer<String> consumer = new FlinkKinesisConsumer<>(streamName, new SimpleStringSchema(), config);
-
- // having the current Kinesis shard we're trying to find the restored state for
- final StreamShardMetadata current = new StreamShardMetadata();
- current.setShardId(shardId);
- current.setStreamName(streamName);
- current.setEndingSequenceNumber(null);
-
- final HashMap<StreamShardMetadata, SequenceNumber> sequenceNumsToRestore = new LinkedHashMap<>();
- assertFalse("Current shard is open, expecting a short-circuit", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
-
- // create some non-matching shards
- final StreamShardMetadata differentStreamName = new StreamShardMetadata();
- differentStreamName.setStreamName("fakeStream2");
- differentStreamName.setShardId(shardId);
-
- final StreamShardMetadata differentShardId = new StreamShardMetadata();
- differentShardId.setStreamName(streamName);
- differentShardId.setShardId("shard-000002");
-
- // create the matching shard
- final StreamShardMetadata match = new StreamShardMetadata();
- match.setStreamName(streamName);
- match.setShardId(shardId);
- // ensure the sequence number isn't set (shard is considered in open state)
- match.setEndingSequenceNumber(null);
-
- sequenceNumsToRestore.put(differentStreamName, new SequenceNumber("123"));
- sequenceNumsToRestore.put(differentShardId, new SequenceNumber("456"));
- sequenceNumsToRestore.put(match, new SequenceNumber("789"));
-
- assertFalse("No ending sequence number was set, so no synchronisation was done.", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
-
- // alter the ending sequence number (indicating the shard is now closed)
- final String endingSequenceNumber = "99999";
- current.setEndingSequenceNumber(endingSequenceNumber);
- assertTrue("Shard was closed (ending number set).", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
- assertEquals(endingSequenceNumber, match.getEndingSequenceNumber());
- assertEquals(current, match);
- assertTrue("Make sure we can still find back our match in the restored sequences ", sequenceNumsToRestore.containsKey(match));
-
- assertFalse("Ending number was already set, no more need for synchronisation", consumer.updateKinesisShardStateWithMissingEndingSequenceNumber(current, sequenceNumsToRestore));
- }
-
private static final class TestingListState<T> implements ListState<T> {
private final List<T> list = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/flink/blob/140bb0c6/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestRuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestRuntimeContext.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestRuntimeContext.java
new file mode 100644
index 0000000..ce0bd97
--- /dev/null
+++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/testutils/TestRuntimeContext.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kinesis.testutils;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.memory.MemoryManager;
+import org.apache.flink.runtime.operators.testutils.MockEnvironment;
+import org.apache.flink.runtime.state.TestTaskStateManager;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
+
+import java.util.Collections;
+
+/**
+ * A mock {@link StreamingRuntimeContext} for testing.
+ */
+public class TestRuntimeContext extends StreamingRuntimeContext {
+
+ private final boolean isCheckpointingEnabled;
+
+ private final int numParallelSubtasks;
+ private final int subtaskIndex;
+
+ public TestRuntimeContext(
+ boolean isCheckpointingEnabled,
+ int numParallelSubtasks,
+ int subtaskIndex) {
+
+ super(
+ new TestStreamOperator(),
+ new MockEnvironment(
+ "mockTask",
+ 4 * MemoryManager.DEFAULT_PAGE_SIZE,
+ null,
+ 16,
+ new TestTaskStateManager()),
+ Collections.emptyMap());
+
+ this.isCheckpointingEnabled = isCheckpointingEnabled;
+ this.numParallelSubtasks = numParallelSubtasks;
+ this.subtaskIndex = subtaskIndex;
+ }
+
+ @Override
+ public MetricGroup getMetricGroup() {
+ return new UnregisteredMetricsGroup();
+ }
+
+ @Override
+ public boolean isCheckpointingEnabled() {
+ return isCheckpointingEnabled;
+ }
+
+ @Override
+ public int getIndexOfThisSubtask() {
+ return subtaskIndex;
+ }
+
+ @Override
+ public int getNumberOfParallelSubtasks() {
+ return numParallelSubtasks;
+ }
+
+ private static class TestStreamOperator extends AbstractStreamOperator<Integer> {
+
+ private static final long serialVersionUID = -2547912462252989589L;
+
+ @Override
+ public ExecutionConfig getExecutionConfig() {
+ return new ExecutionConfig();
+ }
+ }
+}
[12/31] flink git commit: [hotfix][connectors] Fix log format strings
Posted by tz...@apache.org.
[hotfix][connectors] Fix log format strings
Fixes three occurrences of using '%s' instead of '{}' in logging
statements.
This closes #5380.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a03fbf91
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a03fbf91
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a03fbf91
Branch: refs/heads/master
Commit: a03fbf91b4c40d20316bf62e4baa9d75c157e4c5
Parents: 4197b0d
Author: Patrick Lucas <me...@patricklucas.com>
Authored: Mon Jan 29 13:58:09 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../flink/streaming/connectors/kafka/FlinkKafkaProducer011.java | 2 +-
.../java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a03fbf91/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java
index d0f5039..3fe4bc6 100644
--- a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java
+++ b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011.java
@@ -512,7 +512,7 @@ public class FlinkKafkaProducer011<IN>
long timeout = DEFAULT_KAFKA_TRANSACTION_TIMEOUT.toMilliseconds();
checkState(timeout < Integer.MAX_VALUE && timeout > 0, "timeout does not fit into 32 bit integer");
this.producerConfig.put(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, (int) timeout);
- LOG.warn("Property [%s] not specified. Setting it to %s", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_KAFKA_TRANSACTION_TIMEOUT);
+ LOG.warn("Property [{}] not specified. Setting it to {}", ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, DEFAULT_KAFKA_TRANSACTION_TIMEOUT);
}
// Enable transactionTimeoutWarnings to avoid silent data loss
http://git-wip-us.apache.org/repos/asf/flink/blob/a03fbf91/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java b/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
index 2497712..67c7a0e 100644
--- a/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
+++ b/flink-connectors/flink-jdbc/src/main/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormat.java
@@ -116,7 +116,7 @@ public class JDBCOutputFormat extends RichOutputFormat<Row> {
if (typesArray == null) {
// no types provided
for (int index = 0; index < row.getArity(); index++) {
- LOG.warn("Unknown column type for column %s. Best effort approach to set its value: %s.", index + 1, row.getField(index));
+ LOG.warn("Unknown column type for column {}. Best effort approach to set its value: {}.", index + 1, row.getField(index));
upload.setObject(index + 1, row.getField(index));
}
} else {
@@ -181,7 +181,7 @@ public class JDBCOutputFormat extends RichOutputFormat<Row> {
break;
default:
upload.setObject(index + 1, row.getField(index));
- LOG.warn("Unmanaged sql type (%s) for column %s. Best effort approach to set its value: %s.",
+ LOG.warn("Unmanaged sql type ({}) for column {}. Best effort approach to set its value: {}.",
typesArray[index], index + 1, row.getField(index));
// case java.sql.Types.SQLXML
// case java.sql.Types.ARRAY:
[22/31] flink git commit: [FLINK-8421] [core] Allow
InstantiationUtil.deserializeObject() to use FailureTolerantObjectInputStream
Posted by tz...@apache.org.
[FLINK-8421] [core] Allow InstantiationUtil.deserializeObject() to use FailureTolerantObjectInputStream
This makes it possible to use the FailureTolerantObjectInputStream in
InstantiationUtil.deserializeObject. This behaviour is controlled via a
isFailureTolerant flag argument. All previous calls to the method
remains to have identical behaviour (the ClassloaderObjectInputStream).
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9659ffd9
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9659ffd9
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9659ffd9
Branch: refs/heads/master
Commit: 9659ffd9b3ecafb3ca3a71868cdcde301b5e0b9a
Parents: 7a9d7e7
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Wed Jan 24 17:15:08 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../TypeSerializerSerializationUtil.java | 83 +-------------
.../apache/flink/util/InstantiationUtil.java | 110 +++++++++++++++++--
...ckendStateMetaInfoSnapshotReaderWriters.java | 5 +-
3 files changed, 104 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9659ffd9/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java
index c68bb79..3f82530 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSerializationUtil.java
@@ -20,7 +20,6 @@ package org.apache.flink.api.common.typeutils;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.runtime.KryoRegistrationSerializerConfigSnapshot;
import org.apache.flink.core.io.VersionedIOReadableWritable;
import org.apache.flink.core.memory.ByteArrayInputStreamWithPos;
import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos;
@@ -36,14 +35,9 @@ import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.io.InvalidClassException;
-import java.io.ObjectInputStream;
-import java.io.ObjectStreamClass;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
/**
* Utility methods for serialization of {@link TypeSerializer} and {@link TypeSerializerConfigSnapshot}.
@@ -54,79 +48,6 @@ public class TypeSerializerSerializationUtil {
private static final Logger LOG = LoggerFactory.getLogger(TypeSerializerSerializationUtil.class);
/**
- * This is maintained as a temporary workaround for FLINK-6869.
- *
- * <p>Before 1.3, the Scala serializers did not specify the serialVersionUID.
- * Although since 1.3 they are properly specified, we still have to ignore them for now
- * as their previous serialVersionUIDs will vary depending on the Scala version.
- *
- * <p>This can be removed once 1.2 is no longer supported.
- */
- private static Set<String> scalaSerializerClassnames = new HashSet<>();
- static {
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.TraversableSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.CaseClassSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EitherSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EnumValueSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.OptionSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.TrySerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EitherSerializer");
- scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.UnitSerializer");
- }
-
- /**
- * An {@link ObjectInputStream} that ignores serialVersionUID mismatches when deserializing objects of
- * anonymous classes or our Scala serializer classes and also replaces occurrences of GenericData.Array
- * (from Avro) by a dummy class so that the KryoSerializer can still be deserialized without
- * Avro being on the classpath.
- *
- * <p>The {@link TypeSerializerSerializationProxy} uses this specific object input stream to read serializers,
- * so that mismatching serialVersionUIDs of anonymous classes / Scala serializers are ignored.
- * This is a required workaround to maintain backwards compatibility for our pre-1.3 Scala serializers.
- * See FLINK-6869 for details.
- *
- * @see <a href="https://issues.apache.org/jira/browse/FLINK-6869">FLINK-6869</a>
- */
- public static class FailureTolerantObjectInputStream extends InstantiationUtil.ClassLoaderObjectInputStream {
-
- public FailureTolerantObjectInputStream(InputStream in, ClassLoader cl) throws IOException {
- super(in, cl);
- }
-
- @Override
- protected ObjectStreamClass readClassDescriptor() throws IOException, ClassNotFoundException {
- ObjectStreamClass streamClassDescriptor = super.readClassDescriptor();
-
- try {
- Class.forName(streamClassDescriptor.getName(), false, classLoader);
- } catch (ClassNotFoundException e) {
- if (streamClassDescriptor.getName().equals("org.apache.avro.generic.GenericData$Array")) {
- ObjectStreamClass result = ObjectStreamClass.lookup(
- KryoRegistrationSerializerConfigSnapshot.DummyRegisteredClass.class);
- return result;
- }
- }
-
- Class localClass = resolveClass(streamClassDescriptor);
- if (scalaSerializerClassnames.contains(localClass.getName()) || localClass.isAnonymousClass()
- // isAnonymousClass does not work for anonymous Scala classes; additionally check by classname
- || localClass.getName().contains("$anon$") || localClass.getName().contains("$anonfun")) {
-
- ObjectStreamClass localClassDescriptor = ObjectStreamClass.lookup(localClass);
- if (localClassDescriptor != null
- && localClassDescriptor.getSerialVersionUID() != streamClassDescriptor.getSerialVersionUID()) {
- LOG.warn("Ignoring serialVersionUID mismatch for anonymous class {}; was {}, now {}.",
- streamClassDescriptor.getName(), streamClassDescriptor.getSerialVersionUID(), localClassDescriptor.getSerialVersionUID());
-
- streamClassDescriptor = localClassDescriptor;
- }
- }
-
- return streamClassDescriptor;
- }
- }
-
- /**
* Writes a {@link TypeSerializer} to the provided data output view.
*
* <p>It is written with a format that can be later read again using
@@ -447,8 +368,8 @@ public class TypeSerializerSerializationUtil {
ClassLoader previousClassLoader = Thread.currentThread().getContextClassLoader();
try (
- FailureTolerantObjectInputStream ois =
- new FailureTolerantObjectInputStream(new ByteArrayInputStream(buffer), userClassLoader)) {
+ InstantiationUtil.FailureTolerantObjectInputStream ois =
+ new InstantiationUtil.FailureTolerantObjectInputStream(new ByteArrayInputStream(buffer), userClassLoader)) {
Thread.currentThread().setContextClassLoader(userClassLoader);
typeSerializer = (TypeSerializer<T>) ois.readObject();
http://git-wip-us.apache.org/repos/asf/flink/blob/9659ffd9/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
index 4bfec2e..11e3990 100644
--- a/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/util/InstantiationUtil.java
@@ -20,11 +20,16 @@ package org.apache.flink.util;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil;
+import org.apache.flink.api.java.typeutils.runtime.KryoRegistrationSerializerConfigSnapshot;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -37,6 +42,8 @@ import java.io.Serializable;
import java.lang.reflect.Constructor;
import java.lang.reflect.Modifier;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
/**
* Utility class to create instances from class objects and checking failure reasons.
@@ -44,6 +51,8 @@ import java.util.HashMap;
@Internal
public final class InstantiationUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(InstantiationUtil.class);
+
/**
* A custom ObjectInputStream that can load classes using a specific ClassLoader.
*/
@@ -95,6 +104,78 @@ public final class InstantiationUtil {
}
}
+ /**
+ * This is maintained as a temporary workaround for FLINK-6869.
+ *
+ * <p>Before 1.3, the Scala serializers did not specify the serialVersionUID.
+ * Although since 1.3 they are properly specified, we still have to ignore them for now
+ * as their previous serialVersionUIDs will vary depending on the Scala version.
+ *
+ * <p>This can be removed once 1.2 is no longer supported.
+ */
+ private static Set<String> scalaSerializerClassnames = new HashSet<>();
+ static {
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.TraversableSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.CaseClassSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EitherSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EnumValueSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.OptionSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.TrySerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.EitherSerializer");
+ scalaSerializerClassnames.add("org.apache.flink.api.scala.typeutils.UnitSerializer");
+ }
+
+ /**
+ * An {@link ObjectInputStream} that ignores serialVersionUID mismatches when deserializing objects of
+ * anonymous classes or our Scala serializer classes and also replaces occurences of GenericData.Array
+ * (from Avro) by a dummy class so that the KryoSerializer can still be deserialized without
+ * Avro being on the classpath.
+ *
+ * <p>The {@link TypeSerializerSerializationUtil.TypeSerializerSerializationProxy} uses this specific object input stream to read serializers,
+ * so that mismatching serialVersionUIDs of anonymous classes / Scala serializers are ignored.
+ * This is a required workaround to maintain backwards compatibility for our pre-1.3 Scala serializers.
+ * See FLINK-6869 for details.
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/FLINK-6869">FLINK-6869</a>
+ */
+ public static class FailureTolerantObjectInputStream extends InstantiationUtil.ClassLoaderObjectInputStream {
+
+ public FailureTolerantObjectInputStream(InputStream in, ClassLoader cl) throws IOException {
+ super(in, cl);
+ }
+
+ @Override
+ protected ObjectStreamClass readClassDescriptor() throws IOException, ClassNotFoundException {
+ ObjectStreamClass streamClassDescriptor = super.readClassDescriptor();
+
+ try {
+ Class.forName(streamClassDescriptor.getName(), false, classLoader);
+ } catch (ClassNotFoundException e) {
+ if (streamClassDescriptor.getName().equals("org.apache.avro.generic.GenericData$Array")) {
+ ObjectStreamClass result = ObjectStreamClass.lookup(
+ KryoRegistrationSerializerConfigSnapshot.DummyRegisteredClass.class);
+ return result;
+ }
+ }
+
+ Class localClass = resolveClass(streamClassDescriptor);
+ if (scalaSerializerClassnames.contains(localClass.getName()) || localClass.isAnonymousClass()
+ // isAnonymousClass does not work for anonymous Scala classes; additionally check by classname
+ || localClass.getName().contains("$anon$") || localClass.getName().contains("$anonfun")) {
+
+ ObjectStreamClass localClassDescriptor = ObjectStreamClass.lookup(localClass);
+ if (localClassDescriptor != null
+ && localClassDescriptor.getSerialVersionUID() != streamClassDescriptor.getSerialVersionUID()) {
+ LOG.warn("Ignoring serialVersionUID mismatch for anonymous class {}; was {}, now {}.",
+ streamClassDescriptor.getName(), streamClassDescriptor.getSerialVersionUID(), localClassDescriptor.getSerialVersionUID());
+
+ streamClassDescriptor = localClassDescriptor;
+ }
+ }
+
+ return streamClassDescriptor;
+ }
+ }
/**
* Creates a new instance of the given class.
@@ -284,23 +365,30 @@ public final class InstantiationUtil {
@SuppressWarnings("unchecked")
public static <T> T deserializeObject(byte[] bytes, ClassLoader cl) throws IOException, ClassNotFoundException {
- final ClassLoader old = Thread.currentThread().getContextClassLoader();
- try (ObjectInputStream oois = new ClassLoaderObjectInputStream(new ByteArrayInputStream(bytes), cl)) {
- Thread.currentThread().setContextClassLoader(cl);
- return (T) oois.readObject();
- }
- finally {
- Thread.currentThread().setContextClassLoader(old);
- }
+ return deserializeObject(bytes, cl, false);
}
@SuppressWarnings("unchecked")
public static <T> T deserializeObject(InputStream in, ClassLoader cl) throws IOException, ClassNotFoundException {
+ return deserializeObject(in, cl, false);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> T deserializeObject(byte[] bytes, ClassLoader cl, boolean isFailureTolerant)
+ throws IOException, ClassNotFoundException {
+
+ return deserializeObject(new ByteArrayInputStream(bytes), cl, isFailureTolerant);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> T deserializeObject(InputStream in, ClassLoader cl, boolean isFailureTolerant)
+ throws IOException, ClassNotFoundException {
+
final ClassLoader old = Thread.currentThread().getContextClassLoader();
- ObjectInputStream oois;
// not using resource try to avoid AutoClosable's close() on the given stream
- try {
- oois = new ClassLoaderObjectInputStream(in, cl);
+ try (ObjectInputStream oois = isFailureTolerant
+ ? new InstantiationUtil.FailureTolerantObjectInputStream(in, cl)
+ : new InstantiationUtil.ClassLoaderObjectInputStream(in, cl)) {
Thread.currentThread().setContextClassLoader(cl);
return (T) oois.readObject();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9659ffd9/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
index c333397..03fe612 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorBackendStateMetaInfoSnapshotReaderWriters.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -168,8 +169,8 @@ public class OperatorBackendStateMetaInfoSnapshotReaderWriters {
DataInputViewStream dis = new DataInputViewStream(in);
ClassLoader previousClassLoader = Thread.currentThread().getContextClassLoader();
try (
- TypeSerializerSerializationUtil.FailureTolerantObjectInputStream ois =
- new TypeSerializerSerializationUtil.FailureTolerantObjectInputStream(dis, userCodeClassLoader)) {
+ InstantiationUtil.FailureTolerantObjectInputStream ois =
+ new InstantiationUtil.FailureTolerantObjectInputStream(dis, userCodeClassLoader)) {
Thread.currentThread().setContextClassLoader(userCodeClassLoader);
TypeSerializer<S> stateSerializer = (TypeSerializer<S>) ois.readObject();
[24/31] flink git commit: [hotfix] [test] Remove stale savepoint
files no longer used by migration tests
Posted by tz...@apache.org.
[hotfix] [test] Remove stale savepoint files no longer used by migration tests
This includes:
- Removing MigrationVersion.v1_1, since compatilbity for 1.1 is no
longer supported (and no migration tests still test that)
- Remove all 1.1 test savepoint files (which no migration tests still
use)
- Remove all 1.2 test savepoint files that are no longer in use (e.g.,
CEPMigrationTest does not test 1.2 restores anymore)
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/72f0eb1e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/72f0eb1e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/72f0eb1e
Branch: refs/heads/master
Commit: 72f0eb1e722089499c3e02284ee9113520d33a90
Parents: 130ca4e
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 17:45:47 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../rolling-sink-migration-test-flink1.1-snapshot | Bin 1471 -> 0 bytes
.../kafka/FlinkKafkaConsumerBaseMigrationTest.java | 6 +-----
...er-migration-test-flink1.1-empty-state-snapshot | Bin 468 -> 0 bytes
...kafka-consumer-migration-test-flink1.1-snapshot | Bin 738 -> 0 bytes
.../ContinuousFileProcessingMigrationTest.java | 15 ++++-----------
...-migration-test-1482144479339-flink1.1-snapshot | Bin 468 -> 0 bytes
.../reader-migration-test-flink1.1-snapshot | Bin 979 -> 0 bytes
.../streaming/util/migration/MigrationVersion.java | 1 -
...-migration-test-accum-aligned-flink1.1-snapshot | Bin 621 -> 0 bytes
...p-migration-test-aggr-aligned-flink1.1-snapshot | Bin 586 -> 0 bytes
...gration-test-apply-event-time-flink1.1-snapshot | Bin 2427 -> 0 bytes
...on-test-apply-processing-time-flink1.1-snapshot | Bin 2327 -> 0 bytes
...ration-test-reduce-event-time-flink1.1-snapshot | Bin 2457 -> 0 bytes
...n-test-reduce-processing-time-flink1.1-snapshot | Bin 2406 -> 0 bytes
...session-with-stateful-trigger-flink1.1-snapshot | Bin 2988 -> 0 bytes
...on-with-stateful-trigger-mint-flink1.1-snapshot | Bin 538 -> 0 bytes
...udf-migration-itcase-flink1.1-rocksdb-savepoint | Bin 22283 -> 0 bytes
...tateful-udf-migration-itcase-flink1.1-savepoint | Bin 27146 -> 0 bytes
18 files changed, 5 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-connectors/flink-connector-filesystem/src/test/resources/rolling-sink-migration-test-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-filesystem/src/test/resources/rolling-sink-migration-test-flink1.1-snapshot b/flink-connectors/flink-connector-filesystem/src/test/resources/rolling-sink-migration-test-flink1.1-snapshot
deleted file mode 100644
index 2ebd70a..0000000
Binary files a/flink-connectors/flink-connector-filesystem/src/test/resources/rolling-sink-migration-test-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
index 1980ccc..94097e9 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBaseMigrationTest.java
@@ -350,11 +350,7 @@ public class FlinkKafkaConsumerBaseMigrationTest {
fail("Restore from savepoints from version before Flink 1.3.x should have failed if discovery is enabled.");
} catch (Exception e) {
- if (testMigrateVersion == MigrationVersion.v1_1) {
- Assert.assertTrue(e.getCause() instanceof IllegalArgumentException);
- } else {
- Assert.assertTrue(e instanceof IllegalArgumentException);
- }
+ Assert.assertTrue(e instanceof IllegalArgumentException);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-empty-state-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-empty-state-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-empty-state-snapshot
deleted file mode 100644
index f4dd96d..0000000
Binary files a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-empty-state-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-snapshot b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-snapshot
deleted file mode 100644
index 01c9e03..0000000
Binary files a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-consumer-migration-test-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
index 1c2ea6f..30a6119 100644
--- a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
+++ b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileProcessingMigrationTest.java
@@ -203,17 +203,10 @@ public class ContinuousFileProcessingMigrationTest {
// compare if the results contain what they should contain and also if
// they are the same, as they should.
- if (testMigrateVersion == MigrationVersion.v1_1) {
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(createSplitFromTimestampedSplit(split1))));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(createSplitFromTimestampedSplit(split2))));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(createSplitFromTimestampedSplit(split3))));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(createSplitFromTimestampedSplit(split4))));
- } else {
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split1)));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split2)));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split3)));
- Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split4)));
- }
+ Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split1)));
+ Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split2)));
+ Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split3)));
+ Assert.assertTrue(testHarness.getOutput().contains(new StreamRecord<>(split4)));
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1482144479339-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1482144479339-flink1.1-snapshot b/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1482144479339-flink1.1-snapshot
deleted file mode 100644
index 17eba99..0000000
Binary files a/flink-fs-tests/src/test/resources/monitoring-function-migration-test-1482144479339-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-fs-tests/src/test/resources/reader-migration-test-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-fs-tests/src/test/resources/reader-migration-test-flink1.1-snapshot b/flink-fs-tests/src/test/resources/reader-migration-test-flink1.1-snapshot
deleted file mode 100644
index e47ebbd..0000000
Binary files a/flink-fs-tests/src/test/resources/reader-migration-test-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
index 7b5a85e..eac669b 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/migration/MigrationVersion.java
@@ -26,7 +26,6 @@ public enum MigrationVersion {
// NOTE: the version strings must not change,
// as they are used to locate snapshot file paths
- v1_1("1.1"),
v1_2("1.2"),
v1_3("1.3"),
v1_4("1.4"),
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.1-snapshot
deleted file mode 100644
index 08ee86c..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-accum-aligned-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.1-snapshot
deleted file mode 100644
index c086a91..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-aggr-aligned-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.1-snapshot
deleted file mode 100644
index 02f0df2..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-event-time-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.1-snapshot
deleted file mode 100644
index 78963d6..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-apply-processing-time-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.1-snapshot
deleted file mode 100644
index ed428be..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-event-time-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.1-snapshot
deleted file mode 100644
index 9945be6..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-reduce-processing-time-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.1-snapshot
deleted file mode 100644
index 5f42359..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.1-snapshot
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.1-snapshot b/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.1-snapshot
deleted file mode 100644
index 72e8d90..0000000
Binary files a/flink-streaming-java/src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink1.1-snapshot and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-rocksdb-savepoint
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-rocksdb-savepoint b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-rocksdb-savepoint
deleted file mode 100644
index e63038b..0000000
Binary files a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-rocksdb-savepoint and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/72f0eb1e/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-savepoint
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-savepoint b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-savepoint
deleted file mode 100644
index f2f6dcd..0000000
Binary files a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.1-savepoint and /dev/null differ
[03/31] flink git commit: [FLINK-8409] [kafka] Fix offset committing
race condition in KafkaConsumerThread
Posted by tz...@apache.org.
[FLINK-8409] [kafka] Fix offset committing race condition in KafkaConsumerThread
This closes #5329.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/36552007
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/36552007
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/36552007
Branch: refs/heads/master
Commit: 3655200799929409352945a3f4fce0f3e987b9ad
Parents: 3067777
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Sat Jan 13 20:13:20 2018 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../kafka/internal/KafkaConsumerThread.java | 31 ++++++++++----------
1 file changed, 16 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/36552007/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
index 022cf99..8895b42 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/KafkaConsumerThread.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.connectors.kafka.internal;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.connectors.kafka.internals.ClosableBlockingQueue;
import org.apache.flink.streaming.connectors.kafka.internals.KafkaCommitCallback;
@@ -70,8 +71,8 @@ public class KafkaConsumerThread extends Thread {
/** The handover of data and exceptions between the consumer thread and the task thread. */
private final Handover handover;
- /** The next offsets that the main thread should commit. */
- private final AtomicReference<Map<TopicPartition, OffsetAndMetadata>> nextOffsetsToCommit;
+ /** The next offsets that the main thread should commit and the commit callback. */
+ private final AtomicReference<Tuple2<Map<TopicPartition, OffsetAndMetadata>, KafkaCommitCallback>> nextOffsetsToCommit;
/** The configuration for the Kafka consumer. */
private final Properties kafkaProperties;
@@ -112,9 +113,6 @@ public class KafkaConsumerThread extends Thread {
/** Flag tracking whether the latest commit request has completed. */
private volatile boolean commitInProgress;
- /** User callback to be invoked when commits completed. */
- private volatile KafkaCommitCallback offsetCommitCallback;
-
public KafkaConsumerThread(
Logger log,
Handover handover,
@@ -190,9 +188,6 @@ public class KafkaConsumerThread extends Thread {
return;
}
- // The callback invoked by Kafka once an offset commit is complete
- final OffsetCommitCallback offsetCommitCallback = new CommitCallback();
-
// the latest bulk of records. May carry across the loop if the thread is woken up
// from blocking on the handover
ConsumerRecords<byte[], byte[]> records = null;
@@ -208,15 +203,16 @@ public class KafkaConsumerThread extends Thread {
// check if there is something to commit
if (!commitInProgress) {
// get and reset the work-to-be committed, so we don't repeatedly commit the same
- final Map<TopicPartition, OffsetAndMetadata> toCommit = nextOffsetsToCommit.getAndSet(null);
+ final Tuple2<Map<TopicPartition, OffsetAndMetadata>, KafkaCommitCallback> commitOffsetsAndCallback =
+ nextOffsetsToCommit.getAndSet(null);
- if (toCommit != null) {
+ if (commitOffsetsAndCallback != null) {
log.debug("Sending async offset commit request to Kafka broker");
// also record that a commit is already in progress
// the order here matters! first set the flag, then send the commit command.
commitInProgress = true;
- consumer.commitAsync(toCommit, offsetCommitCallback);
+ consumer.commitAsync(commitOffsetsAndCallback.f0, new CommitCallback(commitOffsetsAndCallback.f1));
}
}
@@ -326,12 +322,11 @@ public class KafkaConsumerThread extends Thread {
@Nonnull KafkaCommitCallback commitCallback) {
// record the work to be committed by the main consumer thread and make sure the consumer notices that
- if (nextOffsetsToCommit.getAndSet(offsetsToCommit) != null) {
+ if (nextOffsetsToCommit.getAndSet(Tuple2.of(offsetsToCommit, commitCallback)) != null) {
log.warn("Committing offsets to Kafka takes longer than the checkpoint interval. " +
"Skipping commit of previous offsets because newer complete checkpoint offsets are available. " +
"This does not compromise Flink's checkpoint integrity.");
}
- this.offsetCommitCallback = commitCallback;
// if the consumer is blocked in a poll() or handover operation, wake it up to commit soon
handover.wakeupProducer();
@@ -489,15 +484,21 @@ public class KafkaConsumerThread extends Thread {
private class CommitCallback implements OffsetCommitCallback {
+ private final KafkaCommitCallback internalCommitCallback;
+
+ CommitCallback(KafkaCommitCallback internalCommitCallback) {
+ this.internalCommitCallback = checkNotNull(internalCommitCallback);
+ }
+
@Override
public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception ex) {
commitInProgress = false;
if (ex != null) {
log.warn("Committing offsets to Kafka failed. This does not compromise Flink's checkpoints.", ex);
- offsetCommitCallback.onException(ex);
+ internalCommitCallback.onException(ex);
} else {
- offsetCommitCallback.onSuccess();
+ internalCommitCallback.onSuccess();
}
}
}
[20/31] flink git commit: [FLINK-8472] [DataStream,
test] Extend StatefulJobSavepointMigrationITCase for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [DataStream, test] Extend StatefulJobSavepointMigrationITCase for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/74daaefb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/74daaefb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/74daaefb
Branch: refs/heads/master
Commit: 74daaefbb9b1cb74fd0228d3b81b024bb4e8b4ef
Parents: 8d3a74f
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 18:37:05 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:59 2018 +0100
----------------------------------------------------------------------
.../utils/StatefulJobSavepointMigrationITCase.java | 6 ++++--
.../_metadata | Bin 0 -> 32810 bytes
.../_metadata | Bin 0 -> 32738 bytes
3 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/74daaefb/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
index f83a3c6..93b39f5 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/utils/StatefulJobSavepointMigrationITCase.java
@@ -73,7 +73,9 @@ public class StatefulJobSavepointMigrationITCase extends SavepointMigrationTestB
Tuple2.of(MigrationVersion.v1_2, StateBackendLoader.MEMORY_STATE_BACKEND_NAME),
Tuple2.of(MigrationVersion.v1_2, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME),
Tuple2.of(MigrationVersion.v1_3, StateBackendLoader.MEMORY_STATE_BACKEND_NAME),
- Tuple2.of(MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME));
+ Tuple2.of(MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME),
+ Tuple2.of(MigrationVersion.v1_4, StateBackendLoader.MEMORY_STATE_BACKEND_NAME),
+ Tuple2.of(MigrationVersion.v1_4, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME));
}
/**
@@ -81,7 +83,7 @@ public class StatefulJobSavepointMigrationITCase extends SavepointMigrationTestB
* TODO change these values accordingly, e.g. to generate for 1.3 with RocksDB,
* TODO set as (MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME)
*/
- private final MigrationVersion flinkGenerateSavepointVersion = MigrationVersion.v1_3;
+ private final MigrationVersion flinkGenerateSavepointVersion = MigrationVersion.v1_4;
private final String flinkGenerateSavepointBackendType = StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME;
private final MigrationVersion testMigrateVersion;
http://git-wip-us.apache.org/repos/asf/flink/blob/74daaefb/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata
new file mode 100644
index 0000000..61a2549
Binary files /dev/null and b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata differ
http://git-wip-us.apache.org/repos/asf/flink/blob/74daaefb/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-savepoint/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-savepoint/_metadata b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-savepoint/_metadata
new file mode 100644
index 0000000..5e760c2
Binary files /dev/null and b/flink-tests/src/test/resources/stateful-udf-migration-itcase-flink1.4-savepoint/_metadata differ
[10/31] flink git commit: [FLINK-8472] [scala,
test] Extend StatefulJobSavepointMigrationITCase for Flink 1.4
Posted by tz...@apache.org.
[FLINK-8472] [scala, test] Extend StatefulJobSavepointMigrationITCase for Flink 1.4
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/45c4b664
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/45c4b664
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/45c4b664
Branch: refs/heads/master
Commit: 45c4b66483e7d949452d90682b5f65e1d15d9d49
Parents: c71fb42
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Thu Jan 25 16:20:31 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Tue Feb 6 19:58:58 2018 +0100
----------------------------------------------------------------------
.../_metadata | Bin 0 -> 204543 bytes
.../_metadata | Bin 0 -> 204543 bytes
.../StatefulJobSavepointMigrationITCase.scala | 6 ++++--
3 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/45c4b664/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-jobmanager-savepoint/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-jobmanager-savepoint/_metadata b/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-jobmanager-savepoint/_metadata
new file mode 100644
index 0000000..69aa8d4
Binary files /dev/null and b/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-jobmanager-savepoint/_metadata differ
http://git-wip-us.apache.org/repos/asf/flink/blob/45c4b664/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata b/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata
new file mode 100644
index 0000000..7dcef9c
Binary files /dev/null and b/flink-tests/src/test/resources/stateful-scala2.11-udf-migration-itcase-flink1.4-rocksdb-savepoint/_metadata differ
http://git-wip-us.apache.org/repos/asf/flink/blob/45c4b664/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala
index 435262c..cba31d3 100644
--- a/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/migration/StatefulJobSavepointMigrationITCase.scala
@@ -54,13 +54,15 @@ object StatefulJobSavepointMigrationITCase {
(MigrationVersion.v1_2, StateBackendLoader.MEMORY_STATE_BACKEND_NAME),
(MigrationVersion.v1_2, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME),
(MigrationVersion.v1_3, StateBackendLoader.MEMORY_STATE_BACKEND_NAME),
- (MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME))
+ (MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME),
+ (MigrationVersion.v1_4, StateBackendLoader.MEMORY_STATE_BACKEND_NAME),
+ (MigrationVersion.v1_4, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME))
}
// TODO to generate savepoints for a specific Flink version / backend type,
// TODO change these values accordingly, e.g. to generate for 1.3 with RocksDB,
// TODO set as (MigrationVersion.v1_3, StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME)
- val GENERATE_SAVEPOINT_VER: MigrationVersion = MigrationVersion.v1_3
+ val GENERATE_SAVEPOINT_VER: MigrationVersion = MigrationVersion.v1_4
val GENERATE_SAVEPOINT_BACKEND_TYPE: String = StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME
val SCALA_VERSION: String = {