You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2021/05/26 06:55:47 UTC
[flink] branch benchmark-request updated (fc47d3a -> d83d0e9)
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a change to branch benchmark-request
in repository https://gitbox.apache.org/repos/asf/flink.git.
omit fc47d3a fixup: decreased number of retain buffer
omit 5a64f54 [FLINK-22376][runtime] Replaced MemorySegment inside of BufferBuilder by Buffer
add 9693b11 [FLINK-22745][zk] Trim starting slashes when creating a namespaced CuratorFramework facade
add 5ec4755 [FLINK-22719][table-planner-blink] Fall back to regular join instead of thrown exception if a join does not satisfy conditions to translate into WindowJoin
add 92c2134 [hotfix][python][docs] Add documentation about how to set config options in Python DataStream API program
add 21c4468 [FLINK-22659][docs] Add execution.checkpointing.interval to docs of checkpoint configuration
add 214ba68 [FLINK-18934][core] Extract reusable CombinedWatermark class
add 18a2a8a [FLINK-18934][runtime] Idle stream does not advance watermark in connected stream
add ee9f9b2 [FLINK-18934][runtime] Drop StreamStatusMaintainer & StreamStatusProvider
add a47a3b7 [FLINK-22747] Upgrade to commons-io 2.8.0
add d67064d [FLINK-22639] Override URLClassLoader#getURLs
add 80ad5b3 [FLINK-22464][runtime][tests] Disable a test failing with AdaptiveScheduler tracked in FLINK-22464
add 75a2aaa [FLINK-22725][coordination] SlotManagers unregister metrics in suspend()
add 11f59bf [FLINK-22684][runtime] Added ability to ignore in-flight data during the recovery
add bbc94d7 fixup: Create the new operator states instead of changing old one
add 15f8c95 [FLINK-22319][table][sql-client] Support RESET table option for ALTER TABLE statement
new d83d0e9 Revert IDLE/ACTIVE on records
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (fc47d3a)
\
N -- N -- N refs/heads/benchmark-request (d83d0e9)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/content.zh/docs/deployment/config.md | 1 +
docs/content.zh/docs/dev/python/python_config.md | 31 +-
docs/content.zh/docs/dev/table/config.md | 8 +-
docs/content/docs/deployment/config.md | 1 +
docs/content/docs/dev/python/python_config.md | 31 +-
docs/content/docs/dev/table/config.md | 8 +-
.../execution_checkpointing_configuration.html | 6 +
.../src/main/resources/META-INF/NOTICE | 2 +-
.../common/eventtime/CombinedWatermarkStatus.java | 130 ++++
.../eventtime/IndexedCombinedWatermarkStatus.java | 87 +++
.../eventtime/WatermarkOutputMultiplexer.java | 100 +--
flink-dist/src/main/resources/META-INF/NOTICE | 2 +-
.../src/main/resources/META-INF/NOTICE | 2 +-
.../src/main/resources/META-INF/NOTICE | 2 +-
.../src/main/resources/META-INF/NOTICE | 2 +-
.../flink/state/api/output/BoundedStreamTask.java | 4 +
.../operators/StateBootstrapWrapperOperator.java | 9 +
.../jobmanager/JMXJobManagerMetricTest.java | 1 +
.../PythonTimestampsAndWatermarksOperator.java | 4 +-
.../runtime/checkpoint/CheckpointCoordinator.java | 42 +-
.../runtime/checkpoint/OperatorSubtaskState.java | 12 +
.../channel/RecoveredChannelStateHandler.java | 47 +-
.../channel/SequentialChannelStateReaderImpl.java | 3 +-
.../librarycache/FlinkUserCodeClassLoaders.java | 5 +
.../network/api/serialization/EventSerializer.java | 4 +-
.../runtime/io/network/buffer/BufferBuilder.java | 36 +-
.../runtime/io/network/buffer/BufferConsumer.java | 32 +-
.../runtime/io/network/buffer/BufferProvider.java | 5 -
.../runtime/io/network/buffer/LocalBufferPool.java | 16 +-
.../partition/BufferWritingResultPartition.java | 2 -
.../network/partition/PartitionSortedBuffer.java | 6 +-
.../partition/SortMergeResultPartition.java | 3 +-
.../tasks/CheckpointCoordinatorConfiguration.java | 34 +-
.../slotmanager/DeclarativeSlotManager.java | 3 +-
.../slotmanager/FineGrainedSlotManager.java | 3 +-
.../apache/flink/runtime/util/ClassLoaderUtil.java | 88 ++-
.../apache/flink/runtime/util/ZooKeeperUtils.java | 2 +-
.../runtime/zookeeper/ZooKeeperUtilityFactory.java | 5 +-
.../CheckpointCoordinatorMasterHooksTest.java | 1 +
.../CheckpointCoordinatorRestoringTest.java | 140 ++++-
.../CheckpointSettingsSerializableTest.java | 1 +
.../checkpoint/CheckpointStatsTrackerTest.java | 1 +
.../ExecutionGraphCheckpointCoordinatorTest.java | 1 +
.../FailoverStrategyCheckpointCoordinatorTest.java | 1 +
.../checkpoint/OperatorSubtaskStateTest.java | 54 ++
.../channel/ChannelStateChunkReaderTest.java | 16 -
.../channel/ChannelStateSerializerImplTest.java | 5 +-
.../channel/RecoveredChannelStateHandlerTest.java | 234 -------
.../FlinkUserCodeClassLoadersTest.java | 37 ++
.../executiongraph/ArchivedExecutionGraphTest.java | 1 +
.../DefaultExecutionGraphDeploymentTest.java | 1 +
.../SpanningRecordSerializationTest.java | 8 +-
.../buffer/BufferBuilderAndConsumerTest.java | 58 +-
.../io/network/buffer/BufferBuilderTestUtils.java | 29 +-
.../BufferConsumerWithPartialRecordLengthTest.java | 5 +-
.../io/network/buffer/LocalBufferPoolTest.java | 13 +-
.../runtime/io/network/buffer/NoOpBufferPool.java | 10 -
.../io/network/buffer/UnpooledBufferPool.java | 10 +-
.../BoundedBlockingSubpartitionWriteReadTest.java | 5 +-
.../partition/consumer/LocalInputChannelTest.java | 11 +-
.../runtime/io/network/util/TestBufferFactory.java | 11 +-
.../io/network/util/TestPooledBufferProvider.java | 62 +-
.../io/network/util/TestSubpartitionProducer.java | 5 +-
.../flink/runtime/jobgraph/JobGraphTest.java | 1 +
.../tasks/JobCheckpointingSettingsTest.java | 1 +
.../slotmanager/DeclarativeSlotManagerTest.java | 36 ++
.../slotmanager/FineGrainedSlotManagerTest.java | 46 ++
.../FineGrainedSlotManagerTestBase.java | 19 +-
.../runtime/scheduler/SchedulerTestingUtils.java | 1 +
.../zookeeper/ZooKeeperUtilityFactoryTest.java | 27 +-
.../api/environment/CheckpointConfig.java | 34 ++
.../environment/ExecutionCheckpointingOptions.java | 17 +
.../source/ContinuousFileReaderOperator.java | 1 -
.../api/graph/StreamingJobGraphGenerator.java | 2 +
.../streaming/api/operators/AbstractInput.java | 6 +
.../api/operators/AbstractStreamOperator.java | 54 +-
.../api/operators/AbstractStreamOperatorV2.java | 30 +-
.../streaming/api/operators/CountingOutput.java | 6 +
.../flink/streaming/api/operators/Input.java | 3 +
.../flink/streaming/api/operators/Output.java | 3 +
.../streaming/api/operators/StreamSource.java | 10 +-
.../api/operators/StreamSourceContexts.java | 53 +-
.../api/operators/TimestampedCollector.java | 6 +
.../api/operators/TwoInputStreamOperator.java | 5 +
.../streaming/runtime/io/AbstractDataOutput.java | 46 --
.../streaming/runtime/io/RecordWriterOutput.java | 45 +-
.../io/StreamMultipleInputProcessorFactory.java | 108 +---
.../runtime/io/StreamTwoInputProcessorFactory.java | 62 +-
.../operators/TimestampsAndWatermarksOperator.java | 12 +-
.../runtime/streamstatus/AnnouncedStatus.java | 70 +++
.../streamstatus/StreamStatusMaintainer.java | 34 --
.../runtime/streamstatus/StreamStatusProvider.java | 33 -
.../runtime/tasks/BroadcastingOutputCollector.java | 20 +-
.../streaming/runtime/tasks/ChainingOutput.java | 30 +-
.../tasks/CopyingBroadcastingOutputCollector.java | 6 +-
.../runtime/tasks/CopyingChainingOutput.java | 6 +-
.../runtime/tasks/MultipleInputStreamTask.java | 1 -
.../runtime/tasks/OneInputStreamTask.java | 15 +-
.../streaming/runtime/tasks/OperatorChain.java | 43 +-
.../runtime/tasks/SourceOperatorStreamTask.java | 17 +-
.../streaming/runtime/tasks/SourceStreamTask.java | 2 +-
.../runtime/tasks/StreamIterationTail.java | 4 +
.../flink/streaming/runtime/tasks/StreamTask.java | 5 -
.../runtime/tasks/TwoInputStreamTask.java | 1 -
.../api/operators/AbstractStreamOperatorTest.java | 102 ++++
.../operators/AbstractStreamOperatorV2Test.java | 128 ++++
.../AbstractUdfStreamOperatorLifecycleTest.java | 4 +-
.../api/operators/MockStreamStatusMaintainer.java | 39 --
.../StreamSourceContextIdleDetectionTests.java | 116 ++--
.../DemultiplexingRecordDeserializerTest.java | 20 +-
.../StreamSourceOperatorLatencyMetricsTest.java | 7 +-
.../StreamSourceOperatorWatermarksTest.java | 8 -
.../runtime/tasks/MultipleInputStreamTaskTest.java | 45 +-
.../runtime/tasks/OneInputStreamTaskTest.java | 209 -------
.../streaming/runtime/tasks/OperatorChainTest.java | 7 +-
.../streaming/runtime/tasks/StreamTaskTest.java | 2 -
.../tasks/SubtaskCheckpointCoordinatorTest.java | 4 +
.../runtime/tasks/TwoInputStreamTaskTest.java | 4 +-
.../util/AbstractStreamOperatorTestHarness.java | 10 +-
.../flink/streaming/util/CollectorOutput.java | 6 +
... KeyedMultiInputStreamOperatorTestHarness.java} | 37 +-
.../apache/flink/streaming/util/MockOutput.java | 6 +
.../flink/streaming/util/MockStreamTask.java | 9 -
.../streaming/util/MockStreamTaskBuilder.java | 10 -
.../util/MultiInputStreamOperatorTestHarness.java | 69 +++
.../util/TwoInputStreamOperatorTestHarness.java | 9 +
.../src/test/resources/sql/table.q | 123 +++-
.../src/main/codegen/data/Parser.tdd | 1 +
.../src/main/codegen/includes/parserImpls.ftl | 35 ++
...erTableOptions.java => SqlAlterTableReset.java} | 44 +-
.../flink/sql/parser/FlinkSqlParserImplTest.java | 10 +
.../operations/SqlToOperationConverter.java | 20 +
.../src/main/resources/META-INF/NOTICE | 2 +-
.../plan/metadata/FlinkRelMdWindowProperties.scala | 4 +-
.../physical/stream/StreamPhysicalJoinRule.scala | 4 +-
.../stream/StreamPhysicalTemporalJoinRule.scala | 4 +-
.../stream/StreamPhysicalWindowJoinRule.scala | 6 +-
.../table/planner/plan/utils/WindowJoinUtil.scala | 114 ++--
.../operations/SqlToOperationConverterTest.java | 157 ++---
.../plan/stream/sql/join/WindowJoinTest.xml | 676 +++++++++++++++++++++
.../flink/table/api/TableEnvironmentTest.scala | 131 +++-
.../table/planner/catalog/CatalogTableITCase.scala | 27 +-
.../plan/stream/sql/join/WindowJoinTest.scala | 74 ---
.../multipleinput/input/FirstInputOfTwoInput.java | 6 +
.../operators/multipleinput/input/OneInput.java | 6 +
.../multipleinput/input/SecondInputOfTwoInput.java | 6 +
.../multipleinput/output/BroadcastingOutput.java | 8 +
...gSecondInputOfTwoInputStreamOperatorOutput.java | 10 +
.../FirstInputOfTwoInputStreamOperatorOutput.java | 10 +
.../output/OneInputStreamOperatorOutput.java | 10 +
.../SecondInputOfTwoInputStreamOperatorOutput.java | 10 +
.../wmassigners/WatermarkAssignerOperator.java | 10 +-
.../multipleinput/output/BlackHoleOutput.java | 6 +
.../over/NonBufferOverWindowOperatorTest.java | 6 +
.../wmassigners/WatermarkAssignerOperatorTest.java | 32 +-
.../WatermarkAssignerOperatorTestBase.java | 8 +
.../JobMasterStopWithSavepointITCase.java | 1 +
.../jobmaster/JobMasterTriggerSavepointITCase.java | 1 +
.../OperatorEventSendingCheckpointITCase.java | 3 +
.../checkpointing/IgnoreInFlightDataITCase.java | 189 ++++++
.../runtime/SortingBoundedInputITCase.java | 4 +
pom.xml | 2 +-
162 files changed, 3190 insertions(+), 1841 deletions(-)
create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/eventtime/CombinedWatermarkStatus.java
create mode 100644 flink-core/src/main/java/org/apache/flink/api/common/eventtime/IndexedCombinedWatermarkStatus.java
delete mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandlerTest.java
copy flink-clients/src/test/java/org/apache/flink/client/cli/CliFrontendTestBase.java => flink-runtime/src/test/java/org/apache/flink/runtime/zookeeper/ZooKeeperUtilityFactoryTest.java (52%)
delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/AbstractDataOutput.java
create mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/AnnouncedStatus.java
delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StreamStatusMaintainer.java
delete mode 100644 flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StreamStatusProvider.java
delete mode 100644 flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/MockStreamStatusMaintainer.java
copy flink-streaming-java/src/test/java/org/apache/flink/streaming/util/{KeyedBroadcastOperatorTestHarness.java => KeyedMultiInputStreamOperatorTestHarness.java} (60%)
create mode 100644 flink-streaming-java/src/test/java/org/apache/flink/streaming/util/MultiInputStreamOperatorTestHarness.java
copy flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/{SqlAlterTableOptions.java => SqlAlterTableReset.java} (68%)
create mode 100644 flink-tests/src/test/java/org/apache/flink/test/checkpointing/IgnoreInFlightDataITCase.java
[flink] 01/01: Revert IDLE/ACTIVE on records
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch benchmark-request
in repository https://gitbox.apache.org/repos/asf/flink.git
commit d83d0e92701866ca181c7c8d88a2d438a61e3be0
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Wed May 26 08:51:55 2021 +0200
Revert IDLE/ACTIVE on records
---
.../java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java | 2 +-
.../flink/streaming/runtime/tasks/MultipleInputStreamTaskTest.java | 2 --
2 files changed, 1 insertion(+), 3 deletions(-)
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
index 8d1d38c..05351b8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/RecordWriterOutput.java
@@ -100,7 +100,7 @@ public class RecordWriterOutput<OUT> implements WatermarkGaugeExposingOutput<Str
private <X> void pushToRecordWriter(StreamRecord<X> record) {
// record could've been generated somewhere in the pipeline even though an IDLE status was
// emitted. It might've originated from a timer or just a wrong behaving operator
- try (AutoCloseable ignored = announcedStatus.ensureActive(this::writeStreamStatus)) {
+ try /*(AutoCloseable ignored = announcedStatus.ensureActive(this::writeStreamStatus))*/ {
serializationDelegate.setInstance(record);
recordWriter.emit(serializationDelegate);
} catch (Exception e) {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/MultipleInputStreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/MultipleInputStreamTaskTest.java
index 352557c..5ed7b4a 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/MultipleInputStreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/MultipleInputStreamTaskTest.java
@@ -618,10 +618,8 @@ public class MultipleInputStreamTaskTest {
// FLIP-27 sources do not emit active status on new records, we wrap a record with
// ACTIVE/IDLE sequence
- expectedOutput.add(StreamStatus.ACTIVE);
expectedOutput.add(
new StreamRecord<>("" + (initialTime + 10), TimestampAssigner.NO_TIMESTAMP));
- expectedOutput.add(StreamStatus.IDLE);
expectedOutput.add(StreamStatus.ACTIVE); // activate source on new watermark
expectedOutput.add(new Watermark(initialTime + 10)); // forward W from source
expectedOutput.add(StreamStatus.IDLE); // go idle after reading all records